Auto merge of #39419 - jseyfried:simplify_tokentree, r=nrc

Simplify `TokenTree` and fix `macro_rules!` bugs

This PR
 - fixes #39390, fixes #39403, and fixes #39404 (each is a [breaking-change], see issues for examples),
 - fixes #39889,
 - simplifies and optimizes macro invocation parsing,
 - cleans up `ext::tt::transcribe`,
 - removes `tokenstream::TokenTree::Sequence` and `Token::MatchNt`,
   - instead, adds a new type `ext::tt::quoted::TokenTree` for use by `macro_rules!` (`ext::tt`)
 - removes `parser.quote_depth` and `parser.parsing_token_tree`, and
 - removes `quote_matcher!`.
   - Instead, use `quote_tokens!` and `ext::tt::quoted::parse` the result with `expect_matchers=true`.
   - I found no outside uses of `quote_matcher!` when searching Rust code on Github.

r? @nrc

1 files changed, 5 insertions, 7 deletions

diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 20e80afc115..6fec49b229a 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -46,6 +46,7 @@ pub struct ParseSess {
     pub span_diagnostic: Handler,
     pub unstable_features: UnstableFeatures,
     pub config: CrateConfig,
+    pub missing_fragment_specifiers: RefCell<HashSet<Span>>,
     /// Used to determine and report recursive mod inclusions
     included_mod_stack: RefCell<Vec<PathBuf>>,
     code_map: Rc<CodeMap>,
@@ -66,6 +67,7 @@ impl ParseSess {
             span_diagnostic: handler,
             unstable_features: UnstableFeatures::from_environment(),
             config: HashSet::new(),
+            missing_fragment_specifiers: RefCell::new(HashSet::new()),
             included_mod_stack: RefCell::new(vec![]),
             code_map: code_map
         }
@@ -139,13 +141,9 @@ pub fn parse_stmt_from_source_str<'a>(name: String, source: String, sess: &'a Pa
     new_parser_from_source_str(sess, name, source).parse_stmt()
 }
 
-// Warning: This parses with quote_depth > 0, which is not the default.
 pub fn parse_tts_from_source_str<'a>(name: String, source: String, sess: &'a ParseSess)
-                                     -> PResult<'a, Vec<tokenstream::TokenTree>> {
-    let mut p = new_parser_from_source_str(sess, name, source);
-    p.quote_depth += 1;
-    // right now this is re-creating the token trees from ... token trees.
-    p.parse_all_token_trees()
+                                     -> Vec<tokenstream::TokenTree> {
+    filemap_to_tts(sess, sess.codemap().new_filemap(name, None, source))
 }
 
 // Create a new parser from a source string
@@ -986,7 +984,7 @@ mod tests {
             _ => panic!("not a macro"),
         };
 
-        let span = tts.iter().rev().next().unwrap().get_span();
+        let span = tts.iter().rev().next().unwrap().span();
 
         match sess.codemap().span_to_snippet(span) {
             Ok(s) => assert_eq!(&s[..], "{ body }"),