Auto merge of #39419 - jseyfried:simplify_tokentree, r=nrc

Simplify `TokenTree` and fix `macro_rules!` bugs

This PR
 - fixes #39390, fixes #39403, and fixes #39404 (each is a [breaking-change], see issues for examples),
 - fixes #39889,
 - simplifies and optimizes macro invocation parsing,
 - cleans up `ext::tt::transcribe`,
 - removes `tokenstream::TokenTree::Sequence` and `Token::MatchNt`,
   - instead, adds a new type `ext::tt::quoted::TokenTree` for use by `macro_rules!` (`ext::tt`)
 - removes `parser.quote_depth` and `parser.parsing_token_tree`, and
 - removes `quote_matcher!`.
   - Instead, use `quote_tokens!` and `ext::tt::quoted::parse` the result with `expect_matchers=true`.
   - I found no outside uses of `quote_matcher!` when searching Rust code on Github.

r? @nrc

13 files changed, 79 insertions, 28 deletions

diff --git a/src/test/compile-fail-fulldeps/gated-quote.rs b/src/test/compile-fail-fulldeps/gated-quote.rs
index 726af9864b4..63e1c6f16b3 100644
--- a/src/test/compile-fail-fulldeps/gated-quote.rs
+++ b/src/test/compile-fail-fulldeps/gated-quote.rs
@@ -54,8 +54,6 @@ pub fn main() {
     //~^ ERROR cannot find macro `quote_arm!` in this scope
     let x = quote_stmt!(ecx, 3);
     //~^ ERROR cannot find macro `quote_stmt!` in this scope
-    let x = quote_matcher!(ecx, 3);
-    //~^ ERROR cannot find macro `quote_matcher!` in this scope
     let x = quote_attr!(ecx, 3);
     //~^ ERROR cannot find macro `quote_attr!` in this scope
     let x = quote_arg!(ecx, 3);
diff --git a/src/test/compile-fail/issue-35450.rs b/src/test/compile-fail/issue-35450.rs
index d890d02a910..5f54f269c6c 100644
--- a/src/test/compile-fail/issue-35450.rs
+++ b/src/test/compile-fail/issue-35450.rs
@@ -8,9 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-macro_rules! m { ($t:tt) => { $t } }
+macro_rules! m { ($($t:tt)*) => { $($t)* } }
 
 fn main() {
-    m!($t); //~ ERROR unknown macro variable
-            //~| ERROR expected expression
+    m!($t); //~ ERROR expected expression
 }
diff --git a/src/test/compile-fail/issue-39404.rs b/src/test/compile-fail/issue-39404.rs
new file mode 100644
index 00000000000..0168ae7d910
--- /dev/null
+++ b/src/test/compile-fail/issue-39404.rs
@@ -0,0 +1,18 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![deny(missing_fragment_specifier)] //~ NOTE lint level defined here
+
+macro_rules! m { ($i) => {} }
+//~^ ERROR missing fragment specifier
+//~| WARN previously accepted
+//~| NOTE issue #40107
+
+fn main() {}
diff --git a/src/test/compile-fail/macro-error.rs b/src/test/compile-fail/macro-error.rs
index 78f95e365c4..82a5aa48729 100644
--- a/src/test/compile-fail/macro-error.rs
+++ b/src/test/compile-fail/macro-error.rs
@@ -9,7 +9,7 @@
 // except according to those terms.
 
 macro_rules! foo {
-    ($a:expr) => $a; //~ ERROR macro rhs must be delimited
+    ($a:expr) => a; //~ ERROR macro rhs must be delimited
 }
 
 fn main() {
diff --git a/src/test/compile-fail/macro-tt-matchers.rs b/src/test/compile-fail/macro-tt-matchers.rs
index 969f1500717..7255e7d00b6 100644
--- a/src/test/compile-fail/macro-tt-matchers.rs
+++ b/src/test/compile-fail/macro-tt-matchers.rs
@@ -17,16 +17,5 @@ macro_rules! foo {
 
 foo!(Box);
 
-macro_rules! bar {
-    ($x:tt) => {
-        macro_rules! baz {
-            ($x:tt, $y:tt) => { ($x, $y) }
-        }
-    }
-}
-
 #[rustc_error]
-fn main() { //~ ERROR compilation successful
-    bar!($y);
-    let _: (i8, i16) = baz!(0i8, 0i16);
-}
+fn main() {} //~ ERROR compilation successful
diff --git a/src/test/compile-fail/malformed_macro_lhs.rs b/src/test/compile-fail/malformed_macro_lhs.rs
index 5d81e21f056..0b437be5393 100644
--- a/src/test/compile-fail/malformed_macro_lhs.rs
+++ b/src/test/compile-fail/malformed_macro_lhs.rs
@@ -9,7 +9,7 @@
 // except according to those terms.
 
 macro_rules! my_precioooous {
-    $($t:tt)* => (1); //~ ERROR invalid macro matcher
+    t => (1); //~ ERROR invalid macro matcher
 }
 
 fn main() {
diff --git a/src/test/parse-fail/issue-33569.rs b/src/test/parse-fail/issue-33569.rs
index e3c17af82aa..15d491719a6 100644
--- a/src/test/parse-fail/issue-33569.rs
+++ b/src/test/parse-fail/issue-33569.rs
@@ -12,7 +12,9 @@
 
 macro_rules! foo {
     { $+ } => { //~ ERROR expected identifier, found `+`
+                //~^ ERROR missing fragment specifier
         $(x)(y) //~ ERROR expected `*` or `+`
-                //~^ ERROR no rules expected the token `)`
     }
 }
+
+foo!();
diff --git a/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs b/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs
index 2b3857048f3..3db69f2167c 100644
--- a/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs
+++ b/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs
@@ -23,6 +23,7 @@ use syntax::ast::{Ident, Pat};
 use syntax::tokenstream::{TokenTree};
 use syntax::ext::base::{ExtCtxt, MacResult, MacEager};
 use syntax::ext::build::AstBuilder;
+use syntax::ext::tt::quoted;
 use syntax::ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal};
 use syntax::ext::tt::macro_parser::{Success, Failure, Error};
 use syntax::ext::tt::macro_parser::parse_failure_msg;
@@ -33,7 +34,8 @@ use rustc_plugin::Registry;
 fn expand_mbe_matches(cx: &mut ExtCtxt, _: Span, args: &[TokenTree])
         -> Box<MacResult + 'static> {
 
-    let mbe_matcher = quote_matcher!(cx, $matched:expr, $($pat:pat)|+);
+    let mbe_matcher = quote_tokens!(cx, $$matched:expr, $$($$pat:pat)|+);
+    let mbe_matcher = quoted::parse(&mbe_matcher, true, cx.parse_sess);
     let map = match TokenTree::parse(cx, &mbe_matcher, args) {
         Success(map) => map,
         Failure(_, tok) => {
diff --git a/src/test/run-pass-fulldeps/mbe_matching_test_macro.rs b/src/test/run-pass-fulldeps/mbe_matching_test_macro.rs
index 5383b11cf53..822b2c9b93b 100644
--- a/src/test/run-pass-fulldeps/mbe_matching_test_macro.rs
+++ b/src/test/run-pass-fulldeps/mbe_matching_test_macro.rs
@@ -14,11 +14,7 @@
 #![feature(plugin)]
 #![plugin(procedural_mbe_matching)]
 
-#[no_link]
-extern crate procedural_mbe_matching;
-
 pub fn main() {
-    let abc = 123u32;
     assert_eq!(matches!(Some(123), None | Some(0)), false);
     assert_eq!(matches!(Some(123), None | Some(123)), true);
     assert_eq!(matches!(true, true), true);
diff --git a/src/test/run-pass-fulldeps/proc-macro/auxiliary/issue-39889.rs b/src/test/run-pass-fulldeps/proc-macro/auxiliary/issue-39889.rs
new file mode 100644
index 00000000000..9094310fb3e
--- /dev/null
+++ b/src/test/run-pass-fulldeps/proc-macro/auxiliary/issue-39889.rs
@@ -0,0 +1,27 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+use proc_macro::TokenStream;
+
+#[proc_macro_derive(Issue39889)]
+pub fn f(_input: TokenStream) -> TokenStream {
+    let rules = r#"
+        macro_rules! id {
+            ($($tt:tt)*) => { $($tt)* };
+        }
+    "#;
+    rules.parse().unwrap()
+}
diff --git a/src/test/run-pass-fulldeps/proc-macro/issue-39889.rs b/src/test/run-pass-fulldeps/proc-macro/issue-39889.rs
new file mode 100644
index 00000000000..05610116ad6
--- /dev/null
+++ b/src/test/run-pass-fulldeps/proc-macro/issue-39889.rs
@@ -0,0 +1,22 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// aux-build:issue-39889.rs
+
+#![feature(proc_macro)]
+#![allow(unused)]
+
+extern crate issue_39889;
+use issue_39889::Issue39889;
+
+#[derive(Issue39889)]
+struct S;
+
+fn main() {}
diff --git a/src/test/run-pass-fulldeps/quote-tokens.rs b/src/test/run-pass-fulldeps/quote-tokens.rs
index 9e9b7ce5bf2..8e6a69cb584 100644
--- a/src/test/run-pass-fulldeps/quote-tokens.rs
+++ b/src/test/run-pass-fulldeps/quote-tokens.rs
@@ -37,7 +37,6 @@ fn syntax_extension(cx: &ExtCtxt) {
 
     let _l: P<syntax::ast::Ty> = quote_ty!(cx, &isize);
 
-    let _m: Vec<syntax::tokenstream::TokenTree> = quote_matcher!(cx, $($foo:tt,)* bar);
     let _n: syntax::ast::Attribute = quote_attr!(cx, #![cfg(foo, bar = "baz")]);
 
     let _o: Option<P<syntax::ast::Item>> = quote_item!(cx, fn foo<T: ?Sized>() {});
diff --git a/src/test/compile-fail/issue-39709.rs b/src/test/run-pass/issue-39709.rs
index 0f66fe84393..ebca9312a64 100644
--- a/src/test/compile-fail/issue-39709.rs
+++ b/src/test/run-pass/issue-39709.rs
@@ -9,7 +9,6 @@
 // except according to those terms.
 
 fn main() {
-    println!("{}", { macro_rules! x { ($()*) => {} } 33 });
-    //~^ ERROR no syntax variables matched as repeating at this depth
+    println!("{}", { macro_rules! x { ($(t:tt)*) => {} } 33 });
 }