Auto merge of #73293 - Aaron1011:feature/macro-rules-arg-capture, r=petrochenkov

Always capture tokens for `macro_rules!` arguments

When we invoke a proc-macro, the `TokenStream` we pass to it may contain 'interpolated' AST fragments, represented by `rustc_ast::token::Nonterminal`. In order to correctly, pass a `Nonterminal` to a proc-macro, we need to have 'captured' its `TokenStream` at the time the AST was parsed.

Currently, we perform this capturing when attributes are present on items and expressions, since we will end up using a `Nonterminal` to pass the item/expr to any proc-macro attributes it is annotated with. However, `Nonterminal`s are also introduced by the expansion of metavariables in `macro_rules!` macros. Since these metavariables may be passed to proc-macros, we need to have tokens available to avoid the need to pretty-print and reparse (see https://github.com/rust-lang/rust/issues/43081).

This PR unconditionally performs token capturing for AST items and expressions that are passed to a `macro_rules!` invocation. We cannot know in advance if captured item/expr will be passed to proc-macro, so this is needed to ensure that tokens will always be available when they are needed.

This ensures that proc-macros will receive tokens with proper `Spans` (both location and hygiene) in more cases. Like all work on https://github.com/rust-lang/rust/issues/43081, this will cause regressions in proc-macros that were relying on receiving tokens with dummy spans.

In this case, Crater revealed only one regression: the [Pear](https://github.com/SergioBenitez/Pear) crate (a helper for [rocket](https://github.com/SergioBenitez/Rocket)), which was previously [fixed](https://github.com/SergioBenitez/Pear/pull/25) as part of https://github.com/rust-lang/rust/pull/73084.

This regression manifests itself as the following error:

```
[INFO] [stdout] error: proc macro panicked
[INFO] [stdout]    --> /opt/rustwide/cargo-home/registry/src/github.com-1ecc6299db9ec823/rocket_http-0.4.5/src/parse/uri/parser.rs:119:34
[INFO] [stdout]     |
[INFO] [stdout] 119 |             let path_and_query = pear_try!(path_and_query(is_pchar));
[INFO] [stdout]     |                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[INFO] [stdout]     |
[INFO] [stdout]     = help: message: called `Option::unwrap()` on a `None` value
[INFO] [stdout]     = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
```

It can be fixed by running `cargo update -p pear`, which updates your `Cargo.lock` to use the latest version of Pear (which includes a bugfix for the regression).

Split out from https://github.com/rust-lang/rust/pull/73084/

7 files changed, 179 insertions, 0 deletions

diff --git a/src/test/ui/proc-macro/auxiliary/first-second.rs b/src/test/ui/proc-macro/auxiliary/first-second.rs
new file mode 100644
index 00000000000..6331608fbe5
--- /dev/null
+++ b/src/test/ui/proc-macro/auxiliary/first-second.rs
@@ -0,0 +1,20 @@
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::{TokenStream, TokenTree, Group, Delimiter};
+
+#[proc_macro_attribute]
+pub fn first(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    let tokens: TokenStream = "#[derive(Second)]".parse().unwrap();
+    let wrapped = TokenTree::Group(Group::new(Delimiter::None, item.into_iter().collect()));
+    tokens.into_iter().chain(std::iter::once(wrapped)).collect()
+}
+
+#[proc_macro_derive(Second)]
+pub fn second(item: TokenStream) -> TokenStream {
+    TokenStream::new()
+}
diff --git a/src/test/ui/proc-macro/auxiliary/recollect.rs b/src/test/ui/proc-macro/auxiliary/recollect.rs
new file mode 100644
index 00000000000..d4494a5aff2
--- /dev/null
+++ b/src/test/ui/proc-macro/auxiliary/recollect.rs
@@ -0,0 +1,12 @@
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+use proc_macro::TokenStream;
+
+#[proc_macro]
+pub fn recollect(tokens: TokenStream) -> TokenStream {
+    tokens.into_iter().collect()
+}
diff --git a/src/test/ui/proc-macro/auxiliary/weird-hygiene.rs b/src/test/ui/proc-macro/auxiliary/weird-hygiene.rs
new file mode 100644
index 00000000000..338e436df50
--- /dev/null
+++ b/src/test/ui/proc-macro/auxiliary/weird-hygiene.rs
@@ -0,0 +1,48 @@
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+
+use proc_macro::{TokenStream, TokenTree, Group};
+
+fn find_my_ident(tokens: TokenStream) -> Option<TokenStream> {
+    for token in tokens {
+        if let TokenTree::Ident(ident) = &token {
+            if ident.to_string() == "hidden_ident" {
+                return Some(vec![token].into_iter().collect())
+            }
+        } else if let TokenTree::Group(g) = token {
+            if let Some(stream) = find_my_ident(g.stream()) {
+                return Some(stream)
+            }
+        }
+    }
+    return None;
+}
+
+
+#[proc_macro_derive(WeirdDerive)]
+pub fn weird_derive(item: TokenStream) -> TokenStream {
+    let my_ident = find_my_ident(item).expect("Missing 'my_ident'!");
+    let tokens: TokenStream = "call_it!();".parse().unwrap();
+    let final_call = tokens.into_iter().map(|tree| {
+        if let TokenTree::Group(g) = tree {
+            return Group::new(g.delimiter(), my_ident.clone()).into()
+        } else {
+            return tree
+        }
+    }).collect();
+    final_call
+}
+
+#[proc_macro]
+pub fn recollect(item: TokenStream) -> TokenStream {
+    item.into_iter().collect()
+}
+
+#[proc_macro_attribute]
+pub fn recollect_attr(_attr: TokenStream, mut item: TokenStream) -> TokenStream {
+    item.into_iter().collect()
+}
diff --git a/src/test/ui/proc-macro/capture-macro-rules-invoke.rs b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
new file mode 100644
index 00000000000..a404ddace9b
--- /dev/null
+++ b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
@@ -0,0 +1,22 @@
+// aux-build:test-macros.rs
+// check-pass
+
+extern crate test_macros;
+use test_macros::recollect;
+
+macro_rules! use_expr {
+    ($expr:expr) => {
+        recollect!($expr)
+    }
+}
+
+#[allow(dead_code)]
+struct Foo;
+impl Foo {
+    #[allow(dead_code)]
+    fn use_self(self) {
+        drop(use_expr!(self));
+    }
+}
+
+fn main() {}
diff --git a/src/test/ui/proc-macro/macro-rules-derive.rs b/src/test/ui/proc-macro/macro-rules-derive.rs
new file mode 100644
index 00000000000..5b4d577a1ac
--- /dev/null
+++ b/src/test/ui/proc-macro/macro-rules-derive.rs
@@ -0,0 +1,20 @@
+// aux-build:first-second.rs
+// FIXME: The spans here are bad, see PR #73084
+
+extern crate first_second;
+use first_second::*;
+
+macro_rules! produce_it {
+    ($name:ident) => {
+        #[first] //~ ERROR cannot find type
+        struct $name {
+            field: MissingType
+        }
+    }
+}
+
+produce_it!(MyName);
+
+fn main() {
+    println!("Hello, world!");
+}
diff --git a/src/test/ui/proc-macro/macro-rules-derive.stderr b/src/test/ui/proc-macro/macro-rules-derive.stderr
new file mode 100644
index 00000000000..4b72d29fe8a
--- /dev/null
+++ b/src/test/ui/proc-macro/macro-rules-derive.stderr
@@ -0,0 +1,9 @@
+error[E0412]: cannot find type `MissingType` in this scope
+  --> $DIR/macro-rules-derive.rs:9:9
+   |
+LL |         #[first]
+   |         ^^^^^^^^ not found in this scope
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0412`.
diff --git a/src/test/ui/proc-macro/weird-hygiene.rs b/src/test/ui/proc-macro/weird-hygiene.rs
new file mode 100644
index 00000000000..3f48191b5b2
--- /dev/null
+++ b/src/test/ui/proc-macro/weird-hygiene.rs
@@ -0,0 +1,48 @@
+// aux-build:weird-hygiene.rs
+// check-pass
+// FIXME: This should actually error, see PR #73084
+
+#![feature(stmt_expr_attributes)]
+#![feature(proc_macro_hygiene)]
+
+extern crate weird_hygiene;
+use weird_hygiene::*;
+
+macro_rules! other {
+    ($tokens:expr) => {
+        macro_rules! call_it {
+            ($outer_ident:ident) => {
+                macro_rules! inner {
+                    () => {
+                        $outer_ident;
+                    }
+                }
+            }
+        }
+
+        #[derive(WeirdDerive)]
+        enum MyEnum {
+            Value = (stringify!($tokens + hidden_ident), 1).1
+        }
+
+        inner!();
+    }
+}
+
+macro_rules! invoke_it {
+    ($token:expr) => {
+        #[recollect_attr] {
+            $token;
+            hidden_ident
+        }
+    }
+}
+
+fn main() {
+    // `other` and `invoke_it` are both macro_rules! macros,
+    // so it should be impossible for them to ever see `hidden_ident`,
+    // even if they invoke a proc macro.
+    let hidden_ident = "Hello1";
+    other!(50);
+    invoke_it!(25);
+}