Auto merge of #73084 - Aaron1011:feature/new-recursive-expand, r=petrochenkov

Re-land PR #72388:  Recursively expand `TokenKind::Interpolated` in `probably_equal_for_proc_macro`

PR #72388 allowed us to preserve the original `TokenStream` in more cases during proc-macro expansion, but had to be reverted due to a large number of regressions (See #72545 and #72622). These regressions fell into two categories

1. Missing handling for `Group`s with `Delimiter::None`, which are inserted during `macro_rules!` expansion (but are lost during stringification and re-parsing). A large number of these regressions were due to `syn` and `proc-macro-hack`, but several crates needed changes to their own proc-macro code.
2. Legitimate hygiene issues that were previously being masked by stringification. Some of these were relatively benign (e.g. [a compiliation error](https://github.com/paritytech/parity-scale-codec/pull/210) caused by misusing `quote_spanned!`). However, two crates had intentionally written unhygenic `macro_rules!` macros, which were able to access identifiers that were not passed as arguments (see https://github.com/rust-lang/rust/issues/72622#issuecomment-636402573).

All but one of the Crater regressions have now been fixed upstream (see https://hackmd.io/ItrXWRaSSquVwoJATPx3PQ?both). The remaining crate (which has a PR pending at https://github.com/sammhicks/face-generator/pull/1) is not on `crates.io`, and is a Yew application that seems unlikely to have any reverse dependencies.

As @petrochenkov mentioned in https://github.com/rust-lang/rust/issues/72545#issuecomment-638632434, not re-landing PR #72388 allows more crates to write unhygenic `macro_rules!` macros, which will eventually stop compiling. Since there is only one Crater regression remaining, since additional crates could write unhygenic `macro_rules!` macros in the time it takes that PR to be merged.

11 files changed, 172 insertions, 62 deletions

diff --git a/src/test/ui/proc-macro/group-compat-hack/auxiliary/group-compat-hack.rs b/src/test/ui/proc-macro/group-compat-hack/auxiliary/group-compat-hack.rs
new file mode 100644
index 00000000000..5cd3b40a2e4
--- /dev/null
+++ b/src/test/ui/proc-macro/group-compat-hack/auxiliary/group-compat-hack.rs
@@ -0,0 +1,13 @@
+// force-host
+// no-prefer-dynamic
+
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+use proc_macro::TokenStream;
+
+#[proc_macro_attribute]
+pub fn my_macro(_attr: TokenStream, input: TokenStream) -> TokenStream {
+    println!("Called proc_macro_hack with {:?}", input);
+    input
+}
diff --git a/src/test/ui/proc-macro/group-compat-hack/group-compat-hack.rs b/src/test/ui/proc-macro/group-compat-hack/group-compat-hack.rs
new file mode 100644
index 00000000000..35c101587de
--- /dev/null
+++ b/src/test/ui/proc-macro/group-compat-hack/group-compat-hack.rs
@@ -0,0 +1,30 @@
+// check-pass
+// aux-build:group-compat-hack.rs
+// compile-flags: -Z span-debug
+
+#![no_std] // Don't load unnecessary hygiene information from std
+extern crate std;
+
+#[macro_use] extern crate group_compat_hack;
+
+// Tests the backwards compatibility hack added for certain macros
+// When an attribute macro named `proc_macro_hack` or `wasm_bindgen`
+// has an `NtIdent` named `$name`, we pass a plain `Ident` token in
+// place of a `None`-delimited group. This allows us to maintain
+// backwards compatibility for older versions of these crates.
+
+include!("js-sys/src/lib.rs");
+include!("time-macros-impl/src/lib.rs");
+
+macro_rules! other {
+    ($name:ident) => {
+        #[my_macro] struct Three($name);
+    }
+}
+
+fn main() {
+    struct Foo;
+    impl_macros!(Foo);
+    arrays!(Foo);
+    other!(Foo);
+}
diff --git a/src/test/ui/proc-macro/group-compat-hack/group-compat-hack.stdout b/src/test/ui/proc-macro/group-compat-hack/group-compat-hack.stdout
new file mode 100644
index 00000000000..d519daab1f2
--- /dev/null
+++ b/src/test/ui/proc-macro/group-compat-hack/group-compat-hack.stdout
@@ -0,0 +1,3 @@
+Called proc_macro_hack with TokenStream [Ident { ident: "struct", span: $DIR/time-macros-impl/src/lib.rs:5:21: 5:27 (#5) }, Ident { ident: "One", span: $DIR/time-macros-impl/src/lib.rs:5:28: 5:31 (#5) }, Group { delimiter: Parenthesis, stream: TokenStream [Ident { ident: "Foo", span: $DIR/group-compat-hack.rs:27:18: 27:21 (#0) }], span: $DIR/time-macros-impl/src/lib.rs:5:31: 5:38 (#5) }, Punct { ch: ';', spacing: Alone, span: $DIR/time-macros-impl/src/lib.rs:5:38: 5:39 (#5) }]
+Called proc_macro_hack with TokenStream [Ident { ident: "struct", span: $DIR/js-sys/src/lib.rs:5:21: 5:27 (#9) }, Ident { ident: "Two", span: $DIR/js-sys/src/lib.rs:5:28: 5:31 (#9) }, Group { delimiter: Parenthesis, stream: TokenStream [Ident { ident: "Foo", span: $DIR/group-compat-hack.rs:28:13: 28:16 (#0) }], span: $DIR/js-sys/src/lib.rs:5:31: 5:38 (#9) }, Punct { ch: ';', spacing: Alone, span: $DIR/js-sys/src/lib.rs:5:38: 5:39 (#9) }]
+Called proc_macro_hack with TokenStream [Ident { ident: "struct", span: $DIR/group-compat-hack.rs:21:21: 21:27 (#13) }, Ident { ident: "Three", span: $DIR/group-compat-hack.rs:21:28: 21:33 (#13) }, Group { delimiter: Parenthesis, stream: TokenStream [Group { delimiter: None, stream: TokenStream [Ident { ident: "Foo", span: $DIR/group-compat-hack.rs:29:12: 29:15 (#0) }], span: $DIR/group-compat-hack.rs:21:34: 21:39 (#13) }], span: $DIR/group-compat-hack.rs:21:33: 21:40 (#13) }, Punct { ch: ';', spacing: Alone, span: $DIR/group-compat-hack.rs:21:40: 21:41 (#13) }]
diff --git a/src/test/ui/proc-macro/group-compat-hack/js-sys/src/lib.rs b/src/test/ui/proc-macro/group-compat-hack/js-sys/src/lib.rs
new file mode 100644
index 00000000000..d1a66940ebf
--- /dev/null
+++ b/src/test/ui/proc-macro/group-compat-hack/js-sys/src/lib.rs
@@ -0,0 +1,7 @@
+// ignore-test this is not a test
+
+macro_rules! arrays {
+    ($name:ident) => {
+        #[my_macro] struct Two($name);
+    }
+}
diff --git a/src/test/ui/proc-macro/group-compat-hack/time-macros-impl/src/lib.rs b/src/test/ui/proc-macro/group-compat-hack/time-macros-impl/src/lib.rs
new file mode 100644
index 00000000000..c94c3579209
--- /dev/null
+++ b/src/test/ui/proc-macro/group-compat-hack/time-macros-impl/src/lib.rs
@@ -0,0 +1,7 @@
+// ignore-test this is not a test
+
+macro_rules! impl_macros {
+    ($name:ident) => {
+        #[my_macro] struct One($name);
+    }
+}
diff --git a/src/test/ui/proc-macro/input-interpolated.stdout b/src/test/ui/proc-macro/input-interpolated.stdout
index 9cf33ba4a9d..a9636cfef82 100644
--- a/src/test/ui/proc-macro/input-interpolated.stdout
+++ b/src/test/ui/proc-macro/input-interpolated.stdout
@@ -15,51 +15,63 @@ PRINT-ATTR INPUT (DISPLAY): const A : u8 = 0 ;
 PRINT-ATTR INPUT (DEBUG): TokenStream [
     Ident {
         ident: "const",
-        span: #0 bytes(0..0),
+        span: #3 bytes(416..421),
     },
-    Ident {
-        ident: "A",
-        span: #0 bytes(0..0),
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "A",
+                span: #0 bytes(503..504),
+            },
+        ],
+        span: #3 bytes(422..424),
     },
     Punct {
         ch: ':',
         spacing: Alone,
-        span: #0 bytes(0..0),
+        span: #3 bytes(424..425),
     },
     Ident {
         ident: "u8",
-        span: #0 bytes(0..0),
+        span: #3 bytes(426..428),
     },
     Punct {
         ch: '=',
         spacing: Alone,
-        span: #0 bytes(0..0),
+        span: #3 bytes(429..430),
     },
     Literal {
         kind: Integer,
         symbol: "0",
         suffix: None,
-        span: #0 bytes(0..0),
+        span: #3 bytes(431..432),
     },
     Punct {
         ch: ';',
         spacing: Alone,
-        span: #0 bytes(0..0),
+        span: #3 bytes(432..433),
     },
 ]
 PRINT-DERIVE INPUT (DISPLAY): struct A { }
 PRINT-DERIVE INPUT (DEBUG): TokenStream [
     Ident {
         ident: "struct",
-        span: #0 bytes(0..0),
+        span: #3 bytes(468..474),
     },
-    Ident {
-        ident: "A",
-        span: #0 bytes(0..0),
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "A",
+                span: #0 bytes(503..504),
+            },
+        ],
+        span: #3 bytes(475..477),
     },
     Group {
         delimiter: Brace,
         stream: TokenStream [],
-        span: #0 bytes(0..0),
+        span: #3 bytes(478..480),
     },
 ]
diff --git a/src/test/ui/proc-macro/macro-rules-derive.rs b/src/test/ui/proc-macro/macro-rules-derive.rs
index 5b4d577a1ac..e0c40bbc734 100644
--- a/src/test/ui/proc-macro/macro-rules-derive.rs
+++ b/src/test/ui/proc-macro/macro-rules-derive.rs
@@ -1,14 +1,13 @@
 // aux-build:first-second.rs
-// FIXME: The spans here are bad, see PR #73084
 
 extern crate first_second;
 use first_second::*;
 
 macro_rules! produce_it {
     ($name:ident) => {
-        #[first] //~ ERROR cannot find type
+        #[first]
         struct $name {
-            field: MissingType
+            field: MissingType //~ ERROR cannot find type
         }
     }
 }
diff --git a/src/test/ui/proc-macro/macro-rules-derive.stderr b/src/test/ui/proc-macro/macro-rules-derive.stderr
index 4b72d29fe8a..54a079e4e73 100644
--- a/src/test/ui/proc-macro/macro-rules-derive.stderr
+++ b/src/test/ui/proc-macro/macro-rules-derive.stderr
@@ -1,8 +1,13 @@
 error[E0412]: cannot find type `MissingType` in this scope
-  --> $DIR/macro-rules-derive.rs:9:9
+  --> $DIR/macro-rules-derive.rs:10:20
    |
-LL |         #[first]
-   |         ^^^^^^^^ not found in this scope
+LL |             field: MissingType
+   |                    ^^^^^^^^^^^ not found in this scope
+...
+LL | produce_it!(MyName);
+   | -------------------- in this macro invocation
+   |
+   = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error: aborting due to previous error
 
diff --git a/src/test/ui/proc-macro/nodelim-groups.stdout b/src/test/ui/proc-macro/nodelim-groups.stdout
index 2fcd41f6da0..cdf851b535a 100644
--- a/src/test/ui/proc-macro/nodelim-groups.stdout
+++ b/src/test/ui/proc-macro/nodelim-groups.stdout
@@ -71,7 +71,6 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
     },
 ]
 PRINT-BANG INPUT (DISPLAY): "hi" "hello".len() + "world".len() (1 + 1)
-PRINT-BANG RE-COLLECTED (DISPLAY): "hi" "hello" . len() + "world" . len() (1 + 1)
 PRINT-BANG INPUT (DEBUG): TokenStream [
     Literal {
         kind: Str,
@@ -82,50 +81,62 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
     Group {
         delimiter: None,
         stream: TokenStream [
-            Literal {
-                kind: Str,
-                symbol: "hello",
-                suffix: None,
-                span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
-            },
-            Punct {
-                ch: '.',
-                spacing: Alone,
-                span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
-            },
-            Ident {
-                ident: "len",
-                span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
-            },
             Group {
-                delimiter: Parenthesis,
-                stream: TokenStream [],
-                span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
+                delimiter: None,
+                stream: TokenStream [
+                    Literal {
+                        kind: Str,
+                        symbol: "hello",
+                        suffix: None,
+                        span: $DIR/nodelim-groups.rs:21:17: 21:24 (#0),
+                    },
+                    Punct {
+                        ch: '.',
+                        spacing: Alone,
+                        span: $DIR/nodelim-groups.rs:21:24: 21:25 (#0),
+                    },
+                    Ident {
+                        ident: "len",
+                        span: $DIR/nodelim-groups.rs:21:25: 21:28 (#0),
+                    },
+                    Group {
+                        delimiter: Parenthesis,
+                        stream: TokenStream [],
+                        span: $DIR/nodelim-groups.rs:21:28: 21:30 (#0),
+                    },
+                ],
+                span: $DIR/nodelim-groups.rs:15:49: 15:54 (#7),
             },
             Punct {
                 ch: '+',
                 spacing: Alone,
-                span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
-            },
-            Literal {
-                kind: Str,
-                symbol: "world",
-                suffix: None,
-                span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
-            },
-            Punct {
-                ch: '.',
-                spacing: Alone,
-                span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
-            },
-            Ident {
-                ident: "len",
-                span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
+                span: $DIR/nodelim-groups.rs:15:55: 15:56 (#7),
             },
             Group {
-                delimiter: Parenthesis,
-                stream: TokenStream [],
-                span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
+                delimiter: None,
+                stream: TokenStream [
+                    Literal {
+                        kind: Str,
+                        symbol: "world",
+                        suffix: None,
+                        span: $DIR/nodelim-groups.rs:21:33: 21:40 (#0),
+                    },
+                    Punct {
+                        ch: '.',
+                        spacing: Alone,
+                        span: $DIR/nodelim-groups.rs:21:40: 21:41 (#0),
+                    },
+                    Ident {
+                        ident: "len",
+                        span: $DIR/nodelim-groups.rs:21:41: 21:44 (#0),
+                    },
+                    Group {
+                        delimiter: Parenthesis,
+                        stream: TokenStream [],
+                        span: $DIR/nodelim-groups.rs:21:44: 21:46 (#0),
+                    },
+                ],
+                span: $DIR/nodelim-groups.rs:15:57: 15:62 (#7),
             },
         ],
         span: $DIR/nodelim-groups.rs:16:47: 16:51 (#8),
diff --git a/src/test/ui/proc-macro/weird-hygiene.rs b/src/test/ui/proc-macro/weird-hygiene.rs
index 3f48191b5b2..7ba3f98a7a9 100644
--- a/src/test/ui/proc-macro/weird-hygiene.rs
+++ b/src/test/ui/proc-macro/weird-hygiene.rs
@@ -1,6 +1,4 @@
 // aux-build:weird-hygiene.rs
-// check-pass
-// FIXME: This should actually error, see PR #73084
 
 #![feature(stmt_expr_attributes)]
 #![feature(proc_macro_hygiene)]
@@ -22,7 +20,7 @@ macro_rules! other {
 
         #[derive(WeirdDerive)]
         enum MyEnum {
-            Value = (stringify!($tokens + hidden_ident), 1).1
+            Value = (stringify!($tokens + hidden_ident), 1).1 //~ ERROR cannot find
         }
 
         inner!();
@@ -33,7 +31,7 @@ macro_rules! invoke_it {
     ($token:expr) => {
         #[recollect_attr] {
             $token;
-            hidden_ident
+            hidden_ident //~ ERROR cannot find
         }
     }
 }
diff --git a/src/test/ui/proc-macro/weird-hygiene.stderr b/src/test/ui/proc-macro/weird-hygiene.stderr
new file mode 100644
index 00000000000..b17dc28f840
--- /dev/null
+++ b/src/test/ui/proc-macro/weird-hygiene.stderr
@@ -0,0 +1,25 @@
+error[E0425]: cannot find value `hidden_ident` in this scope
+  --> $DIR/weird-hygiene.rs:23:43
+   |
+LL |             Value = (stringify!($tokens + hidden_ident), 1).1
+   |                                           ^^^^^^^^^^^^ not found in this scope
+...
+LL |     other!(50);
+   |     ----------- in this macro invocation
+   |
+   = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error[E0425]: cannot find value `hidden_ident` in this scope
+  --> $DIR/weird-hygiene.rs:34:13
+   |
+LL |             hidden_ident
+   |             ^^^^^^^^^^^^ not found in this scope
+...
+LL |     invoke_it!(25);
+   |     --------------- in this macro invocation
+   |
+   = note: this error originates in a macro (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0425`.