Auto merge of #72287 - Aaron1011:feature/min-token-collect, r=petrochenkov

Store tokens inside `ast::Expr`

This is a smaller version of #70091.

We now store captured tokens inside `ast::Expr`, which allows us to avoid some reparsing in `nt_to_tokenstream`. To try to mitigate the performance impact, we only collect tokens when we've seen an outer attribute.

This makes progress towards solving #43081. There are still many things left to do:

* Collect tokens for other AST items.
* Come up with a way to handle inner attributes (we need to be collecting tokens by the time we encounter them)
* Avoid re-parsing when a `#[cfg]` attr is used.

However, this is enough to fix spans for a simple example, which I've included as a test case.

3 files changed, 32 insertions, 0 deletions

diff --git a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
index a3d31d25774..cef600bed5f 100644
--- a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
+++ b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
@@ -56,6 +56,7 @@ fn expr(kind: ExprKind) -> P<Expr> {
         kind,
         span: DUMMY_SP,
         attrs: ThinVec::new(),
+        tokens: None
     })
 }
 
@@ -200,6 +201,7 @@ impl MutVisitor for AddParens {
                 kind: ExprKind::Paren(e),
                 span: DUMMY_SP,
                 attrs: ThinVec::new(),
+                tokens: None
             })
         });
     }
diff --git a/src/test/ui/proc-macro/keep-expr-tokens.rs b/src/test/ui/proc-macro/keep-expr-tokens.rs
new file mode 100644
index 00000000000..888785363cf
--- /dev/null
+++ b/src/test/ui/proc-macro/keep-expr-tokens.rs
@@ -0,0 +1,15 @@
+// aux-build:test-macros.rs
+
+#![feature(stmt_expr_attributes)]
+#![feature(proc_macro_hygiene)]
+
+extern crate test_macros;
+
+use test_macros::recollect_attr;
+
+fn main() {
+    #[test_macros::recollect_attr]
+    for item in missing_fn() {} //~ ERROR cannot find
+
+    (#[recollect_attr] #[recollect_attr] ((#[recollect_attr] bad))); //~ ERROR cannot
+}
diff --git a/src/test/ui/proc-macro/keep-expr-tokens.stderr b/src/test/ui/proc-macro/keep-expr-tokens.stderr
new file mode 100644
index 00000000000..2be8c0184da
--- /dev/null
+++ b/src/test/ui/proc-macro/keep-expr-tokens.stderr
@@ -0,0 +1,15 @@
+error[E0425]: cannot find function `missing_fn` in this scope
+  --> $DIR/keep-expr-tokens.rs:12:17
+   |
+LL |     for item in missing_fn() {}
+   |                 ^^^^^^^^^^ not found in this scope
+
+error[E0425]: cannot find value `bad` in this scope
+  --> $DIR/keep-expr-tokens.rs:14:62
+   |
+LL |     (#[recollect_attr] #[recollect_attr] ((#[recollect_attr] bad)));
+   |                                                              ^^^ not found in this scope
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0425`.