Fully integrate token collection for additional AST structs

This commit contains miscellaneous changes that don't fit into any of
the other commits in this PR

5 files changed, 326 insertions, 13 deletions

diff --git a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
index 633d153e391..caf55bec53d 100644
--- a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
+++ b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
@@ -62,7 +62,7 @@ fn expr(kind: ExprKind) -> P<Expr> {
 
 fn make_x() -> P<Expr> {
     let seg = PathSegment::from_ident(Ident::from_str("x"));
-    let path = Path { segments: vec![seg], span: DUMMY_SP };
+    let path = Path { segments: vec![seg], span: DUMMY_SP, tokens: None };
     expr(ExprKind::Path(None, path))
 }
 
@@ -113,6 +113,7 @@ fn iter_exprs(depth: usize, f: &mut dyn FnMut(P<Expr>)) {
                     id: DUMMY_NODE_ID,
                     rules: BlockCheckMode::Default,
                     span: DUMMY_SP,
+                    tokens: None,
                 });
                 iter_exprs(depth - 1, &mut |e| g(ExprKind::If(e, block.clone(), None)));
             },
diff --git a/src/test/ui/ast-json/ast-json-noexpand-output.stdout b/src/test/ui/ast-json/ast-json-noexpand-output.stdout
index 0e5a3a14ac7..3d7d476cf6c 100644
--- a/src/test/ui/ast-json/ast-json-noexpand-output.stdout
+++ b/src/test/ui/ast-json/ast-json-noexpand-output.stdout
@@ -1 +1 @@
-{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]}}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}
+{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}
diff --git a/src/test/ui/ast-json/ast-json-output.stdout b/src/test/ui/ast-json/ast-json-output.stdout
index 8752ed2ae99..a4ce6061b4c 100644
--- a/src/test/ui/ast-json/ast-json-output.stdout
+++ b/src/test/ui/ast-json/ast-json-output.stdout
@@ -1 +1 @@
-{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":"Empty"}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":"Empty"}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}]},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]}}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}
+{"module":{"inner":{"lo":0,"hi":0},"unsafety":"No","items":[{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"prelude_import","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"","span":{"lo":0,"hi":0}},"kind":{"variant":"Use","fields":[{"prefix":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"{{root}}","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"std","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"prelude","span":{"lo":0,"hi":0}},"id":0,"args":null},{"ident":{"name":"v1","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"kind":"Glob","span":{"lo":0,"hi":0}}]},"tokens":null},{"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"macro_use","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":"Empty","tokens":null}]},"id":null,"style":"Outer","span":{"lo":0,"hi":0}}],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"std","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null},{"attrs":[],"id":0,"span":{"lo":0,"hi":0},"vis":{"kind":"Inherited","span":{"lo":0,"hi":0},"tokens":null},"ident":{"name":"core","span":{"lo":0,"hi":0}},"kind":{"variant":"ExternCrate","fields":[null]},"tokens":null}],"inline":true},"attrs":[{"kind":{"variant":"Normal","fields":[{"path":{"span":{"lo":0,"hi":0},"segments":[{"ident":{"name":"crate_type","span":{"lo":0,"hi":0}},"id":0,"args":null}],"tokens":null},"args":{"variant":"Eq","fields":[{"lo":0,"hi":0},{"0":[[{"variant":"Token","fields":[{"kind":{"variant":"Literal","fields":[{"kind":"Str","symbol":"lib","suffix":null}]},"span":{"lo":0,"hi":0}}]},"Alone"]]}]},"tokens":null}]},"id":null,"style":"Inner","span":{"lo":0,"hi":0}}],"span":{"lo":0,"hi":0},"proc_macros":[]}
diff --git a/src/test/ui/proc-macro/capture-macro-rules-invoke.rs b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
index 2ff6ad6d68f..de008a3708a 100644
--- a/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
+++ b/src/test/ui/proc-macro/capture-macro-rules-invoke.rs
@@ -1,10 +1,20 @@
 // aux-build:test-macros.rs
 // check-pass
 // compile-flags: -Z span-debug
-// normalize-stdout-test "#\d+" -> "#CTXT"
+
+#![no_std] // Don't load unnecessary hygiene information from std
+extern crate std;
 
 extern crate test_macros;
-use test_macros::print_bang;
+use test_macros::{print_bang, print_bang_consume};
+
+macro_rules! test_matchers {
+    ($expr:expr, $block:block, $stmt:stmt, $ty:ty, $ident:ident, $lifetime:lifetime,
+     $meta:meta, $path:path, $vis:vis, $tt:tt, $lit:literal) => {
+        print_bang_consume!($expr, $block, $stmt, $ty, $ident,
+                            $lifetime, $meta, $path, $vis, $tt, $lit)
+    }
+}
 
 macro_rules! use_expr {
     ($expr:expr) => {
@@ -24,10 +34,23 @@ impl Foo {
     #[allow(dead_code)]
     fn use_self(self) {
         drop(use_expr!(self));
+        test_matchers!(
+            1 + 1,
+            { "a" },
+            let a = 1,
+            String,
+            my_name,
+            'a,
+            my_val = 30,
+            std::option::Option,
+            pub(in some::path),
+            [ a b c ],
+            -30
+        );
     }
 
     fn with_pat(use_pat!((a, b)): (u32, u32)) {
-        println!("Args: {} {}", a, b);
+        let _ = (a, b);
     }
 }
 
diff --git a/src/test/ui/proc-macro/capture-macro-rules-invoke.stdout b/src/test/ui/proc-macro/capture-macro-rules-invoke.stdout
index 28812e20548..652bf6b6b22 100644
--- a/src/test/ui/proc-macro/capture-macro-rules-invoke.stdout
+++ b/src/test/ui/proc-macro/capture-macro-rules-invoke.stdout
@@ -5,10 +5,299 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
         stream: TokenStream [
             Ident {
                 ident: "self",
-                span: $DIR/capture-macro-rules-invoke.rs:26:24: 26:28 (#CTXT),
+                span: $DIR/capture-macro-rules-invoke.rs:36:24: 36:28 (#0),
             },
         ],
-        span: $DIR/capture-macro-rules-invoke.rs:11:21: 11:26 (#CTXT),
+        span: $DIR/capture-macro-rules-invoke.rs:21:21: 21:26 (#3),
+    },
+]
+PRINT-BANG INPUT (DISPLAY): 1 + 1, { "a" }, let a = 1;, String, my_name, 'a, my_val = 30,
+std::option::Option, pub(in some::path) , [a b c], -30
+PRINT-BANG RE-COLLECTED (DISPLAY): 1 + 1, { "a" }, let a = 1, String, my_name, 'a, my_val = 30,
+std :: option :: Option, pub(in some :: path), [a b c], - 30
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/capture-macro-rules-invoke.rs:38:13: 38:14 (#0),
+            },
+            Punct {
+                ch: '+',
+                spacing: Alone,
+                span: $DIR/capture-macro-rules-invoke.rs:38:15: 38:16 (#0),
+            },
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/capture-macro-rules-invoke.rs:38:17: 38:18 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:14:29: 14:34 (#7),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:14:34: 14:35 (#7),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Group {
+                delimiter: Brace,
+                stream: TokenStream [
+                    Literal {
+                        kind: Str,
+                        symbol: "a",
+                        suffix: None,
+                        span: $DIR/capture-macro-rules-invoke.rs:39:15: 39:18 (#0),
+                    },
+                ],
+                span: $DIR/capture-macro-rules-invoke.rs:39:13: 39:20 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:14:36: 14:42 (#7),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:14:42: 14:43 (#7),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "let",
+                span: $DIR/capture-macro-rules-invoke.rs:40:13: 40:16 (#0),
+            },
+            Ident {
+                ident: "a",
+                span: $DIR/capture-macro-rules-invoke.rs:40:17: 40:18 (#0),
+            },
+            Punct {
+                ch: '=',
+                spacing: Alone,
+                span: $DIR/capture-macro-rules-invoke.rs:40:19: 40:20 (#0),
+            },
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/capture-macro-rules-invoke.rs:40:21: 40:22 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:14:44: 14:49 (#7),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:14:49: 14:50 (#7),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "String",
+                span: $DIR/capture-macro-rules-invoke.rs:41:13: 41:19 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:14:51: 14:54 (#7),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:14:54: 14:55 (#7),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "my_name",
+                span: $DIR/capture-macro-rules-invoke.rs:42:13: 42:20 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:14:56: 14:62 (#7),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:14:62: 14:63 (#7),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Punct {
+                ch: '\'',
+                spacing: Joint,
+                span: $DIR/capture-macro-rules-invoke.rs:43:13: 43:15 (#0),
+            },
+            Ident {
+                ident: "a",
+                span: $DIR/capture-macro-rules-invoke.rs:43:13: 43:15 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:15:29: 15:38 (#7),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:15:38: 15:39 (#7),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "my_val",
+                span: $DIR/capture-macro-rules-invoke.rs:44:13: 44:19 (#0),
+            },
+            Punct {
+                ch: '=',
+                spacing: Alone,
+                span: $DIR/capture-macro-rules-invoke.rs:44:20: 44:21 (#0),
+            },
+            Literal {
+                kind: Integer,
+                symbol: "30",
+                suffix: None,
+                span: $DIR/capture-macro-rules-invoke.rs:44:22: 44:24 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:15:40: 15:45 (#7),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:15:45: 15:46 (#7),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "std",
+                span: $DIR/capture-macro-rules-invoke.rs:45:13: 45:16 (#0),
+            },
+            Punct {
+                ch: ':',
+                spacing: Joint,
+                span: $DIR/capture-macro-rules-invoke.rs:45:16: 45:18 (#0),
+            },
+            Punct {
+                ch: ':',
+                spacing: Alone,
+                span: $DIR/capture-macro-rules-invoke.rs:45:16: 45:18 (#0),
+            },
+            Ident {
+                ident: "option",
+                span: $DIR/capture-macro-rules-invoke.rs:45:18: 45:24 (#0),
+            },
+            Punct {
+                ch: ':',
+                spacing: Joint,
+                span: $DIR/capture-macro-rules-invoke.rs:45:24: 45:26 (#0),
+            },
+            Punct {
+                ch: ':',
+                spacing: Alone,
+                span: $DIR/capture-macro-rules-invoke.rs:45:24: 45:26 (#0),
+            },
+            Ident {
+                ident: "Option",
+                span: $DIR/capture-macro-rules-invoke.rs:45:26: 45:32 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:15:47: 15:52 (#7),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:15:52: 15:53 (#7),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Ident {
+                ident: "pub",
+                span: $DIR/capture-macro-rules-invoke.rs:46:13: 46:16 (#0),
+            },
+            Group {
+                delimiter: Parenthesis,
+                stream: TokenStream [
+                    Ident {
+                        ident: "in",
+                        span: $DIR/capture-macro-rules-invoke.rs:46:17: 46:19 (#0),
+                    },
+                    Ident {
+                        ident: "some",
+                        span: $DIR/capture-macro-rules-invoke.rs:46:20: 46:24 (#0),
+                    },
+                    Punct {
+                        ch: ':',
+                        spacing: Joint,
+                        span: $DIR/capture-macro-rules-invoke.rs:46:24: 46:26 (#0),
+                    },
+                    Punct {
+                        ch: ':',
+                        spacing: Alone,
+                        span: $DIR/capture-macro-rules-invoke.rs:46:24: 46:26 (#0),
+                    },
+                    Ident {
+                        ident: "path",
+                        span: $DIR/capture-macro-rules-invoke.rs:46:26: 46:30 (#0),
+                    },
+                ],
+                span: $DIR/capture-macro-rules-invoke.rs:46:16: 46:31 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:15:54: 15:58 (#7),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:15:58: 15:59 (#7),
+    },
+    Group {
+        delimiter: Bracket,
+        stream: TokenStream [
+            Ident {
+                ident: "a",
+                span: $DIR/capture-macro-rules-invoke.rs:47:15: 47:16 (#0),
+            },
+            Ident {
+                ident: "b",
+                span: $DIR/capture-macro-rules-invoke.rs:47:17: 47:18 (#0),
+            },
+            Ident {
+                ident: "c",
+                span: $DIR/capture-macro-rules-invoke.rs:47:19: 47:20 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:47:13: 47:22 (#0),
+    },
+    Punct {
+        ch: ',',
+        spacing: Alone,
+        span: $DIR/capture-macro-rules-invoke.rs:15:63: 15:64 (#7),
+    },
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Punct {
+                ch: '-',
+                spacing: Alone,
+                span: $DIR/capture-macro-rules-invoke.rs:48:13: 48:14 (#0),
+            },
+            Literal {
+                kind: Integer,
+                symbol: "30",
+                suffix: None,
+                span: $DIR/capture-macro-rules-invoke.rs:48:14: 48:16 (#0),
+            },
+        ],
+        span: $DIR/capture-macro-rules-invoke.rs:15:65: 15:69 (#7),
     },
 ]
 PRINT-BANG INPUT (DISPLAY): (a, b)
@@ -21,21 +310,21 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
                 stream: TokenStream [
                     Ident {
                         ident: "a",
-                        span: $DIR/capture-macro-rules-invoke.rs:29:27: 29:28 (#CTXT),
+                        span: $DIR/capture-macro-rules-invoke.rs:52:27: 52:28 (#0),
                     },
                     Punct {
                         ch: ',',
                         spacing: Alone,
-                        span: $DIR/capture-macro-rules-invoke.rs:29:28: 29:29 (#CTXT),
+                        span: $DIR/capture-macro-rules-invoke.rs:52:28: 52:29 (#0),
                     },
                     Ident {
                         ident: "b",
-                        span: $DIR/capture-macro-rules-invoke.rs:29:30: 29:31 (#CTXT),
+                        span: $DIR/capture-macro-rules-invoke.rs:52:30: 52:31 (#0),
                     },
                 ],
-                span: $DIR/capture-macro-rules-invoke.rs:29:26: 29:32 (#CTXT),
+                span: $DIR/capture-macro-rules-invoke.rs:52:26: 52:32 (#0),
             },
         ],
-        span: $DIR/capture-macro-rules-invoke.rs:17:21: 17:25 (#CTXT),
+        span: $DIR/capture-macro-rules-invoke.rs:27:21: 27:25 (#11),
     },
 ]