Allow skipping extra paren insertion during AST pretty-printing

Fixes #74616
Makes progress towards #43081
Unblocks PR #76130

When pretty-printing an AST node, we may insert additional parenthesis
to ensure that precedence is properly preserved in code we output.
However, the proc macro implementation relies on comparing a
pretty-printed AST node to the captured `TokenStream`. Inserting extra
parenthesis changes the structure of the reparsed `TokenStream`, making
the comparison fail.

This PR refactors the AST pretty-printing code to allow skipping the
insertion of additional parenthesis. Several freestanding methods are
moved to trait methods on `PrintState`, which keep track of an internal
`insert_extra_parens` flag. This flag is normally `true`, but we expose
a public method which allows pretty-printing a nonterminal with
`insert_extra_parens = false`.

To avoid changing the public interface of `rustc_ast_pretty`, the
freestanding `_to_string` methods are changed to delegate to a
newly-crated `State`. The main pretty-printing code is moved to a new
`state` module to ensure that it does not accidentally call any of these
public helper functions (instead, the internal functions with the same
name should be used).

2 files changed, 160 insertions, 0 deletions

diff --git a/src/test/ui/proc-macro/issue-75734-pp-paren.rs b/src/test/ui/proc-macro/issue-75734-pp-paren.rs
new file mode 100644
index 00000000000..faa93787d13
--- /dev/null
+++ b/src/test/ui/proc-macro/issue-75734-pp-paren.rs
@@ -0,0 +1,26 @@
+// Regression test for issue #75734
+// Ensures that we don't lose tokens when pretty-printing would
+// normally insert extra parentheses.
+
+// check-pass
+// aux-build:test-macros.rs
+// compile-flags: -Z span-debug
+
+#![no_std] // Don't load unnecessary hygiene information from std
+extern crate std;
+
+#[macro_use]
+extern crate test_macros;
+
+macro_rules! mul_2 {
+    ($val:expr) => {
+        print_bang!($val * 2);
+    };
+}
+
+
+#[print_attr]
+fn main() {
+    &|_: u8| {};
+    mul_2!(1 + 1);
+}
diff --git a/src/test/ui/proc-macro/issue-75734-pp-paren.stdout b/src/test/ui/proc-macro/issue-75734-pp-paren.stdout
new file mode 100644
index 00000000000..b33b85f1705
--- /dev/null
+++ b/src/test/ui/proc-macro/issue-75734-pp-paren.stdout
@@ -0,0 +1,134 @@
+PRINT-ATTR INPUT (DISPLAY): fn main() { & | _ : u8 | { } ; mul_2 ! (1 + 1) ; }
+PRINT-ATTR INPUT (DEBUG): TokenStream [
+    Ident {
+        ident: "fn",
+        span: $DIR/issue-75734-pp-paren.rs:23:1: 23:3 (#0),
+    },
+    Ident {
+        ident: "main",
+        span: $DIR/issue-75734-pp-paren.rs:23:4: 23:8 (#0),
+    },
+    Group {
+        delimiter: Parenthesis,
+        stream: TokenStream [],
+        span: $DIR/issue-75734-pp-paren.rs:23:8: 23:10 (#0),
+    },
+    Group {
+        delimiter: Brace,
+        stream: TokenStream [
+            Punct {
+                ch: '&',
+                spacing: Joint,
+                span: $DIR/issue-75734-pp-paren.rs:24:5: 24:6 (#0),
+            },
+            Punct {
+                ch: '|',
+                spacing: Alone,
+                span: $DIR/issue-75734-pp-paren.rs:24:6: 24:7 (#0),
+            },
+            Ident {
+                ident: "_",
+                span: $DIR/issue-75734-pp-paren.rs:24:7: 24:8 (#0),
+            },
+            Punct {
+                ch: ':',
+                spacing: Alone,
+                span: $DIR/issue-75734-pp-paren.rs:24:8: 24:9 (#0),
+            },
+            Ident {
+                ident: "u8",
+                span: $DIR/issue-75734-pp-paren.rs:24:10: 24:12 (#0),
+            },
+            Punct {
+                ch: '|',
+                spacing: Alone,
+                span: $DIR/issue-75734-pp-paren.rs:24:12: 24:13 (#0),
+            },
+            Group {
+                delimiter: Brace,
+                stream: TokenStream [],
+                span: $DIR/issue-75734-pp-paren.rs:24:14: 24:16 (#0),
+            },
+            Punct {
+                ch: ';',
+                spacing: Alone,
+                span: $DIR/issue-75734-pp-paren.rs:24:16: 24:17 (#0),
+            },
+            Ident {
+                ident: "mul_2",
+                span: $DIR/issue-75734-pp-paren.rs:25:5: 25:10 (#0),
+            },
+            Punct {
+                ch: '!',
+                spacing: Alone,
+                span: $DIR/issue-75734-pp-paren.rs:25:10: 25:11 (#0),
+            },
+            Group {
+                delimiter: Parenthesis,
+                stream: TokenStream [
+                    Literal {
+                        kind: Integer,
+                        symbol: "1",
+                        suffix: None,
+                        span: $DIR/issue-75734-pp-paren.rs:25:12: 25:13 (#0),
+                    },
+                    Punct {
+                        ch: '+',
+                        spacing: Alone,
+                        span: $DIR/issue-75734-pp-paren.rs:25:14: 25:15 (#0),
+                    },
+                    Literal {
+                        kind: Integer,
+                        symbol: "1",
+                        suffix: None,
+                        span: $DIR/issue-75734-pp-paren.rs:25:16: 25:17 (#0),
+                    },
+                ],
+                span: $DIR/issue-75734-pp-paren.rs:25:11: 25:18 (#0),
+            },
+            Punct {
+                ch: ';',
+                spacing: Alone,
+                span: $DIR/issue-75734-pp-paren.rs:25:18: 25:19 (#0),
+            },
+        ],
+        span: $DIR/issue-75734-pp-paren.rs:23:11: 26:2 (#0),
+    },
+]
+PRINT-BANG INPUT (DISPLAY): 1 + 1 * 2
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/issue-75734-pp-paren.rs:25:12: 25:13 (#0),
+            },
+            Punct {
+                ch: '+',
+                spacing: Alone,
+                span: $DIR/issue-75734-pp-paren.rs:25:14: 25:15 (#0),
+            },
+            Literal {
+                kind: Integer,
+                symbol: "1",
+                suffix: None,
+                span: $DIR/issue-75734-pp-paren.rs:25:16: 25:17 (#0),
+            },
+        ],
+        span: $DIR/issue-75734-pp-paren.rs:17:21: 17:25 (#7),
+    },
+    Punct {
+        ch: '*',
+        spacing: Alone,
+        span: $DIR/issue-75734-pp-paren.rs:17:26: 17:27 (#7),
+    },
+    Literal {
+        kind: Integer,
+        symbol: "2",
+        suffix: None,
+        span: $DIR/issue-75734-pp-paren.rs:17:28: 17:29 (#7),
+    },
+]