Auto merge of #56884 - euclio:codeblock-diagnostics, r=QuietMisdreavus

rustdoc: overhaul code block lexing errors

Fixes #53919.

This PR moves the reporting of code block lexing errors from rendering time to an early pass, so we can use the compiler's error reporting mechanisms. This dramatically improves the diagnostics in this situation: we now de-emphasize the lexing errors as a note under a warning that has a span and suggestion instead of just emitting errors at the top level.

Additionally, this PR generalizes the markdown -> source span calculation function, which should allow other rustdoc warnings to use better spans in the future.

Last, the PR makes sure that the code block is always emitted in the docs, even if it fails to highlight correctly.

Of note:
- The new pass unfortunately adds another pass over the docs to gather the doc blocks for syntax-checking. I wonder if this could be combined with the pass that looks for testable blocks? I'm not familiar with that code, so I don't know how feasible that is.
- `pulldown_cmark` doesn't make it easy to find the spans of the code blocks, so the code that calculates the spans is a little nasty. It works for all the test cases I threw at it, but I wouldn't be surprised if an edge case would break it. Should have a thorough review.
- This PR worsens the state of #56885, since those certain fatal lexing errors are now emitted before docs get generated at all.

3 files changed, 183 insertions, 10 deletions

diff --git a/src/test/rustdoc-ui/invalid-syntax.rs b/src/test/rustdoc-ui/invalid-syntax.rs
index 537816be8d7..924e0386d31 100644
--- a/src/test/rustdoc-ui/invalid-syntax.rs
+++ b/src/test/rustdoc-ui/invalid-syntax.rs
@@ -1,7 +1,66 @@
 // compile-pass
-// compile-flags: --error-format=human
 
 /// ```
 /// \__________pkt->size___________/          \_result->size_/ \__pkt->size__/
 /// ```
 pub fn foo() {}
+
+/// ```
+///    |
+/// LL | use foobar::Baz;
+///    |     ^^^^^^ did you mean `baz::foobar`?
+/// ```
+pub fn bar() {}
+
+/// ```
+/// valid
+/// ```
+///
+/// ```
+/// \_
+/// ```
+///
+/// ```text
+/// "invalid
+/// ```
+pub fn valid_and_invalid() {}
+
+/// This is a normal doc comment, but...
+///
+/// There's a code block with bad syntax in it:
+///
+/// ```rust
+/// \_
+/// ```
+///
+/// Good thing we tested it!
+pub fn baz() {}
+
+/// Indented block start
+///
+///     code with bad syntax
+///     \_
+///
+/// Indented block end
+pub fn quux() {}
+
+/// Unclosed fence
+///
+/// ```
+/// slkdjf
+pub fn xyzzy() {}
+
+/// Indented code that contains a fence
+///
+///     ```
+pub fn blah() {}
+
+/// ```edition2018
+/// \_
+/// ```
+pub fn blargh() {}
+
+#[doc = "```"]
+/// \_
+#[doc = "```"]
+pub fn crazy_attrs() {}
diff --git a/src/test/rustdoc-ui/invalid-syntax.stderr b/src/test/rustdoc-ui/invalid-syntax.stderr
index b5661332e8d..10800380a05 100644
--- a/src/test/rustdoc-ui/invalid-syntax.stderr
+++ b/src/test/rustdoc-ui/invalid-syntax.stderr
@@ -1,10 +1,97 @@
-Output from rustc:
-error: unknown start of token: /
- --> <stdin>:1:1
-  |
-1 | /__________pkt->size___________/          /_result->size_/ /__pkt->size__/
-  | ^
-
-warning: Invalid doc comment starting with: `/__________pkt->size___________/          /_result->size_/ /__pkt->size__/`
-(Ignoring this codeblock)
+warning: could not parse code block as Rust code
+  --> $DIR/invalid-syntax.rs:3:5
+   |
+LL |   /// ```
+   |  _____^
+LL | | /// /__________pkt->size___________/          /_result->size_/ /__pkt->size__/
+LL | | /// ```
+   | |_______^
+   |
+   = note: error from rustc: unknown start of token: /
+help: mark blocks that do not contain Rust code as text
+   |
+LL | /// ```text
+   |     ^^^^^^^
+
+warning: could not parse code block as Rust code
+  --> $DIR/invalid-syntax.rs:8:5
+   |
+LL |   /// ```
+   |  _____^
+LL | | ///    |
+LL | | /// LL | use foobar::Baz;
+LL | | ///    |     ^^^^^^ did you mean `baz::foobar`?
+LL | | /// ```
+   | |_______^
+   |
+   = note: error from rustc: unknown start of token: `
+help: mark blocks that do not contain Rust code as text
+   |
+LL | /// ```text
+   |     ^^^^^^^
+
+warning: could not parse code block as Rust code
+  --> $DIR/invalid-syntax.rs:19:5
+   |
+LL |   /// ```
+   |  _____^
+LL | | /// /_
+LL | | /// ```
+   | |_______^
+   |
+   = note: error from rustc: unknown start of token: /
+help: mark blocks that do not contain Rust code as text
+   |
+LL | /// ```text
+   |     ^^^^^^^
+
+warning: could not parse code block as Rust code
+  --> $DIR/invalid-syntax.rs:32:5
+   |
+LL |   /// ```rust
+   |  _____^
+LL | | /// /_
+LL | | /// ```
+   | |_______^
+   |
+   = note: error from rustc: unknown start of token: /
+
+warning: could not parse code block as Rust code
+  --> $DIR/invalid-syntax.rs:41:9
+   |
+LL |   ///     code with bad syntax
+   |  _________^
+LL | | ///     /_
+   | |__________^
+   |
+   = note: error from rustc: unknown start of token: /
+
+warning: could not parse code block as Rust code
+  --> $DIR/invalid-syntax.rs:55:9
+   |
+LL | ///     ```
+   |         ^^^
+   |
+   = note: error from rustc: unknown start of token: `
+
+warning: could not parse code block as Rust code
+  --> $DIR/invalid-syntax.rs:58:5
+   |
+LL |   /// ```edition2018
+   |  _____^
+LL | | /// /_
+LL | | /// ```
+   | |_______^
+   |
+   = note: error from rustc: unknown start of token: /
+
+warning: doc comment contains an invalid Rust code block
+  --> $DIR/invalid-syntax.rs:63:1
+   |
+LL | / #[doc = "```"]
+LL | | /// /_
+LL | | #[doc = "```"]
+   | |______________^
+   |
+   = help: mark blocks that do not contain Rust code as text: ```text
 
diff --git a/src/test/rustdoc/bad-codeblock-syntax.rs b/src/test/rustdoc/bad-codeblock-syntax.rs
new file mode 100644
index 00000000000..0ab2f68fcde
--- /dev/null
+++ b/src/test/rustdoc/bad-codeblock-syntax.rs
@@ -0,0 +1,27 @@
+// @has bad_codeblock_syntax/fn.foo.html
+// @has - '//*[@class="docblock"]/pre/code' '\_'
+/// ```
+/// \_
+/// ```
+pub fn foo() {}
+
+// @has bad_codeblock_syntax/fn.bar.html
+// @has - '//*[@class="docblock"]/pre/code' '`baz::foobar`'
+/// ```
+/// `baz::foobar`
+/// ```
+pub fn bar() {}
+
+// @has bad_codeblock_syntax/fn.quux.html
+// @has - '//*[@class="docblock"]/pre/code' '\_'
+/// ```rust
+/// \_
+/// ```
+pub fn quux() {}
+
+// @has bad_codeblock_syntax/fn.ok.html
+// @has - '//*[@class="docblock"]/pre/code[@class="language-text"]' '\_'
+/// ```text
+/// \_
+/// ```
+pub fn ok() {}