Rollup merge of #141004 - matthewjasper:unicode-before-expansion, r=davidtwco

Report text_direction_codepoint_in_literal when parsing

The lint is now reported in code that gets removed/modified/duplicated by macro expansion, and spans are more accurate so we don't get ICEs from trying to split a span in the middle of a character.

This removes support for lint level attributes for `text_direction_codepoint_in_literal` except at the crate level, I don't think that there's an easy way around this when the lint can be reported on code that's removed by `cfg` or that is only in the input of a macro.

Fixes #140281

2 files changed, 12 insertions, 0 deletions

diff --git a/src/tools/lint-docs/Cargo.toml b/src/tools/lint-docs/Cargo.toml
index 3578bda8276..f1ffda75ac0 100644
--- a/src/tools/lint-docs/Cargo.toml
+++ b/src/tools/lint-docs/Cargo.toml
@@ -7,6 +7,7 @@ description = "A script to extract the lint documentation for the rustc book."
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+rustc-literal-escaper = "0.0.2"
 serde_json = "1.0.57"
 tempfile = "3.1.0"
 walkdir = "2.3.1"
diff --git a/src/tools/lint-docs/src/lib.rs b/src/tools/lint-docs/src/lib.rs
index cacce01675f..6bb18c2bced 100644
--- a/src/tools/lint-docs/src/lib.rs
+++ b/src/tools/lint-docs/src/lib.rs
@@ -4,6 +4,7 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
+use rustc_literal_escaper::{Mode, unescape_unicode};
 use walkdir::WalkDir;
 
 mod groups;
@@ -214,6 +215,16 @@ impl<'a> LintExtractor<'a> {
                         let line = line.trim();
                         if let Some(text) = line.strip_prefix("/// ") {
                             doc_lines.push(text.to_string());
+                        } else if let Some(text) = line.strip_prefix("#[doc = \"") {
+                            let escaped = text.strip_suffix("\"]").unwrap();
+                            let mut buf = String::new();
+                            unescape_unicode(escaped, Mode::Str, &mut |_, c| match c {
+                                Ok(c) => buf.push(c),
+                                Err(err) => {
+                                    assert!(!err.is_fatal(), "failed to unescape string literal")
+                                }
+                            });
+                            doc_lines.push(buf);
                         } else if line == "///" {
                             doc_lines.push("".to_string());
                         } else if line.starts_with("// ") {