Merge commit '574e23ec508064613783cba3d1833a95fd9a5080' into sync-from-ra

8 files changed, 879 insertions, 15 deletions

diff --git a/src/tools/rust-analyzer/xtask/Cargo.toml b/src/tools/rust-analyzer/xtask/Cargo.toml
index 863a63ac82e..5e758e0190c 100644
--- a/src/tools/rust-analyzer/xtask/Cargo.toml
+++ b/src/tools/rust-analyzer/xtask/Cargo.toml
@@ -14,7 +14,8 @@ xshell.workspace = true
 xflags = "0.3.0"
 time = { version = "0.3", default-features = false }
 zip = { version = "0.6", default-features = false, features = ["deflate", "time"] }
+stdx.workspace = true
 # Avoid adding more dependencies to this crate
 
 [lints]
-workspace = true
\ No newline at end of file
+workspace = true
diff --git a/src/tools/rust-analyzer/xtask/src/codegen.rs b/src/tools/rust-analyzer/xtask/src/codegen.rs
new file mode 100644
index 00000000000..40f872a24ab
--- /dev/null
+++ b/src/tools/rust-analyzer/xtask/src/codegen.rs
@@ -0,0 +1,218 @@
+use std::{
+    fmt, fs, mem,
+    path::{Path, PathBuf},
+};
+
+use xshell::{cmd, Shell};
+
+use crate::{flags, project_root};
+
+pub(crate) mod assists_doc_tests;
+pub(crate) mod diagnostics_docs;
+mod lints;
+
+impl flags::Codegen {
+    pub(crate) fn run(self, _sh: &Shell) -> anyhow::Result<()> {
+        match self.codegen_type.unwrap_or_default() {
+            flags::CodegenType::All => {
+                diagnostics_docs::generate(self.check);
+                assists_doc_tests::generate(self.check);
+                // lints::generate(self.check) Updating clones the rust repo, so don't run it unless
+                // explicitly asked for
+            }
+            flags::CodegenType::AssistsDocTests => assists_doc_tests::generate(self.check),
+            flags::CodegenType::DiagnosticsDocs => diagnostics_docs::generate(self.check),
+            flags::CodegenType::LintDefinitions => lints::generate(self.check),
+        }
+        Ok(())
+    }
+}
+
+fn list_rust_files(dir: &Path) -> Vec<PathBuf> {
+    let mut res = list_files(dir);
+    res.retain(|it| {
+        it.file_name().unwrap_or_default().to_str().unwrap_or_default().ends_with(".rs")
+    });
+    res
+}
+
+fn list_files(dir: &Path) -> Vec<PathBuf> {
+    let mut res = Vec::new();
+    let mut work = vec![dir.to_path_buf()];
+    while let Some(dir) = work.pop() {
+        for entry in dir.read_dir().unwrap() {
+            let entry = entry.unwrap();
+            let file_type = entry.file_type().unwrap();
+            let path = entry.path();
+            let is_hidden =
+                path.file_name().unwrap_or_default().to_str().unwrap_or_default().starts_with('.');
+            if !is_hidden {
+                if file_type.is_dir() {
+                    work.push(path);
+                } else if file_type.is_file() {
+                    res.push(path);
+                }
+            }
+        }
+    }
+    res
+}
+
+#[derive(Clone)]
+pub(crate) struct CommentBlock {
+    pub(crate) id: String,
+    pub(crate) line: usize,
+    pub(crate) contents: Vec<String>,
+    is_doc: bool,
+}
+
+impl CommentBlock {
+    fn extract(tag: &str, text: &str) -> Vec<CommentBlock> {
+        assert!(tag.starts_with(char::is_uppercase));
+
+        let tag = format!("{tag}:");
+        let mut blocks = CommentBlock::extract_untagged(text);
+        blocks.retain_mut(|block| {
+            let first = block.contents.remove(0);
+            let Some(id) = first.strip_prefix(&tag) else {
+                return false;
+            };
+
+            if block.is_doc {
+                panic!("Use plain (non-doc) comments with tags like {tag}:\n    {first}");
+            }
+
+            block.id = id.trim().to_owned();
+            true
+        });
+        blocks
+    }
+
+    fn extract_untagged(text: &str) -> Vec<CommentBlock> {
+        let mut res = Vec::new();
+
+        let lines = text.lines().map(str::trim_start);
+
+        let dummy_block =
+            CommentBlock { id: String::new(), line: 0, contents: Vec::new(), is_doc: false };
+        let mut block = dummy_block.clone();
+        for (line_num, line) in lines.enumerate() {
+            match line.strip_prefix("//") {
+                Some(mut contents) => {
+                    if let Some('/' | '!') = contents.chars().next() {
+                        contents = &contents[1..];
+                        block.is_doc = true;
+                    }
+                    if let Some(' ') = contents.chars().next() {
+                        contents = &contents[1..];
+                    }
+                    block.contents.push(contents.to_owned());
+                }
+                None => {
+                    if !block.contents.is_empty() {
+                        let block = mem::replace(&mut block, dummy_block.clone());
+                        res.push(block);
+                    }
+                    block.line = line_num + 2;
+                }
+            }
+        }
+        if !block.contents.is_empty() {
+            res.push(block);
+        }
+        res
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct Location {
+    pub(crate) file: PathBuf,
+    pub(crate) line: usize,
+}
+
+impl fmt::Display for Location {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let path = self.file.strip_prefix(project_root()).unwrap().display().to_string();
+        let path = path.replace('\\', "/");
+        let name = self.file.file_name().unwrap();
+        write!(
+            f,
+            "https://github.com/rust-lang/rust-analyzer/blob/master/{}#L{}[{}]",
+            path,
+            self.line,
+            name.to_str().unwrap()
+        )
+    }
+}
+
+fn ensure_rustfmt(sh: &Shell) {
+    let version = cmd!(sh, "rustup run stable rustfmt --version").read().unwrap_or_default();
+    if !version.contains("stable") {
+        panic!(
+            "Failed to run rustfmt from toolchain 'stable'. \
+                 Please run `rustup component add rustfmt --toolchain stable` to install it.",
+        );
+    }
+}
+
+fn reformat(text: String) -> String {
+    let sh = Shell::new().unwrap();
+    ensure_rustfmt(&sh);
+    let rustfmt_toml = project_root().join("rustfmt.toml");
+    let mut stdout = cmd!(
+        sh,
+        "rustup run stable rustfmt --config-path {rustfmt_toml} --config fn_single_line=true"
+    )
+    .stdin(text)
+    .read()
+    .unwrap();
+    if !stdout.ends_with('\n') {
+        stdout.push('\n');
+    }
+    stdout
+}
+
+fn add_preamble(generator: &'static str, mut text: String) -> String {
+    let preamble = format!("//! Generated by `{generator}`, do not edit by hand.\n\n");
+    text.insert_str(0, &preamble);
+    text
+}
+
+/// Checks that the `file` has the specified `contents`. If that is not the
+/// case, updates the file and then fails the test.
+#[allow(clippy::print_stderr)]
+fn ensure_file_contents(file: &Path, contents: &str, check: bool) {
+    if let Ok(old_contents) = fs::read_to_string(file) {
+        if normalize_newlines(&old_contents) == normalize_newlines(contents) {
+            // File is already up to date.
+            return;
+        }
+    }
+
+    let display_path = file.strip_prefix(project_root()).unwrap_or(file);
+    if check {
+        panic!(
+            "{} was not up-to-date{}",
+            file.display(),
+            if std::env::var("CI").is_ok() {
+                "\n    NOTE: run `cargo codegen` locally and commit the updated files\n"
+            } else {
+                ""
+            }
+        );
+    } else {
+        eprintln!(
+            "\n\x1b[31;1merror\x1b[0m: {} was not up-to-date, updating\n",
+            display_path.display()
+        );
+
+        if let Some(parent) = file.parent() {
+            let _ = fs::create_dir_all(parent);
+        }
+        fs::write(file, contents).unwrap();
+    }
+}
+
+fn normalize_newlines(s: &str) -> String {
+    s.replace("\r\n", "\n")
+}
diff --git a/src/tools/rust-analyzer/xtask/src/codegen/assists_doc_tests.rs b/src/tools/rust-analyzer/xtask/src/codegen/assists_doc_tests.rs
new file mode 100644
index 00000000000..b2d89dde765
--- /dev/null
+++ b/src/tools/rust-analyzer/xtask/src/codegen/assists_doc_tests.rs
@@ -0,0 +1,197 @@
+//! Generates `assists.md` documentation.
+
+use std::{fmt, fs, path::Path};
+
+use stdx::format_to_acc;
+
+use crate::{
+    codegen::{
+        add_preamble, ensure_file_contents, list_rust_files, reformat, CommentBlock, Location,
+    },
+    project_root,
+};
+
+pub(crate) fn generate(check: bool) {
+    let assists = Assist::collect();
+
+    {
+        // Generate doctests.
+
+        let mut buf = "
+use super::check_doc_test;
+"
+        .to_owned();
+        for assist in assists.iter() {
+            for (idx, section) in assist.sections.iter().enumerate() {
+                let test_id =
+                    if idx == 0 { assist.id.clone() } else { format!("{}_{idx}", &assist.id) };
+                let test = format!(
+                    r######"
+#[test]
+fn doctest_{}() {{
+    check_doc_test(
+        "{}",
+r#####"
+{}"#####, r#####"
+{}"#####)
+}}
+"######,
+                    &test_id,
+                    &assist.id,
+                    reveal_hash_comments(&section.before),
+                    reveal_hash_comments(&section.after)
+                );
+
+                buf.push_str(&test)
+            }
+        }
+        let buf = add_preamble("sourcegen_assists_docs", reformat(buf));
+        ensure_file_contents(
+            &project_root().join("crates/ide-assists/src/tests/generated.rs"),
+            &buf,
+            check,
+        );
+    }
+
+    {
+        // Generate assists manual. Note that we do _not_ commit manual to the
+        // git repo. Instead, `cargo xtask release` runs this test before making
+        // a release.
+
+        let contents = add_preamble(
+            "sourcegen_assists_docs",
+            assists.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n"),
+        );
+        let dst = project_root().join("docs/user/generated_assists.adoc");
+        fs::write(dst, contents).unwrap();
+    }
+}
+
+#[derive(Debug)]
+struct Section {
+    doc: String,
+    before: String,
+    after: String,
+}
+
+#[derive(Debug)]
+struct Assist {
+    id: String,
+    location: Location,
+    sections: Vec<Section>,
+}
+
+impl Assist {
+    fn collect() -> Vec<Assist> {
+        let handlers_dir = project_root().join("crates/ide-assists/src/handlers");
+
+        let mut res = Vec::new();
+        for path in list_rust_files(&handlers_dir) {
+            collect_file(&mut res, path.as_path());
+        }
+        res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id));
+        return res;
+
+        fn collect_file(acc: &mut Vec<Assist>, path: &Path) {
+            let text = fs::read_to_string(path).unwrap();
+            let comment_blocks = CommentBlock::extract("Assist", &text);
+
+            for block in comment_blocks {
+                let id = block.id;
+                assert!(
+                    id.chars().all(|it| it.is_ascii_lowercase() || it == '_'),
+                    "invalid assist id: {id:?}"
+                );
+                let mut lines = block.contents.iter().peekable();
+                let location = Location { file: path.to_path_buf(), line: block.line };
+                let mut assist = Assist { id, location, sections: Vec::new() };
+
+                while lines.peek().is_some() {
+                    let doc = take_until(lines.by_ref(), "```").trim().to_owned();
+                    assert!(
+                        (doc.chars().next().unwrap().is_ascii_uppercase() && doc.ends_with('.'))
+                            || !assist.sections.is_empty(),
+                        "\n\n{}: assist docs should be proper sentences, with capitalization and a full stop at the end.\n\n{}\n\n",
+                        &assist.id,
+                        doc,
+                    );
+
+                    let before = take_until(lines.by_ref(), "```");
+
+                    assert_eq!(lines.next().unwrap().as_str(), "->");
+                    assert_eq!(lines.next().unwrap().as_str(), "```");
+                    let after = take_until(lines.by_ref(), "```");
+
+                    assist.sections.push(Section { doc, before, after });
+                }
+
+                acc.push(assist)
+            }
+        }
+
+        fn take_until<'a>(lines: impl Iterator<Item = &'a String>, marker: &str) -> String {
+            let mut buf = Vec::new();
+            for line in lines {
+                if line == marker {
+                    break;
+                }
+                buf.push(line.clone());
+            }
+            buf.join("\n")
+        }
+    }
+}
+
+impl fmt::Display for Assist {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let _ = writeln!(
+            f,
+            "[discrete]\n=== `{}`
+**Source:** {}",
+            self.id, self.location,
+        );
+
+        for section in &self.sections {
+            let before = section.before.replace("$0", "┃"); // Unicode pseudo-graphics bar
+            let after = section.after.replace("$0", "┃");
+            let _ = writeln!(
+                f,
+                "
+{}
+
+.Before
+```rust
+{}```
+
+.After
+```rust
+{}```",
+                section.doc,
+                hide_hash_comments(&before),
+                hide_hash_comments(&after)
+            );
+        }
+
+        Ok(())
+    }
+}
+
+fn hide_hash_comments(text: &str) -> String {
+    text.split('\n') // want final newline
+        .filter(|&it| !(it.starts_with("# ") || it == "#"))
+        .fold(String::new(), |mut acc, it| format_to_acc!(acc, "{it}\n"))
+}
+
+fn reveal_hash_comments(text: &str) -> String {
+    text.split('\n') // want final newline
+        .map(|it| {
+            if let Some(stripped) = it.strip_prefix("# ") {
+                stripped
+            } else if it == "#" {
+                ""
+            } else {
+                it
+            }
+        })
+        .fold(String::new(), |mut acc, it| format_to_acc!(acc, "{it}\n"))
+}
diff --git a/src/tools/rust-analyzer/xtask/src/codegen/diagnostics_docs.rs b/src/tools/rust-analyzer/xtask/src/codegen/diagnostics_docs.rs
new file mode 100644
index 00000000000..cf30531e7f9
--- /dev/null
+++ b/src/tools/rust-analyzer/xtask/src/codegen/diagnostics_docs.rs
@@ -0,0 +1,77 @@
+//! Generates `assists.md` documentation.
+
+use std::{fmt, fs, io, path::PathBuf};
+
+use crate::{
+    codegen::{add_preamble, list_rust_files, CommentBlock, Location},
+    project_root,
+};
+
+pub(crate) fn generate(check: bool) {
+    let diagnostics = Diagnostic::collect().unwrap();
+    if !check {
+        let contents =
+            diagnostics.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
+        let contents = add_preamble("sourcegen_diagnostic_docs", contents);
+        let dst = project_root().join("docs/user/generated_diagnostic.adoc");
+        fs::write(dst, contents).unwrap();
+    }
+}
+
+#[derive(Debug)]
+struct Diagnostic {
+    id: String,
+    location: Location,
+    doc: String,
+}
+
+impl Diagnostic {
+    fn collect() -> io::Result<Vec<Diagnostic>> {
+        let handlers_dir = project_root().join("crates/ide-diagnostics/src/handlers");
+
+        let mut res = Vec::new();
+        for path in list_rust_files(&handlers_dir) {
+            collect_file(&mut res, path)?;
+        }
+        res.sort_by(|lhs, rhs| lhs.id.cmp(&rhs.id));
+        return Ok(res);
+
+        fn collect_file(acc: &mut Vec<Diagnostic>, path: PathBuf) -> io::Result<()> {
+            let text = fs::read_to_string(&path)?;
+            let comment_blocks = CommentBlock::extract("Diagnostic", &text);
+
+            for block in comment_blocks {
+                let id = block.id;
+                if let Err(msg) = is_valid_diagnostic_name(&id) {
+                    panic!("invalid diagnostic name: {id:?}:\n  {msg}")
+                }
+                let doc = block.contents.join("\n");
+                let location = Location { file: path.clone(), line: block.line };
+                acc.push(Diagnostic { id, location, doc })
+            }
+
+            Ok(())
+        }
+    }
+}
+
+fn is_valid_diagnostic_name(diagnostic: &str) -> Result<(), String> {
+    let diagnostic = diagnostic.trim();
+    if diagnostic.find(char::is_whitespace).is_some() {
+        return Err("Diagnostic names can't contain whitespace symbols".into());
+    }
+    if diagnostic.chars().any(|c| c.is_ascii_uppercase()) {
+        return Err("Diagnostic names can't contain uppercase symbols".into());
+    }
+    if diagnostic.chars().any(|c| !c.is_ascii()) {
+        return Err("Diagnostic can't contain non-ASCII symbols".into());
+    }
+
+    Ok(())
+}
+
+impl fmt::Display for Diagnostic {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        writeln!(f, "=== {}\n**Source:** {}\n{}", self.id, self.location, self.doc)
+    }
+}
diff --git a/src/tools/rust-analyzer/xtask/src/codegen/lints.rs b/src/tools/rust-analyzer/xtask/src/codegen/lints.rs
new file mode 100644
index 00000000000..63abcfc0904
--- /dev/null
+++ b/src/tools/rust-analyzer/xtask/src/codegen/lints.rs
@@ -0,0 +1,342 @@
+//! Generates descriptor structures for unstable features from the unstable book
+//! and lints from rustc, rustdoc, and clippy.
+use std::{borrow::Cow, fs, path::Path};
+
+use stdx::format_to;
+use xshell::{cmd, Shell};
+
+use crate::{
+    codegen::{add_preamble, ensure_file_contents, list_files, reformat},
+    project_root,
+};
+
+const DESTINATION: &str = "crates/ide-db/src/generated/lints.rs";
+
+/// This clones rustc repo, and so is not worth to keep up-to-date on a constant basis.
+pub(crate) fn generate(check: bool) {
+    let sh = &Shell::new().unwrap();
+
+    let rust_repo = project_root().join("./target/rust");
+    if rust_repo.exists() {
+        cmd!(sh, "git -C {rust_repo} pull --rebase").run().unwrap();
+    } else {
+        cmd!(sh, "git clone --depth=1 https://github.com/rust-lang/rust {rust_repo}")
+            .run()
+            .unwrap();
+    }
+    // need submodules for Cargo to parse the workspace correctly
+    cmd!(
+        sh,
+        "git -C {rust_repo} submodule update --init --recursive --depth=1 --
+         compiler library src/tools"
+    )
+    .run()
+    .unwrap();
+
+    let mut contents = String::from(
+        r"
+#[derive(Clone)]
+pub struct Lint {
+    pub label: &'static str,
+    pub description: &'static str,
+}
+
+pub struct LintGroup {
+    pub lint: Lint,
+    pub children: &'static [&'static str],
+}
+
+",
+    );
+
+    generate_lint_descriptor(sh, &mut contents);
+    contents.push('\n');
+
+    let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned());
+    let unstable_book = project_root().join("./target/unstable-book-gen");
+    cmd!(
+        sh,
+        "{cargo} run --manifest-path {rust_repo}/src/tools/unstable-book-gen/Cargo.toml --
+         {rust_repo}/library {rust_repo}/compiler {rust_repo}/src {unstable_book}"
+    )
+    .run()
+    .unwrap();
+    generate_feature_descriptor(&mut contents, &unstable_book.join("src"));
+    contents.push('\n');
+
+    let lints_json = project_root().join("./target/clippy_lints.json");
+    cmd!(
+        sh,
+        "curl https://rust-lang.github.io/rust-clippy/master/lints.json --output {lints_json}"
+    )
+    .run()
+    .unwrap();
+    generate_descriptor_clippy(&mut contents, &lints_json);
+
+    let contents = add_preamble("sourcegen_lints", reformat(contents));
+
+    let destination = project_root().join(DESTINATION);
+    ensure_file_contents(destination.as_path(), &contents, check);
+}
+
+/// Parses the output of `rustdoc -Whelp` and prints `Lint` and `LintGroup` constants into `buf`.
+///
+/// As of writing, the output of `rustc -Whelp` (not rustdoc) has the following format:
+///
+/// ```text
+/// Lint checks provided by rustc:
+///
+/// name  default  meaning
+/// ----  -------  -------
+///
+/// ...
+///
+/// Lint groups provided by rustc:
+///
+/// name  sub-lints
+/// ----  ---------
+///
+/// ...
+/// ```
+///
+/// `rustdoc -Whelp` (and any other custom `rustc` driver) adds another two
+/// tables after the `rustc` ones, with a different title but the same format.
+fn generate_lint_descriptor(sh: &Shell, buf: &mut String) {
+    let stdout = cmd!(sh, "rustdoc -Whelp").read().unwrap();
+    let lints_pat = "----  -------  -------\n";
+    let lint_groups_pat = "----  ---------\n";
+    let lints = find_and_slice(&stdout, lints_pat);
+    let lint_groups = find_and_slice(lints, lint_groups_pat);
+    let lints_rustdoc = find_and_slice(lint_groups, lints_pat);
+    let lint_groups_rustdoc = find_and_slice(lints_rustdoc, lint_groups_pat);
+
+    buf.push_str(r#"pub const DEFAULT_LINTS: &[Lint] = &["#);
+    buf.push('\n');
+
+    let lints = lints.lines().take_while(|l| !l.is_empty()).map(|line| {
+        let (name, rest) = line.trim().split_once(char::is_whitespace).unwrap();
+        let (_default_level, description) = rest.trim().split_once(char::is_whitespace).unwrap();
+        (name.trim(), Cow::Borrowed(description.trim()), vec![])
+    });
+    let lint_groups = lint_groups.lines().take_while(|l| !l.is_empty()).map(|line| {
+        let (name, lints) = line.trim().split_once(char::is_whitespace).unwrap();
+        (
+            name.trim(),
+            format!("lint group for: {}", lints.trim()).into(),
+            lints
+                .split_ascii_whitespace()
+                .map(|s| s.trim().trim_matches(',').replace('-', "_"))
+                .collect(),
+        )
+    });
+
+    let mut lints = lints.chain(lint_groups).collect::<Vec<_>>();
+    lints.sort_by(|(ident, ..), (ident2, ..)| ident.cmp(ident2));
+
+    for (name, description, ..) in &lints {
+        push_lint_completion(buf, &name.replace('-', "_"), description);
+    }
+    buf.push_str("];\n\n");
+
+    buf.push_str(r#"pub const DEFAULT_LINT_GROUPS: &[LintGroup] = &["#);
+    for (name, description, children) in &lints {
+        if !children.is_empty() {
+            // HACK: warnings is emitted with a general description, not with its members
+            if name == &"warnings" {
+                push_lint_group(buf, name, description, &Vec::new());
+                continue;
+            }
+            push_lint_group(buf, &name.replace('-', "_"), description, children);
+        }
+    }
+    buf.push('\n');
+    buf.push_str("];\n");
+
+    // rustdoc
+
+    buf.push('\n');
+    buf.push_str(r#"pub const RUSTDOC_LINTS: &[Lint] = &["#);
+    buf.push('\n');
+
+    let lints_rustdoc = lints_rustdoc.lines().take_while(|l| !l.is_empty()).map(|line| {
+        let (name, rest) = line.trim().split_once(char::is_whitespace).unwrap();
+        let (_default_level, description) = rest.trim().split_once(char::is_whitespace).unwrap();
+        (name.trim(), Cow::Borrowed(description.trim()), vec![])
+    });
+    let lint_groups_rustdoc =
+        lint_groups_rustdoc.lines().take_while(|l| !l.is_empty()).map(|line| {
+            let (name, lints) = line.trim().split_once(char::is_whitespace).unwrap();
+            (
+                name.trim(),
+                format!("lint group for: {}", lints.trim()).into(),
+                lints
+                    .split_ascii_whitespace()
+                    .map(|s| s.trim().trim_matches(',').replace('-', "_"))
+                    .collect(),
+            )
+        });
+
+    let mut lints_rustdoc = lints_rustdoc.chain(lint_groups_rustdoc).collect::<Vec<_>>();
+    lints_rustdoc.sort_by(|(ident, ..), (ident2, ..)| ident.cmp(ident2));
+
+    for (name, description, ..) in &lints_rustdoc {
+        push_lint_completion(buf, &name.replace('-', "_"), description)
+    }
+    buf.push_str("];\n\n");
+
+    buf.push_str(r#"pub const RUSTDOC_LINT_GROUPS: &[LintGroup] = &["#);
+    for (name, description, children) in &lints_rustdoc {
+        if !children.is_empty() {
+            push_lint_group(buf, &name.replace('-', "_"), description, children);
+        }
+    }
+    buf.push('\n');
+    buf.push_str("];\n");
+}
+
+#[track_caller]
+fn find_and_slice<'a>(i: &'a str, p: &str) -> &'a str {
+    let idx = i.find(p).unwrap();
+    &i[idx + p.len()..]
+}
+
+/// Parses the unstable book `src_dir` and prints a constant with the list of
+/// unstable features into `buf`.
+///
+/// It does this by looking for all `.md` files in the `language-features` and
+/// `library-features` directories, and using the file name as the feature
+/// name, and the file contents as the feature description.
+fn generate_feature_descriptor(buf: &mut String, src_dir: &Path) {
+    let mut features = ["language-features", "library-features"]
+        .into_iter()
+        .flat_map(|it| list_files(&src_dir.join(it)))
+        // Get all `.md` files
+        .filter(|path| path.extension() == Some("md".as_ref()))
+        .map(|path| {
+            let feature_ident = path.file_stem().unwrap().to_str().unwrap().replace('-', "_");
+            let doc = fs::read_to_string(path).unwrap();
+            (feature_ident, doc)
+        })
+        .collect::<Vec<_>>();
+    features.sort_by(|(feature_ident, _), (feature_ident2, _)| feature_ident.cmp(feature_ident2));
+
+    buf.push_str(r#"pub const FEATURES: &[Lint] = &["#);
+    for (feature_ident, doc) in features.into_iter() {
+        push_lint_completion(buf, &feature_ident, &doc)
+    }
+    buf.push('\n');
+    buf.push_str("];\n");
+}
+
+#[derive(Default)]
+struct ClippyLint {
+    help: String,
+    id: String,
+}
+
+fn unescape(s: &str) -> String {
+    s.replace(r#"\""#, "").replace(r#"\n"#, "\n").replace(r#"\r"#, "")
+}
+
+#[allow(clippy::print_stderr)]
+fn generate_descriptor_clippy(buf: &mut String, path: &Path) {
+    let file_content = std::fs::read_to_string(path).unwrap();
+    let mut clippy_lints: Vec<ClippyLint> = Vec::new();
+    let mut clippy_groups: std::collections::BTreeMap<String, Vec<String>> = Default::default();
+
+    for line in file_content.lines().map(str::trim) {
+        if let Some(line) = line.strip_prefix(r#""id": ""#) {
+            let clippy_lint = ClippyLint {
+                id: line.strip_suffix(r#"","#).expect("should be suffixed by comma").into(),
+                help: String::new(),
+            };
+            clippy_lints.push(clippy_lint)
+        } else if let Some(line) = line.strip_prefix(r#""group": ""#) {
+            if let Some(group) = line.strip_suffix("\",") {
+                clippy_groups
+                    .entry(group.to_owned())
+                    .or_default()
+                    .push(clippy_lints.last().unwrap().id.clone());
+            }
+        } else if let Some(line) = line.strip_prefix(r#""docs": ""#) {
+            let header = "### What it does";
+            let line = match line.find(header) {
+                Some(idx) => &line[idx + header.len()..],
+                None => {
+                    let id = &clippy_lints.last().unwrap().id;
+                    // these just don't have the common header
+                    let allowed = ["allow_attributes", "read_line_without_trim"];
+                    if allowed.contains(&id.as_str()) {
+                        line
+                    } else {
+                        eprintln!("\nunexpected clippy prefix for {id}, line={line:?}\n",);
+                        continue;
+                    }
+                }
+            };
+            // Only take the description, any more than this is a lot of additional data we would embed into the exe
+            // which seems unnecessary
+            let up_to = line.find(r#"###"#).expect("no second section found?");
+            let line = &line[..up_to];
+
+            let clippy_lint = clippy_lints.last_mut().expect("clippy lint must already exist");
+            clippy_lint.help = unescape(line).trim().to_owned();
+        }
+    }
+    clippy_lints.sort_by(|lint, lint2| lint.id.cmp(&lint2.id));
+
+    buf.push_str(r#"pub const CLIPPY_LINTS: &[Lint] = &["#);
+    buf.push('\n');
+    for clippy_lint in clippy_lints.into_iter() {
+        let lint_ident = format!("clippy::{}", clippy_lint.id);
+        let doc = clippy_lint.help;
+        push_lint_completion(buf, &lint_ident, &doc);
+    }
+    buf.push_str("];\n");
+
+    buf.push_str(r#"pub const CLIPPY_LINT_GROUPS: &[LintGroup] = &["#);
+    for (id, children) in clippy_groups {
+        let children = children.iter().map(|id| format!("clippy::{id}")).collect::<Vec<_>>();
+        if !children.is_empty() {
+            let lint_ident = format!("clippy::{id}");
+            let description = format!("lint group for: {}", children.join(", "));
+            push_lint_group(buf, &lint_ident, &description, &children);
+        }
+    }
+    buf.push('\n');
+    buf.push_str("];\n");
+}
+
+fn push_lint_completion(buf: &mut String, label: &str, description: &str) {
+    format_to!(
+        buf,
+        r###"    Lint {{
+        label: "{}",
+        description: r##"{}"##,
+    }},"###,
+        label,
+        description,
+    );
+}
+
+fn push_lint_group(buf: &mut String, label: &str, description: &str, children: &[String]) {
+    buf.push_str(
+        r###"    LintGroup {
+        lint:
+        "###,
+    );
+
+    push_lint_completion(buf, label, description);
+
+    let children = format!(
+        "&[{}]",
+        children.iter().map(|it| format!("\"{it}\"")).collect::<Vec<_>>().join(", ")
+    );
+    format_to!(
+        buf,
+        r###"
+        children: {},
+        }},"###,
+        children,
+    );
+}
diff --git a/src/tools/rust-analyzer/xtask/src/flags.rs b/src/tools/rust-analyzer/xtask/src/flags.rs
index e234090a07c..681c588bd01 100644
--- a/src/tools/rust-analyzer/xtask/src/flags.rs
+++ b/src/tools/rust-analyzer/xtask/src/flags.rs
@@ -52,6 +52,11 @@ xflags::xflags! {
         cmd bb {
             required suffix: String
         }
+
+        cmd codegen {
+            optional codegen_type: CodegenType
+            optional --check
+        }
     }
 }
 
@@ -73,8 +78,36 @@ pub enum XtaskCmd {
     PublishReleaseNotes(PublishReleaseNotes),
     Metrics(Metrics),
     Bb(Bb),
+    Codegen(Codegen),
+}
+
+#[derive(Debug)]
+pub struct Codegen {
+    pub check: bool,
+    pub codegen_type: Option<CodegenType>,
+}
+
+#[derive(Debug, Default)]
+pub enum CodegenType {
+    #[default]
+    All,
+    AssistsDocTests,
+    DiagnosticsDocs,
+    LintDefinitions,
 }
 
+impl FromStr for CodegenType {
+    type Err = String;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "all" => Ok(Self::All),
+            "assists-doc-tests" => Ok(Self::AssistsDocTests),
+            "diagnostics-docs" => Ok(Self::DiagnosticsDocs),
+            "lints-definitions" => Ok(Self::LintDefinitions),
+            _ => Err("Invalid option".to_owned()),
+        }
+    }
+}
 #[derive(Debug)]
 pub struct Install {
     pub client: bool,
diff --git a/src/tools/rust-analyzer/xtask/src/main.rs b/src/tools/rust-analyzer/xtask/src/main.rs
index df4d9810e6f..9418675a348 100644
--- a/src/tools/rust-analyzer/xtask/src/main.rs
+++ b/src/tools/rust-analyzer/xtask/src/main.rs
@@ -13,6 +13,7 @@
 
 mod flags;
 
+mod codegen;
 mod dist;
 mod install;
 mod metrics;
@@ -20,10 +21,7 @@ mod publish;
 mod release;
 
 use anyhow::bail;
-use std::{
-    env,
-    path::{Path, PathBuf},
-};
+use std::{env, path::PathBuf};
 use xshell::{cmd, Shell};
 
 fn main() -> anyhow::Result<()> {
@@ -40,6 +38,7 @@ fn main() -> anyhow::Result<()> {
         flags::XtaskCmd::Dist(cmd) => cmd.run(sh),
         flags::XtaskCmd::PublishReleaseNotes(cmd) => cmd.run(sh),
         flags::XtaskCmd::Metrics(cmd) => cmd.run(sh),
+        flags::XtaskCmd::Codegen(cmd) => cmd.run(sh),
         flags::XtaskCmd::Bb(cmd) => {
             {
                 let _d = sh.push_dir("./crates/rust-analyzer");
@@ -54,14 +53,11 @@ fn main() -> anyhow::Result<()> {
     }
 }
 
+/// Returns the path to the root directory of `rust-analyzer` project.
 fn project_root() -> PathBuf {
-    Path::new(
-        &env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| env!("CARGO_MANIFEST_DIR").to_owned()),
-    )
-    .ancestors()
-    .nth(1)
-    .unwrap()
-    .to_path_buf()
+    let dir =
+        env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| env!("CARGO_MANIFEST_DIR").to_owned());
+    PathBuf::from(dir).parent().unwrap().to_owned()
 }
 
 fn run_fuzzer(sh: &Shell) -> anyhow::Result<()> {
diff --git a/src/tools/rust-analyzer/xtask/src/release.rs b/src/tools/rust-analyzer/xtask/src/release.rs
index 4a306914778..1a5e6dfb4cc 100644
--- a/src/tools/rust-analyzer/xtask/src/release.rs
+++ b/src/tools/rust-analyzer/xtask/src/release.rs
@@ -2,7 +2,7 @@ mod changelog;
 
 use xshell::{cmd, Shell};
 
-use crate::{date_iso, flags, is_release_tag, project_root};
+use crate::{codegen, date_iso, flags, is_release_tag, project_root};
 
 impl flags::Release {
     pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
@@ -23,8 +23,8 @@ impl flags::Release {
         }
 
         // Generates bits of manual.adoc.
-        cmd!(sh, "cargo test -p ide-assists -p ide-diagnostics -p rust-analyzer -- sourcegen_")
-            .run()?;
+        codegen::diagnostics_docs::generate(false);
+        codegen::assists_doc_tests::generate(false);
 
         let website_root = project_root().join("../rust-analyzer.github.io");
         {