Add a new component, `rust-json-docs`, to distribute the JSON-formatted documentation for std crates in nightly toolchains.

We also add a new flag to `x doc`, `--json`, to render the JSON-formatted version alongside the HTML-formatted one.

1 files changed, 153 insertions, 59 deletions

diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs
index f909ecc0ab8..a54481f39e0 100644
--- a/src/bootstrap/doc.rs
+++ b/src/bootstrap/doc.rs
@@ -7,6 +7,7 @@
 //! Everything here is basically just a shim around calling either `rustbook` or
 //! `rustdoc`.
 
+use std::ffi::OsStr;
 use std::fs;
 use std::io;
 use std::path::{Path, PathBuf};
@@ -431,49 +432,24 @@ impl Step for Std {
     fn run(self, builder: &Builder<'_>) {
         let stage = self.stage;
         let target = self.target;
-        builder.info(&format!("Documenting stage{} std ({})", stage, target));
-        if builder.no_std(target) == Some(true) {
-            panic!(
-                "building std documentation for no_std target {target} is not supported\n\
-                 Set `docs = false` in the config to disable documentation."
-            );
-        }
         let out = builder.doc_out(target);
         t!(fs::create_dir_all(&out));
-        let compiler = builder.compiler(stage, builder.config.build);
-
-        let out_dir = builder.stage_out(compiler, Mode::Std).join(target.triple).join("doc");
-
         t!(fs::copy(builder.src.join("src/doc/rust.css"), out.join("rust.css")));
 
-        let run_cargo_rustdoc_for = |package: &str| {
-            let mut cargo =
-                builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "rustdoc");
-            compile::std_cargo(builder, target, compiler.stage, &mut cargo);
-
-            cargo
-                .arg("-p")
-                .arg(package)
-                .arg("-Zskip-rustdoc-fingerprint")
-                .arg("--")
-                .arg("--markdown-css")
-                .arg("rust.css")
-                .arg("--markdown-no-toc")
-                .arg("-Z")
-                .arg("unstable-options")
-                .arg("--resource-suffix")
-                .arg(&builder.version)
-                .arg("--index-page")
-                .arg(&builder.src.join("src/doc/index.md"));
-
-            if !builder.config.docs_minification {
-                cargo.arg("--disable-minification");
-            }
-
-            builder.run(&mut cargo.into());
-        };
+        let index_page = builder.src.join("src/doc/index.md").into_os_string();
+        let mut extra_args = vec![
+            OsStr::new("--markdown-css"),
+            OsStr::new("rust.css"),
+            OsStr::new("--markdown-no-toc"),
+            OsStr::new("--index-page"),
+            &index_page,
+        ];
+
+        if !builder.config.docs_minification {
+            extra_args.push(OsStr::new("--disable-minification"));
+        }
 
-        let paths = builder
+        let requested_crates = builder
             .paths
             .iter()
             .map(components_simplified)
@@ -491,30 +467,20 @@ impl Step for Std {
             })
             .collect::<Vec<_>>();
 
-        // Only build the following crates. While we could just iterate over the
-        // folder structure, that would also build internal crates that we do
-        // not want to show in documentation. These crates will later be visited
-        // by the rustc step, so internal documentation will show them.
-        //
-        // Note that the order here is important! The crates need to be
-        // processed starting from the leaves, otherwise rustdoc will not
-        // create correct links between crates because rustdoc depends on the
-        // existence of the output directories to know if it should be a local
-        // or remote link.
-        let krates = ["core", "alloc", "std", "proc_macro", "test"];
-        for krate in &krates {
-            run_cargo_rustdoc_for(krate);
-            if paths.iter().any(|p| p == krate) {
-                // No need to document more of the libraries if we have the one we want.
-                break;
-            }
-        }
-        builder.cp_r(&out_dir, &out);
+        doc_std(
+            builder,
+            DocumentationFormat::HTML,
+            stage,
+            target,
+            &out,
+            &extra_args,
+            &requested_crates,
+        );
 
         // Look for library/std, library/core etc in the `x.py doc` arguments and
         // open the corresponding rendered docs.
-        for requested_crate in paths {
-            if krates.iter().any(|k| *k == requested_crate.as_str()) {
+        for requested_crate in requested_crates {
+            if STD_PUBLIC_CRATES.iter().any(|k| *k == requested_crate.as_str()) {
                 let index = out.join(requested_crate).join("index.html");
                 open(builder, &index);
             }
@@ -523,6 +489,134 @@ impl Step for Std {
 }
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct JsonStd {
+    pub stage: u32,
+    pub target: TargetSelection,
+}
+
+impl Step for JsonStd {
+    type Output = ();
+    const DEFAULT: bool = false;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        let default = run.builder.config.docs && run.builder.config.cmd.json();
+        run.all_krates("test").path("library").default_condition(default)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Std { stage: run.builder.top_stage, target: run.target });
+    }
+
+    /// Build JSON documentation for the standard library crates.
+    ///
+    /// This is largely just a wrapper around `cargo doc`.
+    fn run(self, builder: &Builder<'_>) {
+        let stage = self.stage;
+        let target = self.target;
+        let out = builder.json_doc_out(target);
+        t!(fs::create_dir_all(&out));
+        let extra_args = [OsStr::new("--output-format"), OsStr::new("json")];
+        doc_std(builder, DocumentationFormat::JSON, stage, target, &out, &extra_args, &[])
+    }
+}
+
+/// Name of the crates that are visible to consumers of the standard library.
+/// Documentation for internal crates is handled by the rustc step, so internal crates will show
+/// up there.
+///
+/// Order here is important!
+/// Crates need to be processed starting from the leaves, otherwise rustdoc will not
+/// create correct links between crates because rustdoc depends on the
+/// existence of the output directories to know if it should be a local
+/// or remote link.
+const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"];
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+enum DocumentationFormat {
+    HTML,
+    JSON,
+}
+
+impl DocumentationFormat {
+    fn as_str(&self) -> &str {
+        match self {
+            DocumentationFormat::HTML => "HTML",
+            DocumentationFormat::JSON => "JSON",
+        }
+    }
+}
+
+/// Build the documentation for public standard library crates.
+///
+/// `requested_crates` can be used to build only a subset of the crates. If empty, all crates will
+/// be built.
+fn doc_std(
+    builder: &Builder<'_>,
+    format: DocumentationFormat,
+    stage: u32,
+    target: TargetSelection,
+    out: &Path,
+    extra_args: &[&OsStr],
+    requested_crates: &[String],
+) {
+    builder.info(&format!(
+        "Documenting stage{} std ({}) in {} format",
+        stage,
+        target,
+        format.as_str()
+    ));
+    if builder.no_std(target) == Some(true) {
+        panic!(
+            "building std documentation for no_std target {target} is not supported\n\
+             Set `docs = false` in the config to disable documentation."
+        );
+    }
+    let compiler = builder.compiler(stage, builder.config.build);
+    // This is directory where the compiler will place the output of the command.
+    // We will then copy the files from this directory into the final `out` directory, the specified
+    // as a function parameter.
+    let out_dir = builder.stage_out(compiler, Mode::Std).join(target.triple).join("doc");
+    // `cargo` uses the same directory for both JSON docs and HTML docs.
+    // This could lead to cross-contamination when copying files into the specified `out` directory.
+    // For example:
+    // ```bash
+    // x doc std
+    // x doc std --json
+    // ```
+    // could lead to HTML docs being copied into the JSON docs output directory.
+    // To avoid this issue, we clean the doc folder before invoking `cargo`.
+    if out_dir.exists() {
+        builder.remove_dir(&out_dir);
+    }
+
+    let run_cargo_rustdoc_for = |package: &str| {
+        let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "rustdoc");
+        compile::std_cargo(builder, target, compiler.stage, &mut cargo);
+        cargo
+            .arg("-p")
+            .arg(package)
+            .arg("-Zskip-rustdoc-fingerprint")
+            .arg("--")
+            .arg("-Z")
+            .arg("unstable-options")
+            .arg("--resource-suffix")
+            .arg(&builder.version)
+            .args(extra_args);
+        builder.run(&mut cargo.into());
+    };
+
+    for krate in STD_PUBLIC_CRATES {
+        run_cargo_rustdoc_for(krate);
+        if requested_crates.iter().any(|p| p == krate) {
+            // No need to document more of the libraries if we have the one we want.
+            break;
+        }
+    }
+
+    builder.cp_r(&out_dir, &out);
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Rustc {
     pub stage: u32,
     pub target: TargetSelection,