Rollup merge of #143615 - Kobzol:doc-std, r=jieyouxu

Fix handling of no_std targets in `doc::Std` step

The previous logic was wrong for no_std targets, it just didn't do anything. The logic was added there because by default, the `Std` step would otherwise have a list of all std crates to check, but these would fail for no_std targets. What has to happen instead is to select the default set of packages to check/doc/build, which currently happens in the `std_cargo` function, but the `self.crates` list was overriding that.

In general, using `crates: Vec<String>` in the `Std` steps is quite fishy, because it's difficult to distinguish between all crates (either they are all enumerated or `crates` is empty) and the default (e.g. `x <kind> [library]`) vs a subset (e.g. `x <kind> core`). I wanted to improve that using an enum that would distinguish these situations, avoid passing `-p` for all of the crates explicitly, and unify the selection of packages to compile/check/... in `std_cargo`, based on this enum.

However, I found out from some other bootstrap comments that when you pass `-p` explicitly for all crates, cargo behaves differently (apparently for check it will also check targets/examples etc. with `-p`, but not without it). Furthermore, the doc step has a special case where it does not document the `sysroot` package. So as usually, unifying this logic would get into some edge cases... So instead I opted for a seemingly simpler solution, where I try to prefilter only two allowed crates (core and alloc) for no_std targets in the `std_crates_for_run_make` function.

It's not perfect, but I think it's better than the status quo (words to live by when working on bootstrap...).

Fixes [this Zulip topic](https://rust-lang.zulipchat.com/#narrow/channel/326414-t-infra.2Fbootstrap/topic/docs.20for.20non-host.20targets.3F).

r? `@jieyouxu`

5 files changed, 174 insertions, 68 deletions

diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
index 0587d21ecc2..b721911de7e 100644
--- a/src/bootstrap/src/core/build_steps/compile.rs
+++ b/src/bootstrap/src/core/build_steps/compile.rs
@@ -449,26 +449,24 @@ fn copy_self_contained_objects(
     target_deps
 }
 
-/// Resolves standard library crates for `Std::run_make` for any build kind (like check, build, clippy, etc.).
+/// Resolves standard library crates for `Std::run_make` for any build kind (like check, doc,
+/// build, clippy, etc.).
 pub fn std_crates_for_run_make(run: &RunConfig<'_>) -> Vec<String> {
-    // FIXME: Extend builder tests to cover the `crates` field of `Std` instances.
-    if cfg!(test) {
-        return vec![];
-    }
-
-    let has_alias = run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library"));
+    let mut crates = run.make_run_crates(builder::Alias::Library);
+
+    // For no_std targets, we only want to check core and alloc
+    // Regardless of core/alloc being selected explicitly or via the "library" default alias,
+    // we only want to keep these two crates.
+    // The set of no_std crates should be kept in sync with what `Builder::std_cargo` does.
+    // Note: an alternative design would be to return an enum from this function (Default vs Subset)
+    // of crates. However, several steps currently pass `-p <package>` even if all crates are
+    // selected, because Cargo behaves differently in that case. To keep that behavior without
+    // making further changes, we pre-filter the no-std crates here.
     let target_is_no_std = run.builder.no_std(run.target).unwrap_or(false);
-
-    // For no_std targets, do not add any additional crates to the compilation other than what `compile::std_cargo` already adds for no_std targets.
     if target_is_no_std {
-        vec![]
-    }
-    // If the paths include "library", build the entire standard library.
-    else if has_alias {
-        run.make_run_crates(builder::Alias::Library)
-    } else {
-        run.cargo_crates_in_set()
+        crates.retain(|c| c == "core" || c == "alloc");
     }
+    crates
 }
 
 /// Tries to find LLVM's `compiler-rt` source directory, for building `library/profiler_builtins`.
diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs
index f7c4c5ad0bb..48dde4aae68 100644
--- a/src/bootstrap/src/core/build_steps/doc.rs
+++ b/src/bootstrap/src/core/build_steps/doc.rs
@@ -665,7 +665,11 @@ impl Step for Std {
     }
 
     fn metadata(&self) -> Option<StepMetadata> {
-        Some(StepMetadata::doc("std", self.target).stage(self.stage))
+        Some(
+            StepMetadata::doc("std", self.target)
+                .stage(self.stage)
+                .with_metadata(format!("crates=[{}]", self.crates.join(","))),
+        )
     }
 }
 
diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs
index 7464327fde9..4d606953d99 100644
--- a/src/bootstrap/src/core/builder/mod.rs
+++ b/src/bootstrap/src/core/builder/mod.rs
@@ -146,6 +146,8 @@ pub struct StepMetadata {
     target: TargetSelection,
     built_by: Option<Compiler>,
     stage: Option<u32>,
+    /// Additional opaque string printed in the metadata
+    metadata: Option<String>,
 }
 
 impl StepMetadata {
@@ -170,7 +172,7 @@ impl StepMetadata {
     }
 
     fn new(name: &'static str, target: TargetSelection, kind: Kind) -> Self {
-        Self { name, kind, target, built_by: None, stage: None }
+        Self { name, kind, target, built_by: None, stage: None, metadata: None }
     }
 
     pub fn built_by(mut self, compiler: Compiler) -> Self {
@@ -183,6 +185,11 @@ impl StepMetadata {
         self
     }
 
+    pub fn with_metadata(mut self, metadata: String) -> Self {
+        self.metadata = Some(metadata);
+        self
+    }
+
     pub fn get_stage(&self) -> Option<u32> {
         self.stage.or(self
             .built_by
diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs
index bbcb58fca14..68d1885b0a7 100644
--- a/src/bootstrap/src/core/builder/tests.rs
+++ b/src/bootstrap/src/core/builder/tests.rs
@@ -637,8 +637,8 @@ mod snapshot {
 
     use crate::core::build_steps::{compile, dist, doc, test, tool};
     use crate::core::builder::tests::{
-        TEST_TRIPLE_1, TEST_TRIPLE_2, TEST_TRIPLE_3, configure, configure_with_args, first,
-        host_target, render_steps, run_build,
+        RenderConfig, TEST_TRIPLE_1, TEST_TRIPLE_2, TEST_TRIPLE_3, configure, configure_with_args,
+        first, host_target, render_steps, run_build,
     };
     use crate::core::builder::{Builder, Kind, StepDescription, StepMetadata};
     use crate::core::config::TargetSelection;
@@ -994,7 +994,7 @@ mod snapshot {
         [build] llvm <host>
         [build] rustc 0 <host> -> rustc 1 <host>
         [build] rustdoc 0 <host>
-        [doc] std 1 <host>
+        [doc] std 1 <host> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
         ");
     }
 
@@ -1043,12 +1043,12 @@ mod snapshot {
         [build] rustc 1 <host> -> std 1 <host>
         [build] rustc 1 <host> -> rustc 2 <host>
         [build] rustdoc 1 <host>
-        [doc] std 2 <host>
+        [doc] std 2 <host> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
         [build] rustc 2 <host> -> std 2 <host>
         [build] rustc 0 <host> -> LintDocs 1 <host>
         [build] rustc 0 <host> -> RustInstaller 1 <host>
         [dist] docs <host>
-        [doc] std 2 <host>
+        [doc] std 2 <host> crates=[]
         [dist] mingw <host>
         [build] rustc 0 <host> -> GenerateCopyright 1 <host>
         [dist] rustc <host>
@@ -1075,12 +1075,12 @@ mod snapshot {
         [build] rustc 1 <host> -> rustc 2 <host>
         [build] rustc 1 <host> -> WasmComponentLd 2 <host>
         [build] rustdoc 1 <host>
-        [doc] std 2 <host>
+        [doc] std 2 <host> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
         [build] rustc 2 <host> -> std 2 <host>
         [build] rustc 0 <host> -> LintDocs 1 <host>
         [build] rustc 0 <host> -> RustInstaller 1 <host>
         [dist] docs <host>
-        [doc] std 2 <host>
+        [doc] std 2 <host> crates=[]
         [dist] mingw <host>
         [build] rustc 0 <host> -> GenerateCopyright 1 <host>
         [dist] rustc <host>
@@ -1112,15 +1112,15 @@ mod snapshot {
         [build] rustc 1 <host> -> std 1 <host>
         [build] rustc 1 <host> -> rustc 2 <host>
         [build] rustdoc 1 <host>
-        [doc] std 2 <host>
-        [doc] std 2 <target1>
+        [doc] std 2 <host> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
+        [doc] std 2 <target1> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
         [build] rustc 2 <host> -> std 2 <host>
         [build] rustc 0 <host> -> LintDocs 1 <host>
         [build] rustc 0 <host> -> RustInstaller 1 <host>
         [dist] docs <host>
         [dist] docs <target1>
-        [doc] std 2 <host>
-        [doc] std 2 <target1>
+        [doc] std 2 <host> crates=[]
+        [doc] std 2 <target1> crates=[]
         [dist] mingw <host>
         [dist] mingw <target1>
         [build] rustc 0 <host> -> GenerateCopyright 1 <host>
@@ -1149,14 +1149,14 @@ mod snapshot {
         [build] rustc 1 <host> -> std 1 <host>
         [build] rustc 1 <host> -> rustc 2 <host>
         [build] rustdoc 1 <host>
-        [doc] std 2 <host>
+        [doc] std 2 <host> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
         [build] rustc 2 <host> -> std 2 <host>
         [build] rustc 0 <host> -> LintDocs 1 <host>
         [build] rustc 1 <host> -> std 1 <target1>
         [build] rustc 2 <host> -> std 2 <target1>
         [build] rustc 0 <host> -> RustInstaller 1 <host>
         [dist] docs <host>
-        [doc] std 2 <host>
+        [doc] std 2 <host> crates=[]
         [dist] mingw <host>
         [build] rustc 0 <host> -> GenerateCopyright 1 <host>
         [dist] rustc <host>
@@ -1186,8 +1186,8 @@ mod snapshot {
         [build] rustc 1 <host> -> std 1 <host>
         [build] rustc 1 <host> -> rustc 2 <host>
         [build] rustdoc 1 <host>
-        [doc] std 2 <host>
-        [doc] std 2 <target1>
+        [doc] std 2 <host> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
+        [doc] std 2 <target1> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
         [build] rustc 2 <host> -> std 2 <host>
         [build] rustc 0 <host> -> LintDocs 1 <host>
         [build] rustc 1 <host> -> std 1 <target1>
@@ -1195,8 +1195,8 @@ mod snapshot {
         [build] rustc 0 <host> -> RustInstaller 1 <host>
         [dist] docs <host>
         [dist] docs <target1>
-        [doc] std 2 <host>
-        [doc] std 2 <target1>
+        [doc] std 2 <host> crates=[]
+        [doc] std 2 <target1> crates=[]
         [dist] mingw <host>
         [dist] mingw <target1>
         [build] rustc 0 <host> -> GenerateCopyright 1 <host>
@@ -1228,11 +1228,11 @@ mod snapshot {
         [build] rustc 1 <host> -> std 1 <host>
         [build] rustc 1 <host> -> rustc 2 <host>
         [build] rustdoc 1 <host>
-        [doc] std 2 <target1>
+        [doc] std 2 <target1> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
         [build] rustc 2 <host> -> std 2 <host>
         [build] rustc 0 <host> -> RustInstaller 1 <host>
         [dist] docs <target1>
-        [doc] std 2 <target1>
+        [doc] std 2 <target1> crates=[]
         [dist] mingw <target1>
         [build] rustc 2 <host> -> std 2 <target1>
         [dist] rustc 2 <host> -> std 2 <target1>
@@ -1260,14 +1260,14 @@ mod snapshot {
         [build] rustc 1 <host> -> rustc 2 <host>
         [build] rustc 1 <host> -> WasmComponentLd 2 <host>
         [build] rustdoc 1 <host>
-        [doc] std 2 <target1>
+        [doc] std 2 <target1> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
         [build] rustc 2 <host> -> std 2 <host>
         [build] rustc 1 <host> -> std 1 <target1>
         [build] rustc 2 <host> -> std 2 <target1>
         [build] rustc 0 <host> -> LintDocs 1 <host>
         [build] rustc 0 <host> -> RustInstaller 1 <host>
         [dist] docs <target1>
-        [doc] std 2 <target1>
+        [doc] std 2 <target1> crates=[]
         [dist] mingw <target1>
         [build] llvm <target1>
         [build] rustc 1 <host> -> rustc 2 <target1>
@@ -1575,6 +1575,80 @@ mod snapshot {
         steps.assert_contains(StepMetadata::test("CrateLibrustc", host));
         steps.assert_contains_fuzzy(StepMetadata::build("rustc", host));
     }
+
+    #[test]
+    fn doc_library() {
+        let ctx = TestCtx::new();
+        insta::assert_snapshot!(
+            ctx.config("doc")
+                .path("library")
+                .render_steps(), @r"
+        [build] llvm <host>
+        [build] rustc 0 <host> -> rustc 1 <host>
+        [build] rustdoc 0 <host>
+        [doc] std 1 <host> crates=[alloc,compiler_builtins,core,panic_abort,panic_unwind,proc_macro,std,sysroot,test,unwind]
+        ");
+    }
+
+    #[test]
+    fn doc_core() {
+        let ctx = TestCtx::new();
+        insta::assert_snapshot!(
+            ctx.config("doc")
+                .path("core")
+                .render_steps(), @r"
+        [build] llvm <host>
+        [build] rustc 0 <host> -> rustc 1 <host>
+        [build] rustdoc 0 <host>
+        [doc] std 1 <host> crates=[core]
+        ");
+    }
+
+    #[test]
+    fn doc_core_no_std_target() {
+        let ctx = TestCtx::new();
+        insta::assert_snapshot!(
+            ctx.config("doc")
+                .path("core")
+                .override_target_no_std(&host_target())
+                .render_steps(), @r"
+        [build] llvm <host>
+        [build] rustc 0 <host> -> rustc 1 <host>
+        [build] rustdoc 0 <host>
+        [doc] std 1 <host> crates=[core]
+        ");
+    }
+
+    #[test]
+    fn doc_library_no_std_target() {
+        let ctx = TestCtx::new();
+        insta::assert_snapshot!(
+            ctx.config("doc")
+                .path("library")
+                .override_target_no_std(&host_target())
+                .render_steps(), @r"
+        [build] llvm <host>
+        [build] rustc 0 <host> -> rustc 1 <host>
+        [build] rustdoc 0 <host>
+        [doc] std 1 <host> crates=[alloc,core]
+        ");
+    }
+
+    #[test]
+    fn doc_library_no_std_target_cross_compile() {
+        let ctx = TestCtx::new();
+        insta::assert_snapshot!(
+            ctx.config("doc")
+                .path("library")
+                .targets(&[TEST_TRIPLE_1])
+                .override_target_no_std(TEST_TRIPLE_1)
+                .render_steps(), @r"
+        [build] llvm <host>
+        [build] rustc 0 <host> -> rustc 1 <host>
+        [build] rustdoc 0 <host>
+        [doc] std 1 <target1> crates=[alloc,core]
+        ");
+    }
 }
 
 struct ExecutedSteps {
@@ -1583,7 +1657,11 @@ struct ExecutedSteps {
 
 impl ExecutedSteps {
     fn render(&self) -> String {
-        render_steps(&self.steps)
+        self.render_with(RenderConfig::default())
+    }
+
+    fn render_with(&self, config: RenderConfig) -> String {
+        render_steps(&self.steps, config)
     }
 
     #[track_caller]
@@ -1592,7 +1670,7 @@ impl ExecutedSteps {
         if !self.contains(&metadata) {
             panic!(
                 "Metadata `{}` ({metadata:?}) not found in executed steps:\n{}",
-                render_metadata(&metadata),
+                render_metadata(&metadata, &RenderConfig::default()),
                 self.render()
             );
         }
@@ -1607,7 +1685,7 @@ impl ExecutedSteps {
         if !self.contains_fuzzy(&metadata) {
             panic!(
                 "Metadata `{}` ({metadata:?}) not found in executed steps:\n{}",
-                render_metadata(&metadata),
+                render_metadata(&metadata, &RenderConfig::default()),
                 self.render()
             );
         }
@@ -1619,7 +1697,7 @@ impl ExecutedSteps {
         if self.contains(&metadata) {
             panic!(
                 "Metadata `{}` ({metadata:?}) found in executed steps (it should not be there):\n{}",
-                render_metadata(&metadata),
+                render_metadata(&metadata, &RenderConfig::default()),
                 self.render()
             );
         }
@@ -1641,7 +1719,7 @@ impl ExecutedSteps {
 }
 
 fn fuzzy_metadata_eq(executed: &StepMetadata, to_match: &StepMetadata) -> bool {
-    let StepMetadata { name, kind, target, built_by: _, stage: _ } = executed;
+    let StepMetadata { name, kind, target, built_by: _, stage: _, metadata } = executed;
     *name == to_match.name && *kind == to_match.kind && *target == to_match.target
 }
 
@@ -1672,6 +1750,16 @@ impl ConfigBuilder {
     }
 }
 
+struct RenderConfig {
+    normalize_host: bool,
+}
+
+impl Default for RenderConfig {
+    fn default() -> Self {
+        Self { normalize_host: true }
+    }
+}
+
 /// Renders the executed bootstrap steps for usage in snapshot tests with insta.
 /// Only renders certain important steps.
 /// Each value in `steps` should be a tuple of (Step, step output).
@@ -1679,7 +1767,7 @@ impl ConfigBuilder {
 /// The arrow in the rendered output (`X -> Y`) means `X builds Y`.
 /// This is similar to the output printed by bootstrap to stdout, but here it is
 /// generated purely for the purpose of tests.
-fn render_steps(steps: &[ExecutedStep]) -> String {
+fn render_steps(steps: &[ExecutedStep], config: RenderConfig) -> String {
     steps
         .iter()
         .filter_map(|step| {
@@ -1689,32 +1777,35 @@ fn render_steps(steps: &[ExecutedStep]) -> String {
                 return None;
             };
 
-            Some(render_metadata(&metadata))
+            Some(render_metadata(&metadata, &config))
         })
         .collect::<Vec<_>>()
         .join("\n")
 }
 
-fn render_metadata(metadata: &StepMetadata) -> String {
+fn render_metadata(metadata: &StepMetadata, config: &RenderConfig) -> String {
     let mut record = format!("[{}] ", metadata.kind.as_str());
     if let Some(compiler) = metadata.built_by {
-        write!(record, "{} -> ", render_compiler(compiler));
+        write!(record, "{} -> ", render_compiler(compiler, config));
     }
     let stage = metadata.get_stage().map(|stage| format!("{stage} ")).unwrap_or_default();
-    write!(record, "{} {stage}<{}>", metadata.name, normalize_target(metadata.target));
+    write!(record, "{} {stage}<{}>", metadata.name, normalize_target(metadata.target, config));
+    if let Some(metadata) = &metadata.metadata {
+        write!(record, " {metadata}");
+    }
     record
 }
 
-fn normalize_target(target: TargetSelection) -> String {
-    target
-        .to_string()
-        .replace(&host_target(), "host")
-        .replace(TEST_TRIPLE_1, "target1")
-        .replace(TEST_TRIPLE_2, "target2")
+fn normalize_target(target: TargetSelection, config: &RenderConfig) -> String {
+    let mut target = target.to_string();
+    if config.normalize_host {
+        target = target.replace(&host_target(), "host");
+    }
+    target.replace(TEST_TRIPLE_1, "target1").replace(TEST_TRIPLE_2, "target2")
 }
 
-fn render_compiler(compiler: Compiler) -> String {
-    format!("rustc {} <{}>", compiler.stage, normalize_target(compiler.host))
+fn render_compiler(compiler: Compiler, config: &RenderConfig) -> String {
+    format!("rustc {} <{}>", compiler.stage, normalize_target(compiler.host, config))
 }
 
 fn host_target() -> String {
diff --git a/src/bootstrap/src/utils/tests/mod.rs b/src/bootstrap/src/utils/tests/mod.rs
index 5b568c1df5b..ec87e71e0b6 100644
--- a/src/bootstrap/src/utils/tests/mod.rs
+++ b/src/bootstrap/src/utils/tests/mod.rs
@@ -48,17 +48,30 @@ impl ConfigBuilder {
     }
 
     pub fn path(mut self, path: &str) -> Self {
-        self.args.push(path.to_string());
-        self
+        self.arg(path)
     }
 
     pub fn paths(mut self, paths: &[&str]) -> Self {
-        for path in paths {
-            self = self.path(path);
+        self.args(paths)
+    }
+
+    pub fn arg(mut self, arg: &str) -> Self {
+        self.args.push(arg.to_string());
+        self
+    }
+
+    pub fn args(mut self, args: &[&str]) -> Self {
+        for arg in args {
+            self = self.arg(arg);
         }
         self
     }
 
+    /// Set the specified target to be treated as a no_std target.
+    pub fn override_target_no_std(mut self, target: &str) -> Self {
+        self.args(&["--set", &format!("target.{target}.no-std=true")])
+    }
+
     pub fn hosts(mut self, targets: &[&str]) -> Self {
         self.args.push("--host".to_string());
         self.args.push(targets.join(","));
@@ -77,13 +90,6 @@ impl ConfigBuilder {
         self
     }
 
-    pub fn args(mut self, args: &[&str]) -> Self {
-        for arg in args {
-            self.args.push(arg.to_string());
-        }
-        self
-    }
-
     pub fn create_config(mut self) -> Config {
         // Run in dry-check, otherwise the test would be too slow
         self.args.push("--dry-run".to_string());