Auto merge of #146233 - jieyouxu:run-make-fission, r=Kobzol

Split `run-make` into two {`run-make`,`run-make-cargo`} test suites

## Summary

Split `tests/run-make` into two test suites, to make it faster and more convenient for contributors to run run-make tests that do not need in-tree `cargo`.

| New test suites        | Explanation                                                                                                                                                                                                                              |
| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `tests/run-make`       | The "fast path" test suite intended for run-make tests that do not need in-tree `cargo`. These tests may not use `cargo`.                                                                                                                |
| `tests/run-make-cargo` | The "slow path" test suite that requires checking out `cargo` submodule and building in-tree `cargo`, and thus will have access to in-tree `cargo`. In practice, these constitute a very small portion of the original `run-make` tests. |

This PR carries out [MCP 847: Split run-make test suite into slower-building test suite with suitably-staged cargo and faster-building test suite without cargo](https://github.com/rust-lang/compiler-team/issues/847).
Fixes rust-lang/rust#135573 (for the tests that do not need in-tree `cargo`).
Fixes rust-lang/rust#134109.

## Remarks

- I considered if we want to split by in-tree tools previously. However, as discussed rust-lang/rust#134109, in practice `rustdoc` is not very slow to build, but `cargo` takes a good few minutes. So, the partition boundary was determined to be along in-tree `cargo` availability.
- The `run-make` tests previously that wanted to use `cargo` cannot just use the bootstrap `cargo`, otherwise they would run into situations where bootstrap `cargo` can significantly diverge from in-tree `cargo` (see https://github.com/rust-lang/rust/pull/130642).

---

try-job: aarch64-msvc-1
try-job: test-various
try-job: x86_64-gnu-debug
try-job: aarch64-gnu-debug
try-job: aarch64-apple
try-job: dist-various-1

3 files changed, 80 insertions, 21 deletions

diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index cb81d738666..4f839bdf7b8 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -1522,6 +1522,12 @@ test!(Pretty {
 });
 
 test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make", default: true });
+test!(RunMakeCargo {
+    path: "tests/run-make-cargo",
+    mode: "run-make",
+    suite: "run-make-cargo",
+    default: true
+});
 
 test!(AssemblyLlvm {
     path: "tests/assembly-llvm",
@@ -1773,7 +1779,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
                 target,
             });
         }
-        if suite == "run-make" {
+        if mode == "run-make" {
             builder.tool_exe(Tool::RunMakeSupport);
         }
 
@@ -1816,25 +1822,41 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
 
         let is_rustdoc = suite == "rustdoc-ui" || suite == "rustdoc-js";
 
+        // There are (potentially) 2 `cargo`s to consider:
+        //
+        // - A "bootstrap" cargo, which is the same cargo used to build bootstrap itself, and is
+        //   used to build the `run-make` test recipes and the `run-make-support` test library. All
+        //   of these may not use unstable rustc/cargo features.
+        // - An in-tree cargo, which should be considered as under test. The `run-make-cargo` test
+        //   suite is intended to support the use case of testing the "toolchain" (that is, at the
+        //   minimum the interaction between in-tree cargo + rustc) together.
+        //
+        // For build time and iteration purposes, we partition `run-make` tests which needs an
+        // in-tree cargo (a smaller subset) versus `run-make` tests that do not into two test
+        // suites, `run-make` and `run-make-cargo`. That way, contributors who do not need to run
+        // the `run-make` tests that need in-tree cargo do not need to spend time building in-tree
+        // cargo.
         if mode == "run-make" {
-            let cargo_path = if test_compiler.stage == 0 {
-                // If we're using `--stage 0`, we should provide the bootstrap cargo.
-                builder.initial_cargo.clone()
-            } else {
-                builder
-                    .ensure(tool::Cargo::from_build_compiler(
-                        builder.compiler(test_compiler.stage - 1, test_compiler.host),
-                        test_compiler.host,
-                    ))
-                    .tool_path
-            };
-
-            cmd.arg("--cargo-path").arg(cargo_path);
-
             // We need to pass the compiler that was used to compile run-make-support,
             // because we have to use the same compiler to compile rmake.rs recipes.
             let stage0_rustc_path = builder.compiler(0, test_compiler.host);
             cmd.arg("--stage0-rustc-path").arg(builder.rustc(stage0_rustc_path));
+
+            if suite == "run-make-cargo" {
+                let cargo_path = if test_compiler.stage == 0 {
+                    // If we're using `--stage 0`, we should provide the bootstrap cargo.
+                    builder.initial_cargo.clone()
+                } else {
+                    builder
+                        .ensure(tool::Cargo::from_build_compiler(
+                            builder.compiler(test_compiler.stage - 1, test_compiler.host),
+                            test_compiler.host,
+                        ))
+                        .tool_path
+                };
+
+                cmd.arg("--cargo-path").arg(cargo_path);
+            }
         }
 
         // Avoid depending on rustdoc when we don't need it.
diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs
index a818a5c756c..75c8ee36528 100644
--- a/src/bootstrap/src/core/builder/mod.rs
+++ b/src/bootstrap/src/core/builder/mod.rs
@@ -444,6 +444,7 @@ const PATH_REMAP: &[(&str, &[&str])] = &[
             "tests/mir-opt",
             "tests/pretty",
             "tests/run-make",
+            "tests/run-make-cargo",
             "tests/rustdoc",
             "tests/rustdoc-gui",
             "tests/rustdoc-js",
@@ -1128,8 +1129,8 @@ impl<'a> Builder<'a> {
                 test::RustInstaller,
                 test::TestFloatParse,
                 test::CollectLicenseMetadata,
-                // Run run-make last, since these won't pass without make on Windows
                 test::RunMake,
+                test::RunMakeCargo,
             ),
             Kind::Miri => describe!(test::Crate),
             Kind::Bench => describe!(test::Crate, test::CrateLibrustc),
diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs
index 19cfe79be5d..89a0ab7711e 100644
--- a/src/bootstrap/src/core/builder/tests.rs
+++ b/src/bootstrap/src/core/builder/tests.rs
@@ -2152,8 +2152,9 @@ mod snapshot {
         [build] rustc 0 <host> -> HtmlChecker 1 <host>
         [test] html-check <host>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 0 <host> -> cargo 1 <host>
         [test] compiletest-run-make 1 <host>
+        [build] rustc 0 <host> -> cargo 1 <host>
+        [test] compiletest-run-make-cargo 1 <host>
         ");
     }
 
@@ -2171,7 +2172,6 @@ mod snapshot {
         [test] compiletest-ui 1 <host>
         [test] compiletest-ui-fulldeps 1 <host>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 0 <host> -> cargo 1 <host>
         [build] rustdoc 1 <host>
         [test] compiletest-run-make 1 <host>
         [test] compiletest-rustdoc 1 <host>
@@ -2200,7 +2200,6 @@ mod snapshot {
         [build] rustc 2 <host> -> rustc 3 <host>
         [test] compiletest-ui-fulldeps 2 <host>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 1 <host> -> cargo 2 <host>
         [build] rustdoc 2 <host>
         [test] compiletest-run-make 2 <host>
         [test] compiletest-rustdoc 2 <host>
@@ -2234,7 +2233,6 @@ mod snapshot {
         [build] rustc 2 <host> -> rustc 3 <target1>
         [test] compiletest-ui-fulldeps 2 <target1>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 1 <host> -> cargo 2 <host>
         [build] rustdoc 2 <host>
         [test] compiletest-run-make 2 <target1>
         [test] compiletest-rustdoc 2 <target1>
@@ -2329,8 +2327,9 @@ mod snapshot {
         [build] rustc 0 <host> -> HtmlChecker 1 <host>
         [test] html-check <host>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 1 <host> -> cargo 2 <host>
         [test] compiletest-run-make 2 <host>
+        [build] rustc 1 <host> -> cargo 2 <host>
+        [test] compiletest-run-make-cargo 2 <host>
         ");
     }
 
@@ -2464,6 +2463,43 @@ mod snapshot {
         ");
     }
 
+    // Differential snapshots for `./x test run-make` run `./x test run-make-cargo`: only
+    // `run-make-cargo` should build an in-tree cargo, running `./x test run-make` should not.
+    #[test]
+    fn test_run_make_no_cargo() {
+        let ctx = TestCtx::new();
+        insta::assert_snapshot!(
+            ctx.config("test")
+                .path("run-make")
+                .render_steps(), @r"
+        [build] llvm <host>
+        [build] rustc 0 <host> -> rustc 1 <host>
+        [build] rustc 0 <host> -> RunMakeSupport 1 <host>
+        [build] rustc 1 <host> -> std 1 <host>
+        [build] rustc 0 <host> -> Compiletest 1 <host>
+        [build] rustdoc 1 <host>
+        [test] compiletest-run-make 1 <host>
+        ");
+    }
+
+    #[test]
+    fn test_run_make_cargo_builds_cargo() {
+        let ctx = TestCtx::new();
+        insta::assert_snapshot!(
+            ctx.config("test")
+                .path("run-make-cargo")
+                .render_steps(), @r"
+        [build] llvm <host>
+        [build] rustc 0 <host> -> rustc 1 <host>
+        [build] rustc 0 <host> -> RunMakeSupport 1 <host>
+        [build] rustc 1 <host> -> std 1 <host>
+        [build] rustc 0 <host> -> Compiletest 1 <host>
+        [build] rustc 0 <host> -> cargo 1 <host>
+        [build] rustdoc 1 <host>
+        [test] compiletest-run-make-cargo 1 <host>
+        ");
+    }
+
     #[test]
     fn doc_all() {
         let ctx = TestCtx::new();