Auto merge of #108534 - Mark-Simulacrum:compression, r=pietroalbini

Import rust-installer & adjust compression settings

This brings in rust-lang/rust-installer#123, which enables a larger compression window (8 -> 64MB) amongst other changes to the xz compression settings. The net effect should be smaller compressed tarballs which will decrease bandwidth usage for
static.rust-lang.org, download times, and decompression time.

This comes at the cost of higher baseline requirements for running rustup to use these files, which we believe should be largely acceptable (running rustc is likely to use at least this much memory) but if we get specific reports we may explore options to decrease impact (e.g., using the gzip tarballs automatically in rustup).

To simplify iteration on compression settings this also imports the rust-lang/rust-installer submodule, it is now hosted fully inside rust-lang/rust. Once we land this I'll file a followup to add a note to that repo and we can subsequently archive it.

--

CI times for dist-x86_64-linux builds:

* threads=6, master - 2h 50m
* threads=1, new - 3h 40m
* threads=6, new - 2h 50m

3 files changed, 57 insertions, 6 deletions

diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 3d48db8660a..9b5e116aa5c 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -711,6 +711,7 @@ impl<'a> Builder<'a> {
                 test::RustdocUi,
                 test::RustdocJson,
                 test::HtmlCheck,
+                test::RustInstaller,
                 // Run bootstrap close to the end as it's unlikely to fail
                 test::Bootstrap,
                 // Run run-make last, since these won't pass without make on Windows
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index ebd42bcf678..22ddf872215 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -482,12 +482,7 @@ impl Build {
 
             // Make sure we update these before gathering metadata so we don't get an error about missing
             // Cargo.toml files.
-            let rust_submodules = [
-                "src/tools/rust-installer",
-                "src/tools/cargo",
-                "library/backtrace",
-                "library/stdarch",
-            ];
+            let rust_submodules = ["src/tools/cargo", "library/backtrace", "library/stdarch"];
             for s in rust_submodules {
                 build.update_submodule(Path::new(s));
             }
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index b4f1506dc8f..f5d680df113 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -2695,3 +2695,58 @@ impl Step for LintDocs {
         });
     }
 }
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RustInstaller;
+
+impl Step for RustInstaller {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    /// Ensure the version placeholder replacement tool builds
+    fn run(self, builder: &Builder<'_>) {
+        builder.info("test rust-installer");
+
+        let bootstrap_host = builder.config.build;
+        let compiler = builder.compiler(0, bootstrap_host);
+        let cargo = tool::prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolBootstrap,
+            bootstrap_host,
+            "test",
+            "src/tools/rust-installer",
+            SourceType::InTree,
+            &[],
+        );
+        try_run(builder, &mut cargo.into());
+
+        // We currently don't support running the test.sh script outside linux(?) environments.
+        // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a
+        // set of scripts, which will likely allow dropping this if.
+        if bootstrap_host != "x86_64-unknown-linux-gnu" {
+            return;
+        }
+
+        let mut cmd =
+            std::process::Command::new(builder.src.join("src/tools/rust-installer/test.sh"));
+        let tmpdir = testdir(builder, compiler.host).join("rust-installer");
+        let _ = std::fs::remove_dir_all(&tmpdir);
+        let _ = std::fs::create_dir_all(&tmpdir);
+        cmd.current_dir(&tmpdir);
+        cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target"));
+        cmd.env("CARGO", &builder.initial_cargo);
+        cmd.env("RUSTC", &builder.initial_rustc);
+        cmd.env("TMP_DIR", &tmpdir);
+        try_run(builder, &mut cmd);
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/rust-installer")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Self);
+    }
+}