Auto merge of #125166 - lovesegfault:embed-rustc-perf, r=Mark-Simulacrum

refactor: add rustc-perf submodule to src/tools

Currently, it's very challenging to perform a sandboxed `opt-dist`
bootstrap because the tool requires `rustc-perf` to be present, but
there is no proper management/tracking of it. Instead, a specific commit
is hardcoded where it is needed, and a non-checksummed zip is fetched
ad-hoc. This happens in two places:

`src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile`:

```dockerfile
ENV PERF_COMMIT 4f313add609f43e928e98132358e8426ed3969ae
RUN curl -LS -o perf.zip https://ci-mirrors.rust-lang.org/rustc/rustc-perf-$PERF_COMMIT.zip && \
    unzip perf.zip && \
    mv rustc-perf-$PERF_COMMIT rustc-perf && \
    rm perf.zip
```

`src/tools/opt-dist/src/main.rs`

```rust
// FIXME: add some mechanism for synchronization of this commit SHA with
// Linux (which builds rustc-perf in a Dockerfile)
// rustc-perf version from 2023-10-22
const PERF_COMMIT: &str = "4f313add609f43e928e98132358e8426ed3969ae";

let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
let client = reqwest::blocking::Client::builder()
    .timeout(Duration::from_secs(60 * 2))
    .connect_timeout(Duration::from_secs(60 * 2))
    .build()?;
let response = retry_action(
    || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
    "Download rustc-perf archive",
    5,
)?;
```

This causes a few issues:

1. Maintainers need to be careful to bump PERF_COMMIT in both places
   every time
2. In order to run `opt-dist` in a sandbox, you need to provide your own
   `rustc-perf` (https://github.com/rust-lang/rust/pull/125125), but to
   figure out which commit to provide you need to grep the Dockerfile
3. Even if you manage to provide the correct `rustc-perf`, its
   dependencies are not included in the `vendor/` dir created during
   `dist`, so it will fail to build from the published source tarballs
4. It is hard to provide any level of automation around updating the
   `rustc-perf` in use, leading to staleness

Fundamentally, this means `rustc-src` tarballs no longer contain
everything you need to bootstrap Rust, and packagers hoping to leverage
`opt-dist` need to go out of their way to keep track of this "hidden"
dependency on `rustc-perf`.

This change adds rustc-perf as a git submodule, pinned to the current
`PERF_COMMIT` 4f313add609f43e928e98132358e8426ed3969ae. Subsequent
commits ensure the submodule is initialized when necessary, and make use
of it in `opt-dist`.

10 files changed, 64 insertions, 55 deletions

diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index 1f006e1453f..91039d0c8dc 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -1010,6 +1010,9 @@ impl Step for PlainSourceTarball {
         if builder.rust_info().is_managed_git_subrepository()
             || builder.rust_info().is_from_tarball()
         {
+            // FIXME: This code looks _very_ similar to what we have in `src/core/build_steps/vendor.rs`
+            // perhaps it should be removed in favor of making `dist` perform the `vendor` step?
+
             // Ensure we have all submodules from src and other directories checked out.
             for submodule in builder.get_all_submodules() {
                 builder.update_submodule(Path::new(submodule));
@@ -1029,6 +1032,10 @@ impl Step for PlainSourceTarball {
                 .arg(builder.src.join("./compiler/rustc_codegen_gcc/Cargo.toml"))
                 .arg("--sync")
                 .arg(builder.src.join("./src/bootstrap/Cargo.toml"))
+                .arg("--sync")
+                .arg(builder.src.join("./src/tools/opt-dist/Cargo.toml"))
+                .arg("--sync")
+                .arg(builder.src.join("./src/tools/rustc-perf/Cargo.toml"))
                 // Will read the libstd Cargo.toml
                 // which uses the unstable `public-dependency` feature.
                 .env("RUSTC_BOOTSTRAP", "1")
diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs
index 21344a4224e..2db3f8f7936 100644
--- a/src/bootstrap/src/core/build_steps/tool.rs
+++ b/src/bootstrap/src/core/build_steps/tool.rs
@@ -1,6 +1,6 @@
 use std::env;
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 
 use crate::core::build_steps::compile;
@@ -313,10 +313,47 @@ bootstrap_tool!(
     SuggestTests, "src/tools/suggest-tests", "suggest-tests";
     GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys";
     RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test";
-    OptimizedDist, "src/tools/opt-dist", "opt-dist";
     CoverageDump, "src/tools/coverage-dump", "coverage-dump";
 );
 
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct OptimizedDist {
+    pub compiler: Compiler,
+    pub target: TargetSelection,
+}
+
+impl Step for OptimizedDist {
+    type Output = PathBuf;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/opt-dist")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(OptimizedDist {
+            compiler: run.builder.compiler(0, run.builder.config.build),
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> PathBuf {
+        // We need to ensure the rustc-perf submodule is initialized when building opt-dist since
+        // the tool requires it to be in place to run.
+        builder.update_submodule(Path::new("src/tools/rustc-perf"));
+
+        builder.ensure(ToolBuild {
+            compiler: self.compiler,
+            target: self.target,
+            tool: "opt-dist",
+            mode: Mode::ToolBootstrap,
+            path: "src/tools/opt-dist",
+            source_type: SourceType::InTree,
+            extra_features: Vec::new(),
+            allow_features: "",
+        })
+    }
+}
+
 #[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]
 pub struct ErrorIndex {
     pub compiler: Compiler,
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
index fe84c23a11c..4e1aae98221 100644
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
@@ -58,14 +58,6 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/
 RUN ./build-clang.sh
 ENV CC=clang CXX=clang++
 
-# rustc-perf version from 2023-10-22
-# Should also be changed in the opt-dist tool for other environments.
-ENV PERF_COMMIT 4f313add609f43e928e98132358e8426ed3969ae
-RUN curl -LS -o perf.zip https://ci-mirrors.rust-lang.org/rustc/rustc-perf-$PERF_COMMIT.zip && \
-    unzip perf.zip && \
-    mv rustc-perf-$PERF_COMMIT rustc-perf && \
-    rm perf.zip
-
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml
index 1ff410e723a..88e8640d56a 100644
--- a/src/tools/opt-dist/Cargo.toml
+++ b/src/tools/opt-dist/Cargo.toml
@@ -13,8 +13,6 @@ humansize = "2"
 sysinfo = { version = "0.30", default-features = false }
 fs_extra = "1"
 camino = "1"
-reqwest = { version = "0.11", features = ["blocking"] }
-zip = { version = "0.6", default-features = false, features = ["deflate"] }
 tar = "0.4"
 xz = { version = "0.1", package = "xz2" }
 serde = { version = "1", features = ["derive"] }
diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs
index bd0a3815855..a709076f245 100644
--- a/src/tools/opt-dist/src/main.rs
+++ b/src/tools/opt-dist/src/main.rs
@@ -3,10 +3,7 @@ use anyhow::Context;
 use camino::{Utf8Path, Utf8PathBuf};
 use clap::Parser;
 use log::LevelFilter;
-use std::io::Cursor;
-use std::time::Duration;
 use utils::io;
-use zip::ZipArchive;
 
 use crate::environment::{Environment, EnvironmentBuilder};
 use crate::exec::{cmd, Bootstrap};
@@ -17,9 +14,9 @@ use crate::training::{
     rustc_benchmarks,
 };
 use crate::utils::artifact_size::print_binary_sizes;
-use crate::utils::io::{copy_directory, move_directory, reset_directory};
+use crate::utils::io::{copy_directory, reset_directory};
 use crate::utils::{
-    clear_llvm_files, format_env_variables, print_free_disk_space, retry_action, with_log_group,
+    clear_llvm_files, format_env_variables, print_free_disk_space, with_log_group,
     write_timer_to_summary,
 };
 
@@ -69,7 +66,12 @@ enum EnvironmentCmd {
         #[arg(long, default_value = "opt-artifacts")]
         artifact_dir: Utf8PathBuf,
 
-        /// Checkout directory of `rustc-perf`, it will be fetched automatically if unspecified.
+        /// Checkout directory of `rustc-perf`.
+        ///
+        /// If unspecified, defaults to the rustc-perf submodule in the rustc checkout dir
+        /// (`src/tools/rustc-perf`), which should have been initialized when building this tool.
+        // FIXME: Move update_submodule into build_helper, that way we can also ensure the submodule
+        // is updated when _running_ opt-dist, rather than building.
         #[arg(long)]
         rustc_perf_checkout_dir: Option<Utf8PathBuf>,
 
@@ -146,8 +148,6 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)>
                 .host_llvm_dir(Utf8PathBuf::from("/rustroot"))
                 .artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts"))
                 .build_dir(checkout_dir.join("obj"))
-                // /tmp/rustc-perf comes from the x64 dist Dockerfile
-                .prebuilt_rustc_perf(Some(Utf8PathBuf::from("/tmp/rustc-perf")))
                 .shared_llvm(true)
                 .use_bolt(true)
                 .skipped_tests(vec![
@@ -191,9 +191,12 @@ fn execute_pipeline(
 ) -> anyhow::Result<()> {
     reset_directory(&env.artifact_dir())?;
 
-    with_log_group("Building rustc-perf", || match env.prebuilt_rustc_perf() {
-        Some(dir) => copy_rustc_perf(env, &dir),
-        None => download_rustc_perf(env),
+    with_log_group("Building rustc-perf", || {
+        let rustc_perf_checkout_dir = match env.prebuilt_rustc_perf() {
+            Some(dir) => dir,
+            None => env.checkout_path().join("src").join("tools").join("rustc-perf"),
+        };
+        copy_rustc_perf(env, &rustc_perf_checkout_dir)
     })?;
 
     // Stage 1: Build PGO instrumented rustc
@@ -409,36 +412,6 @@ fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> {
     build_rustc_perf(env)
 }
 
-// Download and build rustc-perf into the given environment.
-fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> {
-    reset_directory(&env.rustc_perf_dir())?;
-
-    // FIXME: add some mechanism for synchronization of this commit SHA with
-    // Linux (which builds rustc-perf in a Dockerfile)
-    // rustc-perf version from 2023-10-22
-    const PERF_COMMIT: &str = "4f313add609f43e928e98132358e8426ed3969ae";
-
-    let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
-    let client = reqwest::blocking::Client::builder()
-        .timeout(Duration::from_secs(60 * 2))
-        .connect_timeout(Duration::from_secs(60 * 2))
-        .build()?;
-    let response = retry_action(
-        || Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
-        "Download rustc-perf archive",
-        5,
-    )?;
-
-    let mut archive = ZipArchive::new(Cursor::new(response))?;
-    archive.extract(env.rustc_perf_dir())?;
-    move_directory(
-        &env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
-        &env.rustc_perf_dir(),
-    )?;
-
-    build_rustc_perf(env)
-}
-
 fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> {
     cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"])
         .workdir(&env.rustc_perf_dir())
diff --git a/src/tools/rustc-perf b/src/tools/rustc-perf
new file mode 160000
+Subproject 4f313add609f43e928e98132358e8426ed3969a
diff --git a/src/tools/tidy/config/black.toml b/src/tools/tidy/config/black.toml
index 51a722979f5..e73847a93ba 100644
--- a/src/tools/tidy/config/black.toml
+++ b/src/tools/tidy/config/black.toml
@@ -11,5 +11,6 @@ extend-exclude = """(\
     src/doc/edition-guide/|\
     src/llvm-project/|\
     src/doc/embedded-book/|\
+    src/tools/rustc-perf/|\
     library/backtrace/
     )"""
diff --git a/src/tools/tidy/config/ruff.toml b/src/tools/tidy/config/ruff.toml
index cf08c62648b..cf89ffd9ac7 100644
--- a/src/tools/tidy/config/ruff.toml
+++ b/src/tools/tidy/config/ruff.toml
@@ -26,6 +26,7 @@ extend-exclude = [
     "src/llvm-project/",
     "src/doc/embedded-book/",
     "library/backtrace/",
+    "src/tools/rustc-perf/",
     # Hack: CI runs from a subdirectory under the main checkout
     "../src/doc/nomicon/",
     "../src/tools/cargo/",
@@ -38,4 +39,5 @@ extend-exclude = [
     "../src/llvm-project/",
     "../src/doc/embedded-book/",
     "../library/backtrace/",
+    "../src/tools/rustc-perf/",
 ]
diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index 58f4455ae2f..efcd2a181ff 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -81,12 +81,10 @@ const EXCEPTIONS: ExceptionList = &[
     ("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc
     ("colored", "MPL-2.0"),                                  // rustfmt
     ("dissimilar", "Apache-2.0"),                            // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps)
-    ("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"), // opt-dist
     ("fluent-langneg", "Apache-2.0"),                        // rustc (fluent translations)
     ("fortanix-sgx-abi", "MPL-2.0"),                         // libstd but only for `sgx` target. FIXME: this dependency violates the documentation comment above.
     ("instant", "BSD-3-Clause"),                             // rustc_driver/tracing-subscriber/parking_lot
     ("mdbook", "MPL-2.0"),                                   // mdbook
-    ("openssl", "Apache-2.0"),                               // opt-dist
     ("option-ext", "MPL-2.0"),                               // cargo-miri (via `directories`)
     ("rustc_apfloat", "Apache-2.0 WITH LLVM-exception"),     // rustc (license is the same as LLVM uses)
     ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0                       // cargo/... (because of serde)
diff --git a/src/tools/tidy/src/walk.rs b/src/tools/tidy/src/walk.rs
index 851c21f1c0f..f68b7675c76 100644
--- a/src/tools/tidy/src/walk.rs
+++ b/src/tools/tidy/src/walk.rs
@@ -18,6 +18,7 @@ pub fn filter_dirs(path: &Path) -> bool {
         "src/tools/clippy",
         "src/tools/miri",
         "src/tools/rust-analyzer",
+        "src/tools/rustc-perf",
         "src/tools/rustfmt",
         "src/doc/book",
         "src/doc/edition-guide",