Port PGO/LTO/BOLT optimized build pipeline to Rust

17 files changed, 1337 insertions, 0 deletions

diff --git a/src/tools/build_helper/Cargo.toml b/src/tools/build_helper/Cargo.toml
index 99f6fea2ecf..66894e1abc4 100644
--- a/src/tools/build_helper/Cargo.toml
+++ b/src/tools/build_helper/Cargo.toml
@@ -6,3 +6,5 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+serde = "1"
+serde_derive = "1"
diff --git a/src/tools/build_helper/src/lib.rs b/src/tools/build_helper/src/lib.rs
index 3fa970373b3..575f3677155 100644
--- a/src/tools/build_helper/src/lib.rs
+++ b/src/tools/build_helper/src/lib.rs
@@ -1,3 +1,4 @@
 pub mod ci;
 pub mod git;
+pub mod metrics;
 pub mod util;
diff --git a/src/tools/build_helper/src/metrics.rs b/src/tools/build_helper/src/metrics.rs
new file mode 100644
index 00000000000..2d0c66a8f33
--- /dev/null
+++ b/src/tools/build_helper/src/metrics.rs
@@ -0,0 +1,92 @@
+use serde_derive::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct JsonRoot {
+    #[serde(default)] // For version 0 the field was not present.
+    pub format_version: usize,
+    pub system_stats: JsonInvocationSystemStats,
+    pub invocations: Vec<JsonInvocation>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct JsonInvocation {
+    // Unix timestamp in seconds
+    //
+    // This is necessary to easily correlate this invocation with logs or other data.
+    pub start_time: u64,
+    pub duration_including_children_sec: f64,
+    pub children: Vec<JsonNode>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum JsonNode {
+    RustbuildStep {
+        #[serde(rename = "type")]
+        type_: String,
+        debug_repr: String,
+
+        duration_excluding_children_sec: f64,
+        system_stats: JsonStepSystemStats,
+
+        children: Vec<JsonNode>,
+    },
+    TestSuite(TestSuite),
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TestSuite {
+    pub metadata: TestSuiteMetadata,
+    pub tests: Vec<Test>,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum TestSuiteMetadata {
+    CargoPackage {
+        crates: Vec<String>,
+        target: String,
+        host: String,
+        stage: u32,
+    },
+    Compiletest {
+        suite: String,
+        mode: String,
+        compare_mode: Option<String>,
+        target: String,
+        host: String,
+        stage: u32,
+    },
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Test {
+    pub name: String,
+    #[serde(flatten)]
+    pub outcome: TestOutcome,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(tag = "outcome", rename_all = "snake_case")]
+pub enum TestOutcome {
+    Passed,
+    Failed,
+    Ignored { ignore_reason: Option<String> },
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct JsonInvocationSystemStats {
+    pub cpu_threads_count: usize,
+    pub cpu_model: String,
+
+    pub memory_total_bytes: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub struct JsonStepSystemStats {
+    pub cpu_utilization_percent: f64,
+}
diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml
new file mode 100644
index 00000000000..5a1794d3336
--- /dev/null
+++ b/src/tools/opt-dist/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "opt-dist"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+build_helper = { path = "../build_helper" }
+env_logger = "0.10"
+log = "0.4"
+anyhow = { version = "1", features = ["backtrace"] }
+humantime = "2"
+humansize = "2"
+sysinfo = { version = "0.29", default-features = false }
+fs_extra = "1"
+camino = "1"
+reqwest = { version = "0.11", features = ["blocking"] }
+zip = { version = "0.6", default-features = false, features = ["deflate"] }
+tar = "0.4"
+xz = "0.1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+glob = "0.3"
diff --git a/src/tools/opt-dist/README.md b/src/tools/opt-dist/README.md
new file mode 100644
index 00000000000..05a75870da1
--- /dev/null
+++ b/src/tools/opt-dist/README.md
@@ -0,0 +1,7 @@
+# Optimized build pipeline
+This binary implements a heavily optimized build pipeline for `rustc` and `LLVM` artifacts that are used for both for
+benchmarking using the perf. bot and for final distribution to users.
+
+It uses LTO, PGO and BOLT to optimize the compiler and LLVM as much as possible.
+This logic is not part of bootstrap, because it needs to invoke bootstrap multiple times, force-rebuild various
+artifacts repeatedly and sometimes go around bootstrap's cache mechanism.
diff --git a/src/tools/opt-dist/src/environment/linux.rs b/src/tools/opt-dist/src/environment/linux.rs
new file mode 100644
index 00000000000..d4c55c46f7c
--- /dev/null
+++ b/src/tools/opt-dist/src/environment/linux.rs
@@ -0,0 +1,54 @@
+use crate::environment::Environment;
+use crate::exec::cmd;
+use crate::utils::io::copy_directory;
+use camino::{Utf8Path, Utf8PathBuf};
+
+pub(super) struct LinuxEnvironment;
+
+impl Environment for LinuxEnvironment {
+    fn python_binary(&self) -> &'static str {
+        "python3"
+    }
+
+    fn checkout_path(&self) -> Utf8PathBuf {
+        Utf8PathBuf::from("/checkout")
+    }
+
+    fn downloaded_llvm_dir(&self) -> Utf8PathBuf {
+        Utf8PathBuf::from("/rustroot")
+    }
+
+    fn opt_artifacts(&self) -> Utf8PathBuf {
+        Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")
+    }
+
+    fn build_root(&self) -> Utf8PathBuf {
+        self.checkout_path().join("obj")
+    }
+
+    fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
+        // /tmp/rustc-perf comes from the x64 dist Dockerfile
+        copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?;
+        cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
+            .workdir(&self.rustc_perf_dir())
+            .env("RUSTC", &self.rustc_stage_0().into_string())
+            .env("RUSTC_BOOTSTRAP", "1")
+            .run()?;
+        Ok(())
+    }
+
+    fn supports_bolt(&self) -> bool {
+        true
+    }
+
+    fn executable_extension(&self) -> &'static str {
+        ""
+    }
+
+    fn skipped_tests(&self) -> &'static [&'static str] {
+        &[
+            // Fails because of linker errors, as of June 2023.
+            "tests/ui/process/nofile-limit.rs",
+        ]
+    }
+}
diff --git a/src/tools/opt-dist/src/environment/mod.rs b/src/tools/opt-dist/src/environment/mod.rs
new file mode 100644
index 00000000000..f66b9ab41ea
--- /dev/null
+++ b/src/tools/opt-dist/src/environment/mod.rs
@@ -0,0 +1,75 @@
+use camino::Utf8PathBuf;
+
+#[cfg(target_family = "unix")]
+mod linux;
+#[cfg(target_family = "windows")]
+mod windows;
+
+pub trait Environment {
+    fn host_triple(&self) -> String {
+        std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing")
+    }
+
+    fn python_binary(&self) -> &'static str;
+
+    /// The rustc checkout, where the compiler source is located.
+    fn checkout_path(&self) -> Utf8PathBuf;
+
+    /// Path to the downloaded host LLVM.
+    fn downloaded_llvm_dir(&self) -> Utf8PathBuf;
+
+    /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
+    /// will be stored.
+    fn opt_artifacts(&self) -> Utf8PathBuf;
+
+    /// The main directory where the build occurs.
+    fn build_root(&self) -> Utf8PathBuf;
+
+    fn build_artifacts(&self) -> Utf8PathBuf {
+        self.build_root().join("build").join(self.host_triple())
+    }
+
+    fn cargo_stage_0(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage0")
+            .join("bin")
+            .join(format!("cargo{}", self.executable_extension()))
+    }
+
+    fn rustc_stage_0(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage0")
+            .join("bin")
+            .join(format!("rustc{}", self.executable_extension()))
+    }
+
+    fn rustc_stage_2(&self) -> Utf8PathBuf {
+        self.build_artifacts()
+            .join("stage2")
+            .join("bin")
+            .join(format!("rustc{}", self.executable_extension()))
+    }
+
+    /// Path to the built rustc-perf benchmark suite.
+    fn rustc_perf_dir(&self) -> Utf8PathBuf {
+        self.opt_artifacts().join("rustc-perf")
+    }
+
+    /// Download and/or compile rustc-perf.
+    fn prepare_rustc_perf(&self) -> anyhow::Result<()>;
+
+    fn supports_bolt(&self) -> bool;
+
+    /// What is the extension of binary executables in this environment?
+    fn executable_extension(&self) -> &'static str;
+
+    /// List of test paths that should be skipped when testing the optimized artifacts.
+    fn skipped_tests(&self) -> &'static [&'static str];
+}
+
+pub fn create_environment() -> Box<dyn Environment> {
+    #[cfg(target_family = "unix")]
+    return Box::new(linux::LinuxEnvironment);
+    #[cfg(target_family = "windows")]
+    return Box::new(windows::WindowsEnvironment::new());
+}
diff --git a/src/tools/opt-dist/src/environment/windows.rs b/src/tools/opt-dist/src/environment/windows.rs
new file mode 100644
index 00000000000..36f4c0f29cc
--- /dev/null
+++ b/src/tools/opt-dist/src/environment/windows.rs
@@ -0,0 +1,78 @@
+use crate::environment::Environment;
+use crate::exec::cmd;
+use crate::utils::io::move_directory;
+use camino::Utf8PathBuf;
+use std::io::Cursor;
+use zip::ZipArchive;
+
+pub(super) struct WindowsEnvironment {
+    checkout_dir: Utf8PathBuf,
+}
+
+impl WindowsEnvironment {
+    pub fn new() -> Self {
+        Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap() }
+    }
+}
+
+impl Environment for WindowsEnvironment {
+    fn python_binary(&self) -> &'static str {
+        "python"
+    }
+
+    fn checkout_path(&self) -> Utf8PathBuf {
+        self.checkout_dir.clone()
+    }
+
+    fn downloaded_llvm_dir(&self) -> Utf8PathBuf {
+        self.checkout_path().join("citools").join("clang-rust")
+    }
+
+    fn opt_artifacts(&self) -> Utf8PathBuf {
+        self.checkout_path().join("opt-artifacts")
+    }
+
+    fn build_root(&self) -> Utf8PathBuf {
+        self.checkout_path()
+    }
+
+    fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
+        // FIXME: add some mechanism for synchronization of this commit SHA with
+        // Linux (which builds rustc-perf in a Dockerfile)
+        // rustc-perf version from 2023-05-30
+        const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
+
+        let url = format!("https://github.com/rust-lang/rustc-perf/archive/{PERF_COMMIT}.zip");
+        let response = reqwest::blocking::get(url)?.error_for_status()?.bytes()?.to_vec();
+
+        let mut archive = ZipArchive::new(Cursor::new(response))?;
+        archive.extract(self.rustc_perf_dir())?;
+        move_directory(
+            &self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
+            &self.rustc_perf_dir(),
+        )?;
+
+        cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
+            .workdir(&self.rustc_perf_dir())
+            .env("RUSTC", &self.rustc_stage_0().into_string())
+            .env("RUSTC_BOOTSTRAP", "1")
+            .run()?;
+
+        Ok(())
+    }
+
+    fn supports_bolt(&self) -> bool {
+        false
+    }
+
+    fn executable_extension(&self) -> &'static str {
+        ".exe"
+    }
+
+    fn skipped_tests(&self) -> &'static [&'static str] {
+        &[
+            // Fails as of June 2023.
+            "tests\\codegen\\vec-shrink-panik.rs",
+        ]
+    }
+}
diff --git a/src/tools/opt-dist/src/exec.rs b/src/tools/opt-dist/src/exec.rs
new file mode 100644
index 00000000000..d05ddbd4c0e
--- /dev/null
+++ b/src/tools/opt-dist/src/exec.rs
@@ -0,0 +1,169 @@
+use crate::environment::Environment;
+use crate::metrics::{load_metrics, record_metrics};
+use crate::timer::TimerSection;
+use crate::training::{LlvmBoltProfile, LlvmPGOProfile, RustcPGOProfile};
+use camino::{Utf8Path, Utf8PathBuf};
+use std::collections::BTreeMap;
+use std::fs::File;
+use std::process::{Command, Stdio};
+
+#[derive(Default)]
+pub struct CmdBuilder {
+    args: Vec<String>,
+    env: BTreeMap<String, String>,
+    workdir: Option<Utf8PathBuf>,
+    output: Option<Utf8PathBuf>,
+}
+
+impl CmdBuilder {
+    pub fn arg(mut self, arg: &str) -> Self {
+        self.args.push(arg.to_string());
+        self
+    }
+
+    pub fn env(mut self, name: &str, value: &str) -> Self {
+        self.env.insert(name.to_string(), value.to_string());
+        self
+    }
+
+    pub fn workdir(mut self, path: &Utf8Path) -> Self {
+        self.workdir = Some(path.to_path_buf());
+        self
+    }
+
+    pub fn redirect_output(mut self, path: Utf8PathBuf) -> Self {
+        self.output = Some(path);
+        self
+    }
+
+    pub fn run(self) -> anyhow::Result<()> {
+        let mut cmd_str = String::new();
+        cmd_str.push_str(
+            &self
+                .env
+                .iter()
+                .map(|(key, value)| format!("{key}={value}"))
+                .collect::<Vec<_>>()
+                .join(" "),
+        );
+        if !self.env.is_empty() {
+            cmd_str.push(' ');
+        }
+        cmd_str.push_str(&self.args.join(" "));
+        if let Some(ref path) = self.output {
+            cmd_str.push_str(&format!(" > {path:?}"));
+        }
+        cmd_str.push_str(&format!(
+            " [at {}]",
+            self.workdir
+                .clone()
+                .unwrap_or_else(|| std::env::current_dir().unwrap().try_into().unwrap())
+        ));
+        log::info!("Executing `{cmd_str}`");
+
+        let mut cmd = Command::new(&self.args[0]);
+        cmd.stdin(Stdio::null());
+        cmd.args(self.args.iter().skip(1));
+        for (key, value) in &self.env {
+            cmd.env(key, value);
+        }
+        if let Some(ref output) = self.output {
+            cmd.stdout(File::create(output.clone().into_std_path_buf())?);
+        }
+        if let Some(ref workdir) = self.workdir {
+            cmd.current_dir(workdir.clone().into_std_path_buf());
+        }
+        let exit_status = cmd.spawn()?.wait()?;
+        if !exit_status.success() {
+            Err(anyhow::anyhow!(
+                "Command {cmd_str} has failed with exit code {:?}",
+                exit_status.code(),
+            ))
+        } else {
+            Ok(())
+        }
+    }
+}
+
+pub fn cmd(args: &[&str]) -> CmdBuilder {
+    assert!(!args.is_empty());
+    CmdBuilder { args: args.iter().map(|s| s.to_string()).collect(), ..Default::default() }
+}
+
+pub struct Bootstrap {
+    cmd: CmdBuilder,
+    metrics_path: Utf8PathBuf,
+}
+
+impl Bootstrap {
+    pub fn build(env: &dyn Environment) -> Self {
+        let metrics_path = env.build_root().join("build").join("metrics.json");
+        let cmd = cmd(&[
+            env.python_binary(),
+            env.checkout_path().join("x.py").as_str(),
+            "build",
+            "--target",
+            &env.host_triple(),
+            "--host",
+            &env.host_triple(),
+            "--stage",
+            "2",
+            "library/std",
+        ])
+        .env("RUST_BACKTRACE", "full");
+        Self { cmd, metrics_path }
+    }
+
+    pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self {
+        let metrics_path = env.build_root().join("build").join("metrics.json");
+        let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>())
+            .env("RUST_BACKTRACE", "full");
+        Self { cmd, metrics_path }
+    }
+
+    pub fn llvm_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
+        self.cmd = self
+            .cmd
+            .arg("--llvm-profile-generate")
+            .env("LLVM_PROFILE_DIR", profile_dir.join("prof-%p").as_str());
+        self
+    }
+
+    pub fn llvm_pgo_optimize(mut self, profile: &LlvmPGOProfile) -> Self {
+        self.cmd = self.cmd.arg("--llvm-profile-use").arg(profile.0.as_str());
+        self
+    }
+
+    pub fn rustc_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
+        self.cmd = self.cmd.arg("--rust-profile-generate").arg(profile_dir.as_str());
+        self
+    }
+
+    pub fn rustc_pgo_optimize(mut self, profile: &RustcPGOProfile) -> Self {
+        self.cmd = self.cmd.arg("--rust-profile-use").arg(profile.0.as_str());
+        self
+    }
+
+    pub fn llvm_bolt_instrument(mut self) -> Self {
+        self.cmd = self.cmd.arg("--llvm-bolt-profile-generate");
+        self
+    }
+
+    pub fn llvm_bolt_optimize(mut self, profile: &LlvmBoltProfile) -> Self {
+        self.cmd = self.cmd.arg("--llvm-bolt-profile-use").arg(profile.0.as_str());
+        self
+    }
+
+    /// Do not rebuild rustc, and use a previously built rustc sysroot instead.
+    pub fn avoid_rustc_rebuild(mut self) -> Self {
+        self.cmd = self.cmd.arg("--keep-stage").arg("0").arg("--keep-stage").arg("1");
+        self
+    }
+
+    pub fn run(self, timer: &mut TimerSection) -> anyhow::Result<()> {
+        self.cmd.run()?;
+        let metrics = load_metrics(&self.metrics_path)?;
+        record_metrics(&metrics, timer);
+        Ok(())
+    }
+}
diff --git a/src/tools/opt-dist/src/main.rs b/src/tools/opt-dist/src/main.rs
new file mode 100644
index 00000000000..39fa7e1fb2a
--- /dev/null
+++ b/src/tools/opt-dist/src/main.rs
@@ -0,0 +1,175 @@
+use anyhow::Context;
+use log::LevelFilter;
+
+use crate::environment::{create_environment, Environment};
+use crate::exec::Bootstrap;
+use crate::tests::run_tests;
+use crate::timer::Timer;
+use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles};
+use crate::utils::io::reset_directory;
+use crate::utils::{clear_llvm_files, format_env_variables, print_free_disk_space};
+
+mod environment;
+mod exec;
+mod metrics;
+mod tests;
+mod timer;
+mod training;
+mod utils;
+
+fn is_try_build() -> bool {
+    std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0"
+}
+
+fn execute_pipeline(
+    env: &dyn Environment,
+    timer: &mut Timer,
+    dist_args: Vec<String>,
+) -> anyhow::Result<()> {
+    reset_directory(&env.opt_artifacts())?;
+    env.prepare_rustc_perf()?;
+
+    // Stage 1: Build PGO instrumented rustc
+    // We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
+    // same time can cause issues, because the host and in-tree LLVM versions can diverge.
+    let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| {
+        let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo");
+
+        stage.section("Build PGO instrumented rustc and LLVM", |section| {
+            Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root).run(section)
+        })?;
+
+        let profile = stage
+            .section("Gather profiles", |_| gather_rustc_profiles(env, &rustc_profile_dir_root))?;
+        print_free_disk_space()?;
+
+        stage.section("Build PGO optimized rustc", |section| {
+            Bootstrap::build(env).rustc_pgo_optimize(&profile).run(section)
+        })?;
+
+        Ok(profile)
+    })?;
+
+    // Stage 2: Gather LLVM PGO profiles
+    // Here we build a PGO instrumented LLVM, reusing the previously PGO optimized rustc.
+    // Then we use the instrumented LLVM to gather LLVM PGO profiles.
+    let llvm_pgo_profile = timer.section("Stage 2 (LLVM PGO)", |stage| {
+        // Remove the previous, uninstrumented build of LLVM.
+        clear_llvm_files(env)?;
+
+        let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo");
+
+        stage.section("Build PGO instrumented LLVM", |section| {
+            Bootstrap::build(env)
+                .llvm_pgo_instrument(&llvm_profile_dir_root)
+                .avoid_rustc_rebuild()
+                .run(section)
+        })?;
+
+        let profile = stage
+            .section("Gather profiles", |_| gather_llvm_profiles(env, &llvm_profile_dir_root))?;
+
+        print_free_disk_space()?;
+
+        // Proactively delete the instrumented artifacts, to avoid using them by accident in
+        // follow-up stages.
+        clear_llvm_files(env)?;
+
+        Ok(profile)
+    })?;
+
+    let llvm_bolt_profile = if env.supports_bolt() {
+        // Stage 3: Build BOLT instrumented LLVM
+        // We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles.
+        // Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build.
+        // BOLT instrumentation is performed "on-the-fly" when the LLVM library is copied to the sysroot of rustc,
+        // therefore the LLVM artifacts on disk are not "tainted" with BOLT instrumentation and they can be reused.
+        timer.section("Stage 3 (LLVM BOLT)", |stage| {
+            stage.section("Build BOLT instrumented LLVM", |stage| {
+                Bootstrap::build(env)
+                    .llvm_bolt_instrument()
+                    .llvm_pgo_optimize(&llvm_pgo_profile)
+                    .avoid_rustc_rebuild()
+                    .run(stage)
+            })?;
+
+            let profile = stage.section("Gather profiles", |_| gather_llvm_bolt_profiles(env))?;
+            print_free_disk_space()?;
+
+            // LLVM is not being cleared here, we want to reuse the previous PGO-optimized build
+
+            Ok(Some(profile))
+        })?
+    } else {
+        None
+    };
+
+    let mut dist = Bootstrap::dist(env, &dist_args)
+        .llvm_pgo_optimize(&llvm_pgo_profile)
+        .rustc_pgo_optimize(&rustc_pgo_profile)
+        .avoid_rustc_rebuild();
+
+    if let Some(llvm_bolt_profile) = llvm_bolt_profile {
+        dist = dist.llvm_bolt_optimize(&llvm_bolt_profile);
+    }
+
+    // Final stage: Assemble the dist artifacts
+    // The previous PGO optimized rustc build and PGO optimized LLVM builds should be reused.
+    timer.section("Stage 4 (final build)", |stage| dist.run(stage))?;
+
+    // After dist has finished, run a subset of the test suite on the optimized artifacts to discover
+    // possible regressions.
+    // The tests are not executed for try builds, which can be in various broken states, so we don't
+    // want to gatekeep them with tests.
+    if !is_try_build() {
+        timer.section("Run tests", |_| run_tests(env))?;
+    }
+
+    Ok(())
+}
+
+fn main() -> anyhow::Result<()> {
+    // Make sure that we get backtraces for easier debugging in CI
+    std::env::set_var("RUST_BACKTRACE", "1");
+
+    env_logger::builder()
+        .filter_level(LevelFilter::Info)
+        .format_timestamp_millis()
+        .parse_default_env()
+        .init();
+
+    let mut build_args: Vec<String> = std::env::args().skip(1).collect();
+    log::info!("Running optimized build pipeline with args `{}`", build_args.join(" "));
+    log::info!("Environment values\n{}", format_env_variables());
+
+    if let Ok(config) = std::fs::read_to_string("config.toml") {
+        log::info!("Contents of `config.toml`:\n{config}");
+    }
+
+    // Skip components that are not needed for try builds to speed them up
+    if is_try_build() {
+        log::info!("Skipping building of unimportant components for a try build");
+        for target in [
+            "rust-docs",
+            "rustc-docs",
+            "rust-docs-json",
+            "rust-analyzer",
+            "rustc-src",
+            "clippy",
+            "miri",
+            "rustfmt",
+        ] {
+            build_args.extend(["--exclude".to_string(), target.to_string()]);
+        }
+    }
+
+    let mut timer = Timer::new();
+    let env = create_environment();
+
+    let result = execute_pipeline(env.as_ref(), &mut timer, build_args);
+    log::info!("Timer results\n{}", timer.format_stats());
+
+    print_free_disk_space()?;
+
+    result.context("Optimized build pipeline has failed")
+}
diff --git a/src/tools/opt-dist/src/metrics.rs b/src/tools/opt-dist/src/metrics.rs
new file mode 100644
index 00000000000..cabe07eda32
--- /dev/null
+++ b/src/tools/opt-dist/src/metrics.rs
@@ -0,0 +1,106 @@
+use crate::timer::TimerSection;
+use build_helper::metrics::{JsonNode, JsonRoot};
+use camino::Utf8Path;
+use std::time::Duration;
+
+#[derive(Clone, Debug)]
+pub struct BuildStep {
+    r#type: String,
+    children: Vec<BuildStep>,
+    duration: Duration,
+}
+
+impl BuildStep {
+    pub fn find_all_by_type(&self, r#type: &str) -> Vec<&BuildStep> {
+        let mut result = Vec::new();
+        self.find_by_type(r#type, &mut result);
+        result
+    }
+    fn find_by_type<'a>(&'a self, r#type: &str, result: &mut Vec<&'a BuildStep>) {
+        if self.r#type == r#type {
+            result.push(self);
+        }
+        for child in &self.children {
+            child.find_by_type(r#type, result);
+        }
+    }
+}
+
+/// Loads the metrics of the most recent bootstrap execution from a metrics.json file.
+pub fn load_metrics(path: &Utf8Path) -> anyhow::Result<BuildStep> {
+    let content = std::fs::read(path.as_std_path())?;
+    let mut metrics = serde_json::from_slice::<JsonRoot>(&content)?;
+    let invocation = metrics
+        .invocations
+        .pop()
+        .ok_or_else(|| anyhow::anyhow!("No bootstrap invocation found in metrics file"))?;
+
+    fn parse(node: JsonNode) -> Option<BuildStep> {
+        match node {
+            JsonNode::RustbuildStep {
+                type_: kind,
+                children,
+                duration_excluding_children_sec,
+                ..
+            } => {
+                let children: Vec<_> = children.into_iter().filter_map(parse).collect();
+                let children_duration = children.iter().map(|c| c.duration).sum::<Duration>();
+                Some(BuildStep {
+                    r#type: kind.to_string(),
+                    children,
+                    duration: children_duration
+                        + Duration::from_secs_f64(duration_excluding_children_sec),
+                })
+            }
+            JsonNode::TestSuite(_) => None,
+        }
+    }
+
+    let duration = Duration::from_secs_f64(invocation.duration_including_children_sec);
+    let children: Vec<_> = invocation.children.into_iter().filter_map(parse).collect();
+    Ok(BuildStep { r#type: "root".to_string(), children, duration })
+}
+
+/// Logs the individual metrics in a table and add Rustc and LLVM durations to the passed
+/// timer.
+pub fn record_metrics(metrics: &BuildStep, timer: &mut TimerSection) {
+    let llvm_steps = metrics.find_all_by_type("bootstrap::llvm::Llvm");
+    let llvm_duration: Duration = llvm_steps.into_iter().map(|s| s.duration).sum();
+
+    let rustc_steps = metrics.find_all_by_type("bootstrap::compile::Rustc");
+    let rustc_duration: Duration = rustc_steps.into_iter().map(|s| s.duration).sum();
+
+    // The LLVM step is part of the Rustc step
+    let rustc_duration = rustc_duration.saturating_sub(llvm_duration);
+
+    if !llvm_duration.is_zero() {
+        timer.add_duration("LLVM", llvm_duration);
+    }
+    if !rustc_duration.is_zero() {
+        timer.add_duration("Rustc", rustc_duration);
+    }
+
+    log_metrics(metrics);
+}
+
+fn log_metrics(metrics: &BuildStep) {
+    use std::fmt::Write;
+
+    let mut substeps: Vec<(u32, &BuildStep)> = Vec::new();
+
+    fn visit<'a>(step: &'a BuildStep, level: u32, substeps: &mut Vec<(u32, &'a BuildStep)>) {
+        substeps.push((level, step));
+        for child in &step.children {
+            visit(child, level + 1, substeps);
+        }
+    }
+
+    visit(metrics, 0, &mut substeps);
+
+    let mut output = String::new();
+    for (level, step) in substeps {
+        let label = format!("{}{}", ".".repeat(level as usize), step.r#type);
+        writeln!(output, "{label:<65}{:>8.2}s", step.duration.as_secs_f64()).unwrap();
+    }
+    log::info!("Build step durations\n{output}");
+}
diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs
new file mode 100644
index 00000000000..6b35b13e586
--- /dev/null
+++ b/src/tools/opt-dist/src/tests.rs
@@ -0,0 +1,101 @@
+use crate::environment::Environment;
+use crate::exec::cmd;
+use crate::utils::io::{copy_directory, unpack_archive};
+use anyhow::Context;
+use camino::Utf8PathBuf;
+
+/// Run tests on optimized dist artifacts.
+pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
+    // After `dist` is executed, we extract its archived components into a sysroot directory,
+    // and then use that extracted rustc as a stage0 compiler.
+    // Then we run a subset of tests using that compiler, to have a basic smoke test which checks
+    // whether the optimization pipeline hasn't broken something.
+    let build_dir = env.build_root().join("build");
+    let dist_dir = build_dir.join("dist");
+    let unpacked_dist_dir = build_dir.join("unpacked-dist");
+    std::fs::create_dir_all(&unpacked_dist_dir)?;
+
+    let extract_dist_dir = |name: &str| -> anyhow::Result<Utf8PathBuf> {
+        unpack_archive(&dist_dir.join(format!("{name}.tar.xz")), &unpacked_dist_dir)?;
+        let extracted_path = unpacked_dist_dir.join(name);
+        assert!(extracted_path.is_dir());
+        Ok(extracted_path)
+    };
+    let host_triple = env.host_triple();
+
+    // Extract rustc, libstd, cargo and src archives to create the optimized sysroot
+    let rustc_dir = extract_dist_dir(&format!("rustc-nightly-{host_triple}"))?.join("rustc");
+    let libstd_dir = extract_dist_dir(&format!("rust-std-nightly-{host_triple}"))?
+        .join(format!("rust-std-{host_triple}"));
+    let cargo_dir = extract_dist_dir(&format!("cargo-nightly-{host_triple}"))?.join("cargo");
+    let extracted_src_dir = extract_dist_dir("rust-src-nightly")?.join("rust-src");
+
+    // We need to manually copy libstd to the extracted rustc sysroot
+    copy_directory(
+        &libstd_dir.join("lib").join("rustlib").join(&host_triple).join("lib"),
+        &rustc_dir.join("lib").join("rustlib").join(&host_triple).join("lib"),
+    )?;
+
+    // Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd
+    // sources to the extracted sysroot. We need sources available so that `-Zsimulate-remapped-rust-src-base`
+    // works correctly.
+    copy_directory(
+        &extracted_src_dir.join("lib").join("rustlib").join("src"),
+        &rustc_dir.join("lib").join("rustlib").join("src"),
+    )?;
+
+    let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", env.executable_extension()));
+    assert!(rustc_path.is_file());
+    let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", env.executable_extension()));
+    assert!(cargo_path.is_file());
+
+    // Specify path to a LLVM config so that LLVM is not rebuilt.
+    // It doesn't really matter which LLVM config we choose, because no sysroot will be compiled.
+    let llvm_config = env
+        .build_artifacts()
+        .join("llvm")
+        .join("bin")
+        .join(format!("llvm-config{}", env.executable_extension()));
+    assert!(llvm_config.is_file());
+
+    let config_content = format!(
+        r#"profile = "user"
+changelog-seen = 2
+
+[build]
+rustc = "{rustc}"
+cargo = "{cargo}"
+
+[target.{host_triple}]
+llvm-config = "{llvm_config}"
+"#,
+        rustc = rustc_path.to_string().replace('\\', "/"),
+        cargo = cargo_path.to_string().replace('\\', "/"),
+        llvm_config = llvm_config.to_string().replace('\\', "/")
+    );
+    log::info!("Using following `config.toml` for running tests:\n{config_content}");
+
+    // Simulate a stage 0 compiler with the extracted optimized dist artifacts.
+    std::fs::write("config.toml", config_content)?;
+
+    let x_py = env.checkout_path().join("x.py");
+    let mut args = vec![
+        env.python_binary(),
+        x_py.as_str(),
+        "test",
+        "--stage",
+        "0",
+        "tests/assembly",
+        "tests/codegen",
+        "tests/codegen-units",
+        "tests/incremental",
+        "tests/mir-opt",
+        "tests/pretty",
+        "tests/run-pass-valgrind",
+        "tests/ui",
+    ];
+    for test_path in env.skipped_tests() {
+        args.extend(["--exclude", test_path]);
+    }
+    cmd(&args).env("COMPILETEST_FORCE_STAGE0", "1").run().context("Cannot execute tests")
+}
diff --git a/src/tools/opt-dist/src/timer.rs b/src/tools/opt-dist/src/timer.rs
new file mode 100644
index 00000000000..2b29ba8d59f
--- /dev/null
+++ b/src/tools/opt-dist/src/timer.rs
@@ -0,0 +1,167 @@
+use std::ops::{Deref, DerefMut};
+use std::time::{Duration, SystemTime};
+
+pub struct Timer {
+    root: TimerSection,
+}
+
+impl Timer {
+    pub fn new() -> Self {
+        Timer { root: TimerSection::new(None) }
+    }
+
+    pub fn format_stats(&self) -> String {
+        use std::fmt::Write;
+
+        let mut items = Vec::new();
+        for (name, child) in &self.root.children {
+            match child {
+                SectionEntry::SubSection(section) => {
+                    section.collect_levels(0, name, &mut items);
+                }
+                SectionEntry::Duration(duration) => items.push((0, name, *duration)),
+            }
+        }
+
+        let rows: Vec<(String, Duration)> = items
+            .into_iter()
+            .map(|(level, name, duration)| (format!("{}{name}:", "  ".repeat(level)), duration))
+            .collect();
+
+        let total_duration = self.total_duration();
+        let total_duration_label = "Total duration:".to_string();
+
+        const SPACE_AFTER_LABEL: usize = 2;
+        let max_label_length = 16.max(rows.iter().map(|(label, _)| label.len()).max().unwrap_or(0))
+            + SPACE_AFTER_LABEL;
+
+        let table_width = max_label_length + 23;
+        let divider = "-".repeat(table_width);
+
+        let mut output = String::new();
+        writeln!(output, "{divider}").unwrap();
+        for (label, duration) in rows {
+            let pct = (duration.as_millis() as f64 / total_duration.as_millis() as f64) * 100.0;
+            let duration_fmt = format!("{:>12.2}s ({pct:>5.2}%)", duration.as_secs_f64());
+            writeln!(output, "{label:<0$} {duration_fmt}", max_label_length).unwrap();
+        }
+        output.push('\n');
+
+        let total_duration = Duration::new(total_duration.as_secs(), 0);
+        let total_duration = format!(
+            "{:>1$}",
+            humantime::format_duration(total_duration).to_string(),
+            table_width - total_duration_label.len()
+        );
+        writeln!(output, "{total_duration_label}{total_duration}").unwrap();
+
+        writeln!(output, "{divider}").unwrap();
+        output
+    }
+}
+
+impl Deref for Timer {
+    type Target = TimerSection;
+
+    fn deref(&self) -> &Self::Target {
+        &self.root
+    }
+}
+
+impl DerefMut for Timer {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.root
+    }
+}
+
+pub struct TimerSection {
+    name: Option<String>,
+    children: Vec<(String, SectionEntry)>,
+    duration_excluding_children: Duration,
+}
+
+impl TimerSection {
+    pub fn new(name: Option<String>) -> Self {
+        TimerSection {
+            name,
+            children: Default::default(),
+            duration_excluding_children: Duration::ZERO,
+        }
+    }
+
+    pub fn section<F: FnOnce(&mut TimerSection) -> anyhow::Result<R>, R>(
+        &mut self,
+        name: &str,
+        func: F,
+    ) -> anyhow::Result<R> {
+        let full_name = match &self.name {
+            Some(current_name) => {
+                format!("{current_name} > {name}")
+            }
+            None => name.to_string(),
+        };
+        log::info!("Section `{full_name}` starts");
+        let mut child = TimerSection {
+            name: Some(full_name.clone()),
+            children: Default::default(),
+            duration_excluding_children: Duration::ZERO,
+        };
+
+        let start = SystemTime::now();
+        let result = func(&mut child);
+        let duration = start.elapsed().unwrap();
+
+        let msg = match result {
+            Ok(_) => "OK",
+            Err(_) => "FAIL",
+        };
+
+        child.duration_excluding_children = duration.saturating_sub(child.total_duration());
+
+        log::info!("Section `{full_name}` ended: {msg} ({:.2}s)`", duration.as_secs_f64());
+        self.children.push((name.to_string(), SectionEntry::SubSection(child)));
+        result
+    }
+
+    pub fn add_duration(&mut self, name: &str, duration: Duration) {
+        self.children.push((name.to_string(), SectionEntry::Duration(duration)));
+    }
+
+    fn total_duration(&self) -> Duration {
+        self.duration_excluding_children
+            + self.children.iter().map(|(_, child)| child.total_duration()).sum::<Duration>()
+    }
+
+    fn collect_levels<'a>(
+        &'a self,
+        level: usize,
+        name: &'a str,
+        items: &mut Vec<(usize, &'a str, Duration)>,
+    ) {
+        items.push((level, name, self.total_duration()));
+        for (name, child) in &self.children {
+            match &child {
+                SectionEntry::Duration(duration) => {
+                    items.push((level + 1, name, *duration));
+                }
+                SectionEntry::SubSection(section) => {
+                    section.collect_levels(level + 1, name, items);
+                }
+            }
+        }
+    }
+}
+
+enum SectionEntry {
+    Duration(Duration),
+    SubSection(TimerSection),
+}
+
+impl SectionEntry {
+    fn total_duration(&self) -> Duration {
+        match self {
+            SectionEntry::Duration(duration) => *duration,
+            SectionEntry::SubSection(timer) => timer.total_duration(),
+        }
+    }
+}
diff --git a/src/tools/opt-dist/src/training.rs b/src/tools/opt-dist/src/training.rs
new file mode 100644
index 00000000000..10f4a603695
--- /dev/null
+++ b/src/tools/opt-dist/src/training.rs
@@ -0,0 +1,202 @@
+use crate::environment::Environment;
+use crate::exec::{cmd, CmdBuilder};
+use crate::utils::io::{count_files, delete_directory};
+use anyhow::Context;
+use camino::{Utf8Path, Utf8PathBuf};
+use humansize::BINARY;
+
+const LLVM_PGO_CRATES: &[&str] = &[
+    "syn-1.0.89",
+    "cargo-0.60.0",
+    "serde-1.0.136",
+    "ripgrep-13.0.0",
+    "regex-1.5.5",
+    "clap-3.1.6",
+    "hyper-0.14.18",
+];
+
+const RUSTC_PGO_CRATES: &[&str] = &[
+    "externs",
+    "ctfe-stress-5",
+    "cargo-0.60.0",
+    "token-stream-stress",
+    "match-stress",
+    "tuple-stress",
+    "diesel-1.4.8",
+    "bitmaps-3.1.0",
+];
+
+const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES;
+
+fn init_compiler_benchmarks(
+    env: &dyn Environment,
+    profiles: &[&str],
+    scenarios: &[&str],
+    crates: &[&str],
+) -> CmdBuilder {
+    // Run rustc-perf benchmarks
+    // Benchmark using profile_local with eprintln, which essentially just means
+    // don't actually benchmark -- just make sure we run rustc a bunch of times.
+    cmd(&[
+        env.cargo_stage_0().as_str(),
+        "run",
+        "-p",
+        "collector",
+        "--bin",
+        "collector",
+        "--",
+        "profile_local",
+        "eprintln",
+        env.rustc_stage_2().as_str(),
+        "--id",
+        "Test",
+        "--cargo",
+        env.cargo_stage_0().as_str(),
+        "--profiles",
+        profiles.join(",").as_str(),
+        "--scenarios",
+        scenarios.join(",").as_str(),
+        "--include",
+        crates.join(",").as_str(),
+    ])
+    .env("RUST_LOG", "collector=debug")
+    .env("RUSTC", env.rustc_stage_0().as_str())
+    .env("RUSTC_BOOTSTRAP", "1")
+    .workdir(&env.rustc_perf_dir())
+}
+
+fn merge_llvm_profiles(
+    env: &dyn Environment,
+    merged_path: &Utf8Path,
+    profile_dir: &Utf8Path,
+) -> anyhow::Result<()> {
+    cmd(&[
+        env.downloaded_llvm_dir().join("bin/llvm-profdata").as_str(),
+        "merge",
+        "-o",
+        merged_path.as_str(),
+        profile_dir.as_str(),
+    ])
+    .run()
+    .context("Cannot merge LLVM profiles")?;
+    Ok(())
+}
+
+fn log_profile_stats(
+    name: &str,
+    merged_profile: &Utf8Path,
+    profile_root: &Utf8Path,
+) -> anyhow::Result<()> {
+    log::info!("{name} PGO statistics");
+    log::info!(
+        "{merged_profile}: {}",
+        humansize::format_size(std::fs::metadata(merged_profile.as_std_path())?.len(), BINARY)
+    );
+    log::info!(
+        "{profile_root}: {}",
+        humansize::format_size(fs_extra::dir::get_size(profile_root.as_std_path())?, BINARY)
+    );
+    log::info!("Profile file count: {}", count_files(profile_root)?);
+    Ok(())
+}
+
+pub struct LlvmPGOProfile(pub Utf8PathBuf);
+
+pub fn gather_llvm_profiles(
+    env: &dyn Environment,
+    profile_root: &Utf8Path,
+) -> anyhow::Result<LlvmPGOProfile> {
+    log::info!("Running benchmarks with PGO instrumented LLVM");
+
+    init_compiler_benchmarks(env, &["Debug", "Opt"], &["Full"], LLVM_PGO_CRATES)
+        .run()
+        .context("Cannot gather LLVM PGO profiles")?;
+
+    let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata");
+    log::info!("Merging LLVM PGO profiles to {merged_profile}");
+
+    merge_llvm_profiles(env, &merged_profile, profile_root)?;
+    log_profile_stats("LLVM", &merged_profile, profile_root)?;
+
+    // We don't need the individual .profraw files now that they have been merged
+    // into a final .profdata
+    delete_directory(profile_root)?;
+
+    Ok(LlvmPGOProfile(merged_profile))
+}
+
+pub struct RustcPGOProfile(pub Utf8PathBuf);
+
+pub fn gather_rustc_profiles(
+    env: &dyn Environment,
+    profile_root: &Utf8Path,
+) -> anyhow::Result<RustcPGOProfile> {
+    log::info!("Running benchmarks with PGO instrumented rustc");
+
+    // The profile data is written into a single filepath that is being repeatedly merged when each
+    // rustc invocation ends. Empirically, this can result in some profiling data being lost. That's
+    // why we override the profile path to include the PID. This will produce many more profiling
+    // files, but the resulting profile will produce a slightly faster rustc binary.
+    let profile_template = profile_root.join("default_%m_%p.profraw");
+
+    // Here we're profiling the `rustc` frontend, so we also include `Check`.
+    // The benchmark set includes various stress tests that put the frontend under pressure.
+    init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES)
+        .env("LLVM_PROFILE_FILE", profile_template.as_str())
+        .run()
+        .context("Cannot gather rustc PGO profiles")?;
+
+    let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata");
+    log::info!("Merging Rustc PGO profiles to {merged_profile}");
+
+    merge_llvm_profiles(env, &merged_profile, profile_root)?;
+    log_profile_stats("Rustc", &merged_profile, profile_root)?;
+
+    // We don't need the individual .profraw files now that they have been merged
+    // into a final .profdata
+    delete_directory(profile_root)?;
+
+    Ok(RustcPGOProfile(merged_profile))
+}
+
+pub struct LlvmBoltProfile(pub Utf8PathBuf);
+
+pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBoltProfile> {
+    log::info!("Running benchmarks with BOLT instrumented LLVM");
+
+    init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["Full"], LLVM_BOLT_CRATES)
+        .run()
+        .context("Cannot gather LLVM BOLT profiles")?;
+
+    let merged_profile = env.opt_artifacts().join("bolt.profdata");
+    let profile_root = Utf8PathBuf::from("/tmp/prof.fdata");
+    log::info!("Merging LLVM BOLT profiles to {merged_profile}");
+
+    let profiles: Vec<_> =
+        glob::glob(&format!("{profile_root}*"))?.into_iter().collect::<Result<Vec<_>, _>>()?;
+
+    let mut merge_args = vec!["merge-fdata"];
+    merge_args.extend(profiles.iter().map(|p| p.to_str().unwrap()));
+
+    cmd(&merge_args)
+        .redirect_output(merged_profile.clone())
+        .run()
+        .context("Cannot merge BOLT profiles")?;
+
+    log::info!("LLVM BOLT statistics");
+    log::info!(
+        "{merged_profile}: {}",
+        humansize::format_size(std::fs::metadata(merged_profile.as_std_path())?.len(), BINARY)
+    );
+
+    let size = profiles
+        .iter()
+        .map(|p| std::fs::metadata(p).map(|metadata| metadata.len()))
+        .collect::<Result<Vec<_>, _>>()?
+        .into_iter()
+        .sum::<u64>();
+    log::info!("{profile_root}: {}", humansize::format_size(size, BINARY));
+    log::info!("Profile file count: {}", profiles.len());
+
+    Ok(LlvmBoltProfile(merged_profile))
+}
diff --git a/src/tools/opt-dist/src/utils/io.rs b/src/tools/opt-dist/src/utils/io.rs
new file mode 100644
index 00000000000..43546e5fcfa
--- /dev/null
+++ b/src/tools/opt-dist/src/utils/io.rs
@@ -0,0 +1,48 @@
+use anyhow::Context;
+use camino::Utf8Path;
+use fs_extra::dir::CopyOptions;
+use std::fs::File;
+
+/// Delete and re-create the directory.
+pub fn reset_directory(path: &Utf8Path) -> anyhow::Result<()> {
+    log::info!("Resetting directory {path}");
+    let _ = std::fs::remove_dir(path);
+    std::fs::create_dir_all(path)?;
+    Ok(())
+}
+
+pub fn copy_directory(src: &Utf8Path, dst: &Utf8Path) -> anyhow::Result<()> {
+    log::info!("Copying directory {src} to {dst}");
+    fs_extra::dir::copy(src, dst, &CopyOptions::default().copy_inside(true))?;
+    Ok(())
+}
+
+#[allow(unused)]
+pub fn move_directory(src: &Utf8Path, dst: &Utf8Path) -> anyhow::Result<()> {
+    log::info!("Moving directory {src} to {dst}");
+    fs_extra::dir::move_dir(src, dst, &CopyOptions::default().content_only(true))?;
+    Ok(())
+}
+
+/// Counts all children of a directory (non-recursively).
+pub fn count_files(dir: &Utf8Path) -> anyhow::Result<u64> {
+    Ok(std::fs::read_dir(dir)?.count() as u64)
+}
+
+pub fn delete_directory(path: &Utf8Path) -> anyhow::Result<()> {
+    log::info!("Deleting directory `{path}`");
+    std::fs::remove_dir_all(path.as_std_path())
+        .context(format!("Cannot remove directory {path}"))?;
+    Ok(())
+}
+
+pub fn unpack_archive(path: &Utf8Path, dest_dir: &Utf8Path) -> anyhow::Result<()> {
+    log::info!("Unpacking directory `{path}` into `{dest_dir}`");
+
+    assert!(path.as_str().ends_with(".tar.xz"));
+    let file = File::open(path.as_std_path())?;
+    let file = xz::read::XzDecoder::new(file);
+    let mut archive = tar::Archive::new(file);
+    archive.unpack(dest_dir.as_std_path())?;
+    Ok(())
+}
diff --git a/src/tools/opt-dist/src/utils/mod.rs b/src/tools/opt-dist/src/utils/mod.rs
new file mode 100644
index 00000000000..9305d4989aa
--- /dev/null
+++ b/src/tools/opt-dist/src/utils/mod.rs
@@ -0,0 +1,36 @@
+pub mod io;
+
+use crate::environment::Environment;
+use crate::utils::io::delete_directory;
+use humansize::BINARY;
+use sysinfo::{DiskExt, RefreshKind, System, SystemExt};
+
+pub fn format_env_variables() -> String {
+    let vars = std::env::vars().map(|(key, value)| format!("{key}={value}")).collect::<Vec<_>>();
+    vars.join("\n")
+}
+
+pub fn print_free_disk_space() -> anyhow::Result<()> {
+    let sys = System::new_with_specifics(RefreshKind::default().with_disks_list().with_disks());
+    let available_space: u64 = sys.disks().iter().map(|d| d.available_space()).sum();
+    let total_space: u64 = sys.disks().iter().map(|d| d.total_space()).sum();
+    let used_space = total_space - available_space;
+
+    log::info!(
+        "Free disk space: {} out of total {} ({:.2}% used)",
+        humansize::format_size(available_space, BINARY),
+        humansize::format_size(total_space, BINARY),
+        (used_space as f64 / total_space as f64) * 100.0
+    );
+    Ok(())
+}
+
+pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> {
+    // Bootstrap currently doesn't support rebuilding LLVM when PGO options
+    // change (or any other llvm-related options); so just clear out the relevant
+    // directories ourselves.
+    log::info!("Clearing LLVM build files");
+    delete_directory(&env.build_artifacts().join("llvm"))?;
+    delete_directory(&env.build_artifacts().join("lld"))?;
+    Ok(())
+}
diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index ecc84c1618c..1dea956c117 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -40,10 +40,12 @@ const EXCEPTIONS: &[(&str, &str)] = &[
     ("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc
     ("colored", "MPL-2.0"),                                  // rustfmt
     ("dissimilar", "Apache-2.0"),                            // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps)
+    ("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"), // opt-dist
     ("fluent-langneg", "Apache-2.0"),                        // rustc (fluent translations)
     ("fortanix-sgx-abi", "MPL-2.0"),                         // libstd but only for `sgx` target. FIXME: this dependency violates the documentation comment above.
     ("instant", "BSD-3-Clause"),                             // rustc_driver/tracing-subscriber/parking_lot
     ("mdbook", "MPL-2.0"),                                   // mdbook
+    ("openssl", "Apache-2.0"),                               // opt-dist
     ("ryu", "Apache-2.0 OR BSL-1.0"),                        // cargo/... (because of serde)
     ("self_cell", "Apache-2.0"),                             // rustc (fluent translations)
     ("snap", "BSD-3-Clause"),                                // rustc