Rollup merge of #97828 - ferrocene:pa-config-artifacts, r=jyn514

Allow configuring where artifacts are downloaded from

Bootstrap has support for downloading prebuilt LLVM and rustc artifacts to speed up local builds, but that currently works only for users working on `rust-lang/rust`. Forks of the repository (for example Ferrocene) might have different URLs to download artifacts from, or might use a different email address on merge commits, breaking both LLVM and rustc artifact downloads.

This PR refactors bootstrap to load the download URLs and other constants from `src/stage0.json`, allowing downstream forks to tweak those values. It also future-proofs the download code to easily allow forks to add their own custom protocols (like `s3://`).

This PR is best reviewed commit-by-commit.

5 files changed, 69 insertions, 51 deletions

diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index 635e4f3703b..3b2b507b062 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -1043,7 +1043,7 @@ def bootstrap(help_triggered):
     build.checksums_sha256 = data["checksums_sha256"]
     build.stage0_compiler = Stage0Toolchain(data["compiler"])
 
-    build.set_dist_environment(data["dist_server"])
+    build.set_dist_environment(data["config"]["dist_server"])
 
     build.build = args.build or build.build_triple()
 
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 38d4f15d3c8..7b74c5ccdbb 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -870,20 +870,23 @@ impl<'a> Builder<'a> {
         self.try_run(patchelf.arg(fname));
     }
 
-    pub(crate) fn download_component(
-        &self,
-        base: &str,
-        url: &str,
-        dest_path: &Path,
-        help_on_error: &str,
-    ) {
+    pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) {
         // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
         let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
-        self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error);
+        // While bootstrap itself only supports http and https downloads, downstream forks might
+        // need to download components from other protocols. The match allows them adding more
+        // protocols without worrying about merge conficts if we change the HTTP implementation.
+        match url.split_once("://").map(|(proto, _)| proto) {
+            Some("http") | Some("https") => {
+                self.download_http_with_retries(&tempfile, url, help_on_error)
+            }
+            Some(other) => panic!("unsupported protocol {other} in {url}"),
+            None => panic!("no protocol in {url}"),
+        }
         t!(std::fs::rename(&tempfile, dest_path));
     }
 
-    fn download_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
+    fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
         println!("downloading {}", url);
         // Try curl. If that fails and we are on windows, fallback to PowerShell.
         let mut curl = Command::new("curl");
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 28663af135c..14607741932 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -20,7 +20,6 @@ use crate::channel::GitInfo;
 pub use crate::flags::Subcommand;
 use crate::flags::{Color, Flags};
 use crate::util::{exe, output, program_out_of_date, t};
-use crate::RustfmtMetadata;
 use once_cell::sync::OnceCell;
 use serde::{Deserialize, Deserializer};
 
@@ -73,6 +72,7 @@ pub struct Config {
     pub test_compare_mode: bool,
     pub color: Color,
     pub patch_binaries_for_nix: bool,
+    pub stage0_metadata: Stage0Metadata,
 
     pub on_fail: Option<String>,
     pub stage: u32,
@@ -212,6 +212,28 @@ pub struct Config {
     pub out: PathBuf,
 }
 
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Metadata {
+    pub config: Stage0Config,
+    pub checksums_sha256: HashMap<String, String>,
+    pub rustfmt: Option<RustfmtMetadata>,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct Stage0Config {
+    pub dist_server: String,
+    pub artifacts_server: String,
+    pub artifacts_with_llvm_assertions_server: String,
+    pub git_merge_commit_email: String,
+}
+#[derive(Default, Deserialize)]
+#[cfg_attr(test, derive(Clone))]
+pub struct RustfmtMetadata {
+    pub date: String,
+    pub version: String,
+}
+
 #[derive(Clone, Debug)]
 pub enum RustfmtState {
     SystemToolchain(PathBuf),
@@ -776,6 +798,9 @@ impl Config {
         config.llvm_profile_use = flags.llvm_profile_use;
         config.llvm_profile_generate = flags.llvm_profile_generate;
 
+        let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json")));
+        config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
+
         #[cfg(test)]
         let get_toml = |_| TomlConfig::default();
         #[cfg(not(test))]
@@ -1103,8 +1128,11 @@ impl Config {
             config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
             config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
             config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
-            config.download_rustc_commit =
-                download_ci_rustc_commit(rust.download_rustc, config.verbose > 0);
+            config.download_rustc_commit = download_ci_rustc_commit(
+                &config.stage0_metadata,
+                rust.download_rustc,
+                config.verbose > 0,
+            );
         } else {
             config.rust_profile_use = flags.rust_profile_use;
             config.rust_profile_generate = flags.rust_profile_generate;
@@ -1424,7 +1452,11 @@ fn threads_from_config(v: u32) -> u32 {
 }
 
 /// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
-fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) -> Option<String> {
+fn download_ci_rustc_commit(
+    stage0_metadata: &Stage0Metadata,
+    download_rustc: Option<StringOrBool>,
+    verbose: bool,
+) -> Option<String> {
     // If `download-rustc` is not set, default to rebuilding.
     let if_unchanged = match download_rustc {
         None | Some(StringOrBool::Bool(false)) => return None,
@@ -1443,13 +1475,12 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
 
     // Look for a version to compare to based on the current commit.
     // Only commits merged by bors will have CI artifacts.
-    let merge_base = output(Command::new("git").args(&[
-        "rev-list",
-        "--author=bors@rust-lang.org",
-        "-n1",
-        "--first-parent",
-        "HEAD",
-    ]));
+    let merge_base = output(
+        Command::new("git")
+            .arg("rev-list")
+            .arg(format!("--author={}", stage0_metadata.config.git_merge_commit_email))
+            .args(&["-n1", "--first-parent", "HEAD"]),
+    );
     let commit = merge_base.trim_end();
     if commit.is_empty() {
         println!("error: could not find commit hash for downloading rustc");
@@ -1484,7 +1515,7 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
 }
 
 fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
-    let RustfmtMetadata { date, version } = builder.stage0_metadata.rustfmt.as_ref()?;
+    let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?;
     let channel = format!("{version}-{date}");
 
     let host = builder.config.build;
@@ -1568,13 +1599,13 @@ fn download_component(
     let tarball = cache_dir.join(&filename);
     let (base_url, url, should_verify) = match mode {
         DownloadSource::CI => (
-            "https://ci-artifacts.rust-lang.org/rustc-builds".to_string(),
+            builder.config.stage0_metadata.config.artifacts_server.clone(),
             format!("{key}/{filename}"),
             false,
         ),
         DownloadSource::Dist => {
             let dist_server = env::var("RUSTUP_DIST_SERVER")
-                .unwrap_or(builder.stage0_metadata.dist_server.to_string());
+                .unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string());
             // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
             (dist_server, format!("dist/{key}/{filename}"), true)
         }
@@ -1590,7 +1621,7 @@ fn download_component(
             target at this time, see https://doc.rust-lang.org/nightly\
             /rustc/platform-support.html for more information."
         );
-        let sha256 = builder.stage0_metadata.checksums_sha256.get(&url).expect(&error);
+        let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error);
         if tarball.exists() {
             if builder.verify(&tarball, sha256) {
                 builder.unpack(&tarball, &bin_root, prefix);
@@ -1610,7 +1641,7 @@ fn download_component(
         None
     };
 
-    builder.download_component(&base_url, &url, &tarball, "");
+    builder.download_component(&format!("{base_url}/{url}"), &tarball, "");
     if let Some(sha256) = checksum {
         if !builder.verify(&tarball, sha256) {
             panic!("failed to verify {}", tarball.display());
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index b4333566f07..49096426a98 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -118,7 +118,6 @@ use std::os::windows::fs::symlink_file;
 
 use filetime::FileTime;
 use once_cell::sync::OnceCell;
-use serde::Deserialize;
 
 use crate::builder::Kind;
 use crate::config::{LlvmLibunwind, TargetSelection};
@@ -294,8 +293,6 @@ pub struct Build {
     hosts: Vec<TargetSelection>,
     targets: Vec<TargetSelection>,
 
-    // Stage 0 (downloaded) compiler, lld and cargo or their local rust equivalents
-    stage0_metadata: Stage0Metadata,
     initial_rustc: PathBuf,
     initial_cargo: PathBuf,
     initial_lld: PathBuf,
@@ -322,18 +319,6 @@ pub struct Build {
     metrics: metrics::BuildMetrics,
 }
 
-#[derive(Deserialize)]
-struct Stage0Metadata {
-    dist_server: String,
-    checksums_sha256: HashMap<String, String>,
-    rustfmt: Option<RustfmtMetadata>,
-}
-#[derive(Deserialize)]
-struct RustfmtMetadata {
-    date: String,
-    version: String,
-}
-
 #[derive(Debug)]
 struct Crate {
     name: Interned<String>,
@@ -482,11 +467,7 @@ impl Build {
             bootstrap_out
         };
 
-        let stage0_json = t!(std::fs::read_to_string(&src.join("src").join("stage0.json")));
-        let stage0_metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
-
         let mut build = Build {
-            stage0_metadata,
             initial_rustc: config.initial_rustc.clone(),
             initial_cargo: config.initial_cargo.clone(),
             initial_lld,
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index 18f82d024f8..329bb68672e 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -121,7 +121,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
     let mut rev_list = Command::new("git");
     rev_list.args(&[
         PathBuf::from("rev-list"),
-        "--author=bors@rust-lang.org".into(),
+        format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(),
         "-n1".into(),
         "--first-parent".into(),
         "HEAD".into(),
@@ -170,11 +170,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
     if !rustc_cache.exists() {
         t!(fs::create_dir_all(&rustc_cache));
     }
-    let base = "https://ci-artifacts.rust-lang.org";
-    let url = if llvm_assertions {
-        format!("rustc-builds-alt/{}", llvm_sha)
+    let base = if llvm_assertions {
+        &builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server
     } else {
-        format!("rustc-builds/{}", llvm_sha)
+        &builder.config.stage0_metadata.config.artifacts_server
     };
     let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
     let tarball = rustc_cache.join(&filename);
@@ -187,7 +186,11 @@ help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in
 [llvm]
 download-ci-llvm = false
 ";
-        builder.download_component(base, &format!("{}/{}", url, filename), &tarball, help_on_error);
+        builder.download_component(
+            &format!("{base}/{llvm_sha}/{filename}"),
+            &tarball,
+            help_on_error,
+        );
     }
     let llvm_root = builder.config.ci_llvm_root();
     builder.unpack(&tarball, &llvm_root, "rust-dev");