diff options
| -rw-r--r-- | src/bootstrap/bootstrap.py | 2 | ||||
| -rw-r--r-- | src/bootstrap/builder.rs | 21 | ||||
| -rw-r--r-- | src/bootstrap/config.rs | 63 | ||||
| -rw-r--r-- | src/bootstrap/lib.rs | 19 | ||||
| -rw-r--r-- | src/bootstrap/native.rs | 15 | ||||
| -rw-r--r-- | src/stage0.json | 18 | ||||
| -rw-r--r-- | src/tools/bump-stage0/src/main.rs | 67 |
7 files changed, 132 insertions, 73 deletions
diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 635e4f3703b..3b2b507b062 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -1043,7 +1043,7 @@ def bootstrap(help_triggered): build.checksums_sha256 = data["checksums_sha256"] build.stage0_compiler = Stage0Toolchain(data["compiler"]) - build.set_dist_environment(data["dist_server"]) + build.set_dist_environment(data["config"]["dist_server"]) build.build = args.build or build.build_triple() diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 38d4f15d3c8..7b74c5ccdbb 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -870,20 +870,23 @@ impl<'a> Builder<'a> { self.try_run(patchelf.arg(fname)); } - pub(crate) fn download_component( - &self, - base: &str, - url: &str, - dest_path: &Path, - help_on_error: &str, - ) { + pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) { // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); - self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error); + // While bootstrap itself only supports http and https downloads, downstream forks might + // need to download components from other protocols. The match allows them adding more + // protocols without worrying about merge conficts if we change the HTTP implementation. + match url.split_once("://").map(|(proto, _)| proto) { + Some("http") | Some("https") => { + self.download_http_with_retries(&tempfile, url, help_on_error) + } + Some(other) => panic!("unsupported protocol {other} in {url}"), + None => panic!("no protocol in {url}"), + } t!(std::fs::rename(&tempfile, dest_path)); } - fn download_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { + fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { println!("downloading {}", url); // Try curl. If that fails and we are on windows, fallback to PowerShell. let mut curl = Command::new("curl"); diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 28663af135c..14607741932 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -20,7 +20,6 @@ use crate::channel::GitInfo; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags}; use crate::util::{exe, output, program_out_of_date, t}; -use crate::RustfmtMetadata; use once_cell::sync::OnceCell; use serde::{Deserialize, Deserializer}; @@ -73,6 +72,7 @@ pub struct Config { pub test_compare_mode: bool, pub color: Color, pub patch_binaries_for_nix: bool, + pub stage0_metadata: Stage0Metadata, pub on_fail: Option<String>, pub stage: u32, @@ -212,6 +212,28 @@ pub struct Config { pub out: PathBuf, } +#[derive(Default, Deserialize)] +#[cfg_attr(test, derive(Clone))] +pub struct Stage0Metadata { + pub config: Stage0Config, + pub checksums_sha256: HashMap<String, String>, + pub rustfmt: Option<RustfmtMetadata>, +} +#[derive(Default, Deserialize)] +#[cfg_attr(test, derive(Clone))] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, +} +#[derive(Default, Deserialize)] +#[cfg_attr(test, derive(Clone))] +pub struct RustfmtMetadata { + pub date: String, + pub version: String, +} + #[derive(Clone, Debug)] pub enum RustfmtState { SystemToolchain(PathBuf), @@ -776,6 +798,9 @@ impl Config { config.llvm_profile_use = flags.llvm_profile_use; config.llvm_profile_generate = flags.llvm_profile_generate; + let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); + config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json)); + #[cfg(test)] let get_toml = |_| TomlConfig::default(); #[cfg(not(test))] @@ -1103,8 +1128,11 @@ impl Config { config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); - config.download_rustc_commit = - download_ci_rustc_commit(rust.download_rustc, config.verbose > 0); + config.download_rustc_commit = download_ci_rustc_commit( + &config.stage0_metadata, + rust.download_rustc, + config.verbose > 0, + ); } else { config.rust_profile_use = flags.rust_profile_use; config.rust_profile_generate = flags.rust_profile_generate; @@ -1424,7 +1452,11 @@ fn threads_from_config(v: u32) -> u32 { } /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. -fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) -> Option<String> { +fn download_ci_rustc_commit( + stage0_metadata: &Stage0Metadata, + download_rustc: Option<StringOrBool>, + verbose: bool, +) -> Option<String> { // If `download-rustc` is not set, default to rebuilding. let if_unchanged = match download_rustc { None | Some(StringOrBool::Bool(false)) => return None, @@ -1443,13 +1475,12 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) // Look for a version to compare to based on the current commit. // Only commits merged by bors will have CI artifacts. - let merge_base = output(Command::new("git").args(&[ - "rev-list", - "--author=bors@rust-lang.org", - "-n1", - "--first-parent", - "HEAD", - ])); + let merge_base = output( + Command::new("git") + .arg("rev-list") + .arg(format!("--author={}", stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); let commit = merge_base.trim_end(); if commit.is_empty() { println!("error: could not find commit hash for downloading rustc"); @@ -1484,7 +1515,7 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) } fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> { - let RustfmtMetadata { date, version } = builder.stage0_metadata.rustfmt.as_ref()?; + let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?; let channel = format!("{version}-{date}"); let host = builder.config.build; @@ -1568,13 +1599,13 @@ fn download_component( let tarball = cache_dir.join(&filename); let (base_url, url, should_verify) = match mode { DownloadSource::CI => ( - "https://ci-artifacts.rust-lang.org/rustc-builds".to_string(), + builder.config.stage0_metadata.config.artifacts_server.clone(), format!("{key}/{filename}"), false, ), DownloadSource::Dist => { let dist_server = env::var("RUSTUP_DIST_SERVER") - .unwrap_or(builder.stage0_metadata.dist_server.to_string()); + .unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string()); // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json (dist_server, format!("dist/{key}/{filename}"), true) } @@ -1590,7 +1621,7 @@ fn download_component( target at this time, see https://doc.rust-lang.org/nightly\ /rustc/platform-support.html for more information." ); - let sha256 = builder.stage0_metadata.checksums_sha256.get(&url).expect(&error); + let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error); if tarball.exists() { if builder.verify(&tarball, sha256) { builder.unpack(&tarball, &bin_root, prefix); @@ -1610,7 +1641,7 @@ fn download_component( None }; - builder.download_component(&base_url, &url, &tarball, ""); + builder.download_component(&format!("{base_url}/{url}"), &tarball, ""); if let Some(sha256) = checksum { if !builder.verify(&tarball, sha256) { panic!("failed to verify {}", tarball.display()); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index b4333566f07..49096426a98 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -118,7 +118,6 @@ use std::os::windows::fs::symlink_file; use filetime::FileTime; use once_cell::sync::OnceCell; -use serde::Deserialize; use crate::builder::Kind; use crate::config::{LlvmLibunwind, TargetSelection}; @@ -294,8 +293,6 @@ pub struct Build { hosts: Vec<TargetSelection>, targets: Vec<TargetSelection>, - // Stage 0 (downloaded) compiler, lld and cargo or their local rust equivalents - stage0_metadata: Stage0Metadata, initial_rustc: PathBuf, initial_cargo: PathBuf, initial_lld: PathBuf, @@ -322,18 +319,6 @@ pub struct Build { metrics: metrics::BuildMetrics, } -#[derive(Deserialize)] -struct Stage0Metadata { - dist_server: String, - checksums_sha256: HashMap<String, String>, - rustfmt: Option<RustfmtMetadata>, -} -#[derive(Deserialize)] -struct RustfmtMetadata { - date: String, - version: String, -} - #[derive(Debug)] struct Crate { name: Interned<String>, @@ -482,11 +467,7 @@ impl Build { bootstrap_out }; - let stage0_json = t!(std::fs::read_to_string(&src.join("src").join("stage0.json"))); - let stage0_metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json)); - let mut build = Build { - stage0_metadata, initial_rustc: config.initial_rustc.clone(), initial_cargo: config.initial_cargo.clone(), initial_lld, diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 18f82d024f8..329bb68672e 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -121,7 +121,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { let mut rev_list = Command::new("git"); rev_list.args(&[ PathBuf::from("rev-list"), - "--author=bors@rust-lang.org".into(), + format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(), "-n1".into(), "--first-parent".into(), "HEAD".into(), @@ -170,11 +170,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { if !rustc_cache.exists() { t!(fs::create_dir_all(&rustc_cache)); } - let base = "https://ci-artifacts.rust-lang.org"; - let url = if llvm_assertions { - format!("rustc-builds-alt/{}", llvm_sha) + let base = if llvm_assertions { + &builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server } else { - format!("rustc-builds/{}", llvm_sha) + &builder.config.stage0_metadata.config.artifacts_server }; let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple); let tarball = rustc_cache.join(&filename); @@ -187,7 +186,11 @@ help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in [llvm] download-ci-llvm = false "; - builder.download_component(base, &format!("{}/{}", url, filename), &tarball, help_on_error); + builder.download_component( + &format!("{base}/{llvm_sha}/{filename}"), + &tarball, + help_on_error, + ); } let llvm_root = builder.config.ci_llvm_root(); builder.unpack(&tarball, &llvm_root, "rust-dev"); diff --git a/src/stage0.json b/src/stage0.json index 6371b9eae59..b6b502f4cf0 100644 --- a/src/stage0.json +++ b/src/stage0.json @@ -1,6 +1,20 @@ { - "__comment": "Generated by `./x.py run src/tools/bump-stage0`. Run that command again to update the bootstrap compiler.", - "dist_server": "https://static.rust-lang.org", + "config": { + "dist_server": "https://static.rust-lang.org", + "artifacts_server": "https://ci-artifacts.rust-lang.org/rustc-builds", + "artifacts_with_llvm_assertions_server": "https://ci-artifacts.rust-lang.org/rustc-builds-alt", + "git_merge_commit_email": "bors@rust-lang.org" + }, + "__comments": [ + "The configuration above this comment is editable, and can be changed", + "by forks of the repository if they have alternate values.", + "", + "The section below is generated by `./x.py run src/tools/bump-stage0`,", + "run that command again to update the bootstrap compiler.", + "", + "All changes below this comment will be overridden the next time the", + "tool is executed." + ], "compiler": { "date": "2022-05-20", "version": "beta" diff --git a/src/tools/bump-stage0/src/main.rs b/src/tools/bump-stage0/src/main.rs index d6364e28fef..1c839fdc00a 100644 --- a/src/tools/bump-stage0/src/main.rs +++ b/src/tools/bump-stage0/src/main.rs @@ -4,11 +4,14 @@ use indexmap::IndexMap; use std::collections::HashMap; use std::convert::TryInto; -const DIST_SERVER: &str = "https://static.rust-lang.org"; +const PATH: &str = "src/stage0.json"; const COMPILER_COMPONENTS: &[&str] = &["rustc", "rust-std", "cargo"]; const RUSTFMT_COMPONENTS: &[&str] = &["rustfmt-preview"]; struct Tool { + config: Config, + comments: Vec<String>, + channel: Channel, version: [u16; 3], checksums: IndexMap<String, String>, @@ -32,18 +35,23 @@ impl Tool { .try_into() .map_err(|_| anyhow::anyhow!("failed to parse version"))?; - Ok(Self { channel, version, checksums: IndexMap::new() }) + let existing: Stage0 = serde_json::from_slice(&std::fs::read(PATH)?)?; + + Ok(Self { + channel, + version, + config: existing.config, + comments: existing.comments, + checksums: IndexMap::new(), + }) } fn update_json(mut self) -> Result<(), Error> { std::fs::write( - "src/stage0.json", + PATH, format!( "{}\n", serde_json::to_string_pretty(&Stage0 { - comment: "Generated by `./x.py run src/tools/bump-stage0`. \ - Run that command again to update the bootstrap compiler.", - dist_server: DIST_SERVER.into(), compiler: self.detect_compiler()?, rustfmt: self.detect_rustfmt()?, checksums_sha256: { @@ -51,7 +59,9 @@ impl Tool { // are added while filling the other struct fields just above this block. self.checksums.sort_keys(); self.checksums - } + }, + config: self.config, + comments: self.comments, })? ), )?; @@ -74,7 +84,7 @@ impl Tool { Channel::Nightly => "beta".to_string(), }; - let manifest = fetch_manifest(&channel)?; + let manifest = fetch_manifest(&self.config, &channel)?; self.collect_checksums(&manifest, COMPILER_COMPONENTS)?; Ok(Stage0Toolchain { date: manifest.date, @@ -100,13 +110,13 @@ impl Tool { return Ok(None); } - let manifest = fetch_manifest("nightly")?; + let manifest = fetch_manifest(&self.config, "nightly")?; self.collect_checksums(&manifest, RUSTFMT_COMPONENTS)?; Ok(Some(Stage0Toolchain { date: manifest.date, version: "nightly".into() })) } fn collect_checksums(&mut self, manifest: &Manifest, components: &[&str]) -> Result<(), Error> { - let prefix = format!("{}/", DIST_SERVER); + let prefix = format!("{}/", self.config.dist_server); for component in components { let pkg = manifest .pkg @@ -136,10 +146,10 @@ fn main() -> Result<(), Error> { Ok(()) } -fn fetch_manifest(channel: &str) -> Result<Manifest, Error> { +fn fetch_manifest(config: &Config, channel: &str) -> Result<Manifest, Error> { Ok(toml::from_slice(&http_get(&format!( "{}/dist/channel-rust-{}.toml", - DIST_SERVER, channel + config.dist_server, channel ))?)?) } @@ -166,35 +176,52 @@ enum Channel { Nightly, } -#[derive(Debug, serde::Serialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] struct Stage0 { - #[serde(rename = "__comment")] - comment: &'static str, - dist_server: String, + config: Config, + // Comments are explicitly below the config, do not move them above. + // + // Downstream forks of the compiler codebase can change the configuration values defined above, + // but doing so would risk merge conflicts whenever they import new changes that include a + // bootstrap compiler bump. + // + // To lessen the pain, a big block of comments is placed between the configuration and the + // auto-generated parts of the file, preventing git diffs of the config to include parts of the + // auto-generated content and vice versa. This should prevent merge conflicts. + #[serde(rename = "__comments")] + comments: Vec<String>, compiler: Stage0Toolchain, rustfmt: Option<Stage0Toolchain>, checksums_sha256: IndexMap<String, String>, } -#[derive(Debug, serde::Serialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] +struct Config { + dist_server: String, + artifacts_server: String, + artifacts_with_llvm_assertions_server: String, + git_merge_commit_email: String, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] struct Stage0Toolchain { date: String, version: String, } -#[derive(Debug, serde::Deserialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] struct Manifest { date: String, pkg: HashMap<String, ManifestPackage>, } -#[derive(Debug, serde::Deserialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] struct ManifestPackage { version: String, target: HashMap<String, ManifestTargetPackage>, } -#[derive(Debug, serde::Deserialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize)] struct ManifestTargetPackage { url: Option<String>, hash: Option<String>, |