diff options
| -rw-r--r-- | Cargo.lock | 2 | ||||
| -rw-r--r-- | src/bootstrap/Cargo.toml | 2 | ||||
| -rw-r--r-- | src/bootstrap/bootstrap.py | 84 | ||||
| -rw-r--r-- | src/bootstrap/builder.rs | 30 | ||||
| -rw-r--r-- | src/bootstrap/config.rs | 182 | ||||
| -rw-r--r-- | src/bootstrap/format.rs | 18 | ||||
| -rw-r--r-- | src/bootstrap/lib.rs | 20 | ||||
| -rw-r--r-- | src/bootstrap/test.rs | 4 |
8 files changed, 233 insertions, 109 deletions
diff --git a/Cargo.lock b/Cargo.lock index 4a9b35fe4ab..b6077574ee6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,6 +216,7 @@ dependencies = [ "cmake", "filetime", "getopts", + "hex 0.4.2", "ignore", "libc", "once_cell", @@ -223,6 +224,7 @@ dependencies = [ "pretty_assertions 0.7.2", "serde", "serde_json", + "sha2", "sysinfo", "tar", "toml", diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 5027a45e0ad..0e54837610a 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -40,8 +40,10 @@ filetime = "0.2" getopts = "0.2.19" cc = "1.0.69" libc = "0.2" +hex = "0.4" serde = { version = "1.0.8", features = ["derive"] } serde_json = "1.0.2" +sha2 = "0.10" tar = "0.4" toml = "0.5" ignore = "0.4.10" diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index d81874bfe7e..635e4f3703b 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -63,31 +63,30 @@ def support_xz(): except tarfile.CompressionError: return False -def get(base, url, path, checksums, verbose=False, do_verify=True): +def get(base, url, path, checksums, verbose=False): with tempfile.NamedTemporaryFile(delete=False) as temp_file: temp_path = temp_file.name try: - if do_verify: - if url not in checksums: - raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. " - "Pre-built artifacts might not available for this " - "target at this time, see https://doc.rust-lang.org/nightly" - "/rustc/platform-support.html for more information.") - .format(url)) - sha256 = checksums[url] - if os.path.exists(path): - if verify(path, sha256, False): - if verbose: - print("using already-download file", path) - return - else: - if verbose: - print("ignoring already-download file", - path, "due to failed verification") - os.unlink(path) + if url not in checksums: + raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. " + "Pre-built artifacts might not be available for this " + "target at this time, see https://doc.rust-lang.org/nightly" + "/rustc/platform-support.html for more information.") + .format(url)) + sha256 = checksums[url] + if os.path.exists(path): + if verify(path, sha256, False): + if verbose: + print("using already-download file", path) + return + else: + if verbose: + print("ignoring already-download file", + path, "due to failed verification") + os.unlink(path) download(temp_path, "{}/{}".format(base, url), True, verbose) - if do_verify and not verify(temp_path, sha256, verbose): + if not verify(temp_path, sha256, verbose): raise RuntimeError("failed verification") if verbose: print("moving {} to {}".format(temp_path, path)) @@ -430,7 +429,6 @@ class RustBuild(object): def __init__(self): self.checksums_sha256 = {} self.stage0_compiler = None - self.stage0_rustfmt = None self._download_url = '' self.build = '' self.build_dir = '' @@ -484,31 +482,10 @@ class RustBuild(object): with output(self.rustc_stamp()) as rust_stamp: rust_stamp.write(key) - if self.rustfmt() and self.rustfmt().startswith(bin_root) and ( - not os.path.exists(self.rustfmt()) - or self.program_out_of_date( - self.rustfmt_stamp(), - "" if self.stage0_rustfmt is None else self.stage0_rustfmt.channel() - ) - ): - if self.stage0_rustfmt is not None: - tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' - filename = "rustfmt-{}-{}{}".format( - self.stage0_rustfmt.version, self.build, tarball_suffix, - ) - self._download_component_helper( - filename, "rustfmt-preview", tarball_suffix, key=self.stage0_rustfmt.date - ) - self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root)) - self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root)) - with output(self.rustfmt_stamp()) as rustfmt_stamp: - rustfmt_stamp.write(self.stage0_rustfmt.channel()) - def _download_component_helper( - self, filename, pattern, tarball_suffix, key=None + self, filename, pattern, tarball_suffix, ): - if key is None: - key = self.stage0_compiler.date + key = self.stage0_compiler.date cache_dst = os.path.join(self.build_dir, "cache") rustc_cache = os.path.join(cache_dst, key) if not os.path.exists(rustc_cache): @@ -524,7 +501,6 @@ class RustBuild(object): tarball, self.checksums_sha256, verbose=self.verbose, - do_verify=True, ) unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) @@ -634,16 +610,6 @@ class RustBuild(object): """ return os.path.join(self.bin_root(), '.rustc-stamp') - def rustfmt_stamp(self): - """Return the path for .rustfmt-stamp - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp") - True - """ - return os.path.join(self.bin_root(), '.rustfmt-stamp') - def program_out_of_date(self, stamp_path, key): """Check if the given program stamp is out of date""" if not os.path.exists(stamp_path) or self.clean: @@ -717,12 +683,6 @@ class RustBuild(object): """Return config path for rustc""" return self.program_config('rustc') - def rustfmt(self): - """Return config path for rustfmt""" - if self.stage0_rustfmt is None: - return None - return self.program_config('rustfmt') - def program_config(self, program): """Return config path for the given program at the given stage @@ -1082,8 +1042,6 @@ def bootstrap(help_triggered): data = json.load(f) build.checksums_sha256 = data["checksums_sha256"] build.stage0_compiler = Stage0Toolchain(data["compiler"]) - if data.get("rustfmt") is not None: - build.stage0_rustfmt = Stage0Toolchain(data["rustfmt"]) build.set_dist_environment(data["dist_server"]) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index da13374cee7..38d4f15d3c8 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -728,7 +728,8 @@ impl<'a> Builder<'a> { Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]), Subcommand::Install { ref paths } => (Kind::Install, &paths[..]), Subcommand::Run { ref paths } => (Kind::Run, &paths[..]), - Subcommand::Format { .. } | Subcommand::Clean { .. } | Subcommand::Setup { .. } => { + Subcommand::Format { .. } => (Kind::Format, &[][..]), + Subcommand::Clean { .. } | Subcommand::Setup { .. } => { panic!() } }; @@ -878,7 +879,6 @@ impl<'a> Builder<'a> { ) { // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); - // FIXME: support `do_verify` (only really needed for nightly rustfmt) self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error); t!(std::fs::rename(&tempfile, dest_path)); } @@ -970,6 +970,28 @@ impl<'a> Builder<'a> { t!(fs::remove_dir_all(dst.join(directory_prefix))); } + /// Returns whether the SHA256 checksum of `path` matches `expected`. + pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool { + use sha2::Digest; + + self.verbose(&format!("verifying {}", path.display())); + let mut hasher = sha2::Sha256::new(); + // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components. + // Consider using streaming IO instead? + let contents = if self.config.dry_run { vec![] } else { t!(fs::read(path)) }; + hasher.update(&contents); + let found = hex::encode(hasher.finalize().as_slice()); + let verified = found == expected; + if !verified && !self.config.dry_run { + println!( + "invalid checksum: \n\ + found: {found}\n\ + expected: {expected}", + ); + } + return verified; + } + /// Obtain a compiler at a given stage and for a given host. Explicitly does /// not take `Compiler` since all `Compiler` instances are meant to be /// obtained through this function, since it ensures that they are valid @@ -1192,6 +1214,10 @@ impl<'a> Builder<'a> { Config::download_rustc(self) } + pub(crate) fn initial_rustfmt(&self) -> Option<PathBuf> { + Config::initial_rustfmt(self) + } + /// Prepares an invocation of `cargo` to be run. /// /// This will create a `Command` that represents a pending execution of diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 6cb0bd518e2..99b69ee9a4f 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -3,7 +3,7 @@ //! This module implements parsing `config.toml` configuration files to tweak //! how the build runs. -use std::cell::Cell; +use std::cell::{Cell, RefCell}; use std::cmp; use std::collections::{HashMap, HashSet}; use std::env; @@ -20,6 +20,7 @@ use crate::channel::GitInfo; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags}; use crate::util::{exe, output, program_out_of_date, t}; +use crate::RustfmtMetadata; use once_cell::sync::OnceCell; use serde::{Deserialize, Deserializer}; @@ -204,10 +205,27 @@ pub struct Config { // These are either the stage0 downloaded binaries or the locally installed ones. pub initial_cargo: PathBuf, pub initial_rustc: PathBuf, - pub initial_rustfmt: Option<PathBuf>, + #[cfg(not(test))] + initial_rustfmt: RefCell<RustfmtState>, + #[cfg(test)] + pub initial_rustfmt: RefCell<RustfmtState>, pub out: PathBuf, } +#[derive(Clone, Debug)] +pub enum RustfmtState { + SystemToolchain(PathBuf), + Downloaded(PathBuf), + Unavailable, + LazyEvaluated, +} + +impl Default for RustfmtState { + fn default() -> Self { + RustfmtState::LazyEvaluated + } +} + #[derive(Debug, Clone, Copy, PartialEq)] pub enum LlvmLibunwind { No, @@ -859,9 +877,6 @@ impl Config { set(&mut config.full_bootstrap, build.full_bootstrap); set(&mut config.extended, build.extended); config.tools = build.tools; - if build.rustfmt.is_some() { - config.initial_rustfmt = build.rustfmt; - } set(&mut config.verbose, build.verbose); set(&mut config.sanitizers, build.sanitizers); set(&mut config.profiler, build.profiler); @@ -1154,18 +1169,22 @@ impl Config { set(&mut config.missing_tools, t.missing_tools); } - config.initial_rustfmt = config.initial_rustfmt.or_else({ - let build = config.build; - let initial_rustc = &config.initial_rustc; - - move || { - // Cargo does not provide a RUSTFMT environment variable, so we - // synthesize it manually. - let rustfmt = initial_rustc.with_file_name(exe("rustfmt", build)); - - if rustfmt.exists() { Some(rustfmt) } else { None } + if let Some(r) = build.rustfmt { + *config.initial_rustfmt.borrow_mut() = if r.exists() { + RustfmtState::SystemToolchain(r) + } else { + RustfmtState::Unavailable + }; + } else { + // If using a system toolchain for bootstrapping, see if that has rustfmt available. + let host = config.build; + let rustfmt_path = config.initial_rustc.with_file_name(exe("rustfmt", host)); + let bin_root = config.out.join(host.triple).join("stage0"); + if !rustfmt_path.starts_with(&bin_root) { + // Using a system-provided toolchain; we shouldn't download rustfmt. + *config.initial_rustfmt.borrow_mut() = RustfmtState::SystemToolchain(rustfmt_path); } - }); + } // Now that we've reached the end of our configuration, infer the // default values for all options that we haven't otherwise stored yet. @@ -1335,6 +1354,25 @@ impl Config { }) } + pub(crate) fn initial_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> { + match &mut *builder.config.initial_rustfmt.borrow_mut() { + RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), + RustfmtState::Unavailable => None, + r @ RustfmtState::LazyEvaluated => { + if builder.config.dry_run { + return Some(PathBuf::new()); + } + let path = maybe_download_rustfmt(builder); + *r = if let Some(p) = &path { + RustfmtState::Downloaded(p.clone()) + } else { + RustfmtState::Unavailable + }; + path + } + } + } + pub fn verbose(&self) -> bool { self.verbose > 0 } @@ -1445,6 +1483,28 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) Some(commit.to_string()) } +fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> { + let RustfmtMetadata { date, version } = builder.stage0_metadata.rustfmt.as_ref()?; + let channel = format!("{version}-{date}"); + + let host = builder.config.build; + let rustfmt_path = builder.config.initial_rustc.with_file_name(exe("rustfmt", host)); + let bin_root = builder.config.out.join(host.triple).join("stage0"); + let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); + if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { + return Some(rustfmt_path); + } + + let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple); + download_component(builder, DownloadSource::Dist, filename, "rustfmt-preview", &date, "stage0"); + + builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); + builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); + + builder.create(&rustfmt_stamp, &channel); + Some(rustfmt_path) +} + fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); // FIXME: support downloading artifacts from the beta channel @@ -1459,12 +1519,12 @@ fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { } let filename = format!("rust-std-{CHANNEL}-{host}.tar.xz"); let pattern = format!("rust-std-{host}"); - download_component(builder, filename, &pattern, commit); + download_ci_component(builder, filename, &pattern, commit); let filename = format!("rustc-{CHANNEL}-{host}.tar.xz"); - download_component(builder, filename, "rustc", commit); + download_ci_component(builder, filename, "rustc", commit); // download-rustc doesn't need its own cargo, it can just use beta's. let filename = format!("rustc-dev-{CHANNEL}-{host}.tar.xz"); - download_component(builder, filename, "rustc-dev", commit); + download_ci_component(builder, filename, "rustc-dev", commit); builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); @@ -1479,21 +1539,83 @@ fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { } } +pub(crate) enum DownloadSource { + CI, + Dist, +} + /// Download a single component of a CI-built toolchain (not necessarily a published nightly). // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident -fn download_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) { +fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) { + download_component(builder, DownloadSource::CI, filename, prefix, commit, "ci-rustc") +} + +fn download_component( + builder: &Builder<'_>, + mode: DownloadSource, + filename: String, + prefix: &str, + key: &str, + destination: &str, +) { let cache_dst = builder.out.join("cache"); - let rustc_cache = cache_dst.join(commit); - if !rustc_cache.exists() { - t!(fs::create_dir_all(&rustc_cache)); + let cache_dir = cache_dst.join(key); + if !cache_dir.exists() { + t!(fs::create_dir_all(&cache_dir)); } - let base = "https://ci-artifacts.rust-lang.org"; - let url = format!("rustc-builds/{commit}"); - let tarball = rustc_cache.join(&filename); - if !tarball.exists() { - builder.download_component(base, &format!("{url}/{filename}"), &tarball, ""); + let bin_root = builder.out.join(builder.config.build.triple).join(destination); + let tarball = cache_dir.join(&filename); + let (base_url, url, should_verify) = match mode { + DownloadSource::CI => ( + "https://ci-artifacts.rust-lang.org/rustc-builds".to_string(), + format!("{key}/{filename}"), + false, + ), + DownloadSource::Dist => { + let dist_server = env::var("RUSTUP_DIST_SERVER") + .unwrap_or(builder.stage0_metadata.dist_server.to_string()); + // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json + (dist_server, format!("dist/{key}/{filename}"), true) + } + }; + + // For the beta compiler, put special effort into ensuring the checksums are valid. + // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update + // this on each and every nightly ... + let checksum = if should_verify { + let error = format!( + "src/stage0.json doesn't contain a checksum for {url}. \ + Pre-built artifacts might not be available for this \ + target at this time, see https://doc.rust-lang.org/nightly\ + /rustc/platform-support.html for more information." + ); + let sha256 = builder.stage0_metadata.checksums_sha256.get(&url).expect(&error); + if tarball.exists() { + if builder.verify(&tarball, sha256) { + builder.unpack(&tarball, &bin_root, prefix); + return; + } else { + builder.verbose(&format!( + "ignoring cached file {} due to failed verification", + tarball.display() + )); + builder.remove(&tarball); + } + } + Some(sha256) + } else if tarball.exists() { + return; + } else { + None + }; + + builder.download_component(&base_url, &url, &tarball, ""); + if let Some(sha256) = checksum { + if !builder.verify(&tarball, sha256) { + panic!("failed to verify {}", tarball.display()); + } } - let bin_root = builder.out.join(builder.config.build.triple).join("ci-rustc"); - builder.unpack(&tarball, &bin_root, prefix) + + builder.unpack(&tarball, &bin_root, prefix); } diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs index d1a450f1bff..60a53c28686 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/format.rs @@ -1,7 +1,7 @@ //! Runs rustfmt on the repository. +use crate::builder::Builder; use crate::util::{output, t}; -use crate::Build; use ignore::WalkBuilder; use std::collections::VecDeque; use std::path::{Path, PathBuf}; @@ -42,7 +42,7 @@ struct RustfmtConfig { ignore: Vec<String>, } -pub fn format(build: &Build, check: bool, paths: &[PathBuf]) { +pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { if build.config.dry_run { return; } @@ -112,15 +112,11 @@ pub fn format(build: &Build, check: bool, paths: &[PathBuf]) { } let ignore_fmt = ignore_fmt.build().unwrap(); - let rustfmt_path = build - .config - .initial_rustfmt - .as_ref() - .unwrap_or_else(|| { - eprintln!("./x.py fmt is not supported on this channel"); - std::process::exit(1); - }) - .to_path_buf(); + let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { + eprintln!("./x.py fmt is not supported on this channel"); + std::process::exit(1); + }); + assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); let src = build.src.clone(); let (tx, rx): (SyncSender<PathBuf>, _) = std::sync::mpsc::sync_channel(128); let walker = match paths.get(0) { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index fab6168bf38..022f2e0fc13 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -118,6 +118,7 @@ use std::os::windows::fs::symlink_file; use filetime::FileTime; use once_cell::sync::OnceCell; +use serde::Deserialize; use crate::builder::Kind; use crate::config::{LlvmLibunwind, TargetSelection}; @@ -294,6 +295,7 @@ pub struct Build { targets: Vec<TargetSelection>, // Stage 0 (downloaded) compiler, lld and cargo or their local rust equivalents + stage0_metadata: Stage0Metadata, initial_rustc: PathBuf, initial_cargo: PathBuf, initial_lld: PathBuf, @@ -320,6 +322,18 @@ pub struct Build { metrics: metrics::BuildMetrics, } +#[derive(Deserialize)] +struct Stage0Metadata { + dist_server: String, + checksums_sha256: HashMap<String, String>, + rustfmt: Option<RustfmtMetadata>, +} +#[derive(Deserialize)] +struct RustfmtMetadata { + date: String, + version: String, +} + #[derive(Debug)] struct Crate { name: Interned<String>, @@ -468,7 +482,11 @@ impl Build { bootstrap_out }; + let stage0_json = t!(std::fs::read_to_string(&src.join("src").join("stage0.json"))); + let stage0_metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json)); + let mut build = Build { + stage0_metadata, initial_rustc: config.initial_rustc.clone(), initial_cargo: config.initial_cargo.clone(), initial_lld, @@ -661,7 +679,7 @@ impl Build { self.maybe_update_submodules(); if let Subcommand::Format { check, paths } = &self.config.cmd { - return format::format(self, *check, &paths); + return format::format(&builder::Builder::new(&self), *check, &paths); } if let Subcommand::Clean { all } = self.config.cmd { diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 8a236ec5130..fdce078bbed 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1010,7 +1010,7 @@ impl Step for Tidy { if builder.config.channel == "dev" || builder.config.channel == "nightly" { builder.info("fmt check"); - if builder.config.initial_rustfmt.is_none() { + if builder.initial_rustfmt().is_none() { let inferred_rustfmt_dir = builder.config.initial_rustc.parent().unwrap(); eprintln!( "\ @@ -1023,7 +1023,7 @@ help: to skip test's attempt to check tidiness, pass `--exclude src/tools/tidy` ); std::process::exit(1); } - crate::format::format(&builder.build, !builder.config.cmd.bless(), &[]); + crate::format::format(&builder, !builder.config.cmd.bless(), &[]); } } |