diff options
| author | 许杰友 Jieyou Xu (Joe) <39484203+jieyouxu@users.noreply.github.com> | 2025-07-04 17:12:10 +0800 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-07-04 17:12:10 +0800 |
| commit | ad9759701a565735b1e645762f0c9b3a7a8b02fe (patch) | |
| tree | 3fa8e6c5426eb1fa41b4ac3e6ebeaa322ab89688 /src/bootstrap | |
| parent | 3fd66139ef9fe27b56d8f3821a988161e9ff443b (diff) | |
| parent | c33dd1b306db360f92c46f1cab79e68e6a845bf5 (diff) | |
Merge pull request #2489 from Kobzol/pull
Diffstat (limited to 'src/bootstrap')
36 files changed, 1807 insertions, 1093 deletions
diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index a6ca699e282..2434a278d55 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -12,6 +12,15 @@ dependencies = [ ] [[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] name = "anstyle" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -62,8 +71,8 @@ dependencies = [ "toml", "tracing", "tracing-chrome", + "tracing-forest", "tracing-subscriber", - "tracing-tree", "walkdir", "windows", "xz2", @@ -450,15 +459,6 @@ dependencies = [ ] [[package]] -name = "nu-ansi-term" -version = "0.50.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] name = "objc2-core-foundation" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -776,6 +776,26 @@ dependencies = [ ] [[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "thread_local" version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -838,6 +858,19 @@ dependencies = [ ] [[package]] +name = "tracing-forest" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee40835db14ddd1e3ba414292272eddde9dad04d3d4b65509656414d1c42592f" +dependencies = [ + "ansi_term", + "smallvec", + "thiserror", + "tracing", + "tracing-subscriber", +] + +[[package]] name = "tracing-log" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -855,7 +888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", - "nu-ansi-term 0.46.0", + "nu-ansi-term", "once_cell", "regex", "sharded-slab", @@ -867,18 +900,6 @@ dependencies = [ ] [[package]] -name = "tracing-tree" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f459ca79f1b0d5f71c54ddfde6debfc59c8b6eeb46808ae492077f739dc7b49c" -dependencies = [ - "nu-ansi-term 0.50.1", - "tracing-core", - "tracing-log", - "tracing-subscriber", -] - -[[package]] name = "typenum" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 9785a306c9b..073cebdcae2 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -7,7 +7,7 @@ default-run = "bootstrap" [features] build-metrics = ["sysinfo"] -tracing = ["dep:tracing", "dep:tracing-chrome", "dep:tracing-subscriber", "dep:tracing-tree"] +tracing = ["dep:tracing", "dep:tracing-chrome", "dep:tracing-subscriber", "dep:tracing-forest"] [lib] path = "src/lib.rs" @@ -64,7 +64,7 @@ sysinfo = { version = "0.35.0", default-features = false, optional = true, featu tracing = { version = "0.1", optional = true, features = ["attributes"] } tracing-chrome = { version = "0.7", optional = true } tracing-subscriber = { version = "0.3", optional = true, features = ["env-filter", "fmt", "registry", "std"] } -tracing-tree = { version = "0.4.0", optional = true } +tracing-forest = { version = "0.1.6", optional = true, default-features = false, features = ["smallvec", "ansi", "env-filter"] } [target.'cfg(windows)'.dependencies.junction] version = "1.0.0" diff --git a/src/bootstrap/defaults/bootstrap.dist.toml b/src/bootstrap/defaults/bootstrap.dist.toml index f0cb34eb458..9daf9faac14 100644 --- a/src/bootstrap/defaults/bootstrap.dist.toml +++ b/src/bootstrap/defaults/bootstrap.dist.toml @@ -20,7 +20,6 @@ download-ci-llvm = false channel = "auto-detect" # Never download a rustc, distributions must build a fresh compiler. download-rustc = false -lld = true # Build the llvm-bitcode-linker llvm-bitcode-linker = true diff --git a/src/bootstrap/defaults/bootstrap.library.toml b/src/bootstrap/defaults/bootstrap.library.toml index 895e50b9a20..6a867093b78 100644 --- a/src/bootstrap/defaults/bootstrap.library.toml +++ b/src/bootstrap/defaults/bootstrap.library.toml @@ -1,7 +1,6 @@ # These defaults are meant for contributors to the standard library and documentation. [build] bench-stage = 1 -build-stage = 1 check-stage = 1 test-stage = 1 diff --git a/src/bootstrap/src/bin/main.rs b/src/bootstrap/src/bin/main.rs index 833f8027951..e1862a451f2 100644 --- a/src/bootstrap/src/bin/main.rs +++ b/src/bootstrap/src/bin/main.rs @@ -217,12 +217,11 @@ fn check_version(config: &Config) -> Option<String> { // "tracing", instrument(..))]`. #[cfg(feature = "tracing")] fn setup_tracing() -> impl Drop { + use tracing_forest::ForestLayer; use tracing_subscriber::EnvFilter; use tracing_subscriber::layer::SubscriberExt; let filter = EnvFilter::from_env("BOOTSTRAP_TRACING"); - // cf. <https://docs.rs/tracing-tree/latest/tracing_tree/struct.HierarchicalLayer.html>. - let layer = tracing_tree::HierarchicalLayer::default().with_targets(true).with_indent_amount(2); let mut chrome_layer = tracing_chrome::ChromeLayerBuilder::new().include_args(true); @@ -233,7 +232,8 @@ fn setup_tracing() -> impl Drop { let (chrome_layer, _guard) = chrome_layer.build(); - let registry = tracing_subscriber::registry().with(filter).with(layer).with(chrome_layer); + let registry = + tracing_subscriber::registry().with(filter).with(ForestLayer::default()).with(chrome_layer); tracing::subscriber::set_global_default(registry).unwrap(); _guard diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs index fcd4f4078ad..6c5f70b2f43 100644 --- a/src/bootstrap/src/core/build_steps/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -1,12 +1,11 @@ //! Implementation of compiling the compiler and standard library, in "check"-based modes. -use crate::core::build_steps::compile; use crate::core::build_steps::compile::{ add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make, }; use crate::core::build_steps::tool::{COMPILETEST_ALLOW_FEATURES, SourceType, prepare_tool_cargo}; use crate::core::builder::{ - self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, crate_description, + self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, StepMetadata, crate_description, }; use crate::core::config::TargetSelection; use crate::utils::build_stamp::{self, BuildStamp}; @@ -67,8 +66,6 @@ impl Step for Std { return; } - builder.require_submodule("library/stdarch", None); - let stage = self.custom_stage.unwrap_or(builder.top_stage); let target = self.target; @@ -89,7 +86,7 @@ impl Step for Std { } // Reuse the stage0 libstd - builder.ensure(compile::Std::new(compiler, target)); + builder.std(compiler, target); return; } @@ -170,6 +167,10 @@ impl Step for Std { let _guard = builder.msg_check("library test/bench/example targets", target, Some(stage)); run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::check("std", self.target)) + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -223,8 +224,8 @@ impl Step for Rustc { // the sysroot for the compiler to find. Otherwise, we're going to // fail when building crates that need to generate code (e.g., build // scripts and their dependencies). - builder.ensure(crate::core::build_steps::compile::Std::new(compiler, compiler.host)); - builder.ensure(crate::core::build_steps::compile::Std::new(compiler, target)); + builder.std(compiler, compiler.host); + builder.std(compiler, target); } else { builder.ensure(Std::new(target)); } @@ -261,6 +262,10 @@ impl Step for Rustc { let hostdir = builder.sysroot_target_libdir(compiler, compiler.host); add_to_sysroot(builder, &libdir, &hostdir, &stamp); } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::check("rustc", self.target)) + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -318,6 +323,10 @@ impl Step for CodegenBackend { run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::check(self.backend, self.target)) + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -376,6 +385,10 @@ impl Step for RustAnalyzer { let _guard = builder.msg_check("rust-analyzer artifacts", target, None); run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::check("rust-analyzer", self.target)) + } } /// Compiletest is implicitly "checked" when it gets built in order to run tests, @@ -435,6 +448,10 @@ impl Step for Compiletest { let _guard = builder.msg_check("compiletest artifacts", self.target, None); run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::check("compiletest", self.target)) + } } macro_rules! tool_check_step { @@ -470,6 +487,10 @@ macro_rules! tool_check_step { let Self { target } = self; run_tool_check_step(builder, target, stringify!($name), $path); } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::check(stringify!($name), self.target)) + } } } } diff --git a/src/bootstrap/src/core/build_steps/clean.rs b/src/bootstrap/src/core/build_steps/clean.rs index 882fcd08780..f67569d1486 100644 --- a/src/bootstrap/src/core/build_steps/clean.rs +++ b/src/bootstrap/src/core/build_steps/clean.rs @@ -181,7 +181,7 @@ fn rm_rf(path: &Path) { panic!("failed to get metadata for file {}: {}", path.display(), e); } Ok(metadata) => { - if metadata.file_type().is_file() || metadata.file_type().is_symlink() { + if !metadata.file_type().is_dir() { do_op(path, "remove file", |p| match fs::remove_file(p) { #[cfg(windows)] Err(e) diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs index ebf0caccfbc..1e44b5b67a4 100644 --- a/src/bootstrap/src/core/build_steps/clippy.rs +++ b/src/bootstrap/src/core/build_steps/clippy.rs @@ -1,8 +1,8 @@ //! Implementation of running clippy on the compiler, standard library and various tools. +use super::check; use super::compile::{run_cargo, rustc_cargo, std_cargo}; use super::tool::{SourceType, prepare_tool_cargo}; -use super::{check, compile}; use crate::builder::{Builder, ShouldRun}; use crate::core::build_steps::compile::std_crates_for_run_make; use crate::core::builder; @@ -141,8 +141,6 @@ impl Step for Std { } fn run(self, builder: &Builder<'_>) { - builder.require_submodule("library/stdarch", None); - let target = self.target; let compiler = builder.compiler(builder.top_stage, builder.config.host_target); @@ -214,8 +212,8 @@ impl Step for Rustc { // the sysroot for the compiler to find. Otherwise, we're going to // fail when building crates that need to generate code (e.g., build // scripts and their dependencies). - builder.ensure(compile::Std::new(compiler, compiler.host)); - builder.ensure(compile::Std::new(compiler, target)); + builder.std(compiler, compiler.host); + builder.std(compiler, target); } else { builder.ensure(check::Std::new(target)); } diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index f6efb23e8d8..c3a3eddd161 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -197,8 +197,6 @@ impl Step for Std { return; } - builder.require_submodule("library/stdarch", None); - let mut target_deps = builder.ensure(StartupObjects { compiler, target }); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); @@ -213,7 +211,7 @@ impl Step for Std { { trace!(?compiler_to_use, ?compiler, "compiler != compiler_to_use, uplifting library"); - builder.ensure(Std::new(compiler_to_use, target)); + builder.std(compiler_to_use, target); let msg = if compiler_to_use.host == target { format!( "Uplifting library (stage{} -> stage{})", @@ -308,11 +306,7 @@ impl Step for Std { } fn metadata(&self) -> Option<StepMetadata> { - Some( - StepMetadata::build("std", self.target) - .built_by(self.compiler) - .stage(self.compiler.stage), - ) + Some(StepMetadata::build("std", self.target).built_by(self.compiler)) } } @@ -690,7 +684,7 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct StdLink { +pub struct StdLink { pub compiler: Compiler, pub target_compiler: Compiler, pub target: TargetSelection, @@ -701,7 +695,7 @@ struct StdLink { } impl StdLink { - fn from_std(std: Std, host_compiler: Compiler) -> Self { + pub fn from_std(std: Std, host_compiler: Compiler) -> Self { Self { compiler: host_compiler, target_compiler: std.compiler, @@ -1022,6 +1016,12 @@ impl Step for Rustc { } fn make_run(run: RunConfig<'_>) { + // If only `compiler` was passed, do not run this step. + // Instead the `Assemble` step will take care of compiling Rustc. + if run.builder.paths == vec![PathBuf::from("compiler")] { + return; + } + let crates = run.cargo_crates_in_set(); run.builder.ensure(Rustc { build_compiler: run @@ -1067,7 +1067,7 @@ impl Step for Rustc { // Build a standard library for `target` using the `build_compiler`. // This will be the standard library that the rustc which we build *links to*. - builder.ensure(Std::new(build_compiler, target)); + builder.std(build_compiler, target); if builder.config.keep_stage.contains(&build_compiler.stage) { trace!(stage = build_compiler.stage, "`keep-stage` requested"); @@ -1108,10 +1108,10 @@ impl Step for Rustc { // build scripts and proc macros. // If we are not cross-compiling, the Std build above will be the same one as the one we // prepare here. - builder.ensure(Std::new( + builder.std( builder.compiler(self.build_compiler.stage, builder.config.host_target), builder.config.host_target, - )); + ); let mut cargo = builder::Cargo::new( builder, @@ -1182,11 +1182,7 @@ impl Step for Rustc { } fn metadata(&self) -> Option<StepMetadata> { - Some( - StepMetadata::build("rustc", self.target) - .built_by(self.build_compiler) - .stage(self.build_compiler.stage + 1), - ) + Some(StepMetadata::build("rustc", self.target).built_by(self.build_compiler)) } } @@ -2079,7 +2075,7 @@ impl Step for Assemble { if builder.download_rustc() { trace!("`download-rustc` requested, reusing CI compiler for stage > 0"); - builder.ensure(Std::new(target_compiler, target_compiler.host)); + builder.std(target_compiler, target_compiler.host); let sysroot = builder.ensure(Sysroot { compiler: target_compiler, force_recompile: false }); // Ensure that `libLLVM.so` ends up in the newly created target directory, @@ -2087,7 +2083,7 @@ impl Step for Assemble { dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); // Lower stages use `ci-rustc-sysroot`, not stageN if target_compiler.stage == builder.top_stage { - builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage=target_compiler.stage)); + builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage = target_compiler.stage)); } let mut precompiled_compiler = target_compiler; @@ -2237,7 +2233,7 @@ impl Step for Assemble { debug!("copying codegen backends to sysroot"); copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); - if builder.config.lld_enabled && !builder.config.is_system_llvm(target_compiler.host) { + if builder.config.lld_enabled { builder.ensure(crate::core::build_steps::tool::LldWrapper { build_compiler, target_compiler, diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index e0f632eda0e..25b7e5a1b5d 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -23,7 +23,7 @@ use crate::core::build_steps::doc::DocumentationFormat; use crate::core::build_steps::tool::{self, Tool}; use crate::core::build_steps::vendor::{VENDOR_DIR, Vendor}; use crate::core::build_steps::{compile, llvm}; -use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step, StepMetadata}; use crate::core::config::TargetSelection; use crate::utils::build_stamp::{self, BuildStamp}; use crate::utils::channel::{self, Info}; @@ -84,6 +84,10 @@ impl Step for Docs { tarball.add_file(builder.src.join("src/doc/robots.txt"), dest, FileType::Regular); Some(tarball.generate()) } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::dist("docs", self.host)) + } } #[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] @@ -354,6 +358,10 @@ impl Step for Mingw { Some(tarball.generate()) } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::dist("mingw", self.host)) + } } #[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] @@ -540,6 +548,10 @@ impl Step for Rustc { } } } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::dist("rustc", self.compiler.host)) + } } #[derive(Debug, Clone, Hash, PartialEq, Eq)] @@ -711,7 +723,7 @@ impl Step for Std { return None; } - builder.ensure(compile::Std::new(compiler, target)); + builder.std(compiler, target); let mut tarball = Tarball::new(builder, "rust-std", &target.triple); tarball.include_target_in_component_name(true); @@ -723,6 +735,10 @@ impl Step for Std { Some(tarball.generate()) } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::dist("std", self.target).built_by(self.compiler)) + } } /// Tarball containing the compiler that gets downloaded and used by @@ -1002,6 +1018,10 @@ impl Step for Src { tarball.generate() } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::dist("src", TargetSelection::default())) + } } #[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] @@ -1036,18 +1056,18 @@ impl Step for PlainSourceTarball { let src_files = [ // tidy-alphabetical-start ".gitmodules", - "bootstrap.example.toml", - "Cargo.lock", - "Cargo.toml", - "configure", "CONTRIBUTING.md", "COPYRIGHT", + "Cargo.lock", + "Cargo.toml", "LICENSE-APACHE", - "license-metadata.json", "LICENSE-MIT", "README.md", "RELEASES.md", "REUSE.toml", + "bootstrap.example.toml", + "configure", + "license-metadata.json", "x", "x.ps1", "x.py", diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 215c155651a..f7c4c5ad0bb 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -14,7 +14,8 @@ use std::{env, fs, mem}; use crate::core::build_steps::compile; use crate::core::build_steps::tool::{self, SourceType, Tool, prepare_tool_cargo}; use crate::core::builder::{ - self, Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step, crate_description, + self, Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step, StepMetadata, + crate_description, }; use crate::core::config::{Config, TargetSelection}; use crate::helpers::{submodule_path_of, symlink_dir, t, up_to_date}; @@ -662,6 +663,10 @@ impl Step for Std { } } } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::doc("std", self.target).stage(self.stage)) + } } /// Name of the crates that are visible to consumers of the standard library. @@ -804,7 +809,7 @@ impl Step for Rustc { // Build the standard library, so that proc-macros can use it. // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.) let compiler = builder.compiler(stage, builder.config.host_target); - builder.ensure(compile::Std::new(compiler, builder.config.host_target)); + builder.std(compiler, builder.config.host_target); let _guard = builder.msg_sysroot_tool( Kind::Doc, @@ -947,7 +952,7 @@ macro_rules! tool_doc { t!(fs::create_dir_all(&out)); let compiler = builder.compiler(stage, builder.config.host_target); - builder.ensure(compile::Std::new(compiler, target)); + builder.std(compiler, target); if true $(&& $rustc_tool)? { // Build rustc docs so that we generate relative links. @@ -1195,7 +1200,7 @@ impl Step for RustcBook { let rustc = builder.rustc(self.compiler); // The tool runs `rustc` for extracting output examples, so it needs a // functional sysroot. - builder.ensure(compile::Std::new(self.compiler, self.target)); + builder.std(self.compiler, self.target); let mut cmd = builder.tool_cmd(Tool::LintDocs); cmd.arg("--src"); cmd.arg(builder.src.join("compiler")); @@ -1272,7 +1277,7 @@ impl Step for Reference { // This is needed for generating links to the standard library using // the mdbook-spec plugin. - builder.ensure(compile::Std::new(self.compiler, builder.config.host_target)); + builder.std(self.compiler, builder.config.host_target); // Run rustbook/mdbook to generate the HTML pages. builder.ensure(RustbookSrc { diff --git a/src/bootstrap/src/core/build_steps/format.rs b/src/bootstrap/src/core/build_steps/format.rs index 61268df7336..d487995e98a 100644 --- a/src/bootstrap/src/core/build_steps/format.rs +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -346,6 +346,6 @@ pub fn format(build: &Builder<'_>, check: bool, all: bool, paths: &[PathBuf]) { // since last merge. // // NOTE: Because of the exit above, this is only reachable if formatting / format checking - // succeeded. So we are not commiting the version if formatting was not good. + // succeeded. So we are not committing the version if formatting was not good. update_rustfmt_version(build); } diff --git a/src/bootstrap/src/core/build_steps/perf.rs b/src/bootstrap/src/core/build_steps/perf.rs index c43043b48f4..4d61b38c876 100644 --- a/src/bootstrap/src/core/build_steps/perf.rs +++ b/src/bootstrap/src/core/build_steps/perf.rs @@ -1,7 +1,7 @@ use std::env::consts::EXE_EXTENSION; use std::fmt::{Display, Formatter}; -use crate::core::build_steps::compile::{Std, Sysroot}; +use crate::core::build_steps::compile::Sysroot; use crate::core::build_steps::tool::{RustcPerf, Rustdoc}; use crate::core::builder::Builder; use crate::core::config::DebuginfoLevel; @@ -152,7 +152,7 @@ Consider setting `rust.debuginfo-level = 1` in `bootstrap.toml`."#); } let compiler = builder.compiler(builder.top_stage, builder.config.host_target); - builder.ensure(Std::new(compiler, builder.config.host_target)); + builder.std(compiler, builder.config.host_target); if let Some(opts) = args.cmd.shared_opts() && opts.profiles.contains(&Profile::Doc) diff --git a/src/bootstrap/src/core/build_steps/setup.rs b/src/bootstrap/src/core/build_steps/setup.rs index 9ce81ff9a22..37fc85518e0 100644 --- a/src/bootstrap/src/core/build_steps/setup.rs +++ b/src/bootstrap/src/core/build_steps/setup.rs @@ -272,7 +272,7 @@ fn rustup_installed(builder: &Builder<'_>) -> bool { let mut rustup = command("rustup"); rustup.arg("--version"); - rustup.allow_failure().run_always().run_capture_stdout(builder).is_success() + rustup.allow_failure().run_in_dry_run().run_capture_stdout(builder).is_success() } fn stage_dir_exists(stage_path: &str) -> bool { diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 419839067f9..2d4d9e53598 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -10,7 +10,7 @@ use std::{env, fs, iter}; use clap_complete::shells; -use crate::core::build_steps::compile::run_cargo; +use crate::core::build_steps::compile::{Std, run_cargo}; use crate::core::build_steps::doc::DocumentationFormat; use crate::core::build_steps::gcc::{Gcc, add_cg_gcc_cargo_flags}; use crate::core::build_steps::llvm::get_llvm_version; @@ -19,7 +19,8 @@ use crate::core::build_steps::tool::{self, COMPILETEST_ALLOW_FEATURES, SourceTyp use crate::core::build_steps::toolstate::ToolState; use crate::core::build_steps::{compile, dist, llvm}; use crate::core::builder::{ - self, Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step, crate_description, + self, Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step, StepMetadata, + crate_description, }; use crate::core::config::TargetSelection; use crate::core::config::flags::{Subcommand, get_completion}; @@ -544,7 +545,7 @@ impl Step for Miri { // We also need sysroots, for Miri and for the host (the latter for build scripts). // This is for the tests so everything is done with the target compiler. let miri_sysroot = Miri::build_miri_sysroot(builder, target_compiler, target); - builder.ensure(compile::Std::new(target_compiler, host)); + builder.std(target_compiler, host); let host_sysroot = builder.sysroot(target_compiler); // Miri has its own "target dir" for ui test dependencies. Make sure it gets cleared when @@ -709,7 +710,7 @@ impl Step for CompiletestTest { // We need `ToolStd` for the locally-built sysroot because // compiletest uses unstable features of the `test` crate. - builder.ensure(compile::Std::new(compiler, host)); + builder.std(compiler, host); let mut cargo = tool::prepare_tool_cargo( builder, compiler, @@ -1009,7 +1010,7 @@ impl Step for RustdocGUI { } fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.target)); + builder.std(self.compiler, self.target); let mut cmd = builder.tool_cmd(Tool::RustdocGUITest); @@ -1107,7 +1108,9 @@ impl Step for Tidy { if builder.config.cmd.bless() { cmd.arg("--bless"); } - if let Some(s) = builder.config.cmd.extra_checks() { + if let Some(s) = + builder.config.cmd.extra_checks().or(builder.config.tidy_extra_checks.as_deref()) + { cmd.arg(format!("--extra-checks={s}")); } let mut args = std::env::args_os(); @@ -1174,6 +1177,10 @@ HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to fn make_run(run: RunConfig<'_>) { run.builder.ensure(Tidy); } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::test("tidy", TargetSelection::default())) + } } fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf { @@ -1236,6 +1243,12 @@ macro_rules! test { }), }) } + + fn metadata(&self) -> Option<StepMetadata> { + Some( + StepMetadata::test(stringify!($name), self.target) + ) + } } }; } @@ -1634,7 +1647,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the if suite == "mir-opt" { builder.ensure(compile::Std::new(compiler, compiler.host).is_for_mir_opt_tests(true)); } else { - builder.ensure(compile::Std::new(compiler, compiler.host)); + builder.std(compiler, compiler.host); } let mut cmd = builder.tool_cmd(Tool::Compiletest); @@ -1642,7 +1655,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the if suite == "mir-opt" { builder.ensure(compile::Std::new(compiler, target).is_for_mir_opt_tests(true)); } else { - builder.ensure(compile::Std::new(compiler, target)); + builder.std(compiler, target); } builder.ensure(RemoteCopyLibs { compiler, target }); @@ -2177,7 +2190,7 @@ impl BookTest { fn run_ext_doc(self, builder: &Builder<'_>) { let compiler = self.compiler; - builder.ensure(compile::Std::new(compiler, compiler.host)); + builder.std(compiler, compiler.host); // mdbook just executes a binary named "rustdoc", so we need to update // PATH so that it points to our rustdoc. @@ -2263,7 +2276,7 @@ impl BookTest { let compiler = self.compiler; let host = self.compiler.host; - builder.ensure(compile::Std::new(compiler, host)); + builder.std(compiler, host); let _guard = builder.msg(Kind::Test, compiler.stage, format!("book {}", self.name), host, host); @@ -2410,7 +2423,7 @@ impl Step for ErrorIndex { drop(guard); // The tests themselves need to link to std, so make sure it is // available. - builder.ensure(compile::Std::new(compiler, compiler.host)); + builder.std(compiler, compiler.host); markdown_test(builder, compiler, &output); } } @@ -2473,7 +2486,7 @@ impl Step for CrateLibrustc { } fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.target)); + builder.std(self.compiler, self.target); // To actually run the tests, delegate to a copy of the `Crate` step. builder.ensure(Crate { @@ -2483,6 +2496,10 @@ impl Step for CrateLibrustc { crates: self.crates, }); } + + fn metadata(&self) -> Option<StepMetadata> { + Some(StepMetadata::test("CrateLibrustc", self.target)) + } } /// Given a `cargo test` subcommand, add the appropriate flags and run it. @@ -2641,7 +2658,7 @@ impl Step for Crate { // Prepare sysroot // See [field@compile::Std::force_recompile]. - builder.ensure(compile::Std::new(compiler, compiler.host).force_recompile(true)); + builder.ensure(Std::new(compiler, compiler.host).force_recompile(true)); // If we're not doing a full bootstrap but we're testing a stage2 // version of libstd, then what we're actually testing is the libstd @@ -2767,7 +2784,7 @@ impl Step for CrateRustdoc { // using `download-rustc`, the rustc_private artifacts may be in a *different sysroot* from // the target rustdoc (`ci-rustc-sysroot` vs `stage2`). In that case, we need to ensure this // explicitly to make sure it ends up in the stage2 sysroot. - builder.ensure(compile::Std::new(compiler, target)); + builder.std(compiler, target); builder.ensure(compile::Rustc::new(compiler, target)); let mut cargo = tool::prepare_tool_cargo( @@ -2911,7 +2928,7 @@ impl Step for RemoteCopyLibs { return; } - builder.ensure(compile::Std::new(compiler, target)); + builder.std(compiler, target); builder.info(&format!("REMOTE copy libs to emulator ({target})")); @@ -3101,7 +3118,7 @@ impl Step for TierCheck { /// Tests the Platform Support page in the rustc book. fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); + builder.std(self.compiler, self.compiler.host); let mut cargo = tool::prepare_tool_cargo( builder, self.compiler, @@ -3334,7 +3351,7 @@ impl Step for CodegenCranelift { let compiler = self.compiler; let target = self.target; - builder.ensure(compile::Std::new(compiler, target)); + builder.std(compiler, target); // If we're not doing a full bootstrap but we're testing a stage2 // version of libstd, then what we're actually testing is the libstd diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 248ee4196b8..ad3f8d89767 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -20,7 +20,7 @@ use crate::core::build_steps::toolstate::ToolState; use crate::core::build_steps::{compile, llvm}; use crate::core::builder; use crate::core::builder::{ - Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step, cargo_profile_var, + Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step, StepMetadata, cargo_profile_var, }; use crate::core::config::{DebuginfoLevel, RustcLto, TargetSelection}; use crate::utils::exec::{BootstrapCommand, command}; @@ -120,16 +120,16 @@ impl Step for ToolBuild { match self.mode { Mode::ToolRustc => { - // If compiler was forced, its artifacts should be prepared earlier. + // If compiler was forced, its artifacts should have been prepared earlier. if !self.compiler.is_forced_compiler() { - builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); + builder.std(self.compiler, self.compiler.host); builder.ensure(compile::Rustc::new(self.compiler, target)); } } Mode::ToolStd => { - // If compiler was forced, its artifacts should be prepared earlier. + // If compiler was forced, its artifacts should have been prepared earlier. if !self.compiler.is_forced_compiler() { - builder.ensure(compile::Std::new(self.compiler, target)) + builder.std(self.compiler, target) } } Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs @@ -479,6 +479,13 @@ macro_rules! bootstrap_tool { } }) } + + fn metadata(&self) -> Option<StepMetadata> { + Some( + StepMetadata::build(stringify!($name), self.target) + .built_by(self.compiler) + ) + } } )+ } @@ -779,6 +786,16 @@ impl Step for Rustdoc { ToolBuildResult { tool_path, build_compiler, target_compiler } } } + + fn metadata(&self) -> Option<StepMetadata> { + Some( + StepMetadata::build("rustdoc", self.compiler.host) + // rustdoc is ToolRustc, so stage N rustdoc is built by stage N-1 rustc + // FIXME: make this stage deduction automatic somehow + // FIXME: log the compiler that actually built ToolRustc steps + .stage(self.compiler.stage.saturating_sub(1)), + ) + } } #[derive(Debug, Clone, Hash, PartialEq, Eq)] @@ -1171,6 +1188,15 @@ macro_rules! tool_extended { None $( .or(Some($add_features)) )?, ) } + + fn metadata(&self) -> Option<StepMetadata> { + // FIXME: refactor extended tool steps to make the build_compiler explicit, + // it is offset by one now for rustc tools + Some( + StepMetadata::build($tool_name, self.target) + .built_by(self.compiler.with_stage(self.compiler.stage.saturating_sub(1))) + ) + } } } } diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index 99044e2a253..deb7106f185 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -3,8 +3,8 @@ use std::ffi::{OsStr, OsString}; use std::path::{Path, PathBuf}; use super::{Builder, Kind}; +use crate::core::build_steps::test; use crate::core::build_steps::tool::SourceType; -use crate::core::build_steps::{compile, test}; use crate::core::config::SplitDebuginfo; use crate::core::config::flags::Color; use crate::utils::build_stamp; @@ -131,7 +131,10 @@ impl Cargo { } pub fn into_cmd(self) -> BootstrapCommand { - self.into() + let mut cmd: BootstrapCommand = self.into(); + // Disable caching for commands originating from Cargo-related operations. + cmd.do_not_cache(); + cmd } /// Same as [`Cargo::new`] except this one doesn't configure the linker with @@ -843,7 +846,7 @@ impl Builder<'_> { // If this is for `miri-test`, prepare the sysroots. if cmd_kind == Kind::MiriTest { - self.ensure(compile::Std::new(compiler, compiler.host)); + self.std(compiler, compiler.host); let host_sysroot = self.sysroot(compiler); let miri_sysroot = test::Miri::build_miri_sysroot(self, compiler, target); cargo.env("MIRI_SYSROOT", &miri_sysroot); diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index b26f47a3171..b96a988cde3 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -15,14 +15,14 @@ use tracing::instrument; pub use self::cargo::{Cargo, cargo_profile_var}; pub use crate::Compiler; +use crate::core::build_steps::compile::{Std, StdLink}; use crate::core::build_steps::{ check, clean, clippy, compile, dist, doc, gcc, install, llvm, run, setup, test, tool, vendor, }; use crate::core::config::flags::Subcommand; use crate::core::config::{DryRun, TargetSelection}; use crate::utils::cache::Cache; -use crate::utils::exec::{BootstrapCommand, command}; -use crate::utils::execution_context::ExecutionContext; +use crate::utils::exec::{BootstrapCommand, ExecutionContext, command}; use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t}; use crate::{Build, Crate, trace}; @@ -139,7 +139,7 @@ pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { /// Metadata that describes an executed step, mostly for testing and tracing. #[allow(unused)] -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub struct StepMetadata { name: &'static str, kind: Kind, @@ -150,7 +150,27 @@ pub struct StepMetadata { impl StepMetadata { pub fn build(name: &'static str, target: TargetSelection) -> Self { - Self { name, kind: Kind::Build, target, built_by: None, stage: None } + Self::new(name, target, Kind::Build) + } + + pub fn check(name: &'static str, target: TargetSelection) -> Self { + Self::new(name, target, Kind::Check) + } + + pub fn doc(name: &'static str, target: TargetSelection) -> Self { + Self::new(name, target, Kind::Doc) + } + + pub fn dist(name: &'static str, target: TargetSelection) -> Self { + Self::new(name, target, Kind::Dist) + } + + pub fn test(name: &'static str, target: TargetSelection) -> Self { + Self::new(name, target, Kind::Test) + } + + fn new(name: &'static str, target: TargetSelection, kind: Kind) -> Self { + Self { name, kind, target, built_by: None, stage: None } } pub fn built_by(mut self, compiler: Compiler) -> Self { @@ -162,6 +182,14 @@ impl StepMetadata { self.stage = Some(stage); self } + + pub fn get_stage(&self) -> Option<u32> { + self.stage.or(self + .built_by + // For std, its stage corresponds to the stage of the compiler that builds it. + // For everything else, a stage N things gets built by a stage N-1 compiler. + .map(|compiler| if self.name == "std" { compiler.stage } else { compiler.stage + 1 })) + } } pub struct RunConfig<'a> { @@ -1350,6 +1378,49 @@ impl<'a> Builder<'a> { resolved_compiler } + /// Obtain a standard library for the given target that will be built by the passed compiler. + /// The standard library will be linked to the sysroot of the passed compiler. + /// + /// Prefer using this method rather than manually invoking `Std::new`. + #[cfg_attr( + feature = "tracing", + instrument( + level = "trace", + name = "Builder::std", + target = "STD", + skip_all, + fields( + compiler = ?compiler, + target = ?target, + ), + ), + )] + pub fn std(&self, compiler: Compiler, target: TargetSelection) { + // FIXME: make the `Std` step return some type-level "proof" that std was indeed built, + // and then require passing that to all Cargo invocations that we do. + + // The "stage 0" std is always precompiled and comes with the stage0 compiler, so we have + // special logic for it, to avoid creating needless and confusing Std steps that don't + // actually build anything. + if compiler.stage == 0 { + if target != compiler.host { + panic!( + r"It is not possible to build the standard library for `{target}` using the stage0 compiler. +You have to build a stage1 compiler for `{}` first, and then use it to build a standard library for `{target}`. +", + compiler.host + ) + } + + // We still need to link the prebuilt standard library into the ephemeral stage0 sysroot + self.ensure(StdLink::from_std(Std::new(compiler, target), compiler)); + } else { + // This step both compiles the std and links it into the compiler's sysroot. + // Yes, it's quite magical and side-effecty.. would be nice to refactor later. + self.ensure(Std::new(compiler, target)); + } + } + pub fn sysroot(&self, compiler: Compiler) -> PathBuf { self.ensure(compile::Sysroot::new(compiler)) } diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index f1af2b285a2..1c5267cb75e 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -10,6 +10,7 @@ use crate::core::build_steps::doc::DocumentationFormat; use crate::core::config::Config; use crate::utils::cache::ExecutedStep; use crate::utils::helpers::get_host_target; +use crate::utils::tests::ConfigBuilder; use crate::utils::tests::git::{GitCtx, git_test}; static TEST_TRIPLE_1: &str = "i686-unknown-haiku"; @@ -193,58 +194,6 @@ fn check_missing_paths_for_x_test_tests() { } #[test] -fn test_exclude() { - let mut config = configure("test", &[TEST_TRIPLE_1], &[TEST_TRIPLE_1]); - config.skip = vec!["src/tools/tidy".into()]; - let cache = run_build(&[], config); - - // Ensure we have really excluded tidy - assert!(!cache.contains::<test::Tidy>()); - - // Ensure other tests are not affected. - assert!(cache.contains::<test::RustdocUi>()); -} - -#[test] -fn test_exclude_kind() { - let path = PathBuf::from("compiler/rustc_data_structures"); - - let mut config = configure("test", &[TEST_TRIPLE_1], &[TEST_TRIPLE_1]); - // Ensure our test is valid, and `test::Rustc` would be run without the exclude. - assert!(run_build(&[], config.clone()).contains::<test::CrateLibrustc>()); - // Ensure tests for rustc are not skipped. - config.skip = vec![path.clone()]; - assert!(run_build(&[], config.clone()).contains::<test::CrateLibrustc>()); - // Ensure builds for rustc are not skipped. - assert!(run_build(&[], config).contains::<compile::Rustc>()); -} - -/// Ensure that if someone passes both a single crate and `library`, all library crates get built. -#[test] -fn alias_and_path_for_library() { - let mut cache = run_build( - &["library".into(), "core".into()], - configure("build", &[TEST_TRIPLE_1], &[TEST_TRIPLE_1]), - ); - assert_eq!( - first(cache.all::<compile::Std>()), - &[ - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1) - ] - ); - - let mut cache = run_build( - &["library".into(), "core".into()], - configure("doc", &[TEST_TRIPLE_1], &[TEST_TRIPLE_1]), - ); - assert_eq!( - first(cache.all::<doc::Std>()), - &[doc_std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1)] - ); -} - -#[test] fn ci_rustc_if_unchanged_invalidate_on_compiler_changes() { git_test(|ctx| { prepare_rustc_checkout(ctx); @@ -316,101 +265,6 @@ mod defaults { use crate::core::builder::*; #[test] - fn build_default() { - let mut cache = run_build(&[], configure("build", &[TEST_TRIPLE_1], &[TEST_TRIPLE_1])); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - assert_eq!( - first(cache.all::<compile::Std>()), - &[ - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1), - ] - ); - assert!(!cache.all::<compile::Assemble>().is_empty()); - // Make sure rustdoc is only built once. - assert_eq!( - first(cache.all::<tool::Rustdoc>()), - // Recall that rustdoc stages are off-by-one - // - this is the compiler it's _linked_ to, not built with. - &[tool::Rustdoc { compiler: Compiler::new(1, a) }], - ); - assert_eq!( - first(cache.all::<compile::Rustc>()), - &[rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0)], - ); - } - - #[test] - fn build_stage_0() { - let config = Config { stage: 0, ..configure("build", &[TEST_TRIPLE_1], &[TEST_TRIPLE_1]) }; - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - assert_eq!( - first(cache.all::<compile::Std>()), - &[std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0)] - ); - assert!(!cache.all::<compile::Assemble>().is_empty()); - assert_eq!( - first(cache.all::<tool::Rustdoc>()), - // This is the beta rustdoc. - // Add an assert here to make sure this is the only rustdoc built. - &[tool::Rustdoc { compiler: Compiler::new(0, a) }], - ); - assert!(cache.all::<compile::Rustc>().is_empty()); - } - - #[test] - fn build_cross_compile() { - let config = Config { - stage: 1, - ..configure("build", &[TEST_TRIPLE_1, TEST_TRIPLE_2], &[TEST_TRIPLE_1, TEST_TRIPLE_2]) - }; - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - let b = TargetSelection::from_user(TEST_TRIPLE_2); - - // Ideally, this build wouldn't actually have `target: a` - // rustdoc/rustcc/std here (the user only requested a host=B build, so - // there's not really a need for us to build for target A in this case - // (since we're producing stage 1 libraries/binaries). But currently - // bootstrap is just a bit buggy here; this should be fixed though. - assert_eq!( - first(cache.all::<compile::Std>()), - &[ - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 0), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1), - ] - ); - assert_eq!( - first(cache.all::<compile::Assemble>()), - &[ - compile::Assemble { target_compiler: Compiler::new(0, a) }, - compile::Assemble { target_compiler: Compiler::new(1, a) }, - compile::Assemble { target_compiler: Compiler::new(1, b) }, - ] - ); - assert_eq!( - first(cache.all::<tool::Rustdoc>()), - &[ - tool::Rustdoc { compiler: Compiler::new(1, a) }, - tool::Rustdoc { compiler: Compiler::new(1, b) }, - ], - ); - assert_eq!( - first(cache.all::<compile::Rustc>()), - &[ - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 0), - ] - ); - } - - #[test] fn doc_default() { let mut config = configure("doc", &[TEST_TRIPLE_1], &[TEST_TRIPLE_1]); config.compiler_docs = true; @@ -447,326 +301,6 @@ mod dist { } #[test] - fn dist_baseline() { - let mut cache = run_build(&[], configure(&[TEST_TRIPLE_1], &[TEST_TRIPLE_1])); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - - assert_eq!(first(cache.all::<dist::Docs>()), &[dist::Docs { host: a },]); - assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: a },]); - assert_eq!( - first(cache.all::<dist::Rustc>()), - &[dist::Rustc { compiler: Compiler::new(2, a) },] - ); - assert_eq!( - first(cache.all::<dist::Std>()), - &[dist::Std { compiler: Compiler::new(1, a), target: a },] - ); - assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); - // Make sure rustdoc is only built once. - assert_eq!( - first(cache.all::<tool::Rustdoc>()), - &[tool::Rustdoc { compiler: Compiler::new(2, a) },] - ); - } - - #[test] - fn dist_with_targets() { - let mut cache = - run_build(&[], configure(&[TEST_TRIPLE_1], &[TEST_TRIPLE_1, TEST_TRIPLE_2])); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - let b = TargetSelection::from_user(TEST_TRIPLE_2); - - assert_eq!( - first(cache.all::<dist::Docs>()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(cache.all::<dist::Mingw>()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(cache.all::<dist::Rustc>()), - &[dist::Rustc { compiler: Compiler::new(2, a) },] - ); - assert_eq!( - first(cache.all::<dist::Std>()), - &[ - dist::Std { compiler: Compiler::new(1, a), target: a }, - dist::Std { compiler: Compiler::new(2, a), target: b }, - ] - ); - assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); - } - - #[test] - fn dist_with_hosts() { - let mut cache = run_build( - &[], - configure(&[TEST_TRIPLE_1, TEST_TRIPLE_2], &[TEST_TRIPLE_1, TEST_TRIPLE_2]), - ); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - let b = TargetSelection::from_user(TEST_TRIPLE_2); - - assert_eq!( - first(cache.all::<dist::Docs>()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(cache.all::<dist::Mingw>()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(cache.all::<dist::Rustc>()), - &[ - dist::Rustc { compiler: Compiler::new(2, a) }, - dist::Rustc { compiler: Compiler::new(2, b) }, - ] - ); - assert_eq!( - first(cache.all::<dist::Std>()), - &[ - dist::Std { compiler: Compiler::new(1, a), target: a }, - dist::Std { compiler: Compiler::new(1, a), target: b }, - ] - ); - assert_eq!( - first(cache.all::<compile::Std>()), - &[ - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 2), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 2), - ], - ); - assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); - } - - #[test] - fn dist_only_cross_host() { - let b = TargetSelection::from_user(TEST_TRIPLE_2); - let mut config = - configure(&[TEST_TRIPLE_1, TEST_TRIPLE_2], &[TEST_TRIPLE_1, TEST_TRIPLE_2]); - config.docs = false; - config.extended = true; - config.hosts = vec![b]; - let mut cache = run_build(&[], config); - - assert_eq!( - first(cache.all::<dist::Rustc>()), - &[dist::Rustc { compiler: Compiler::new(2, b) },] - ); - assert_eq!( - first(cache.all::<compile::Rustc>()), - &[ - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1), - ] - ); - } - - #[test] - fn dist_with_targets_and_hosts() { - let mut cache = run_build( - &[], - configure( - &[TEST_TRIPLE_1, TEST_TRIPLE_2], - &[TEST_TRIPLE_1, TEST_TRIPLE_2, TEST_TRIPLE_3], - ), - ); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - let b = TargetSelection::from_user(TEST_TRIPLE_2); - let c = TargetSelection::from_user(TEST_TRIPLE_3); - - assert_eq!( - first(cache.all::<dist::Docs>()), - &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] - ); - assert_eq!( - first(cache.all::<dist::Mingw>()), - &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] - ); - assert_eq!( - first(cache.all::<dist::Rustc>()), - &[ - dist::Rustc { compiler: Compiler::new(2, a) }, - dist::Rustc { compiler: Compiler::new(2, b) }, - ] - ); - assert_eq!( - first(cache.all::<dist::Std>()), - &[ - dist::Std { compiler: Compiler::new(1, a), target: a }, - dist::Std { compiler: Compiler::new(1, a), target: b }, - dist::Std { compiler: Compiler::new(2, a), target: c }, - ] - ); - assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); - } - - #[test] - fn dist_with_empty_host() { - let config = configure(&[], &[TEST_TRIPLE_3]); - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - let c = TargetSelection::from_user(TEST_TRIPLE_3); - - assert_eq!(first(cache.all::<dist::Docs>()), &[dist::Docs { host: c },]); - assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: c },]); - assert_eq!( - first(cache.all::<dist::Std>()), - &[dist::Std { compiler: Compiler::new(2, a), target: c },] - ); - } - - #[test] - fn dist_with_same_targets_and_hosts() { - let mut cache = run_build( - &[], - configure(&[TEST_TRIPLE_1, TEST_TRIPLE_2], &[TEST_TRIPLE_1, TEST_TRIPLE_2]), - ); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - let b = TargetSelection::from_user(TEST_TRIPLE_2); - - assert_eq!( - first(cache.all::<dist::Docs>()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(cache.all::<dist::Mingw>()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(cache.all::<dist::Rustc>()), - &[ - dist::Rustc { compiler: Compiler::new(2, a) }, - dist::Rustc { compiler: Compiler::new(2, b) }, - ] - ); - assert_eq!( - first(cache.all::<dist::Std>()), - &[ - dist::Std { compiler: Compiler::new(1, a), target: a }, - dist::Std { compiler: Compiler::new(1, a), target: b }, - ] - ); - assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); - assert_eq!( - first(cache.all::<compile::Std>()), - &[ - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 2), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 2), - ] - ); - assert_eq!( - first(cache.all::<compile::Assemble>()), - &[ - compile::Assemble { target_compiler: Compiler::new(0, a) }, - compile::Assemble { target_compiler: Compiler::new(1, a) }, - compile::Assemble { target_compiler: Compiler::new(2, a) }, - compile::Assemble { target_compiler: Compiler::new(2, b) }, - ] - ); - } - - /// This also serves as an important regression test for <https://github.com/rust-lang/rust/issues/138123> - /// and <https://github.com/rust-lang/rust/issues/138004>. - #[test] - fn dist_all_cross() { - let cmd_args = - &["dist", "--stage", "2", "--dry-run", "--config=/does/not/exist"].map(str::to_owned); - let config_str = r#" - [rust] - channel = "nightly" - - [build] - extended = true - - build = "i686-unknown-haiku" - host = ["i686-unknown-netbsd"] - target = ["i686-unknown-netbsd"] - "#; - let config = Config::parse_inner(Flags::parse(cmd_args), |&_| toml::from_str(config_str)); - let mut cache = run_build(&[], config); - - // Stage 2 `compile::Rustc` should **NEVER** be cached here. - assert_eq!( - first(cache.all::<compile::Rustc>()), - &[ - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1), - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_3, stage = 1), - ] - ); - } - - #[test] - fn build_all() { - let build = Build::new(configure( - &[TEST_TRIPLE_1, TEST_TRIPLE_2], - &[TEST_TRIPLE_1, TEST_TRIPLE_2, TEST_TRIPLE_3], - )); - let mut builder = Builder::new(&build); - builder.run_step_descriptions( - &Builder::get_step_descriptions(Kind::Build), - &["compiler/rustc".into(), "library".into()], - ); - - assert_eq!(builder.config.stage, 2); - - // `compile::Rustc` includes one-stage-off compiler information as the target compiler - // artifacts get copied from there to the target stage sysroot. - // For example, `stage2/bin/rustc` gets copied from the `stage1-rustc` build directory. - assert_eq!( - first(builder.cache.all::<compile::Rustc>()), - &[ - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1), - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1), - ] - ); - - assert_eq!( - first(builder.cache.all::<compile::Std>()), - &[ - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 2), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 2), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_3, stage = 2), - ] - ); - - assert_eq!( - first(builder.cache.all::<compile::Assemble>()), - &[ - compile::Assemble { - target_compiler: Compiler::new(0, TargetSelection::from_user(TEST_TRIPLE_1),) - }, - compile::Assemble { - target_compiler: Compiler::new(1, TargetSelection::from_user(TEST_TRIPLE_1),) - }, - compile::Assemble { - target_compiler: Compiler::new(2, TargetSelection::from_user(TEST_TRIPLE_1),) - }, - compile::Assemble { - target_compiler: Compiler::new(2, TargetSelection::from_user(TEST_TRIPLE_2),) - }, - ] - ); - } - - #[test] fn llvm_out_behaviour() { let mut config = configure(&[TEST_TRIPLE_1], &[TEST_TRIPLE_2]); config.llvm_from_ci = true; @@ -784,85 +318,6 @@ mod dist { } #[test] - fn build_with_empty_host() { - let config = configure(&[], &[TEST_TRIPLE_3]); - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); - - let a = TargetSelection::from_user(TEST_TRIPLE_1); - - assert_eq!( - first(builder.cache.all::<compile::Std>()), - &[ - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1), - std!(TEST_TRIPLE_1 => TEST_TRIPLE_3, stage = 2), - ] - ); - assert_eq!( - first(builder.cache.all::<compile::Assemble>()), - &[ - compile::Assemble { target_compiler: Compiler::new(0, a) }, - compile::Assemble { target_compiler: Compiler::new(1, a) }, - compile::Assemble { target_compiler: Compiler::new(2, a) }, - ] - ); - assert_eq!( - first(builder.cache.all::<compile::Rustc>()), - &[ - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), - rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 1), - ] - ); - } - - #[test] - fn test_with_no_doc_stage0() { - let mut config = configure(&[TEST_TRIPLE_1], &[TEST_TRIPLE_1]); - config.stage = 0; - config.paths = vec!["library/std".into()]; - config.cmd = Subcommand::Test { - test_args: vec![], - compiletest_rustc_args: vec![], - no_fail_fast: false, - no_doc: true, - doc: false, - bless: false, - force_rerun: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - run: None, - only_modified: false, - extra_checks: None, - no_capture: false, - }; - - let build = Build::new(config); - let mut builder = Builder::new(&build); - - let host = TargetSelection::from_user(TEST_TRIPLE_1); - - builder.run_step_descriptions( - &[StepDescription::from::<test::Crate>(Kind::Test)], - &["library/std".into()], - ); - - // Ensure we don't build any compiler artifacts. - assert!(!builder.cache.contains::<compile::Rustc>()); - assert_eq!( - first(builder.cache.all::<test::Crate>()), - &[test::Crate { - compiler: Compiler::new(0, host), - target: host, - mode: crate::Mode::Std, - crates: vec!["std".to_owned()], - },] - ); - } - - #[test] fn doc_ci() { let mut config = configure(&[TEST_TRIPLE_1], &[TEST_TRIPLE_1]); config.compiler_docs = true; @@ -889,65 +344,6 @@ mod dist { &[tool::Rustdoc { compiler: Compiler::new(2, a) },] ); } - - #[test] - fn test_docs() { - // Behavior of `x.py test` doing various documentation tests. - let mut config = configure(&[TEST_TRIPLE_1], &[TEST_TRIPLE_1]); - config.cmd = Subcommand::Test { - test_args: vec![], - compiletest_rustc_args: vec![], - no_fail_fast: false, - doc: true, - no_doc: false, - bless: false, - force_rerun: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - run: None, - only_modified: false, - extra_checks: None, - no_capture: false, - }; - // Make sure rustfmt binary not being found isn't an error. - config.channel = "beta".to_string(); - let build = Build::new(config); - let mut builder = Builder::new(&build); - - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); - let a = TargetSelection::from_user(TEST_TRIPLE_1); - - // error_index_generator uses stage 1 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!( - first(builder.cache.all::<doc::ErrorIndex>()), - &[doc::ErrorIndex { target: a },] - ); - assert_eq!( - first(builder.cache.all::<tool::ErrorIndex>()), - &[tool::ErrorIndex { compiler: Compiler::new(1, a) }] - ); - // Unfortunately rustdoc is built twice. Once from stage1 for compiletest - // (and other things), and once from stage0 for std crates. Ideally it - // would only be built once. If someone wants to fix this, it might be - // worth investigating if it would be possible to test std from stage1. - // Note that the stages here are +1 than what they actually are because - // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is - // not 0. - // - // The stage 0 copy is the one downloaded for bootstrapping. It is - // (currently) needed to run "cargo test" on the linkchecker, and - // should be relatively "free". - assert_eq!( - first(builder.cache.all::<tool::Rustdoc>()), - &[ - tool::Rustdoc { compiler: Compiler::new(0, a) }, - tool::Rustdoc { compiler: Compiler::new(1, a) }, - tool::Rustdoc { compiler: Compiler::new(2, a) }, - ] - ); - } } mod sysroot_target_dirs { @@ -1234,15 +630,104 @@ fn any_debug() { assert_eq!(x.downcast_ref::<MyStruct>(), Some(&MyStruct { x: 7 })); } -/// The staging tests use insta for snapshot testing. +/// These tests use insta for snapshot testing. /// See bootstrap's README on how to bless the snapshots. -mod staging { - use crate::Build; - use crate::core::builder::Builder; +mod snapshot { + use std::path::PathBuf; + + use crate::core::build_steps::{compile, dist, doc, test, tool}; use crate::core::builder::tests::{ - TEST_TRIPLE_1, configure, configure_with_args, render_steps, run_build, + TEST_TRIPLE_1, TEST_TRIPLE_2, TEST_TRIPLE_3, configure, configure_with_args, first, + host_target, render_steps, run_build, }; + use crate::core::builder::{Builder, Kind, StepDescription, StepMetadata}; + use crate::core::config::TargetSelection; + use crate::utils::cache::Cache; + use crate::utils::helpers::get_host_target; use crate::utils::tests::{ConfigBuilder, TestCtx}; + use crate::{Build, Compiler, Config, Flags, Subcommand}; + + #[test] + fn build_default() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustdoc 0 <host> + "); + } + + #[test] + fn build_cross_compile() { + let ctx = TestCtx::new(); + + insta::assert_snapshot!( + ctx.config("build") + // Cross-compilation fails on stage 1, as we don't have a stage0 std available + // for non-host targets. + .stage(2) + .hosts(&[&host_target(), TEST_TRIPLE_1]) + .targets(&[&host_target(), TEST_TRIPLE_1]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 1 <host> -> std 1 <target1> + [build] rustc 2 <host> -> std 2 <target1> + [build] rustdoc 1 <host> + [build] llvm <target1> + [build] rustc 1 <host> -> rustc 2 <target1> + [build] rustdoc 1 <target1> + "); + } + + #[test] + fn build_with_empty_host() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx + .config("build") + .hosts(&[]) + .targets(&[TEST_TRIPLE_1]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <target1> + " + ); + } + + #[test] + fn build_compiler_no_explicit_stage() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("compiler") + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + "); + + insta::assert_snapshot!( + ctx.config("build") + .path("rustc") + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + "); + } + + #[test] + #[should_panic] + fn build_compiler_stage_0() { + let ctx = TestCtx::new(); + ctx.config("build").path("compiler").stage(0).run(); + } #[test] fn build_compiler_stage_1() { @@ -1251,25 +736,907 @@ mod staging { ctx.config("build") .path("compiler") .stage(1) - .get_steps(), @r" - [build] rustc 0 <host> -> std 0 <host> + .render_steps(), @r" [build] llvm <host> [build] rustc 0 <host> -> rustc 1 <host> + "); + } + + #[test] + fn build_compiler_stage_2() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("compiler") + .stage(2) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + "); + } + + #[test] + fn build_library_no_explicit_stage() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("library") + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + "); + } + + #[test] + #[should_panic] + fn build_library_stage_0() { + let ctx = TestCtx::new(); + ctx.config("build").path("library").stage(0).run(); + } + + #[test] + fn build_library_stage_1() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("library") + .stage(1) + .render_steps(), @r" + [build] llvm <host> [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> "); } - impl ConfigBuilder { - fn get_steps(self) -> String { - let config = self.create_config(); + #[test] + fn build_library_stage_2() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("library") + .stage(2) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustc 2 <host> -> std 2 <host> + "); + } + + #[test] + fn build_miri_no_explicit_stage() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("miri") + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 0 <host> -> miri 1 <host> + "); + } - let kind = config.cmd.kind(); - let build = Build::new(config); - let builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(kind), &builder.paths); - render_steps(&builder.cache.into_executed_steps()) + #[test] + #[should_panic] + fn build_miri_stage_0() { + let ctx = TestCtx::new(); + ctx.config("build").path("miri").stage(0).run(); + } + + #[test] + fn build_miri_stage_1() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("miri") + .stage(1) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 0 <host> -> miri 1 <host> + "); + } + + #[test] + fn build_miri_stage_2() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("miri") + .stage(2) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustc 1 <host> -> miri 2 <host> + "); + } + + #[test] + fn build_bootstrap_tool_no_explicit_stage() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("opt-dist") + .render_steps(), @"[build] rustc 0 <host> -> OptimizedDist 1 <host>"); + } + + #[test] + #[should_panic] + fn build_bootstrap_tool_stage_0() { + let ctx = TestCtx::new(); + ctx.config("build").path("opt-dist").stage(0).run(); + } + + #[test] + fn build_bootstrap_tool_stage_1() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("opt-dist") + .stage(1) + .render_steps(), @"[build] rustc 0 <host> -> OptimizedDist 1 <host>"); + } + + #[test] + fn build_bootstrap_tool_stage_2() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .path("opt-dist") + .stage(2) + .render_steps(), @"[build] rustc 0 <host> -> OptimizedDist 1 <host>"); + } + + #[test] + fn build_default_stage() { + let ctx = TestCtx::new(); + assert_eq!(ctx.config("build").path("compiler").create_config().stage, 1); + } + + /// Ensure that if someone passes both a single crate and `library`, all + /// library crates get built. + #[test] + fn alias_and_path_for_library() { + let ctx = TestCtx::new(); + insta::assert_snapshot!(ctx.config("build") + .paths(&["library", "core"]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + "); + + insta::assert_snapshot!(ctx.config("build") + .paths(&["std"]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + "); + + insta::assert_snapshot!(ctx.config("build") + .paths(&["core"]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + "); + + insta::assert_snapshot!(ctx.config("build") + .paths(&["alloc"]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + "); + + insta::assert_snapshot!(ctx.config("doc") + .paths(&["library", "core"]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustdoc 0 <host> + [doc] std 1 <host> + "); + } + + #[test] + fn build_all() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("build") + .stage(2) + .paths(&["compiler/rustc", "library"]) + .hosts(&[&host_target(), TEST_TRIPLE_1]) + .targets(&[&host_target(), TEST_TRIPLE_1, TEST_TRIPLE_2]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] llvm <target1> + [build] rustc 1 <host> -> std 1 <target1> + [build] rustc 1 <host> -> rustc 2 <target1> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 2 <host> -> std 2 <target1> + [build] rustc 2 <host> -> std 2 <target2> + "); + } + + #[test] + fn dist_default_stage() { + let ctx = TestCtx::new(); + assert_eq!(ctx.config("dist").path("compiler").create_config().stage, 2); + } + + #[test] + fn dist_baseline() { + let ctx = TestCtx::new(); + // Note that stdlib is uplifted, that is why `[dist] rustc 1 <host> -> std <host>` is in + // the output. + insta::assert_snapshot!( + ctx + .config("dist") + .render_steps(), @r" + [build] rustc 0 <host> -> UnstableBookGen 1 <host> + [build] rustc 0 <host> -> Rustbook 1 <host> + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustdoc 1 <host> + [doc] std 2 <host> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 0 <host> -> LintDocs 1 <host> + [build] rustc 0 <host> -> RustInstaller 1 <host> + [dist] docs <host> + [doc] std 2 <host> + [dist] mingw <host> + [build] rustc 0 <host> -> GenerateCopyright 1 <host> + [dist] rustc <host> + [dist] rustc 1 <host> -> std 1 <host> + [dist] src <> + " + ); + } + + #[test] + fn dist_extended() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx + .config("dist") + .args(&["--set", "build.extended=true"]) + .render_steps(), @r" + [build] rustc 0 <host> -> UnstableBookGen 1 <host> + [build] rustc 0 <host> -> Rustbook 1 <host> + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 0 <host> -> WasmComponentLd 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustc 1 <host> -> WasmComponentLd 2 <host> + [build] rustdoc 1 <host> + [doc] std 2 <host> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 0 <host> -> LintDocs 1 <host> + [build] rustc 0 <host> -> RustInstaller 1 <host> + [dist] docs <host> + [doc] std 2 <host> + [dist] mingw <host> + [build] rustc 0 <host> -> GenerateCopyright 1 <host> + [dist] rustc <host> + [dist] rustc 1 <host> -> std 1 <host> + [dist] src <> + [build] rustc 0 <host> -> rustfmt 1 <host> + [build] rustc 0 <host> -> cargo-fmt 1 <host> + [build] rustc 0 <host> -> clippy-driver 1 <host> + [build] rustc 0 <host> -> cargo-clippy 1 <host> + [build] rustc 0 <host> -> miri 1 <host> + [build] rustc 0 <host> -> cargo-miri 1 <host> + "); + } + + #[test] + fn dist_with_targets() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx + .config("dist") + .hosts(&[&host_target()]) + .targets(&[&host_target(), TEST_TRIPLE_1]) + .render_steps(), @r" + [build] rustc 0 <host> -> UnstableBookGen 1 <host> + [build] rustc 0 <host> -> Rustbook 1 <host> + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustdoc 1 <host> + [doc] std 2 <host> + [doc] std 2 <target1> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 0 <host> -> LintDocs 1 <host> + [build] rustc 0 <host> -> RustInstaller 1 <host> + [dist] docs <host> + [dist] docs <target1> + [doc] std 2 <host> + [doc] std 2 <target1> + [dist] mingw <host> + [dist] mingw <target1> + [build] rustc 0 <host> -> GenerateCopyright 1 <host> + [dist] rustc <host> + [dist] rustc 1 <host> -> std 1 <host> + [build] rustc 2 <host> -> std 2 <target1> + [dist] rustc 2 <host> -> std 2 <target1> + [dist] src <> + " + ); + } + + #[test] + fn dist_with_hosts() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx + .config("dist") + .hosts(&[&host_target(), TEST_TRIPLE_1]) + .targets(&[&host_target()]) + .render_steps(), @r" + [build] rustc 0 <host> -> UnstableBookGen 1 <host> + [build] rustc 0 <host> -> Rustbook 1 <host> + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustdoc 1 <host> + [doc] std 2 <host> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 0 <host> -> LintDocs 1 <host> + [build] rustc 1 <host> -> std 1 <target1> + [build] rustc 2 <host> -> std 2 <target1> + [build] rustc 0 <host> -> RustInstaller 1 <host> + [dist] docs <host> + [doc] std 2 <host> + [dist] mingw <host> + [build] rustc 0 <host> -> GenerateCopyright 1 <host> + [dist] rustc <host> + [build] llvm <target1> + [build] rustc 1 <host> -> rustc 2 <target1> + [build] rustdoc 1 <target1> + [dist] rustc <target1> + [dist] rustc 1 <host> -> std 1 <host> + [dist] src <> + " + ); + } + + #[test] + fn dist_with_targets_and_hosts() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx + .config("dist") + .hosts(&[&host_target(), TEST_TRIPLE_1]) + .targets(&[&host_target(), TEST_TRIPLE_1]) + .render_steps(), @r" + [build] rustc 0 <host> -> UnstableBookGen 1 <host> + [build] rustc 0 <host> -> Rustbook 1 <host> + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustdoc 1 <host> + [doc] std 2 <host> + [doc] std 2 <target1> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 0 <host> -> LintDocs 1 <host> + [build] rustc 1 <host> -> std 1 <target1> + [build] rustc 2 <host> -> std 2 <target1> + [build] rustc 0 <host> -> RustInstaller 1 <host> + [dist] docs <host> + [dist] docs <target1> + [doc] std 2 <host> + [doc] std 2 <target1> + [dist] mingw <host> + [dist] mingw <target1> + [build] rustc 0 <host> -> GenerateCopyright 1 <host> + [dist] rustc <host> + [build] llvm <target1> + [build] rustc 1 <host> -> rustc 2 <target1> + [build] rustdoc 1 <target1> + [dist] rustc <target1> + [dist] rustc 1 <host> -> std 1 <host> + [dist] rustc 1 <host> -> std 1 <target1> + [dist] src <> + " + ); + } + + #[test] + fn dist_with_empty_host() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx + .config("dist") + .hosts(&[]) + .targets(&[TEST_TRIPLE_1]) + .render_steps(), @r" + [build] rustc 0 <host> -> UnstableBookGen 1 <host> + [build] rustc 0 <host> -> Rustbook 1 <host> + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustdoc 1 <host> + [doc] std 2 <target1> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 0 <host> -> RustInstaller 1 <host> + [dist] docs <target1> + [doc] std 2 <target1> + [dist] mingw <target1> + [build] rustc 2 <host> -> std 2 <target1> + [dist] rustc 2 <host> -> std 2 <target1> + "); + } + + /// This also serves as an important regression test for <https://github.com/rust-lang/rust/issues/138123> + /// and <https://github.com/rust-lang/rust/issues/138004>. + #[test] + fn dist_all_cross() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx + .config("dist") + .hosts(&[TEST_TRIPLE_1]) + .targets(&[TEST_TRIPLE_1]) + .args(&["--set", "rust.channel=nightly", "--set", "build.extended=true"]) + .render_steps(), @r" + [build] rustc 0 <host> -> UnstableBookGen 1 <host> + [build] rustc 0 <host> -> Rustbook 1 <host> + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 0 <host> -> WasmComponentLd 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustc 1 <host> -> WasmComponentLd 2 <host> + [build] rustdoc 1 <host> + [doc] std 2 <target1> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 1 <host> -> std 1 <target1> + [build] rustc 2 <host> -> std 2 <target1> + [build] rustc 0 <host> -> LintDocs 1 <host> + [build] rustc 0 <host> -> RustInstaller 1 <host> + [dist] docs <target1> + [doc] std 2 <target1> + [dist] mingw <target1> + [build] llvm <target1> + [build] rustc 1 <host> -> rustc 2 <target1> + [build] rustc 1 <host> -> WasmComponentLd 2 <target1> + [build] rustdoc 1 <target1> + [build] rustc 0 <host> -> GenerateCopyright 1 <host> + [dist] rustc <target1> + [dist] rustc 1 <host> -> std 1 <target1> + [dist] src <> + [build] rustc 0 <host> -> rustfmt 1 <target1> + [build] rustc 0 <host> -> cargo-fmt 1 <target1> + [build] rustc 0 <host> -> clippy-driver 1 <target1> + [build] rustc 0 <host> -> cargo-clippy 1 <target1> + [build] rustc 0 <host> -> miri 1 <target1> + [build] rustc 0 <host> -> cargo-miri 1 <target1> + "); + } + + #[test] + fn check_compiler_no_explicit_stage() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("compiler") + .render_steps(), @r" + [check] std <host> + [build] llvm <host> + [check] rustc <host> + [check] cranelift <host> + [check] gcc <host> + "); + + insta::assert_snapshot!( + ctx.config("check") + .path("rustc") + .render_steps(), @r" + [check] std <host> + [build] llvm <host> + [check] rustc <host> + "); + } + + #[test] + fn check_compiler_stage_0() { + let ctx = TestCtx::new(); + ctx.config("check").path("compiler").stage(0).run(); + } + + #[test] + fn check_compiler_stage_1() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("compiler") + .stage(1) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [check] rustc <host> + [check] cranelift <host> + [check] gcc <host> + "); + } + + #[test] + fn check_compiler_stage_2() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("compiler") + .stage(2) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustc 2 <host> -> std 2 <host> + [check] rustc <host> + [check] cranelift <host> + [check] gcc <host> + "); + } + + #[test] + fn check_cross_compile() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .stage(2) + .targets(&[TEST_TRIPLE_1]) + .hosts(&[TEST_TRIPLE_1]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustc 2 <host> -> std 2 <host> + [build] rustc 1 <host> -> std 1 <target1> + [build] rustc 2 <host> -> std 2 <target1> + [check] rustc <target1> + [check] Rustdoc <target1> + [check] cranelift <target1> + [check] gcc <target1> + [check] Clippy <target1> + [check] Miri <target1> + [check] CargoMiri <target1> + [check] MiroptTestTools <target1> + [check] Rustfmt <target1> + [check] rust-analyzer <target1> + [check] TestFloatParse <target1> + [check] FeaturesStatusDump <target1> + [check] std <target1> + "); + } + + #[test] + fn check_library_no_explicit_stage() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("library") + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [check] std <host> + "); + } + + #[test] + fn check_library_stage_0() { + let ctx = TestCtx::new(); + ctx.config("check").path("library").stage(0).run(); + } + + #[test] + fn check_library_stage_1() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("library") + .stage(1) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [check] std <host> + "); + } + + #[test] + fn check_library_stage_2() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("library") + .stage(2) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [check] std <host> + "); + } + + #[test] + fn check_library_cross_compile() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .paths(&["core", "alloc", "std"]) + .targets(&[TEST_TRIPLE_1, TEST_TRIPLE_2]) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [check] std <target1> + [check] std <target2> + "); + } + + #[test] + fn check_miri_no_explicit_stage() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("miri") + .render_steps(), @r" + [check] std <host> + [build] llvm <host> + [check] rustc <host> + [check] Miri <host> + "); + } + + #[test] + fn check_miri_stage_0() { + let ctx = TestCtx::new(); + ctx.config("check").path("miri").stage(0).run(); + } + + #[test] + fn check_miri_stage_1() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("miri") + .stage(1) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [check] rustc <host> + [check] Miri <host> + "); + } + + #[test] + fn check_miri_stage_2() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("miri") + .stage(2) + .render_steps(), @r" + [build] llvm <host> + [build] rustc 0 <host> -> rustc 1 <host> + [build] rustc 1 <host> -> std 1 <host> + [build] rustc 1 <host> -> rustc 2 <host> + [build] rustc 2 <host> -> std 2 <host> + [check] rustc <host> + [check] Miri <host> + "); + } + + #[test] + fn check_compiletest() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("compiletest") + .render_steps(), @"[check] compiletest <host>"); + } + + #[test] + fn check_compiletest_stage1_libtest() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("compiletest") + .args(&["--set", "build.compiletest-use-stage0-libtest=false"]) + .render_steps(), @r" + [check] std <host> + [build] llvm <host> + [check] rustc <host> + [check] compiletest <host> + "); + } + + #[test] + fn check_codegen() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("rustc_codegen_cranelift") + .render_steps(), @r" + [check] std <host> + [build] llvm <host> + [check] rustc <host> + [check] cranelift <host> + [check] gcc <host> + "); + } + + #[test] + fn check_rust_analyzer() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("rust-analyzer") + .render_steps(), @r" + [check] std <host> + [build] llvm <host> + [check] rustc <host> + [check] rust-analyzer <host> + "); + } + + #[test] + fn check_bootstrap_tool() { + let ctx = TestCtx::new(); + insta::assert_snapshot!( + ctx.config("check") + .path("run-make-support") + .render_steps(), @r" + [check] std <host> + [build] llvm <host> + [check] rustc <host> + [check] RunMakeSupport <host> + "); + } + + #[test] + fn test_exclude() { + let ctx = TestCtx::new(); + let steps = ctx.config("test").args(&["--skip", "src/tools/tidy"]).get_steps(); + + let host = TargetSelection::from_user(&host_target()); + steps.assert_contains(StepMetadata::test("RustdocUi", host)); + steps.assert_not_contains(test::Tidy); + } + + #[test] + fn test_exclude_kind() { + let ctx = TestCtx::new(); + let host = TargetSelection::from_user(&host_target()); + + let get_steps = |args: &[&str]| ctx.config("test").args(args).get_steps(); + + // Ensure our test is valid, and `test::Rustc` would be run without the exclude. + get_steps(&[]).assert_contains(StepMetadata::test("CrateLibrustc", host)); + + let steps = get_steps(&["--skip", "compiler/rustc_data_structures"]); + + // Ensure tests for rustc are not skipped. + steps.assert_contains(StepMetadata::test("CrateLibrustc", host)); + steps.assert_contains_fuzzy(StepMetadata::build("rustc", host)); + } +} + +struct ExecutedSteps { + steps: Vec<ExecutedStep>, +} + +impl ExecutedSteps { + fn render(&self) -> String { + render_steps(&self.steps) + } + + #[track_caller] + fn assert_contains<M: Into<StepMetadata>>(&self, metadata: M) { + let metadata = metadata.into(); + if !self.contains(&metadata) { + panic!( + "Metadata `{}` ({metadata:?}) not found in executed steps:\n{}", + render_metadata(&metadata), + self.render() + ); + } + } + + /// Try to match metadata by similarity, it does not need to match exactly. + /// Stages (and built_by compiler) do not need to match, but name, target and + /// kind has to match. + #[track_caller] + fn assert_contains_fuzzy<M: Into<StepMetadata>>(&self, metadata: M) { + let metadata = metadata.into(); + if !self.contains_fuzzy(&metadata) { + panic!( + "Metadata `{}` ({metadata:?}) not found in executed steps:\n{}", + render_metadata(&metadata), + self.render() + ); + } + } + + #[track_caller] + fn assert_not_contains<M: Into<StepMetadata>>(&self, metadata: M) { + let metadata = metadata.into(); + if self.contains(&metadata) { + panic!( + "Metadata `{}` ({metadata:?}) found in executed steps (it should not be there):\n{}", + render_metadata(&metadata), + self.render() + ); } } + + fn contains(&self, metadata: &StepMetadata) -> bool { + self.steps + .iter() + .filter_map(|s| s.metadata.as_ref()) + .any(|executed_metadata| executed_metadata == metadata) + } + + fn contains_fuzzy(&self, metadata: &StepMetadata) -> bool { + self.steps + .iter() + .filter_map(|s| s.metadata.as_ref()) + .any(|executed_metadata| fuzzy_metadata_eq(executed_metadata, metadata)) + } +} + +fn fuzzy_metadata_eq(executed: &StepMetadata, to_match: &StepMetadata) -> bool { + let StepMetadata { name, kind, target, built_by: _, stage: _ } = executed; + *name == to_match.name && *kind == to_match.kind && *target == to_match.target +} + +impl<S: Step> From<S> for StepMetadata { + fn from(step: S) -> Self { + step.metadata().expect("step has no metadata") + } +} + +impl ConfigBuilder { + fn run(self) -> Cache { + let config = self.create_config(); + + let kind = config.cmd.kind(); + let build = Build::new(config); + let builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(kind), &builder.paths); + builder.cache + } + + fn get_steps(self) -> ExecutedSteps { + let cache = self.run(); + ExecutedSteps { steps: cache.into_executed_steps() } + } + + fn render_steps(self) -> String { + self.get_steps().render() + } } /// Renders the executed bootstrap steps for usage in snapshot tests with insta. @@ -1289,23 +1656,34 @@ fn render_steps(steps: &[ExecutedStep]) -> String { return None; }; - let mut record = format!("[{}] ", metadata.kind.as_str()); - if let Some(compiler) = metadata.built_by { - write!(record, "{} -> ", render_compiler(compiler)); - } - let stage = - if let Some(stage) = metadata.stage { format!("{stage} ") } else { "".to_string() }; - write!(record, "{} {stage}<{}>", metadata.name, normalize_target(metadata.target)); - Some(record) + Some(render_metadata(&metadata)) }) .collect::<Vec<_>>() .join("\n") } +fn render_metadata(metadata: &StepMetadata) -> String { + let mut record = format!("[{}] ", metadata.kind.as_str()); + if let Some(compiler) = metadata.built_by { + write!(record, "{} -> ", render_compiler(compiler)); + } + let stage = metadata.get_stage().map(|stage| format!("{stage} ")).unwrap_or_default(); + write!(record, "{} {stage}<{}>", metadata.name, normalize_target(metadata.target)); + record +} + fn normalize_target(target: TargetSelection) -> String { - target.to_string().replace(&get_host_target().to_string(), "host") + target + .to_string() + .replace(&host_target(), "host") + .replace(TEST_TRIPLE_1, "target1") + .replace(TEST_TRIPLE_2, "target2") } fn render_compiler(compiler: Compiler) -> String { format!("rustc {} <{}>", compiler.stage, normalize_target(compiler.host)) } + +fn host_target() -> String { + get_host_target().to_string() +} diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index d3393afcae0..49b3fec4c35 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -47,8 +47,7 @@ use crate::core::config::{ }; use crate::core::download::is_download_ci_available; use crate::utils::channel; -use crate::utils::exec::command; -use crate::utils::execution_context::ExecutionContext; +use crate::utils::exec::{ExecutionContext, command}; use crate::utils::helpers::{exe, get_host_target}; use crate::{GitInfo, OnceLock, TargetSelection, check_ci_llvm, helpers, t}; @@ -298,7 +297,8 @@ pub struct Config { /// Whether to use the precompiled stage0 libtest with compiletest. pub compiletest_use_stage0_libtest: bool, - + /// Default value for `--extra-checks` + pub tidy_extra_checks: Option<String>, pub is_running_on_ci: bool, /// Cache for determining path modifications @@ -745,6 +745,7 @@ impl Config { jobs, compiletest_diff_tool, compiletest_use_stage0_libtest, + tidy_extra_checks, mut ccache, exclude, } = toml.build.unwrap_or_default(); @@ -809,7 +810,7 @@ impl Config { config.initial_sysroot = t!(PathBuf::from_str( command(&config.initial_rustc) .args(["--print", "sysroot"]) - .run_always() + .run_in_dry_run() .run_capture_stdout(&config) .stdout() .trim() @@ -1004,15 +1005,14 @@ impl Config { } if config.lld_enabled && config.is_system_llvm(config.host_target) { - eprintln!( - "Warning: LLD is enabled when using external llvm-config. LLD will not be built and copied to the sysroot." - ); + panic!("Cannot enable LLD with `rust.lld = true` when using external llvm-config."); } config.optimized_compiler_builtins = optimized_compiler_builtins.unwrap_or(config.channel != "dev"); config.compiletest_diff_tool = compiletest_diff_tool; config.compiletest_use_stage0_libtest = compiletest_use_stage0_libtest.unwrap_or(true); + config.tidy_extra_checks = tidy_extra_checks; let download_rustc = config.download_rustc_commit.is_some(); config.explicit_stage_from_cli = flags_stage.is_some(); @@ -1023,7 +1023,7 @@ impl Config { || install_stage.is_some() || check_stage.is_some() || bench_stage.is_some(); - // See https://github.com/rust-lang/compiler-team/issues/326 + config.stage = match config.cmd { Subcommand::Check { .. } => flags_stage.or(check_stage).unwrap_or(0), Subcommand::Clippy { .. } | Subcommand::Fix => flags_stage.or(check_stage).unwrap_or(1), @@ -1051,6 +1051,12 @@ impl Config { | Subcommand::Vendor { .. } => flags_stage.unwrap_or(0), }; + // Now check that the selected stage makes sense, and if not, print a warning and end + if let (0, Subcommand::Build) = (config.stage, &config.cmd) { + eprintln!("WARNING: cannot build anything on stage 0. Use at least stage 1."); + exit!(1); + } + // CI should always run stage 2 builds, unless it specifically states otherwise #[cfg(not(test))] if flags_stage.is_none() && config.is_running_on_ci { @@ -1385,11 +1391,11 @@ impl Config { // all the git commands below are actually executed, because some follow-up code // in bootstrap might depend on the submodules being checked out. Furthermore, not all // the command executions below work with an empty output (produced during dry run). - // Therefore, all commands below are marked with `run_always()`, so that they also run in + // Therefore, all commands below are marked with `run_in_dry_run()`, so that they also run in // dry run mode. let submodule_git = || { let mut cmd = helpers::git(Some(&absolute_path)); - cmd.run_always(); + cmd.run_in_dry_run(); cmd }; @@ -1399,7 +1405,7 @@ impl Config { let checked_out_hash = checked_out_hash.trim_end(); // Determine commit that the submodule *should* have. let recorded = helpers::git(Some(&self.src)) - .run_always() + .run_in_dry_run() .args(["ls-tree", "HEAD"]) .arg(relative_path) .run_capture_stdout(self) @@ -1419,7 +1425,7 @@ impl Config { helpers::git(Some(&self.src)) .allow_failure() - .run_always() + .run_in_dry_run() .args(["submodule", "-q", "sync"]) .arg(relative_path) .run(self); @@ -1430,12 +1436,12 @@ impl Config { // even though that has no relation to the upstream for the submodule. let current_branch = helpers::git(Some(&self.src)) .allow_failure() - .run_always() + .run_in_dry_run() .args(["symbolic-ref", "--short", "HEAD"]) .run_capture(self); let mut git = helpers::git(Some(&self.src)).allow_failure(); - git.run_always(); + git.run_in_dry_run(); if current_branch.is_success() { // If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name. // This syntax isn't accepted by `branch.{branch}`. Strip it. diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs index bb21d033458..79275db6486 100644 --- a/src/bootstrap/src/core/config/flags.rs +++ b/src/bootstrap/src/core/config/flags.rs @@ -382,7 +382,7 @@ pub enum Subcommand { bless: bool, #[arg(long)] /// comma-separated list of other files types to check (accepts py, py:lint, - /// py:fmt, shell) + /// py:fmt, shell, shell:lint, cpp, cpp:fmt, spellcheck, spellcheck:fix) extra_checks: Option<String>, #[arg(long)] /// rerun tests even if the inputs are unchanged diff --git a/src/bootstrap/src/core/config/toml/build.rs b/src/bootstrap/src/core/config/toml/build.rs index 98e1194de72..4d29691f38b 100644 --- a/src/bootstrap/src/core/config/toml/build.rs +++ b/src/bootstrap/src/core/config/toml/build.rs @@ -69,6 +69,7 @@ define_config! { jobs: Option<u32> = "jobs", compiletest_diff_tool: Option<String> = "compiletest-diff-tool", compiletest_use_stage0_libtest: Option<bool> = "compiletest-use-stage0-libtest", + tidy_extra_checks: Option<String> = "tidy-extra-checks", ccache: Option<StringOrBool> = "ccache", exclude: Option<Vec<PathBuf>> = "exclude", } diff --git a/src/bootstrap/src/core/config/toml/rust.rs b/src/bootstrap/src/core/config/toml/rust.rs index 642f2f2271d..ac5eaea3bcb 100644 --- a/src/bootstrap/src/core/config/toml/rust.rs +++ b/src/bootstrap/src/core/config/toml/rust.rs @@ -393,6 +393,27 @@ pub fn check_incompatible_options_for_ci_rustc( Ok(()) } +pub(crate) const VALID_CODEGEN_BACKENDS: &[&str] = &["llvm", "cranelift", "gcc"]; + +pub(crate) fn validate_codegen_backends(backends: Vec<String>, section: &str) -> Vec<String> { + for backend in &backends { + if let Some(stripped) = backend.strip_prefix(CODEGEN_BACKEND_PREFIX) { + panic!( + "Invalid value '{backend}' for '{section}.codegen-backends'. \ + Codegen backends are defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + Please, use '{stripped}' instead." + ) + } + if !VALID_CODEGEN_BACKENDS.contains(&backend.as_str()) { + println!( + "HELP: '{backend}' for '{section}.codegen-backends' might fail. \ + List of known good values: {VALID_CODEGEN_BACKENDS:?}" + ); + } + } + backends +} + impl Config { pub fn apply_rust_config( &mut self, @@ -571,24 +592,10 @@ impl Config { set(&mut self.ehcont_guard, ehcont_guard); self.llvm_libunwind_default = llvm_libunwind.map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); - - if let Some(ref backends) = codegen_backends { - let available_backends = ["llvm", "cranelift", "gcc"]; - - self.rust_codegen_backends = backends.iter().map(|s| { - if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { - if available_backends.contains(&backend) { - panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); - } else { - println!("HELP: '{s}' for 'rust.codegen-backends' might fail. \ - Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ - In this case, it would be referred to as '{backend}'."); - } - } - - s.clone() - }).collect(); - } + set( + &mut self.rust_codegen_backends, + codegen_backends.map(|backends| validate_codegen_backends(backends, "rust")), + ); self.rust_codegen_units = codegen_units.map(threads_from_config); self.rust_codegen_units_std = codegen_units_std.map(threads_from_config); diff --git a/src/bootstrap/src/core/config/toml/target.rs b/src/bootstrap/src/core/config/toml/target.rs index b9f6780ca3f..337276948b3 100644 --- a/src/bootstrap/src/core/config/toml/target.rs +++ b/src/bootstrap/src/core/config/toml/target.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use serde::{Deserialize, Deserializer}; -use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; +use crate::core::config::toml::rust::validate_codegen_backends; use crate::core::config::{LlvmLibunwind, Merge, ReplaceOpt, SplitDebuginfo, StringOrBool}; use crate::{Config, HashSet, PathBuf, TargetSelection, define_config, exit}; @@ -142,23 +142,9 @@ impl Config { target.rpath = cfg.rpath; target.optimized_compiler_builtins = cfg.optimized_compiler_builtins; target.jemalloc = cfg.jemalloc; - - if let Some(ref backends) = cfg.codegen_backends { - let available_backends = ["llvm", "cranelift", "gcc"]; - - target.codegen_backends = Some(backends.iter().map(|s| { - if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { - if available_backends.contains(&backend) { - panic!("Invalid value '{s}' for 'target.{triple}.codegen-backends'. Instead, please use '{backend}'."); - } else { - println!("HELP: '{s}' for 'target.{triple}.codegen-backends' might fail. \ - Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ - In this case, it would be referred to as '{backend}'."); - } - } - - s.clone() - }).collect()); + if let Some(backends) = cfg.codegen_backends { + target.codegen_backends = + Some(validate_codegen_backends(backends, &format!("target.{triple}"))) } target.split_debuginfo = cfg.split_debuginfo.as_ref().map(|v| { diff --git a/src/bootstrap/src/core/metadata.rs b/src/bootstrap/src/core/metadata.rs index 2706aba5ffc..c79fbbeb55c 100644 --- a/src/bootstrap/src/core/metadata.rs +++ b/src/bootstrap/src/core/metadata.rs @@ -88,7 +88,7 @@ fn workspace_members(build: &Build) -> Vec<Package> { .arg("--no-deps") .arg("--manifest-path") .arg(build.src.join(manifest_path)); - let metadata_output = cargo.run_always().run_capture_stdout(build).stdout(); + let metadata_output = cargo.run_in_dry_run().run_capture_stdout(build).stdout(); let Output { packages, .. } = t!(serde_json::from_str(&metadata_output)); packages }; diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs index 493f73b21fe..f2119e84cce 100644 --- a/src/bootstrap/src/core/sanity.rs +++ b/src/bootstrap/src/core/sanity.rs @@ -34,8 +34,6 @@ pub struct Finder { // Targets can be removed from this list once they are present in the stage0 compiler (usually by updating the beta compiler of the bootstrap). const STAGE0_MISSING_TARGETS: &[&str] = &[ // just a dummy comment so the list doesn't get onelined - "loongarch32-unknown-none", - "loongarch32-unknown-none-softfloat", ]; /// Minimum version threshold for libstdc++ required when using prebuilt LLVM @@ -202,7 +200,7 @@ than building it. let stage0_supported_target_list: HashSet<String> = command(&build.config.initial_rustc) .args(["--print", "target-list"]) - .run_always() + .run_in_dry_run() .run_capture_stdout(&build) .stdout() .lines() @@ -366,7 +364,7 @@ than building it. // Cygwin. The Cygwin build does not have generators for Visual // Studio, so detect that here and error. let out = - command("cmake").arg("--help").run_always().run_capture_stdout(&build).stdout(); + command("cmake").arg("--help").run_in_dry_run().run_capture_stdout(&build).stdout(); if !out.contains("Visual Studio") { panic!( " diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index f44fe4548a1..ef5c28272b8 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -31,12 +31,12 @@ use cc::Tool; use termcolor::{ColorChoice, StandardStream, WriteColor}; use utils::build_stamp::BuildStamp; use utils::channel::GitInfo; -use utils::execution_context::ExecutionContext; +use utils::exec::ExecutionContext; use crate::core::builder; use crate::core::builder::Kind; use crate::core::config::{DryRun, LldMode, LlvmLibunwind, TargetSelection, flags}; -use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, command}; +use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{ self, dir_is_empty, exe, libdir, set_file_times, split_debuginfo, symlink_dir, }; @@ -383,7 +383,7 @@ impl Build { let in_tree_gcc_info = config.in_tree_gcc_info.clone(); let initial_target_libdir = command(&config.initial_rustc) - .run_always() + .run_in_dry_run() .args(["--print", "target-libdir"]) .run_capture_stdout(&config) .stdout() @@ -490,7 +490,7 @@ impl Build { // If local-rust is the same major.minor as the current version, then force a // local-rebuild let local_version_verbose = command(&build.initial_rustc) - .run_always() + .run_in_dry_run() .args(["--version", "--verbose"]) .run_capture_stdout(&build) .stdout(); @@ -518,7 +518,7 @@ impl Build { // Make sure we update these before gathering metadata so we don't get an error about missing // Cargo.toml files. - let rust_submodules = ["library/backtrace", "library/stdarch"]; + let rust_submodules = ["library/backtrace"]; for s in rust_submodules { build.require_submodule( s, @@ -949,7 +949,7 @@ impl Build { static SYSROOT_CACHE: OnceLock<PathBuf> = OnceLock::new(); SYSROOT_CACHE.get_or_init(|| { command(&self.initial_rustc) - .run_always() + .run_in_dry_run() .args(["--print", "sysroot"]) .run_capture_stdout(self) .stdout() @@ -1512,7 +1512,7 @@ impl Build { "refs/remotes/origin/{}..HEAD", self.config.stage0_metadata.config.nightly_branch )) - .run_always() + .run_in_dry_run() .run_capture(self) .stdout() }); diff --git a/src/bootstrap/src/utils/cache/tests.rs b/src/bootstrap/src/utils/cache/tests.rs index 8562a35b3e0..fd0a7cccd60 100644 --- a/src/bootstrap/src/utils/cache/tests.rs +++ b/src/bootstrap/src/utils/cache/tests.rs @@ -1,12 +1,13 @@ use std::path::PathBuf; -use crate::utils::cache::{INTERNER, Internable, TyIntern}; +use crate::utils::cache::{INTERNER, Internable, Interner, TyIntern}; #[test] fn test_string_interning() { - let s1 = INTERNER.intern_str("Hello"); - let s2 = INTERNER.intern_str("Hello"); - let s3 = INTERNER.intern_str("world"); + let interner = Interner::default(); + let s1 = interner.intern_str("Hello"); + let s2 = interner.intern_str("Hello"); + let s3 = interner.intern_str("world"); assert_eq!(s1, s2, "Same strings should be interned to the same instance"); assert_ne!(s1, s3, "Different strings should have different interned values"); @@ -14,6 +15,8 @@ fn test_string_interning() { #[test] fn test_interned_equality() { + // Because we compare with &str, and the Deref impl accesses the global + // INTERNER variable, we cannot use a local Interner variable here. let s1 = INTERNER.intern_str("test"); let s2 = INTERNER.intern_str("test"); diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 93e01a58077..68312a503ee 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -149,7 +149,7 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ ChangeInfo { change_id: 121976, severity: ChangeSeverity::Info, - summary: "A new `boostrap-cache-path` option has been introduced which can be utilized to modify the cache path for bootstrap.", + summary: "A new `bootstrap-cache-path` option has been introduced which can be utilized to modify the cache path for bootstrap.", }, ChangeInfo { change_id: 122108, @@ -404,7 +404,7 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ ChangeInfo { change_id: 140438, severity: ChangeSeverity::Info, - summary: "Added a new option `rust.debug-assertions-tools` to control debug asssertions for tools.", + summary: "Added a new option `rust.debug-assertions-tools` to control debug assertions for tools.", }, ChangeInfo { change_id: 140732, @@ -426,4 +426,24 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Info, summary: "Added new option `tool.TOOL_NAME.features` to specify the features to compile a tool with", }, + ChangeInfo { + change_id: 142581, + severity: ChangeSeverity::Warning, + summary: "It is no longer possible to `x build` with stage 0. All build commands have to be on stage 1+.", + }, + ChangeInfo { + change_id: 143175, + severity: ChangeSeverity::Info, + summary: "It is no longer possible to combine `rust.lld = true` with configuring external LLVM using `llvm.llvm-config`.", + }, + ChangeInfo { + change_id: 143255, + severity: ChangeSeverity::Warning, + summary: "`llvm.lld` is no longer enabled by default for the dist profile.", + }, + ChangeInfo { + change_id: 143251, + severity: ChangeSeverity::Info, + summary: "Added new option `build.tidy-extra-checks` to specify a default value for the --extra-checks cli flag.", + }, ]; diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs index b28ab573774..21b4257e54d 100644 --- a/src/bootstrap/src/utils/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -8,7 +8,7 @@ use std::fs; use std::path::Path; -use super::execution_context::ExecutionContext; +use super::exec::ExecutionContext; use super::helpers; use crate::Build; use crate::utils::helpers::t; @@ -66,19 +66,22 @@ impl GitInfo { .arg("-1") .arg("--date=short") .arg("--pretty=format:%cd") - .run_always() + .run_in_dry_run() .start_capture_stdout(&exec_ctx); let mut git_hash_cmd = helpers::git(Some(dir)); - let ver_hash = - git_hash_cmd.arg("rev-parse").arg("HEAD").run_always().start_capture_stdout(&exec_ctx); + let ver_hash = git_hash_cmd + .arg("rev-parse") + .arg("HEAD") + .run_in_dry_run() + .start_capture_stdout(&exec_ctx); let mut git_short_hash_cmd = helpers::git(Some(dir)); let short_ver_hash = git_short_hash_cmd .arg("rev-parse") .arg("--short=9") .arg("HEAD") - .run_always() + .run_in_dry_run() .start_capture_stdout(&exec_ctx); GitInfo::Present(Some(Info { diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 78b28ac1828..d092765ef76 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -1,16 +1,29 @@ //! Command Execution Module //! -//! This module provides a structured way to execute and manage commands efficiently, -//! ensuring controlled failure handling and output management. -use std::ffi::OsStr; +//! Provides a structured interface for executing and managing commands during bootstrap, +//! with support for controlled failure handling and output management. +//! +//! This module defines the [`ExecutionContext`] type, which encapsulates global configuration +//! relevant to command execution in the bootstrap process. This includes settings such as +//! dry-run mode, verbosity level, and failure behavior. + +use std::collections::HashMap; +use std::ffi::{OsStr, OsString}; use std::fmt::{Debug, Formatter}; +use std::hash::Hash; +use std::panic::Location; use std::path::Path; -use std::process::{Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; +use std::process::{Child, Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; +use std::sync::{Arc, Mutex}; use build_helper::ci::CiEnv; use build_helper::drop_bomb::DropBomb; +use build_helper::exit; -use super::execution_context::{DeferredCommand, ExecutionContext}; +use crate::PathBuf; +use crate::core::config::DryRun; +#[cfg(feature = "tracing")] +use crate::trace_cmd; /// What should be done when the command fails. #[derive(Debug, Copy, Clone)] @@ -49,6 +62,14 @@ impl OutputMode { } } +#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] +pub struct CommandCacheKey { + program: OsString, + args: Vec<OsString>, + envs: Vec<(OsString, Option<OsString>)>, + cwd: Option<PathBuf>, +} + /// Wrapper around `std::process::Command`. /// /// By default, the command will exit bootstrap if it fails. @@ -60,16 +81,20 @@ impl OutputMode { /// /// Bootstrap will print a debug log to stdout if the command fails and failure is not allowed. /// +/// By default, command executions are cached based on their workdir, program, arguments, and environment variables. +/// This avoids re-running identical commands unnecessarily, unless caching is explicitly disabled. +/// /// [allow_failure]: BootstrapCommand::allow_failure /// [delay_failure]: BootstrapCommand::delay_failure pub struct BootstrapCommand { command: Command, pub failure_behavior: BehaviorOnFailure, // Run the command even during dry run - pub run_always: bool, + pub run_in_dry_run: bool, // This field makes sure that each command is executed (or disarmed) before it is dropped, // to avoid forgetting to execute a command. drop_bomb: DropBomb, + should_cache: bool, } impl<'a> BootstrapCommand { @@ -77,12 +102,16 @@ impl<'a> BootstrapCommand { pub fn new<S: AsRef<OsStr>>(program: S) -> Self { Command::new(program).into() } - pub fn arg<S: AsRef<OsStr>>(&mut self, arg: S) -> &mut Self { self.command.arg(arg.as_ref()); self } + pub fn do_not_cache(&mut self) -> &mut Self { + self.should_cache = false; + self + } + pub fn args<I, S>(&mut self, args: I) -> &mut Self where I: IntoIterator<Item = S>, @@ -138,8 +167,8 @@ impl<'a> BootstrapCommand { Self { failure_behavior: BehaviorOnFailure::Ignore, ..self } } - pub fn run_always(&mut self) -> &mut Self { - self.run_always = true; + pub fn run_in_dry_run(&mut self) -> &mut Self { + self.run_in_dry_run = true; self } @@ -183,9 +212,11 @@ impl<'a> BootstrapCommand { /// Provides access to the stdlib Command inside. /// FIXME: This function should be eventually removed from bootstrap. pub fn as_command_mut(&mut self) -> &mut Command { - // We don't know what will happen with the returned command, so we need to mark this - // command as executed proactively. + // We proactively mark this command as executed since we can't be certain how the returned + // command will be handled. Caching must also be avoided here, as the inner command could be + // modified externally without us being aware. self.mark_as_executed(); + self.do_not_cache(); &mut self.command } @@ -211,6 +242,22 @@ impl<'a> BootstrapCommand { self.env("TERM", "xterm").args(["--color", "always"]); } } + + pub fn cache_key(&self) -> Option<CommandCacheKey> { + if !self.should_cache { + return None; + } + let command = &self.command; + Some(CommandCacheKey { + program: command.get_program().into(), + args: command.get_args().map(OsStr::to_os_string).collect(), + envs: command + .get_envs() + .map(|(k, v)| (k.to_os_string(), v.map(|val| val.to_os_string()))) + .collect(), + cwd: command.get_current_dir().map(Path::to_path_buf), + }) + } } impl Debug for BootstrapCommand { @@ -224,17 +271,18 @@ impl From<Command> for BootstrapCommand { #[track_caller] fn from(command: Command) -> Self { let program = command.get_program().to_owned(); - Self { + should_cache: true, command, failure_behavior: BehaviorOnFailure::Exit, - run_always: false, + run_in_dry_run: false, drop_bomb: DropBomb::arm(program), } } } /// Represents the current status of `BootstrapCommand`. +#[derive(Clone, PartialEq)] enum CommandStatus { /// The command has started and finished with some status. Finished(ExitStatus), @@ -251,6 +299,7 @@ pub fn command<S: AsRef<OsStr>>(program: S) -> BootstrapCommand { } /// Represents the output of an executed process. +#[derive(Clone, PartialEq)] pub struct CommandOutput { status: CommandStatus, stdout: Option<Vec<u8>>, @@ -373,3 +422,329 @@ impl FormatShortCmd for Command { line.join(" ") } } + +#[derive(Clone, Default)] +pub struct ExecutionContext { + dry_run: DryRun, + verbose: u8, + pub fail_fast: bool, + delayed_failures: Arc<Mutex<Vec<String>>>, + command_cache: Arc<CommandCache>, +} + +#[derive(Default)] +pub struct CommandCache { + cache: Mutex<HashMap<CommandCacheKey, CommandOutput>>, +} + +enum CommandState<'a> { + Cached(CommandOutput), + Deferred { + process: Option<Result<Child, std::io::Error>>, + command: &'a mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + executed_at: &'a Location<'a>, + cache_key: Option<CommandCacheKey>, + }, +} + +#[must_use] +pub struct DeferredCommand<'a> { + state: CommandState<'a>, +} + +impl CommandCache { + pub fn get(&self, key: &CommandCacheKey) -> Option<CommandOutput> { + self.cache.lock().unwrap().get(key).cloned() + } + + pub fn insert(&self, key: CommandCacheKey, output: CommandOutput) { + self.cache.lock().unwrap().insert(key, output); + } +} + +impl ExecutionContext { + pub fn new() -> Self { + ExecutionContext::default() + } + + pub fn dry_run(&self) -> bool { + match self.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } + } + + pub fn get_dry_run(&self) -> &DryRun { + &self.dry_run + } + + pub fn verbose(&self, f: impl Fn()) { + if self.is_verbose() { + f() + } + } + + pub fn is_verbose(&self) -> bool { + self.verbose > 0 + } + + pub fn fail_fast(&self) -> bool { + self.fail_fast + } + + pub fn set_dry_run(&mut self, value: DryRun) { + self.dry_run = value; + } + + pub fn set_verbose(&mut self, value: u8) { + self.verbose = value; + } + + pub fn set_fail_fast(&mut self, value: bool) { + self.fail_fast = value; + } + + pub fn add_to_delay_failure(&self, message: String) { + self.delayed_failures.lock().unwrap().push(message); + } + + pub fn report_failures_and_exit(&self) { + let failures = self.delayed_failures.lock().unwrap(); + if failures.is_empty() { + return; + } + eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); + for failure in &*failures { + eprintln!(" - {failure}"); + } + exit!(1); + } + + /// Execute a command and return its output. + /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to + /// execute commands. They internally call this method. + #[track_caller] + pub fn start<'a>( + &self, + command: &'a mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + ) -> DeferredCommand<'a> { + let cache_key = command.cache_key(); + + if let Some(cached_output) = cache_key.as_ref().and_then(|key| self.command_cache.get(key)) + { + command.mark_as_executed(); + self.verbose(|| println!("Cache hit: {command:?}")); + return DeferredCommand { state: CommandState::Cached(cached_output) }; + } + + let created_at = command.get_created_location(); + let executed_at = std::panic::Location::caller(); + + if self.dry_run() && !command.run_in_dry_run { + return DeferredCommand { + state: CommandState::Deferred { + process: None, + command, + stdout, + stderr, + executed_at, + cache_key, + }, + }; + } + + #[cfg(feature = "tracing")] + let _run_span = trace_cmd!(command); + + self.verbose(|| { + println!("running: {command:?} (created at {created_at}, executed at {executed_at})") + }); + + let cmd = &mut command.command; + cmd.stdout(stdout.stdio()); + cmd.stderr(stderr.stdio()); + + let child = cmd.spawn(); + + DeferredCommand { + state: CommandState::Deferred { + process: Some(child), + command, + stdout, + stderr, + executed_at, + cache_key, + }, + } + } + + /// Execute a command and return its output. + /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to + /// execute commands. They internally call this method. + #[track_caller] + pub fn run( + &self, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + ) -> CommandOutput { + self.start(command, stdout, stderr).wait_for_output(self) + } + + fn fail(&self, message: &str, output: CommandOutput) -> ! { + if self.is_verbose() { + println!("{message}"); + } else { + let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); + // If the command captures output, the user would not see any indication that + // it has failed. In this case, print a more verbose error, since to provide more + // context. + if stdout.is_some() || stderr.is_some() { + if let Some(stdout) = output.stdout_if_present().take_if(|s| !s.trim().is_empty()) { + println!("STDOUT:\n{stdout}\n"); + } + if let Some(stderr) = output.stderr_if_present().take_if(|s| !s.trim().is_empty()) { + println!("STDERR:\n{stderr}\n"); + } + println!("Command has failed. Rerun with -v to see more details."); + } else { + println!("Command has failed. Rerun with -v to see more details."); + } + } + exit!(1); + } +} + +impl AsRef<ExecutionContext> for ExecutionContext { + fn as_ref(&self) -> &ExecutionContext { + self + } +} + +impl<'a> DeferredCommand<'a> { + pub fn wait_for_output(self, exec_ctx: impl AsRef<ExecutionContext>) -> CommandOutput { + match self.state { + CommandState::Cached(output) => output, + CommandState::Deferred { process, command, stdout, stderr, executed_at, cache_key } => { + let exec_ctx = exec_ctx.as_ref(); + + let output = + Self::finish_process(process, command, stdout, stderr, executed_at, exec_ctx); + + if (!exec_ctx.dry_run() || command.run_in_dry_run) + && let (Some(cache_key), Some(_)) = (&cache_key, output.status()) + { + exec_ctx.command_cache.insert(cache_key.clone(), output.clone()); + } + + output + } + } + } + + pub fn finish_process( + mut process: Option<Result<Child, std::io::Error>>, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + executed_at: &'a std::panic::Location<'a>, + exec_ctx: &ExecutionContext, + ) -> CommandOutput { + command.mark_as_executed(); + + let process = match process.take() { + Some(p) => p, + None => return CommandOutput::default(), + }; + + let created_at = command.get_created_location(); + + let mut message = String::new(); + + let output = match process { + Ok(child) => match child.wait_with_output() { + Ok(result) if result.status.success() => { + // Successful execution + CommandOutput::from_output(result, stdout, stderr) + } + Ok(result) => { + // Command ran but failed + use std::fmt::Write; + + writeln!( + message, + r#" +Command {command:?} did not execute successfully. +Expected success, got {} +Created at: {created_at} +Executed at: {executed_at}"#, + result.status, + ) + .unwrap(); + + let output = CommandOutput::from_output(result, stdout, stderr); + + if stdout.captures() { + writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); + } + if stderr.captures() { + writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); + } + + output + } + Err(e) => { + // Failed to wait for output + use std::fmt::Write; + + writeln!( + message, + "\n\nCommand {command:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}" + ) + .unwrap(); + + CommandOutput::did_not_start(stdout, stderr) + } + }, + Err(e) => { + // Failed to spawn the command + use std::fmt::Write; + + writeln!( + message, + "\n\nCommand {command:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}" + ) + .unwrap(); + + CommandOutput::did_not_start(stdout, stderr) + } + }; + + if !output.is_success() { + match command.failure_behavior { + BehaviorOnFailure::DelayFail => { + if exec_ctx.fail_fast { + exec_ctx.fail(&message, output); + } + exec_ctx.add_to_delay_failure(message); + } + BehaviorOnFailure::Exit => { + exec_ctx.fail(&message, output); + } + BehaviorOnFailure::Ignore => { + // If failures are allowed, either the error has been printed already + // (OutputMode::Print) or the user used a capture output mode and wants to + // handle the error output on their own. + } + } + } + + output + } +} diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs deleted file mode 100644 index 5b9fef3f824..00000000000 --- a/src/bootstrap/src/utils/execution_context.rs +++ /dev/null @@ -1,265 +0,0 @@ -//! Shared execution context for running bootstrap commands. -//! -//! This module provides the [`ExecutionContext`] type, which holds global configuration -//! relevant during the execution of commands in bootstrap. This includes dry-run -//! mode, verbosity level, and behavior on failure. -use std::panic::Location; -use std::process::Child; -use std::sync::{Arc, Mutex}; - -use crate::core::config::DryRun; -#[cfg(feature = "tracing")] -use crate::trace_cmd; -use crate::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, exit}; - -#[derive(Clone, Default)] -pub struct ExecutionContext { - dry_run: DryRun, - verbose: u8, - pub fail_fast: bool, - delayed_failures: Arc<Mutex<Vec<String>>>, -} - -impl ExecutionContext { - pub fn new() -> Self { - ExecutionContext::default() - } - - pub fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } - - pub fn get_dry_run(&self) -> &DryRun { - &self.dry_run - } - - pub fn verbose(&self, f: impl Fn()) { - if self.is_verbose() { - f() - } - } - - pub fn is_verbose(&self) -> bool { - self.verbose > 0 - } - - pub fn fail_fast(&self) -> bool { - self.fail_fast - } - - pub fn set_dry_run(&mut self, value: DryRun) { - self.dry_run = value; - } - - pub fn set_verbose(&mut self, value: u8) { - self.verbose = value; - } - - pub fn set_fail_fast(&mut self, value: bool) { - self.fail_fast = value; - } - - pub fn add_to_delay_failure(&self, message: String) { - self.delayed_failures.lock().unwrap().push(message); - } - - pub fn report_failures_and_exit(&self) { - let failures = self.delayed_failures.lock().unwrap(); - if failures.is_empty() { - return; - } - eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); - for failure in &*failures { - eprintln!(" - {failure}"); - } - exit!(1); - } - - /// Execute a command and return its output. - /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to - /// execute commands. They internally call this method. - #[track_caller] - pub fn start<'a>( - &self, - command: &'a mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - ) -> DeferredCommand<'a> { - command.mark_as_executed(); - - let created_at = command.get_created_location(); - let executed_at = std::panic::Location::caller(); - - if self.dry_run() && !command.run_always { - return DeferredCommand { process: None, stdout, stderr, command, executed_at }; - } - - #[cfg(feature = "tracing")] - let _run_span = trace_cmd!(command); - - self.verbose(|| { - println!("running: {command:?} (created at {created_at}, executed at {executed_at})") - }); - - let cmd = command.as_command_mut(); - cmd.stdout(stdout.stdio()); - cmd.stderr(stderr.stdio()); - - let child = cmd.spawn(); - - DeferredCommand { process: Some(child), stdout, stderr, command, executed_at } - } - - /// Execute a command and return its output. - /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to - /// execute commands. They internally call this method. - #[track_caller] - pub fn run( - &self, - command: &mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - ) -> CommandOutput { - self.start(command, stdout, stderr).wait_for_output(self) - } - - fn fail(&self, message: &str, output: CommandOutput) -> ! { - if self.is_verbose() { - println!("{message}"); - } else { - let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); - // If the command captures output, the user would not see any indication that - // it has failed. In this case, print a more verbose error, since to provide more - // context. - if stdout.is_some() || stderr.is_some() { - if let Some(stdout) = output.stdout_if_present().take_if(|s| !s.trim().is_empty()) { - println!("STDOUT:\n{stdout}\n"); - } - if let Some(stderr) = output.stderr_if_present().take_if(|s| !s.trim().is_empty()) { - println!("STDERR:\n{stderr}\n"); - } - println!("Command has failed. Rerun with -v to see more details."); - } else { - println!("Command has failed. Rerun with -v to see more details."); - } - } - exit!(1); - } -} - -impl AsRef<ExecutionContext> for ExecutionContext { - fn as_ref(&self) -> &ExecutionContext { - self - } -} - -pub struct DeferredCommand<'a> { - process: Option<Result<Child, std::io::Error>>, - command: &'a mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - executed_at: &'a Location<'a>, -} - -impl<'a> DeferredCommand<'a> { - pub fn wait_for_output(mut self, exec_ctx: impl AsRef<ExecutionContext>) -> CommandOutput { - let exec_ctx = exec_ctx.as_ref(); - - let process = match self.process.take() { - Some(p) => p, - None => return CommandOutput::default(), - }; - - let created_at = self.command.get_created_location(); - let executed_at = self.executed_at; - - let mut message = String::new(); - - let output = match process { - Ok(child) => match child.wait_with_output() { - Ok(result) if result.status.success() => { - // Successful execution - CommandOutput::from_output(result, self.stdout, self.stderr) - } - Ok(result) => { - // Command ran but failed - use std::fmt::Write; - - writeln!( - message, - r#" -Command {:?} did not execute successfully. -Expected success, got {} -Created at: {created_at} -Executed at: {executed_at}"#, - self.command, result.status, - ) - .unwrap(); - - let output = CommandOutput::from_output(result, self.stdout, self.stderr); - - if self.stdout.captures() { - writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); - } - if self.stderr.captures() { - writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); - } - - output - } - Err(e) => { - // Failed to wait for output - use std::fmt::Write; - - writeln!( - message, - "\n\nCommand {:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}", - self.command - ) - .unwrap(); - - CommandOutput::did_not_start(self.stdout, self.stderr) - } - }, - Err(e) => { - // Failed to spawn the command - use std::fmt::Write; - - writeln!( - message, - "\n\nCommand {:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}", - self.command - ) - .unwrap(); - - CommandOutput::did_not_start(self.stdout, self.stderr) - } - }; - - if !output.is_success() { - match self.command.failure_behavior { - BehaviorOnFailure::DelayFail => { - if exec_ctx.fail_fast { - exec_ctx.fail(&message, output); - } - exec_ctx.add_to_delay_failure(message); - } - BehaviorOnFailure::Exit => { - exec_ctx.fail(&message, output); - } - BehaviorOnFailure::Ignore => { - // If failures are allowed, either the error has been printed already - // (OutputMode::Print) or the user used a capture output mode and wants to - // handle the error output on their own. - } - } - } - - output - } -} diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs index 5a0b90801e7..169fcec303e 100644 --- a/src/bootstrap/src/utils/mod.rs +++ b/src/bootstrap/src/utils/mod.rs @@ -8,7 +8,6 @@ pub(crate) mod cc_detect; pub(crate) mod change_tracker; pub(crate) mod channel; pub(crate) mod exec; -pub(crate) mod execution_context; pub(crate) mod helpers; pub(crate) mod job; pub(crate) mod render_tests; diff --git a/src/bootstrap/src/utils/proc_macro_deps.rs b/src/bootstrap/src/utils/proc_macro_deps.rs index b61fa3bb8d6..21c7fc89d7d 100644 --- a/src/bootstrap/src/utils/proc_macro_deps.rs +++ b/src/bootstrap/src/utils/proc_macro_deps.rs @@ -30,7 +30,6 @@ pub static CRATES: &[&str] = &[ "memchr", "minimal-lexical", "nom", - "once_cell", "pest", "pest_generator", "pest_meta", diff --git a/src/bootstrap/src/utils/render_tests.rs b/src/bootstrap/src/utils/render_tests.rs index 77e645a9e3c..051d7dd9fd4 100644 --- a/src/bootstrap/src/utils/render_tests.rs +++ b/src/bootstrap/src/utils/render_tests.rs @@ -202,7 +202,9 @@ impl<'a> Renderer<'a> { } fn render_test_outcome_terse(&mut self, outcome: Outcome<'_>, test: &TestOutcome) { - if self.terse_tests_in_line != 0 && self.terse_tests_in_line % TERSE_TESTS_PER_LINE == 0 { + if self.terse_tests_in_line != 0 + && self.terse_tests_in_line.is_multiple_of(TERSE_TESTS_PER_LINE) + { if let Some(total) = self.tests_count { let total = total.to_string(); let executed = format!("{:>width$}", self.executed_tests - 1, width = total.len()); diff --git a/src/bootstrap/src/utils/tests/mod.rs b/src/bootstrap/src/utils/tests/mod.rs index 91877fd0da4..b8984d1f3aa 100644 --- a/src/bootstrap/src/utils/tests/mod.rs +++ b/src/bootstrap/src/utils/tests/mod.rs @@ -51,12 +51,38 @@ impl ConfigBuilder { self } + pub fn paths(mut self, paths: &[&str]) -> Self { + for path in paths { + self = self.path(path); + } + self + } + + pub fn hosts(mut self, targets: &[&str]) -> Self { + self.args.push("--host".to_string()); + self.args.push(targets.join(",")); + self + } + + pub fn targets(mut self, targets: &[&str]) -> Self { + self.args.push("--target".to_string()); + self.args.push(targets.join(",")); + self + } + pub fn stage(mut self, stage: u32) -> Self { self.args.push("--stage".to_string()); self.args.push(stage.to_string()); self } + pub fn args(mut self, args: &[&str]) -> Self { + for arg in args { + self.args.push(arg.to_string()); + } + self + } + pub fn create_config(mut self) -> Config { // Run in dry-check, otherwise the test would be too slow self.args.push("--dry-run".to_string()); |
