diff options
Diffstat (limited to 'src/bootstrap')
36 files changed, 2828 insertions, 2339 deletions
diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index c60c6b8db64..d8c6be78247 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -394,6 +394,7 @@ def default_build_triple(verbose): "i686": "i686", "i686-AT386": "i686", "i786": "i686", + "loongarch32": "loongarch32", "loongarch64": "loongarch64", "m68k": "m68k", "csky": "csky", diff --git a/src/bootstrap/defaults/bootstrap.library.toml b/src/bootstrap/defaults/bootstrap.library.toml index 6edd00922ae..895e50b9a20 100644 --- a/src/bootstrap/defaults/bootstrap.library.toml +++ b/src/bootstrap/defaults/bootstrap.library.toml @@ -1,8 +1,9 @@ # These defaults are meant for contributors to the standard library and documentation. [build] +bench-stage = 1 build-stage = 1 +check-stage = 1 test-stage = 1 -bench-stage = 1 [rust] # This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower. diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 08a288170fa..82c05092dfa 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -51,16 +51,18 @@ check-aux: $(Q)$(BOOTSTRAP) test --stage 2 \ src/tools/cargo \ src/tools/cargotest \ - src/etc/test-float-parse \ + src/tools/test-float-parse \ $(BOOTSTRAP_ARGS) # Run standard library tests in Miri. - $(Q)$(BOOTSTRAP) miri --stage 2 \ + $(Q)MIRIFLAGS="-Zmiri-strict-provenance" \ + $(BOOTSTRAP) miri --stage 2 \ library/coretests \ library/alloctests \ library/alloc \ $(BOOTSTRAP_ARGS) \ --no-doc - # Some doctests use file system operations to demonstrate dealing with `Result`. + # Some doctests use file system operations to demonstrate dealing with `Result`, + # so we have to run them with isolation disabled. $(Q)MIRIFLAGS="-Zmiri-disable-isolation" \ $(BOOTSTRAP) miri --stage 2 \ library/coretests \ @@ -70,22 +72,19 @@ check-aux: --doc # In `std` we cannot test everything, so we skip some modules. $(Q)MIRIFLAGS="-Zmiri-disable-isolation" \ - $(BOOTSTRAP) miri --stage 2 library/std \ - $(BOOTSTRAP_ARGS) \ - --no-doc -- \ - --skip fs:: --skip net:: --skip process:: --skip sys::fd:: --skip sys::pal:: - $(Q)MIRIFLAGS="-Zmiri-disable-isolation" \ - $(BOOTSTRAP) miri --stage 2 library/std \ + $(BOOTSTRAP) miri --stage 2 \ + library/std \ $(BOOTSTRAP_ARGS) \ - --doc -- \ + -- \ --skip fs:: --skip net:: --skip process:: --skip sys::fd:: --skip sys::pal:: # Also test some very target-specific modules on other targets # (making sure to cover an i686 target as well). $(Q)MIRIFLAGS="-Zmiri-disable-isolation" BOOTSTRAP_SKIP_TARGET_SANITY=1 \ - $(BOOTSTRAP) miri --stage 2 library/std \ - $(BOOTSTRAP_ARGS) \ + $(BOOTSTRAP) miri --stage 2 \ + library/std \ --target aarch64-apple-darwin,i686-pc-windows-msvc \ - --no-doc -- \ + $(BOOTSTRAP_ARGS) \ + -- \ time:: sync:: thread:: env:: dist: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs index 922578f309a..911a51b0e16 100644 --- a/src/bootstrap/src/core/build_steps/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -28,11 +28,16 @@ pub struct Std { /// passing `Builder::kind` to cargo invocations would run clippy on the entire compiler and library, /// which is not useful if we only want to lint a few crates with specific rules. override_build_kind: Option<Kind>, + /// Never use this from outside calls. It is intended for internal use only within `check::Std::make_run` + /// and `check::Std::run`. + custom_stage: Option<u32>, } impl Std { + const CRATE_OR_DEPS: &[&str] = &["sysroot", "coretests", "alloctests"]; + pub fn new(target: TargetSelection) -> Self { - Self { target, crates: vec![], override_build_kind: None } + Self { target, crates: vec![], override_build_kind: None, custom_stage: None } } pub fn build_kind(mut self, kind: Option<Kind>) -> Self { @@ -46,26 +51,60 @@ impl Step for Std { const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let stage = run.builder.top_stage; - run.crate_or_deps("sysroot") - .crate_or_deps("coretests") - .crate_or_deps("alloctests") - .path("library") - .default_condition(stage != 0) + let mut run = run; + for c in Std::CRATE_OR_DEPS { + run = run.crate_or_deps(c); + } + + run.path("library") } fn make_run(run: RunConfig<'_>) { let crates = std_crates_for_run_make(&run); - run.builder.ensure(Std { target: run.target, crates, override_build_kind: None }); + + let stage = if run.builder.config.is_explicit_stage() || run.builder.top_stage >= 1 { + run.builder.top_stage + } else { + 1 + }; + + run.builder.ensure(Std { + target: run.target, + crates, + override_build_kind: None, + custom_stage: Some(stage), + }); } fn run(self, builder: &Builder<'_>) { + if !builder.download_rustc() && builder.config.skip_std_check_if_no_download_rustc { + eprintln!( + "WARNING: `--skip-std-check-if-no-download-rustc` flag was passed and `rust.download-rustc` is not available. Skipping." + ); + return; + } + builder.require_submodule("library/stdarch", None); + let stage = self.custom_stage.unwrap_or(builder.top_stage); + let target = self.target; - let compiler = builder.compiler(builder.top_stage, builder.config.build); + let compiler = builder.compiler(stage, builder.config.build); + + if stage == 0 { + let mut is_explicitly_called = + builder.paths.iter().any(|p| p.starts_with("library") || p.starts_with("std")); + + if !is_explicitly_called { + for c in Std::CRATE_OR_DEPS { + is_explicitly_called = builder.paths.iter().any(|p| p.starts_with(c)); + } + } + + if is_explicitly_called { + eprintln!("WARNING: stage 0 std is precompiled and does nothing during `x check`."); + } - if builder.top_stage == 0 { // Reuse the stage0 libstd builder.ensure(compile::Std::new(compiler, target)); return; @@ -93,6 +132,7 @@ impl Step for Std { let _guard = builder.msg_check( format_args!("library artifacts{}", crate_description(&self.crates)), target, + Some(stage), ); let stamp = build_stamp::libstd_stamp(builder, compiler, target).with_prefix("check"); @@ -145,7 +185,7 @@ impl Step for Std { } let stamp = build_stamp::libstd_stamp(builder, compiler, target).with_prefix("check-test"); - let _guard = builder.msg_check("library test/bench/example targets", target); + let _guard = builder.msg_check("library test/bench/example targets", target, Some(stage)); run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } } @@ -246,6 +286,7 @@ impl Step for Rustc { let _guard = builder.msg_check( format_args!("compiler artifacts{}", crate_description(&self.crates)), target, + None, ); let stamp = build_stamp::librustc_stamp(builder, compiler, target).with_prefix("check"); @@ -306,7 +347,7 @@ impl Step for CodegenBackend { .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - let _guard = builder.msg_check(backend, target); + let _guard = builder.msg_check(backend, target, None); let stamp = build_stamp::codegen_backend_stamp(builder, compiler, target, backend) .with_prefix("check"); @@ -373,7 +414,7 @@ impl Step for RustAnalyzer { let stamp = BuildStamp::new(&builder.cargo_out(compiler, Mode::ToolRustc, target)) .with_prefix("rust-analyzer-check"); - let _guard = builder.msg_check("rust-analyzer artifacts", target); + let _guard = builder.msg_check("rust-analyzer artifacts", target, None); run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } } @@ -436,7 +477,7 @@ impl Step for Compiletest { let stamp = BuildStamp::new(&builder.cargo_out(compiler, mode, self.target)) .with_prefix("compiletest-check"); - let _guard = builder.msg_check("compiletest artifacts", self.target); + let _guard = builder.msg_check("compiletest artifacts", self.target, None); run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } } @@ -514,7 +555,7 @@ fn run_tool_check_step( let stamp = BuildStamp::new(&builder.cargo_out(compiler, Mode::ToolRustc, target)) .with_prefix(&format!("{}-check", step_type_name.to_lowercase())); - let _guard = builder.msg_check(format!("{display_name} artifacts"), target); + let _guard = builder.msg_check(format!("{display_name} artifacts"), target, None); run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } @@ -528,7 +569,7 @@ tool_check_step!(Miri { path: "src/tools/miri" }); tool_check_step!(CargoMiri { path: "src/tools/miri/cargo-miri" }); tool_check_step!(Rustfmt { path: "src/tools/rustfmt" }); tool_check_step!(MiroptTestTools { path: "src/tools/miropt-test-tools" }); -tool_check_step!(TestFloatParse { path: "src/etc/test-float-parse" }); +tool_check_step!(TestFloatParse { path: "src/tools/test-float-parse" }); tool_check_step!(FeaturesStatusDump { path: "src/tools/features-status-dump" }); tool_check_step!(Bootstrap { path: "src/bootstrap", default: false }); diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs index 5e8d6bba841..0652c08ff49 100644 --- a/src/bootstrap/src/core/build_steps/clippy.rs +++ b/src/bootstrap/src/core/build_steps/clippy.rs @@ -355,7 +355,7 @@ lint_any!( Rustfmt, "src/tools/rustfmt", "rustfmt"; RustInstaller, "src/tools/rust-installer", "rust-installer"; Tidy, "src/tools/tidy", "tidy"; - TestFloatParse, "src/etc/test-float-parse", "test-float-parse"; + TestFloatParse, "src/tools/test-float-parse", "test-float-parse"; ); #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index a782d0f6b1a..5ecce31fe15 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -668,7 +668,8 @@ pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, car // Enable frame pointers by default for the library. Note that they are still controlled by a // separate setting for the compiler. - cargo.rustflag("-Cforce-frame-pointers=yes"); + cargo.rustflag("-Zunstable-options"); + cargo.rustflag("-Cforce-frame-pointers=non-leaf"); let html_root = format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),); @@ -1278,6 +1279,17 @@ pub fn rustc_cargo( )); } + // The stage0 compiler changes infrequently and does not directly depend on code + // in the current working directory. Therefore, caching it with sccache should be + // useful. + // This is only performed for non-incremental builds, as ccache cannot deal with these. + if let Some(ref ccache) = builder.config.ccache + && compiler.stage == 0 + && !builder.config.incremental + { + cargo.env("RUSTC_WRAPPER", ccache); + } + rustc_cargo_env(builder, cargo, target, compiler.stage); } @@ -1867,23 +1879,27 @@ impl Step for Sysroot { // so that any tools relying on `rust-src` also work for local builds, // and also for translating the virtual `/rustc/$hash` back to the real // directory (for running tests with `rust.remap-debuginfo = true`). - let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); - t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); - let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); - if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { - eprintln!( - "ERROR: creating symbolic link `{}` to `{}` failed with {}", - sysroot_lib_rustlib_src_rust.display(), - builder.src.display(), - e, - ); - if builder.config.rust_remap_debuginfo { + if compiler.stage != 0 { + let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); + t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); + let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); + if let Err(e) = + symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) + { eprintln!( - "ERROR: some `tests/ui` tests will fail when lacking `{}`", + "ERROR: creating symbolic link `{}` to `{}` failed with {}", sysroot_lib_rustlib_src_rust.display(), + builder.src.display(), + e, ); + if builder.config.rust_remap_debuginfo { + eprintln!( + "ERROR: some `tests/ui` tests will fail when lacking `{}`", + sysroot_lib_rustlib_src_rust.display(), + ); + } + build_helper::exit!(1); } - build_helper::exit!(1); } // rustc-src component is already part of CI rustc's sysroot diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 587fca80374..7aa9e6cc2b5 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1681,7 +1681,8 @@ impl Step for Extended { cmd.run(builder); } - if target.is_windows() { + // FIXME(mati865): `gnullvm` here is temporary, remove it once it can host itself + if target.is_windows() && !target.contains("gnullvm") { let exe = tmp.join("exe"); let _ = fs::remove_dir_all(&exe); diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 20a4d1a1515..2b7703000cb 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -443,23 +443,26 @@ impl Step for Llvm { // See https://github.com/rust-lang/rust/pull/50104 cfg.define("LLVM_ENABLE_LIBXML2", "OFF"); - if !enabled_llvm_projects.is_empty() { - enabled_llvm_projects.sort(); - enabled_llvm_projects.dedup(); - cfg.define("LLVM_ENABLE_PROJECTS", enabled_llvm_projects.join(";")); - } - let mut enabled_llvm_runtimes = Vec::new(); if helpers::forcing_clang_based_tests() { enabled_llvm_runtimes.push("compiler-rt"); } + // This is an experimental flag, which likely builds more than necessary. + // We will optimize it when we get closer to releasing it on nightly. if builder.config.llvm_offload { enabled_llvm_runtimes.push("offload"); //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp. //Remove this line once they achieved it. enabled_llvm_runtimes.push("openmp"); + enabled_llvm_projects.push("compiler-rt"); + } + + if !enabled_llvm_projects.is_empty() { + enabled_llvm_projects.sort(); + enabled_llvm_projects.dedup(); + cfg.define("LLVM_ENABLE_PROJECTS", enabled_llvm_projects.join(";")); } if !enabled_llvm_runtimes.is_empty() { diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 22ab3e56a9c..dddce8fe05d 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -2707,16 +2707,6 @@ impl Step for Crate { .arg(builder.src.join("library/sysroot/Cargo.toml")); } else { compile::std_cargo(builder, target, compiler.stage, &mut cargo); - // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, - // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. - // Override it. - if builder.download_rustc() && compiler.stage > 0 { - let sysroot = builder - .out - .join(compiler.host) - .join(format!("stage{}-test-sysroot", compiler.stage)); - cargo.env("RUSTC_SYSROOT", sysroot); - } } } Mode::Rustc => { @@ -3544,7 +3534,7 @@ impl Step for CodegenGCC { } /// Test step that does two things: -/// - Runs `cargo test` for the `src/etc/test-float-parse` tool. +/// - Runs `cargo test` for the `src/tools/test-float-parse` tool. /// - Invokes the `test-float-parse` tool to test the standard library's /// float parsing routines. #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -3559,7 +3549,7 @@ impl Step for TestFloatParse { const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/etc/test-float-parse") + run.path("src/tools/test-float-parse") } fn make_run(run: RunConfig<'_>) { diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 173b3ff0816..717accb399a 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -23,7 +23,6 @@ use crate::core::builder::{ Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step, cargo_profile_var, }; use crate::core::config::{DebuginfoLevel, RustcLto, TargetSelection}; -use crate::utils::channel::GitInfo; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{add_dylib_path, exe, t}; use crate::{Compiler, FileType, Kind, Mode, gha}; @@ -148,6 +147,17 @@ impl Step for ToolBuild { &self.extra_features, ); + // The stage0 compiler changes infrequently and does not directly depend on code + // in the current working directory. Therefore, caching it with sccache should be + // useful. + // This is only performed for non-incremental builds, as ccache cannot deal with these. + if let Some(ref ccache) = builder.config.ccache + && matches!(self.mode, Mode::ToolBootstrap) + && !builder.config.incremental + { + cargo.env("RUSTC_WRAPPER", ccache); + } + // Rustc tools (miri, clippy, cargo, rustfmt, rust-analyzer) // could use the additional optimizations. if self.mode == Mode::ToolRustc && is_lto_stage(&self.compiler) { @@ -267,7 +277,7 @@ pub fn prepare_tool_cargo( cargo.env("CFG_VER_DESCRIPTION", description); } - let info = GitInfo::new(builder.config.omit_git_hash, &dir); + let info = builder.config.git_info(builder.config.omit_git_hash, &dir); if let Some(sha) = info.sha() { cargo.env("CFG_COMMIT_HASH", sha); } @@ -1269,7 +1279,7 @@ impl Step for TestFloatParse { const DEFAULT: bool = false; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/etc/test-float-parse") + run.path("src/tools/test-float-parse") } fn run(self, builder: &Builder<'_>) -> ToolBuildResult { @@ -1281,7 +1291,7 @@ impl Step for TestFloatParse { target: bootstrap_host, tool: "test-float-parse", mode: Mode::ToolStd, - path: "src/etc/test-float-parse", + path: "src/tools/test-float-parse", source_type: SourceType::InTree, extra_features: Vec::new(), allow_features: Self::ALLOW_FEATURES, @@ -1302,10 +1312,8 @@ impl Builder<'_> { // // Notably this munges the dynamic library lookup path to point to the // right location to run `compiler`. - let mut lib_paths: Vec<PathBuf> = vec![ - self.build.rustc_snapshot_libdir(), - self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps"), - ]; + let mut lib_paths: Vec<PathBuf> = + vec![self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps")]; // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make // mode) and that C compiler may need some extra PATH modification. Do diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index 1e9af68a92d..c6cfdb69356 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -11,7 +11,7 @@ use crate::utils::build_stamp; use crate::utils::helpers::{self, LldThreads, check_cfg_arg, linker_args, linker_flags}; use crate::{ BootstrapCommand, CLang, Compiler, Config, DocTests, DryRun, EXTRA_CHECK_CFGS, GitRepo, Mode, - TargetSelection, command, prepare_behaviour_dump_dir, t, + RemapScheme, TargetSelection, command, prepare_behaviour_dump_dir, t, }; /// Represents flag values in `String` form with whitespace delimiter to pass it to the compiler @@ -551,7 +551,7 @@ impl Builder<'_> { let libdir = self.rustc_libdir(compiler); let sysroot_str = sysroot.as_os_str().to_str().expect("sysroot should be UTF-8"); - if self.is_verbose() && !matches!(self.config.dry_run, DryRun::SelfCheck) { + if self.is_verbose() && !matches!(self.config.get_dry_run(), DryRun::SelfCheck) { println!("using sysroot {sysroot_str}"); } @@ -636,6 +636,15 @@ impl Builder<'_> { for (restricted_mode, name, values) in EXTRA_CHECK_CFGS { if restricted_mode.is_none() || *restricted_mode == Some(mode) { rustflags.arg(&check_cfg_arg(name, *values)); + + if *name == "bootstrap" { + // Cargo doesn't pass RUSTFLAGS to proc_macros: + // https://github.com/rust-lang/cargo/issues/4423 + // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. + // We also declare that the flag is expected, which we need to do to not + // get warnings about it being unexpected. + hostflags.arg(check_cfg_arg(name, *values)); + } } } @@ -645,13 +654,6 @@ impl Builder<'_> { if stage == 0 { hostflags.arg("--cfg=bootstrap"); } - // Cargo doesn't pass RUSTFLAGS to proc_macros: - // https://github.com/rust-lang/cargo/issues/4423 - // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. - // We also declare that the flag is expected, which we need to do to not - // get warnings about it being unexpected. - hostflags.arg("-Zunstable-options"); - hostflags.arg("--check-cfg=cfg(bootstrap)"); // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`, // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See @@ -920,13 +922,46 @@ impl Builder<'_> { hostflags.arg(format!("-Ctarget-feature={sign}crt-static")); } - if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) { - let map = format!("{}={}", self.build.src.display(), map_to); - cargo.env("RUSTC_DEBUGINFO_MAP", map); + // `rustc` needs to know the remapping scheme, in order to know how to reverse it (unremap) + // later. Two env vars are set and made available to the compiler + // + // - `CFG_VIRTUAL_RUST_SOURCE_BASE_DIR`: `rust-src` remap scheme (`NonCompiler`) + // - `CFG_VIRTUAL_RUSTC_DEV_SOURCE_BASE_DIR`: `rustc-dev` remap scheme (`Compiler`) + // + // Keep this scheme in sync with `rustc_metadata::rmeta::decoder`'s + // `try_to_translate_virtual_to_real`. + // + // `RUSTC_DEBUGINFO_MAP` is used to pass through to the underlying rustc + // `--remap-path-prefix`. + match mode { + Mode::Rustc | Mode::Codegen => { + if let Some(ref map_to) = + self.build.debuginfo_map_to(GitRepo::Rustc, RemapScheme::NonCompiler) + { + cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); + } - // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to, - // in order to opportunistically reverse it later. - cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); + if let Some(ref map_to) = + self.build.debuginfo_map_to(GitRepo::Rustc, RemapScheme::Compiler) + { + // When building compiler sources, we want to apply the compiler remap scheme. + cargo.env( + "RUSTC_DEBUGINFO_MAP", + format!("{}={}", self.build.src.display(), map_to), + ); + cargo.env("CFG_VIRTUAL_RUSTC_DEV_SOURCE_BASE_DIR", map_to); + } + } + Mode::Std | Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd => { + if let Some(ref map_to) = + self.build.debuginfo_map_to(GitRepo::Rustc, RemapScheme::NonCompiler) + { + cargo.env( + "RUSTC_DEBUGINFO_MAP", + format!("{}={}", self.build.src.display(), map_to), + ); + } + } } if self.config.rust_remap_debuginfo { diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index af3e3cc37b9..8b1520de3a8 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -22,6 +22,7 @@ use crate::core::config::flags::Subcommand; use crate::core::config::{DryRun, TargetSelection}; use crate::utils::cache::Cache; use crate::utils::exec::{BootstrapCommand, command}; +use crate::utils::execution_context::ExecutionContext; use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t}; use crate::{Build, Crate, trace}; @@ -442,13 +443,15 @@ impl StepDescription { fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool { if builder.config.skip.iter().any(|e| pathset.has(e, builder.kind)) { - if !matches!(builder.config.dry_run, DryRun::SelfCheck) { + if !matches!(builder.config.get_dry_run(), DryRun::SelfCheck) { println!("Skipping {pathset:?} because it is excluded"); } return true; } - if !builder.config.skip.is_empty() && !matches!(builder.config.dry_run, DryRun::SelfCheck) { + if !builder.config.skip.is_empty() + && !matches!(builder.config.get_dry_run(), DryRun::SelfCheck) + { builder.verbose(|| { println!( "{:?} not skipped for {:?} -- not in {:?}", @@ -945,7 +948,6 @@ impl<'a> Builder<'a> { clippy::CI, ), Kind::Check | Kind::Fix => describe!( - check::Std, check::Rustc, check::Rustdoc, check::CodegenBackend, @@ -961,6 +963,13 @@ impl<'a> Builder<'a> { check::Compiletest, check::FeaturesStatusDump, check::CoverageDump, + // This has special staging logic, it may run on stage 1 while others run on stage 0. + // It takes quite some time to build stage 1, so put this at the end. + // + // FIXME: This also helps bootstrap to not interfere with stage 0 builds. We should probably fix + // that issue somewhere else, but we still want to keep `check::Std` at the end so that the + // quicker steps run before this. + check::Std, ), Kind::Test => describe!( crate::core::build_steps::toolstate::ToolStateCheck, @@ -1627,4 +1636,14 @@ impl<'a> Builder<'a> { self.info(&format!("{err}\n")); } } + + pub fn exec_ctx(&self) -> &ExecutionContext { + &self.config.exec_ctx + } +} + +impl<'a> AsRef<ExecutionContext> for Builder<'a> { + fn as_ref(&self) -> &ExecutionContext { + self.exec_ctx() + } } diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index baa22fc7f72..a264d772c56 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -22,7 +22,7 @@ fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config let mut config = Config::parse(Flags::parse(cmd)); // don't save toolstates config.save_toolstates = None; - config.dry_run = DryRun::SelfCheck; + config.set_dry_run(DryRun::SelfCheck); // Ignore most submodules, since we don't need them for a dry run, and the // tests run much faster without them. diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 899ffde8adf..860aa120326 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -1,36 +1,56 @@ -//! Serialized configuration of a build. +//! This module defines the central `Config` struct, which aggregates all components +//! of the bootstrap configuration into a single unit. //! -//! This module implements parsing `bootstrap.toml` configuration files to tweak -//! how the build runs. +//! It serves as the primary public interface for accessing the bootstrap configuration. +//! The module coordinates the overall configuration parsing process using logic from `parsing.rs` +//! and provides top-level methods such as `Config::parse()` for initialization, as well as +//! utility methods for querying and manipulating the complete configuration state. +//! +//! Additionally, this module contains the core logic for parsing, validating, and inferring +//! the final `Config` from various raw inputs. +//! +//! It manages the process of reading command-line arguments, environment variables, +//! and the `bootstrap.toml` file—merging them, applying defaults, and performing +//! cross-component validation. The main `parse_inner` function and its supporting +//! helpers reside here, transforming raw `Toml` data into the structured `Config` type. use std::cell::Cell; use std::collections::{BTreeSet, HashMap, HashSet}; -use std::fmt::{self, Display}; -use std::hash::Hash; use std::io::IsTerminal; use std::path::{Path, PathBuf, absolute}; -use std::process::Command; use std::str::FromStr; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::{Arc, Mutex}; use std::{cmp, env, fs}; use build_helper::ci::CiEnv; use build_helper::exit; -use build_helper::git::{GitConfig, PathFreshness, check_path_modifications, output_result}; -use serde::{Deserialize, Deserializer}; -use serde_derive::Deserialize; +use build_helper::git::{GitConfig, PathFreshness, check_path_modifications}; +use serde::Deserialize; #[cfg(feature = "tracing")] use tracing::{instrument, span}; -use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; use crate::core::build_steps::llvm; use crate::core::build_steps::llvm::LLVM_INVALIDATION_PATHS; pub use crate::core::config::flags::Subcommand; -use crate::core::config::flags::{Color, Flags, Warnings}; +use crate::core::config::flags::{Color, Flags}; +use crate::core::config::target_selection::TargetSelectionList; +use crate::core::config::toml::TomlConfig; +use crate::core::config::toml::build::Build; +use crate::core::config::toml::change_id::ChangeId; +use crate::core::config::toml::rust::{ + LldMode, RustOptimize, check_incompatible_options_for_ci_rustc, +}; +use crate::core::config::toml::target::Target; +use crate::core::config::{ + DebuginfoLevel, DryRun, GccCiMode, LlvmLibunwind, Merge, ReplaceOpt, RustcLto, SplitDebuginfo, + StringOrBool, set, threads_from_config, +}; use crate::core::download::is_download_ci_available; -use crate::utils::cache::{INTERNER, Interned}; -use crate::utils::channel::{self, GitInfo}; -use crate::utils::helpers::{self, exe, output, t}; +use crate::utils::channel; +use crate::utils::exec::command; +use crate::utils::execution_context::ExecutionContext; +use crate::utils::helpers::exe; +use crate::{GitInfo, OnceLock, TargetSelection, check_ci_llvm, helpers, t}; /// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic. /// This means they can be modified and changes to these paths should never trigger a compiler build @@ -44,7 +64,7 @@ use crate::utils::helpers::{self, exe, output, t}; /// For example, "src/bootstrap" should never be included in this list as it plays a crucial role in the /// final output/compiler, which can be significantly affected by changes made to the bootstrap sources. #[rustfmt::skip] // We don't want rustfmt to oneline this list -pub(crate) const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ +pub const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ ":!library", ":!src/tools", ":!src/librustdoc", @@ -53,138 +73,6 @@ pub(crate) const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ ":!triagebot.toml", ]; -macro_rules! check_ci_llvm { - ($name:expr) => { - assert!( - $name.is_none(), - "setting {} is incompatible with download-ci-llvm.", - stringify!($name).replace("_", "-") - ); - }; -} - -/// This file is embedded in the overlay directory of the tarball sources. It is -/// useful in scenarios where developers want to see how the tarball sources were -/// generated. -/// -/// We also use this file to compare the host's bootstrap.toml against the CI rustc builder -/// configuration to detect any incompatible options. -pub(crate) const BUILDER_CONFIG_FILENAME: &str = "builder-config"; - -#[derive(Clone, Default)] -pub enum DryRun { - /// This isn't a dry run. - #[default] - Disabled, - /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. - SelfCheck, - /// This is a dry run enabled by the `--dry-run` flag. - UserSelected, -} - -#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)] -pub enum DebuginfoLevel { - #[default] - None, - LineDirectivesOnly, - LineTablesOnly, - Limited, - Full, -} - -// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only -// deserializes i64, and derive() only generates visit_u64 -impl<'de> Deserialize<'de> for DebuginfoLevel { - fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> - where - D: Deserializer<'de>, - { - use serde::de::Error; - - Ok(match Deserialize::deserialize(deserializer)? { - StringOrInt::String(s) if s == "none" => DebuginfoLevel::None, - StringOrInt::Int(0) => DebuginfoLevel::None, - StringOrInt::String(s) if s == "line-directives-only" => { - DebuginfoLevel::LineDirectivesOnly - } - StringOrInt::String(s) if s == "line-tables-only" => DebuginfoLevel::LineTablesOnly, - StringOrInt::String(s) if s == "limited" => DebuginfoLevel::Limited, - StringOrInt::Int(1) => DebuginfoLevel::Limited, - StringOrInt::String(s) if s == "full" => DebuginfoLevel::Full, - StringOrInt::Int(2) => DebuginfoLevel::Full, - StringOrInt::Int(n) => { - let other = serde::de::Unexpected::Signed(n); - return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); - } - StringOrInt::String(s) => { - let other = serde::de::Unexpected::Str(&s); - return Err(D::Error::invalid_value( - other, - &"expected none, line-tables-only, limited, or full", - )); - } - }) - } -} - -/// Suitable for passing to `-C debuginfo` -impl Display for DebuginfoLevel { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use DebuginfoLevel::*; - f.write_str(match self { - None => "0", - LineDirectivesOnly => "line-directives-only", - LineTablesOnly => "line-tables-only", - Limited => "1", - Full => "2", - }) - } -} - -/// LLD in bootstrap works like this: -/// - Self-contained lld: use `rust-lld` from the compiler's sysroot -/// - External: use an external `lld` binary -/// -/// It is configured depending on the target: -/// 1) Everything except MSVC -/// - Self-contained: `-Clinker-flavor=gnu-lld-cc -Clink-self-contained=+linker` -/// - External: `-Clinker-flavor=gnu-lld-cc` -/// 2) MSVC -/// - Self-contained: `-Clinker=<path to rust-lld>` -/// - External: `-Clinker=lld` -#[derive(Copy, Clone, Default, Debug, PartialEq)] -pub enum LldMode { - /// Do not use LLD - #[default] - Unused, - /// Use `rust-lld` from the compiler's sysroot - SelfContained, - /// Use an externally provided `lld` binary. - /// Note that the linker name cannot be overridden, the binary has to be named `lld` and it has - /// to be in $PATH. - External, -} - -impl LldMode { - pub fn is_used(&self) -> bool { - match self { - LldMode::SelfContained | LldMode::External => true, - LldMode::Unused => false, - } - } -} - -/// Determines how will GCC be provided. -#[derive(Default, Clone)] -pub enum GccCiMode { - /// Build GCC from the local `src/gcc` submodule. - #[default] - BuildLocally, - /// Try to download GCC from CI. - /// If it is not available on CI, it will be built locally instead. - DownloadFromCi, -} - /// Global configuration for the entire build and/or bootstrap. /// /// This structure is parsed from `bootstrap.toml`, and some of the fields are inferred from `git` or build-time parameters. @@ -243,16 +131,12 @@ pub struct Config { pub jobs: Option<u32>, pub cmd: Subcommand, pub incremental: bool, - pub dry_run: DryRun, pub dump_bootstrap_shims: bool, /// Arguments appearing after `--` to be forwarded to tools, /// e.g. `--fix-broken` or test arguments. pub free_args: Vec<String>, /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. - #[cfg(not(test))] - download_rustc_commit: Option<String>, - #[cfg(test)] pub download_rustc_commit: Option<String>, pub deny_warnings: bool, @@ -269,10 +153,6 @@ pub struct Config { pub llvm_release_debuginfo: bool, pub llvm_static_stdcpp: bool, pub llvm_libzstd: bool, - /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. - #[cfg(not(test))] - llvm_link_shared: Cell<Option<bool>>, - #[cfg(test)] pub llvm_link_shared: Cell<Option<bool>>, pub llvm_clang_cl: Option<String>, pub llvm_targets: Option<String>, @@ -345,9 +225,6 @@ pub struct Config { pub hosts: Vec<TargetSelection>, pub targets: Vec<TargetSelection>, pub local_rebuild: bool, - #[cfg(not(test))] - jemalloc: bool, - #[cfg(test)] pub jemalloc: bool, pub control_flow_guard: bool, pub ehcont_guard: bool, @@ -423,932 +300,13 @@ pub struct Config { /// Cache for determining path modifications pub path_modification_cache: Arc<Mutex<HashMap<Vec<&'static str>, PathFreshness>>>, -} - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub enum LlvmLibunwind { - #[default] - No, - InTree, - System, -} - -impl FromStr for LlvmLibunwind { - type Err = String; - - fn from_str(value: &str) -> Result<Self, Self::Err> { - match value { - "no" => Ok(Self::No), - "in-tree" => Ok(Self::InTree), - "system" => Ok(Self::System), - invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), - } - } -} -#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum SplitDebuginfo { - Packed, - Unpacked, - #[default] - Off, -} - -impl std::str::FromStr for SplitDebuginfo { - type Err = (); + /// Skip checking the standard library if `rust.download-rustc` isn't available. + /// This is mostly for RA as building the stage1 compiler to check the library tree + /// on each code change might be too much for some computers. + pub skip_std_check_if_no_download_rustc: bool, - fn from_str(s: &str) -> Result<Self, Self::Err> { - match s { - "packed" => Ok(SplitDebuginfo::Packed), - "unpacked" => Ok(SplitDebuginfo::Unpacked), - "off" => Ok(SplitDebuginfo::Off), - _ => Err(()), - } - } -} - -impl SplitDebuginfo { - /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for - /// `rust.split-debuginfo` in `bootstrap.example.toml`. - fn default_for_platform(target: TargetSelection) -> Self { - if target.contains("apple") { - SplitDebuginfo::Unpacked - } else if target.is_windows() { - SplitDebuginfo::Packed - } else { - SplitDebuginfo::Off - } - } -} - -/// LTO mode used for compiling rustc itself. -#[derive(Default, Clone, PartialEq, Debug)] -pub enum RustcLto { - Off, - #[default] - ThinLocal, - Thin, - Fat, -} - -impl std::str::FromStr for RustcLto { - type Err = String; - - fn from_str(s: &str) -> Result<Self, Self::Err> { - match s { - "thin-local" => Ok(RustcLto::ThinLocal), - "thin" => Ok(RustcLto::Thin), - "fat" => Ok(RustcLto::Fat), - "off" => Ok(RustcLto::Off), - _ => Err(format!("Invalid value for rustc LTO: {s}")), - } - } -} - -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -// N.B.: This type is used everywhere, and the entire codebase relies on it being Copy. -// Making !Copy is highly nontrivial! -pub struct TargetSelection { - pub triple: Interned<String>, - file: Option<Interned<String>>, - synthetic: bool, -} - -/// Newtype over `Vec<TargetSelection>` so we can implement custom parsing logic -#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] -pub struct TargetSelectionList(Vec<TargetSelection>); - -pub fn target_selection_list(s: &str) -> Result<TargetSelectionList, String> { - Ok(TargetSelectionList( - s.split(',').filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), - )) -} - -impl TargetSelection { - pub fn from_user(selection: &str) -> Self { - let path = Path::new(selection); - - let (triple, file) = if path.exists() { - let triple = path - .file_stem() - .expect("Target specification file has no file stem") - .to_str() - .expect("Target specification file stem is not UTF-8"); - - (triple, Some(selection)) - } else { - (selection, None) - }; - - let triple = INTERNER.intern_str(triple); - let file = file.map(|f| INTERNER.intern_str(f)); - - Self { triple, file, synthetic: false } - } - - pub fn create_synthetic(triple: &str, file: &str) -> Self { - Self { - triple: INTERNER.intern_str(triple), - file: Some(INTERNER.intern_str(file)), - synthetic: true, - } - } - - pub fn rustc_target_arg(&self) -> &str { - self.file.as_ref().unwrap_or(&self.triple) - } - - pub fn contains(&self, needle: &str) -> bool { - self.triple.contains(needle) - } - - pub fn starts_with(&self, needle: &str) -> bool { - self.triple.starts_with(needle) - } - - pub fn ends_with(&self, needle: &str) -> bool { - self.triple.ends_with(needle) - } - - // See src/bootstrap/synthetic_targets.rs - pub fn is_synthetic(&self) -> bool { - self.synthetic - } - - pub fn is_msvc(&self) -> bool { - self.contains("msvc") - } - - pub fn is_windows(&self) -> bool { - self.contains("windows") - } - - pub fn is_windows_gnu(&self) -> bool { - self.ends_with("windows-gnu") - } - - pub fn is_cygwin(&self) -> bool { - self.is_windows() && - // ref. https://cygwin.com/pipermail/cygwin/2022-February/250802.html - env::var("OSTYPE").is_ok_and(|v| v.to_lowercase().contains("cygwin")) - } - - pub fn needs_crt_begin_end(&self) -> bool { - self.contains("musl") && !self.contains("unikraft") - } - - /// Path to the file defining the custom target, if any. - pub fn filepath(&self) -> Option<&Path> { - self.file.as_ref().map(Path::new) - } -} - -impl fmt::Display for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.triple)?; - if let Some(file) = self.file { - write!(f, "({file})")?; - } - Ok(()) - } -} - -impl fmt::Debug for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self}") - } -} - -impl PartialEq<&str> for TargetSelection { - fn eq(&self, other: &&str) -> bool { - self.triple == *other - } -} - -// Targets are often used as directory names throughout bootstrap. -// This impl makes it more ergonomics to use them as such. -impl AsRef<Path> for TargetSelection { - fn as_ref(&self) -> &Path { - self.triple.as_ref() - } -} - -/// Per-target configuration stored in the global configuration structure. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct Target { - /// Some(path to llvm-config) if using an external LLVM. - pub llvm_config: Option<PathBuf>, - pub llvm_has_rust_patches: Option<bool>, - /// Some(path to FileCheck) if one was specified. - pub llvm_filecheck: Option<PathBuf>, - pub llvm_libunwind: Option<LlvmLibunwind>, - pub cc: Option<PathBuf>, - pub cxx: Option<PathBuf>, - pub ar: Option<PathBuf>, - pub ranlib: Option<PathBuf>, - pub default_linker: Option<PathBuf>, - pub linker: Option<PathBuf>, - pub split_debuginfo: Option<SplitDebuginfo>, - pub sanitizers: Option<bool>, - pub profiler: Option<StringOrBool>, - pub rpath: Option<bool>, - pub crt_static: Option<bool>, - pub musl_root: Option<PathBuf>, - pub musl_libdir: Option<PathBuf>, - pub wasi_root: Option<PathBuf>, - pub qemu_rootfs: Option<PathBuf>, - pub runner: Option<String>, - pub no_std: bool, - pub codegen_backends: Option<Vec<String>>, - pub optimized_compiler_builtins: Option<bool>, - pub jemalloc: Option<bool>, -} - -impl Target { - pub fn from_triple(triple: &str) -> Self { - let mut target: Self = Default::default(); - if triple.contains("-none") || triple.contains("nvptx") || triple.contains("switch") { - target.no_std = true; - } - if triple.contains("emscripten") { - target.runner = Some("node".into()); - } - target - } -} -/// Structure of the `bootstrap.toml` file that configuration is read from. -/// -/// This structure uses `Decodable` to automatically decode a TOML configuration -/// file into this format, and then this is traversed and written into the above -/// `Config` structure. -#[derive(Deserialize, Default)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] -pub(crate) struct TomlConfig { - #[serde(flatten)] - change_id: ChangeIdWrapper, - build: Option<Build>, - install: Option<Install>, - llvm: Option<Llvm>, - gcc: Option<Gcc>, - rust: Option<Rust>, - target: Option<HashMap<String, TomlTarget>>, - dist: Option<Dist>, - profile: Option<String>, - include: Option<Vec<PathBuf>>, -} - -/// This enum is used for deserializing change IDs from TOML, allowing both numeric values and the string `"ignore"`. -#[derive(Clone, Debug, PartialEq)] -pub enum ChangeId { - Ignore, - Id(usize), -} - -/// Since we use `#[serde(deny_unknown_fields)]` on `TomlConfig`, we need a wrapper type -/// for the "change-id" field to parse it even if other fields are invalid. This ensures -/// that if deserialization fails due to other fields, we can still provide the changelogs -/// to allow developers to potentially find the reason for the failure in the logs.. -#[derive(Deserialize, Default)] -pub(crate) struct ChangeIdWrapper { - #[serde(alias = "change-id", default, deserialize_with = "deserialize_change_id")] - pub(crate) inner: Option<ChangeId>, -} - -fn deserialize_change_id<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result<Option<ChangeId>, D::Error> { - let value = toml::Value::deserialize(deserializer)?; - Ok(match value { - toml::Value::String(s) if s == "ignore" => Some(ChangeId::Ignore), - toml::Value::Integer(i) => Some(ChangeId::Id(i as usize)), - _ => { - return Err(serde::de::Error::custom( - "expected \"ignore\" or an integer for change-id", - )); - } - }) -} - -/// Describes how to handle conflicts in merging two [`TomlConfig`] -#[derive(Copy, Clone, Debug)] -enum ReplaceOpt { - /// Silently ignore a duplicated value - IgnoreDuplicate, - /// Override the current value, even if it's `Some` - Override, - /// Exit with an error on duplicate values - ErrorOnDuplicate, -} - -trait Merge { - fn merge( - &mut self, - parent_config_path: Option<PathBuf>, - included_extensions: &mut HashSet<PathBuf>, - other: Self, - replace: ReplaceOpt, - ); -} - -impl Merge for TomlConfig { - fn merge( - &mut self, - parent_config_path: Option<PathBuf>, - included_extensions: &mut HashSet<PathBuf>, - TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id, include }: Self, - replace: ReplaceOpt, - ) { - fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) { - if let Some(new) = y { - if let Some(original) = x { - original.merge(None, &mut Default::default(), new, replace); - } else { - *x = Some(new); - } - } - } - - self.change_id.inner.merge(None, &mut Default::default(), change_id.inner, replace); - self.profile.merge(None, &mut Default::default(), profile, replace); - - do_merge(&mut self.build, build, replace); - do_merge(&mut self.install, install, replace); - do_merge(&mut self.llvm, llvm, replace); - do_merge(&mut self.gcc, gcc, replace); - do_merge(&mut self.rust, rust, replace); - do_merge(&mut self.dist, dist, replace); - - match (self.target.as_mut(), target) { - (_, None) => {} - (None, Some(target)) => self.target = Some(target), - (Some(original_target), Some(new_target)) => { - for (triple, new) in new_target { - if let Some(original) = original_target.get_mut(&triple) { - original.merge(None, &mut Default::default(), new, replace); - } else { - original_target.insert(triple, new); - } - } - } - } - - let parent_dir = parent_config_path - .as_ref() - .and_then(|p| p.parent().map(ToOwned::to_owned)) - .unwrap_or_default(); - - // `include` handled later since we ignore duplicates using `ReplaceOpt::IgnoreDuplicate` to - // keep the upper-level configuration to take precedence. - for include_path in include.clone().unwrap_or_default().iter().rev() { - let include_path = parent_dir.join(include_path); - let include_path = include_path.canonicalize().unwrap_or_else(|e| { - eprintln!("ERROR: Failed to canonicalize '{}' path: {e}", include_path.display()); - exit!(2); - }); - - let included_toml = Config::get_toml_inner(&include_path).unwrap_or_else(|e| { - eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display()); - exit!(2); - }); - - assert!( - included_extensions.insert(include_path.clone()), - "Cyclic inclusion detected: '{}' is being included again before its previous inclusion was fully processed.", - include_path.display() - ); - - self.merge( - Some(include_path.clone()), - included_extensions, - included_toml, - // Ensures that parent configuration always takes precedence - // over child configurations. - ReplaceOpt::IgnoreDuplicate, - ); - - included_extensions.remove(&include_path); - } - } -} - -// We are using a decl macro instead of a derive proc macro here to reduce the compile time of bootstrap. -macro_rules! define_config { - ($(#[$attr:meta])* struct $name:ident { - $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* - }) => { - $(#[$attr])* - struct $name { - $($field: Option<$field_ty>,)* - } - - impl Merge for $name { - fn merge( - &mut self, - _parent_config_path: Option<PathBuf>, - _included_extensions: &mut HashSet<PathBuf>, - other: Self, - replace: ReplaceOpt - ) { - $( - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.$field.is_none() { - self.$field = other.$field; - } - }, - ReplaceOpt::Override => { - if other.$field.is_some() { - self.$field = other.$field; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.$field.is_some() { - if self.$field.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option: `{}`", stringify!($field)); - exit!(2); - } - } else { - self.$field = other.$field; - } - } - } - } - )* - } - } - - // The following is a trimmed version of what serde_derive generates. All parts not relevant - // for toml deserialization have been removed. This reduces the binary size and improves - // compile time of bootstrap. - impl<'de> Deserialize<'de> for $name { - fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> - where - D: Deserializer<'de>, - { - struct Field; - impl<'de> serde::de::Visitor<'de> for Field { - type Value = $name; - fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(concat!("struct ", stringify!($name))) - } - - #[inline] - fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error> - where - A: serde::de::MapAccess<'de>, - { - $(let mut $field: Option<$field_ty> = None;)* - while let Some(key) = - match serde::de::MapAccess::next_key::<String>(&mut map) { - Ok(val) => val, - Err(err) => { - return Err(err); - } - } - { - match &*key { - $($field_key => { - if $field.is_some() { - return Err(<A::Error as serde::de::Error>::duplicate_field( - $field_key, - )); - } - $field = match serde::de::MapAccess::next_value::<$field_ty>( - &mut map, - ) { - Ok(val) => Some(val), - Err(err) => { - return Err(err); - } - }; - })* - key => { - return Err(serde::de::Error::unknown_field(key, FIELDS)); - } - } - } - Ok($name { $($field),* }) - } - } - const FIELDS: &'static [&'static str] = &[ - $($field_key,)* - ]; - Deserializer::deserialize_struct( - deserializer, - stringify!($name), - FIELDS, - Field, - ) - } - } - } -} - -impl<T> Merge for Option<T> { - fn merge( - &mut self, - _parent_config_path: Option<PathBuf>, - _included_extensions: &mut HashSet<PathBuf>, - other: Self, - replace: ReplaceOpt, - ) { - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.is_none() { - *self = other; - } - } - ReplaceOpt::Override => { - if other.is_some() { - *self = other; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.is_some() { - if self.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option"); - exit!(2); - } - } else { - *self = other; - } - } - } - } - } -} - -define_config! { - /// TOML representation of various global build decisions. - #[derive(Default)] - struct Build { - build: Option<String> = "build", - description: Option<String> = "description", - host: Option<Vec<String>> = "host", - target: Option<Vec<String>> = "target", - build_dir: Option<String> = "build-dir", - cargo: Option<PathBuf> = "cargo", - rustc: Option<PathBuf> = "rustc", - rustfmt: Option<PathBuf> = "rustfmt", - cargo_clippy: Option<PathBuf> = "cargo-clippy", - docs: Option<bool> = "docs", - compiler_docs: Option<bool> = "compiler-docs", - library_docs_private_items: Option<bool> = "library-docs-private-items", - docs_minification: Option<bool> = "docs-minification", - submodules: Option<bool> = "submodules", - gdb: Option<String> = "gdb", - lldb: Option<String> = "lldb", - nodejs: Option<String> = "nodejs", - npm: Option<String> = "npm", - python: Option<String> = "python", - reuse: Option<String> = "reuse", - locked_deps: Option<bool> = "locked-deps", - vendor: Option<bool> = "vendor", - full_bootstrap: Option<bool> = "full-bootstrap", - bootstrap_cache_path: Option<PathBuf> = "bootstrap-cache-path", - extended: Option<bool> = "extended", - tools: Option<HashSet<String>> = "tools", - verbose: Option<usize> = "verbose", - sanitizers: Option<bool> = "sanitizers", - profiler: Option<bool> = "profiler", - cargo_native_static: Option<bool> = "cargo-native-static", - low_priority: Option<bool> = "low-priority", - configure_args: Option<Vec<String>> = "configure-args", - local_rebuild: Option<bool> = "local-rebuild", - print_step_timings: Option<bool> = "print-step-timings", - print_step_rusage: Option<bool> = "print-step-rusage", - check_stage: Option<u32> = "check-stage", - doc_stage: Option<u32> = "doc-stage", - build_stage: Option<u32> = "build-stage", - test_stage: Option<u32> = "test-stage", - install_stage: Option<u32> = "install-stage", - dist_stage: Option<u32> = "dist-stage", - bench_stage: Option<u32> = "bench-stage", - patch_binaries_for_nix: Option<bool> = "patch-binaries-for-nix", - // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally - metrics: Option<bool> = "metrics", - android_ndk: Option<PathBuf> = "android-ndk", - optimized_compiler_builtins: Option<bool> = "optimized-compiler-builtins", - jobs: Option<u32> = "jobs", - compiletest_diff_tool: Option<String> = "compiletest-diff-tool", - compiletest_use_stage0_libtest: Option<bool> = "compiletest-use-stage0-libtest", - ccache: Option<StringOrBool> = "ccache", - exclude: Option<Vec<PathBuf>> = "exclude", - } -} - -define_config! { - /// TOML representation of various global install decisions. - struct Install { - prefix: Option<String> = "prefix", - sysconfdir: Option<String> = "sysconfdir", - docdir: Option<String> = "docdir", - bindir: Option<String> = "bindir", - libdir: Option<String> = "libdir", - mandir: Option<String> = "mandir", - datadir: Option<String> = "datadir", - } -} - -define_config! { - /// TOML representation of how the LLVM build is configured. - struct Llvm { - optimize: Option<bool> = "optimize", - thin_lto: Option<bool> = "thin-lto", - release_debuginfo: Option<bool> = "release-debuginfo", - assertions: Option<bool> = "assertions", - tests: Option<bool> = "tests", - enzyme: Option<bool> = "enzyme", - plugins: Option<bool> = "plugins", - // FIXME: Remove this field at Q2 2025, it has been replaced by build.ccache - ccache: Option<StringOrBool> = "ccache", - static_libstdcpp: Option<bool> = "static-libstdcpp", - libzstd: Option<bool> = "libzstd", - ninja: Option<bool> = "ninja", - targets: Option<String> = "targets", - experimental_targets: Option<String> = "experimental-targets", - link_jobs: Option<u32> = "link-jobs", - link_shared: Option<bool> = "link-shared", - version_suffix: Option<String> = "version-suffix", - clang_cl: Option<String> = "clang-cl", - cflags: Option<String> = "cflags", - cxxflags: Option<String> = "cxxflags", - ldflags: Option<String> = "ldflags", - use_libcxx: Option<bool> = "use-libcxx", - use_linker: Option<String> = "use-linker", - allow_old_toolchain: Option<bool> = "allow-old-toolchain", - offload: Option<bool> = "offload", - polly: Option<bool> = "polly", - clang: Option<bool> = "clang", - enable_warnings: Option<bool> = "enable-warnings", - download_ci_llvm: Option<StringOrBool> = "download-ci-llvm", - build_config: Option<HashMap<String, String>> = "build-config", - } -} - -define_config! { - /// TOML representation of how the GCC build is configured. - struct Gcc { - download_ci_gcc: Option<bool> = "download-ci-gcc", - } -} - -define_config! { - struct Dist { - sign_folder: Option<String> = "sign-folder", - upload_addr: Option<String> = "upload-addr", - src_tarball: Option<bool> = "src-tarball", - compression_formats: Option<Vec<String>> = "compression-formats", - compression_profile: Option<String> = "compression-profile", - include_mingw_linker: Option<bool> = "include-mingw-linker", - vendor: Option<bool> = "vendor", - } -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum StringOrBool { - String(String), - Bool(bool), -} - -impl Default for StringOrBool { - fn default() -> StringOrBool { - StringOrBool::Bool(false) - } -} - -impl StringOrBool { - fn is_string_or_true(&self) -> bool { - matches!(self, Self::String(_) | Self::Bool(true)) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RustOptimize { - String(String), - Int(u8), - Bool(bool), -} - -impl Default for RustOptimize { - fn default() -> RustOptimize { - RustOptimize::Bool(false) - } -} - -impl<'de> Deserialize<'de> for RustOptimize { - fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> - where - D: Deserializer<'de>, - { - deserializer.deserialize_any(OptimizeVisitor) - } -} - -struct OptimizeVisitor; - -impl serde::de::Visitor<'_> for OptimizeVisitor { - type Value = RustOptimize; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) - } - - fn visit_str<E>(self, value: &str) -> Result<Self::Value, E> - where - E: serde::de::Error, - { - if matches!(value, "s" | "z") { - Ok(RustOptimize::String(value.to_string())) - } else { - Err(serde::de::Error::custom(format_optimize_error_msg(value))) - } - } - - fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E> - where - E: serde::de::Error, - { - if matches!(value, 0..=3) { - Ok(RustOptimize::Int(value as u8)) - } else { - Err(serde::de::Error::custom(format_optimize_error_msg(value))) - } - } - - fn visit_bool<E>(self, value: bool) -> Result<Self::Value, E> - where - E: serde::de::Error, - { - Ok(RustOptimize::Bool(value)) - } -} - -fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { - format!( - r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# - ) -} - -impl RustOptimize { - pub(crate) fn is_release(&self) -> bool { - match &self { - RustOptimize::Bool(true) | RustOptimize::String(_) => true, - RustOptimize::Int(i) => *i > 0, - RustOptimize::Bool(false) => false, - } - } - - pub(crate) fn get_opt_level(&self) -> Option<String> { - match &self { - RustOptimize::String(s) => Some(s.clone()), - RustOptimize::Int(i) => Some(i.to_string()), - RustOptimize::Bool(_) => None, - } - } -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum StringOrInt { - String(String), - Int(i64), -} - -impl<'de> Deserialize<'de> for LldMode { - fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> - where - D: Deserializer<'de>, - { - struct LldModeVisitor; - - impl serde::de::Visitor<'_> for LldModeVisitor { - type Value = LldMode; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str("one of true, 'self-contained' or 'external'") - } - - fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E> - where - E: serde::de::Error, - { - Ok(if v { LldMode::External } else { LldMode::Unused }) - } - - fn visit_str<E>(self, v: &str) -> Result<Self::Value, E> - where - E: serde::de::Error, - { - match v { - "external" => Ok(LldMode::External), - "self-contained" => Ok(LldMode::SelfContained), - _ => Err(E::custom(format!("unknown mode {v}"))), - } - } - } - - deserializer.deserialize_any(LldModeVisitor) - } -} - -define_config! { - /// TOML representation of how the Rust build is configured. - struct Rust { - optimize: Option<RustOptimize> = "optimize", - debug: Option<bool> = "debug", - codegen_units: Option<u32> = "codegen-units", - codegen_units_std: Option<u32> = "codegen-units-std", - rustc_debug_assertions: Option<bool> = "debug-assertions", - randomize_layout: Option<bool> = "randomize-layout", - std_debug_assertions: Option<bool> = "debug-assertions-std", - tools_debug_assertions: Option<bool> = "debug-assertions-tools", - overflow_checks: Option<bool> = "overflow-checks", - overflow_checks_std: Option<bool> = "overflow-checks-std", - debug_logging: Option<bool> = "debug-logging", - debuginfo_level: Option<DebuginfoLevel> = "debuginfo-level", - debuginfo_level_rustc: Option<DebuginfoLevel> = "debuginfo-level-rustc", - debuginfo_level_std: Option<DebuginfoLevel> = "debuginfo-level-std", - debuginfo_level_tools: Option<DebuginfoLevel> = "debuginfo-level-tools", - debuginfo_level_tests: Option<DebuginfoLevel> = "debuginfo-level-tests", - backtrace: Option<bool> = "backtrace", - incremental: Option<bool> = "incremental", - default_linker: Option<String> = "default-linker", - channel: Option<String> = "channel", - // FIXME: Remove this field at Q2 2025, it has been replaced by build.description - description: Option<String> = "description", - musl_root: Option<String> = "musl-root", - rpath: Option<bool> = "rpath", - strip: Option<bool> = "strip", - frame_pointers: Option<bool> = "frame-pointers", - stack_protector: Option<String> = "stack-protector", - verbose_tests: Option<bool> = "verbose-tests", - optimize_tests: Option<bool> = "optimize-tests", - codegen_tests: Option<bool> = "codegen-tests", - omit_git_hash: Option<bool> = "omit-git-hash", - dist_src: Option<bool> = "dist-src", - save_toolstates: Option<String> = "save-toolstates", - codegen_backends: Option<Vec<String>> = "codegen-backends", - llvm_bitcode_linker: Option<bool> = "llvm-bitcode-linker", - lld: Option<bool> = "lld", - lld_mode: Option<LldMode> = "use-lld", - llvm_tools: Option<bool> = "llvm-tools", - deny_warnings: Option<bool> = "deny-warnings", - backtrace_on_ice: Option<bool> = "backtrace-on-ice", - verify_llvm_ir: Option<bool> = "verify-llvm-ir", - thin_lto_import_instr_limit: Option<u32> = "thin-lto-import-instr-limit", - remap_debuginfo: Option<bool> = "remap-debuginfo", - jemalloc: Option<bool> = "jemalloc", - test_compare_mode: Option<bool> = "test-compare-mode", - llvm_libunwind: Option<String> = "llvm-libunwind", - control_flow_guard: Option<bool> = "control-flow-guard", - ehcont_guard: Option<bool> = "ehcont-guard", - new_symbol_mangling: Option<bool> = "new-symbol-mangling", - profile_generate: Option<String> = "profile-generate", - profile_use: Option<String> = "profile-use", - // ignored; this is set from an env var set by bootstrap.py - download_rustc: Option<StringOrBool> = "download-rustc", - lto: Option<String> = "lto", - validate_mir_opts: Option<u32> = "validate-mir-opts", - std_features: Option<BTreeSet<String>> = "std-features", - } -} - -define_config! { - /// TOML representation of how each build target is configured. - struct TomlTarget { - cc: Option<String> = "cc", - cxx: Option<String> = "cxx", - ar: Option<String> = "ar", - ranlib: Option<String> = "ranlib", - default_linker: Option<PathBuf> = "default-linker", - linker: Option<String> = "linker", - split_debuginfo: Option<String> = "split-debuginfo", - llvm_config: Option<String> = "llvm-config", - llvm_has_rust_patches: Option<bool> = "llvm-has-rust-patches", - llvm_filecheck: Option<String> = "llvm-filecheck", - llvm_libunwind: Option<String> = "llvm-libunwind", - sanitizers: Option<bool> = "sanitizers", - profiler: Option<StringOrBool> = "profiler", - rpath: Option<bool> = "rpath", - crt_static: Option<bool> = "crt-static", - musl_root: Option<String> = "musl-root", - musl_libdir: Option<String> = "musl-libdir", - wasi_root: Option<String> = "wasi-root", - qemu_rootfs: Option<String> = "qemu-rootfs", - no_std: Option<bool> = "no-std", - codegen_backends: Option<Vec<String>> = "codegen-backends", - runner: Option<String> = "runner", - optimized_compiler_builtins: Option<bool> = "optimized-compiler-builtins", - jemalloc: Option<bool> = "jemalloc", - } + pub exec_ctx: ExecutionContext, } impl Config { @@ -1405,45 +363,12 @@ impl Config { } } - pub(crate) fn get_builder_toml(&self, build_name: &str) -> Result<TomlConfig, toml::de::Error> { - if self.dry_run() { - return Ok(TomlConfig::default()); - } - - let builder_config_path = - self.out.join(self.build.triple).join(build_name).join(BUILDER_CONFIG_FILENAME); - Self::get_toml(&builder_config_path) - } - - pub(crate) fn get_toml(file: &Path) -> Result<TomlConfig, toml::de::Error> { - #[cfg(test)] - return Ok(TomlConfig::default()); - - #[cfg(not(test))] - Self::get_toml_inner(file) + pub fn set_dry_run(&mut self, dry_run: DryRun) { + self.exec_ctx.set_dry_run(dry_run); } - fn get_toml_inner(file: &Path) -> Result<TomlConfig, toml::de::Error> { - let contents = - t!(fs::read_to_string(file), format!("config file {} not found", file.display())); - // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of - // TomlConfig and sub types to be monomorphized 5x by toml. - toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - .inspect_err(|_| { - if let Ok(ChangeIdWrapper { inner: Some(ChangeId::Id(id)) }) = - toml::from_str::<toml::Value>(&contents) - .and_then(|table: toml::Value| ChangeIdWrapper::deserialize(table)) - { - let changes = crate::find_recent_config_change_ids(id); - if !changes.is_empty() { - println!( - "WARNING: There have been changes to x.py since you last updated:\n{}", - crate::human_readable_changes(changes) - ); - } - } - }) + pub fn get_dry_run(&self) -> &DryRun { + self.exec_ctx.get_dry_run() } #[cfg_attr( @@ -1468,6 +393,11 @@ impl Config { get_toml: impl Fn(&Path) -> Result<TomlConfig, toml::de::Error>, ) -> Config { let mut config = Config::default_opts(); + let mut exec_ctx = ExecutionContext::new(); + exec_ctx.set_verbose(flags.verbose); + exec_ctx.set_fail_fast(flags.cmd.fail_fast()); + + config.exec_ctx = exec_ctx; // Set flags. config.paths = std::mem::take(&mut flags.paths); @@ -1496,7 +426,7 @@ impl Config { config.on_fail = flags.on_fail; config.cmd = flags.cmd; config.incremental = flags.incremental; - config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; + config.set_dry_run(if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }); config.dump_bootstrap_shims = flags.dump_bootstrap_shims; config.keep_stage = flags.keep_stage; config.keep_stage_std = flags.keep_stage_std; @@ -1507,6 +437,7 @@ impl Config { config.enable_bolt_settings = flags.enable_bolt_settings; config.bypass_bootstrap_lock = flags.bypass_bootstrap_lock; config.is_running_on_ci = flags.ci.unwrap_or(CiEnv::is_ci()); + config.skip_std_check_if_no_download_rustc = flags.skip_std_check_if_no_download_rustc; // Infer the rest of the configuration. @@ -1525,14 +456,9 @@ impl Config { // has already been (kinda-cross-)compiled to Windows land, we require a normal Windows path. cmd.arg("rev-parse").arg("--show-cdup"); // Discard stderr because we expect this to fail when building from a tarball. - let output = cmd - .as_command_mut() - .stderr(std::process::Stdio::null()) - .output() - .ok() - .and_then(|output| if output.status.success() { Some(output) } else { None }); - if let Some(output) = output { - let git_root_relative = String::from_utf8(output.stdout).unwrap(); + let output = cmd.allow_failure().run_capture_stdout(&config); + if output.is_success() { + let git_root_relative = output.stdout(); // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes, // and to resolve any relative components. let git_root = env::current_dir() @@ -1627,7 +553,7 @@ impl Config { build.cargo = build.cargo.take().or(std::env::var_os("CARGO").map(|p| p.into())); } - if GitInfo::new(false, &config.src).is_from_tarball() && toml.profile.is_none() { + if config.git_info(false, &config.src).is_from_tarball() && toml.profile.is_none() { toml.profile = Some("dist".into()); } @@ -1834,7 +760,12 @@ impl Config { }; config.initial_sysroot = t!(PathBuf::from_str( - output(Command::new(&config.initial_rustc).args(["--print", "sysroot"])).trim() + command(&config.initial_rustc) + .args(["--print", "sysroot"]) + .run_always() + .run_capture_stdout(&config) + .stdout() + .trim() )); config.initial_cargo_clippy = cargo_clippy; @@ -1906,42 +837,11 @@ impl Config { // Verbose flag is a good default for `rust.verbose-tests`. config.verbose_tests = config.is_verbose(); - if let Some(install) = toml.install { - let Install { prefix, sysconfdir, docdir, bindir, libdir, mandir, datadir } = install; - config.prefix = prefix.map(PathBuf::from); - config.sysconfdir = sysconfdir.map(PathBuf::from); - config.datadir = datadir.map(PathBuf::from); - config.docdir = docdir.map(PathBuf::from); - set(&mut config.bindir, bindir.map(PathBuf::from)); - config.libdir = libdir.map(PathBuf::from); - config.mandir = mandir.map(PathBuf::from); - } + config.apply_install_config(toml.install); config.llvm_assertions = toml.llvm.as_ref().is_some_and(|llvm| llvm.assertions.unwrap_or(false)); - // Store off these values as options because if they're not provided - // we'll infer default values for them later - let mut llvm_tests = None; - let mut llvm_enzyme = None; - let mut llvm_offload = None; - let mut llvm_plugins = None; - let mut debug = None; - let mut rustc_debug_assertions = None; - let mut std_debug_assertions = None; - let mut tools_debug_assertions = None; - let mut overflow_checks = None; - let mut overflow_checks_std = None; - let mut debug_logging = None; - let mut debuginfo_level = None; - let mut debuginfo_level_rustc = None; - let mut debuginfo_level_std = None; - let mut debuginfo_level_tools = None; - let mut debuginfo_level_tests = None; - let mut optimize = None; - let mut lld_enabled = None; - let mut std_features = None; - let file_content = t!(fs::read_to_string(config.src.join("src/ci/channel"))); let ci_channel = file_content.trim_end(); @@ -1961,19 +861,21 @@ impl Config { let default = config.channel == "dev"; config.omit_git_hash = toml.rust.as_ref().and_then(|r| r.omit_git_hash).unwrap_or(default); - config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); - config.cargo_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/cargo")); + config.rust_info = config.git_info(config.omit_git_hash, &config.src); + config.cargo_info = + config.git_info(config.omit_git_hash, &config.src.join("src/tools/cargo")); config.rust_analyzer_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rust-analyzer")); + config.git_info(config.omit_git_hash, &config.src.join("src/tools/rust-analyzer")); config.clippy_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/clippy")); - config.miri_info = GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/miri")); + config.git_info(config.omit_git_hash, &config.src.join("src/tools/clippy")); + config.miri_info = + config.git_info(config.omit_git_hash, &config.src.join("src/tools/miri")); config.rustfmt_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/rustfmt")); + config.git_info(config.omit_git_hash, &config.src.join("src/tools/rustfmt")); config.enzyme_info = - GitInfo::new(config.omit_git_hash, &config.src.join("src/tools/enzyme")); - config.in_tree_llvm_info = GitInfo::new(false, &config.src.join("src/llvm-project")); - config.in_tree_gcc_info = GitInfo::new(false, &config.src.join("src/gcc")); + config.git_info(config.omit_git_hash, &config.src.join("src/tools/enzyme")); + config.in_tree_llvm_info = config.git_info(false, &config.src.join("src/llvm-project")); + config.in_tree_gcc_info = config.git_info(false, &config.src.join("src/gcc")); config.vendor = vendor.unwrap_or( config.rust_info.is_from_tarball() @@ -1985,190 +887,10 @@ impl Config { config.channel = ci_channel.into(); } - if let Some(rust) = toml.rust { - let Rust { - optimize: optimize_toml, - debug: debug_toml, - codegen_units, - codegen_units_std, - rustc_debug_assertions: rustc_debug_assertions_toml, - std_debug_assertions: std_debug_assertions_toml, - tools_debug_assertions: tools_debug_assertions_toml, - overflow_checks: overflow_checks_toml, - overflow_checks_std: overflow_checks_std_toml, - debug_logging: debug_logging_toml, - debuginfo_level: debuginfo_level_toml, - debuginfo_level_rustc: debuginfo_level_rustc_toml, - debuginfo_level_std: debuginfo_level_std_toml, - debuginfo_level_tools: debuginfo_level_tools_toml, - debuginfo_level_tests: debuginfo_level_tests_toml, - backtrace, - incremental, - randomize_layout, - default_linker, - channel: _, // already handled above - description: rust_description, - musl_root, - rpath, - verbose_tests, - optimize_tests, - codegen_tests, - omit_git_hash: _, // already handled above - dist_src, - save_toolstates, - codegen_backends, - lld: lld_enabled_toml, - llvm_tools, - llvm_bitcode_linker, - deny_warnings, - backtrace_on_ice, - verify_llvm_ir, - thin_lto_import_instr_limit, - remap_debuginfo, - jemalloc, - test_compare_mode, - llvm_libunwind, - control_flow_guard, - ehcont_guard, - new_symbol_mangling, - profile_generate, - profile_use, - download_rustc, - lto, - validate_mir_opts, - frame_pointers, - stack_protector, - strip, - lld_mode, - std_features: std_features_toml, - } = rust; - - // FIXME(#133381): alt rustc builds currently do *not* have rustc debug assertions - // enabled. We should not download a CI alt rustc if we need rustc to have debug - // assertions (e.g. for crashes test suite). This can be changed once something like - // [Enable debug assertions on alt - // builds](https://github.com/rust-lang/rust/pull/131077) lands. - // - // Note that `rust.debug = true` currently implies `rust.debug-assertions = true`! - // - // This relies also on the fact that the global default for `download-rustc` will be - // `false` if it's not explicitly set. - let debug_assertions_requested = matches!(rustc_debug_assertions_toml, Some(true)) - || (matches!(debug_toml, Some(true)) - && !matches!(rustc_debug_assertions_toml, Some(false))); - - if debug_assertions_requested - && let Some(ref opt) = download_rustc - && opt.is_string_or_true() - { - eprintln!( - "WARN: currently no CI rustc builds have rustc debug assertions \ - enabled. Please either set `rust.debug-assertions` to `false` if you \ - want to use download CI rustc or set `rust.download-rustc` to `false`." - ); - } + config.rust_profile_use = flags.rust_profile_use; + config.rust_profile_generate = flags.rust_profile_generate; - config.download_rustc_commit = config.download_ci_rustc_commit( - download_rustc, - debug_assertions_requested, - config.llvm_assertions, - ); - - debug = debug_toml; - rustc_debug_assertions = rustc_debug_assertions_toml; - std_debug_assertions = std_debug_assertions_toml; - tools_debug_assertions = tools_debug_assertions_toml; - overflow_checks = overflow_checks_toml; - overflow_checks_std = overflow_checks_std_toml; - debug_logging = debug_logging_toml; - debuginfo_level = debuginfo_level_toml; - debuginfo_level_rustc = debuginfo_level_rustc_toml; - debuginfo_level_std = debuginfo_level_std_toml; - debuginfo_level_tools = debuginfo_level_tools_toml; - debuginfo_level_tests = debuginfo_level_tests_toml; - lld_enabled = lld_enabled_toml; - std_features = std_features_toml; - - optimize = optimize_toml; - config.rust_new_symbol_mangling = new_symbol_mangling; - set(&mut config.rust_optimize_tests, optimize_tests); - set(&mut config.codegen_tests, codegen_tests); - set(&mut config.rust_rpath, rpath); - set(&mut config.rust_strip, strip); - set(&mut config.rust_frame_pointers, frame_pointers); - config.rust_stack_protector = stack_protector; - set(&mut config.jemalloc, jemalloc); - set(&mut config.test_compare_mode, test_compare_mode); - set(&mut config.backtrace, backtrace); - if rust_description.is_some() { - eprintln!( - "Warning: rust.description is deprecated. Use build.description instead." - ); - } - description = description.or(rust_description); - set(&mut config.rust_dist_src, dist_src); - set(&mut config.verbose_tests, verbose_tests); - // in the case "false" is set explicitly, do not overwrite the command line args - if let Some(true) = incremental { - config.incremental = true; - } - set(&mut config.lld_mode, lld_mode); - set(&mut config.llvm_bitcode_linker_enabled, llvm_bitcode_linker); - - config.rust_randomize_layout = randomize_layout.unwrap_or_default(); - config.llvm_tools_enabled = llvm_tools.unwrap_or(true); - - config.llvm_enzyme = - llvm_enzyme.unwrap_or(config.channel == "dev" || config.channel == "nightly"); - config.rustc_default_linker = default_linker; - config.musl_root = musl_root.map(PathBuf::from); - config.save_toolstates = save_toolstates.map(PathBuf::from); - set( - &mut config.deny_warnings, - match flags.warnings { - Warnings::Deny => Some(true), - Warnings::Warn => Some(false), - Warnings::Default => deny_warnings, - }, - ); - set(&mut config.backtrace_on_ice, backtrace_on_ice); - set(&mut config.rust_verify_llvm_ir, verify_llvm_ir); - config.rust_thin_lto_import_instr_limit = thin_lto_import_instr_limit; - set(&mut config.rust_remap_debuginfo, remap_debuginfo); - set(&mut config.control_flow_guard, control_flow_guard); - set(&mut config.ehcont_guard, ehcont_guard); - config.llvm_libunwind_default = - llvm_libunwind.map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); - - if let Some(ref backends) = codegen_backends { - let available_backends = ["llvm", "cranelift", "gcc"]; - - config.rust_codegen_backends = backends.iter().map(|s| { - if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { - if available_backends.contains(&backend) { - panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); - } else { - println!("HELP: '{s}' for 'rust.codegen-backends' might fail. \ - Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ - In this case, it would be referred to as '{backend}'."); - } - } - - s.clone() - }).collect(); - } - - config.rust_codegen_units = codegen_units.map(threads_from_config); - config.rust_codegen_units_std = codegen_units_std.map(threads_from_config); - config.rust_profile_use = flags.rust_profile_use.or(profile_use); - config.rust_profile_generate = flags.rust_profile_generate.or(profile_generate); - config.rust_lto = - lto.as_deref().map(|value| RustcLto::from_str(value).unwrap()).unwrap_or_default(); - config.rust_validate_mir_opts = validate_mir_opts; - } else { - config.rust_profile_use = flags.rust_profile_use; - config.rust_profile_generate = flags.rust_profile_generate; - } + config.apply_rust_config(toml.rust, flags.warnings, &mut description); config.reproducible_artifacts = flags.reproducible_artifact; config.description = description; @@ -2189,206 +911,11 @@ impl Config { config.channel = channel; } - if let Some(llvm) = toml.llvm { - let Llvm { - optimize: optimize_toml, - thin_lto, - release_debuginfo, - assertions: _, - tests, - enzyme, - plugins, - ccache: llvm_ccache, - static_libstdcpp, - libzstd, - ninja, - targets, - experimental_targets, - link_jobs, - link_shared, - version_suffix, - clang_cl, - cflags, - cxxflags, - ldflags, - use_libcxx, - use_linker, - allow_old_toolchain, - offload, - polly, - clang, - enable_warnings, - download_ci_llvm, - build_config, - } = llvm; - if llvm_ccache.is_some() { - eprintln!("Warning: llvm.ccache is deprecated. Use build.ccache instead."); - } - - ccache = ccache.or(llvm_ccache); - set(&mut config.ninja_in_file, ninja); - llvm_tests = tests; - llvm_enzyme = enzyme; - llvm_offload = offload; - llvm_plugins = plugins; - set(&mut config.llvm_optimize, optimize_toml); - set(&mut config.llvm_thin_lto, thin_lto); - set(&mut config.llvm_release_debuginfo, release_debuginfo); - set(&mut config.llvm_static_stdcpp, static_libstdcpp); - set(&mut config.llvm_libzstd, libzstd); - if let Some(v) = link_shared { - config.llvm_link_shared.set(Some(v)); - } - config.llvm_targets.clone_from(&targets); - config.llvm_experimental_targets.clone_from(&experimental_targets); - config.llvm_link_jobs = link_jobs; - config.llvm_version_suffix.clone_from(&version_suffix); - config.llvm_clang_cl.clone_from(&clang_cl); - - config.llvm_cflags.clone_from(&cflags); - config.llvm_cxxflags.clone_from(&cxxflags); - config.llvm_ldflags.clone_from(&ldflags); - set(&mut config.llvm_use_libcxx, use_libcxx); - config.llvm_use_linker.clone_from(&use_linker); - config.llvm_allow_old_toolchain = allow_old_toolchain.unwrap_or(false); - config.llvm_offload = offload.unwrap_or(false); - config.llvm_polly = polly.unwrap_or(false); - config.llvm_clang = clang.unwrap_or(false); - config.llvm_enable_warnings = enable_warnings.unwrap_or(false); - config.llvm_build_config = build_config.clone().unwrap_or(Default::default()); - - config.llvm_from_ci = - config.parse_download_ci_llvm(download_ci_llvm, config.llvm_assertions); - - if config.llvm_from_ci { - let warn = |option: &str| { - println!( - "WARNING: `{option}` will only be used on `compiler/rustc_llvm` build, not for the LLVM build." - ); - println!( - "HELP: To use `{option}` for LLVM builds, set `download-ci-llvm` option to false." - ); - }; - - if static_libstdcpp.is_some() { - warn("static-libstdcpp"); - } - - if link_shared.is_some() { - warn("link-shared"); - } - - // FIXME(#129153): instead of all the ad-hoc `download-ci-llvm` checks that follow, - // use the `builder-config` present in tarballs since #128822 to compare the local - // config to the ones used to build the LLVM artifacts on CI, and only notify users - // if they've chosen a different value. - - if libzstd.is_some() { - println!( - "WARNING: when using `download-ci-llvm`, the local `llvm.libzstd` option, \ - like almost all `llvm.*` options, will be ignored and set by the LLVM CI \ - artifacts builder config." - ); - println!( - "HELP: To use `llvm.libzstd` for LLVM/LLD builds, set `download-ci-llvm` option to false." - ); - } - } - - if !config.llvm_from_ci && config.llvm_thin_lto && link_shared.is_none() { - // If we're building with ThinLTO on, by default we want to link - // to LLVM shared, to avoid re-doing ThinLTO (which happens in - // the link step) with each stage. - config.llvm_link_shared.set(Some(true)); - } - } else { - config.llvm_from_ci = config.parse_download_ci_llvm(None, false); - } - - if let Some(gcc) = toml.gcc { - config.gcc_ci_mode = match gcc.download_ci_gcc { - Some(value) => match value { - true => GccCiMode::DownloadFromCi, - false => GccCiMode::BuildLocally, - }, - None => GccCiMode::default(), - }; - } + config.apply_llvm_config(toml.llvm, &mut ccache); - if let Some(t) = toml.target { - for (triple, cfg) in t { - let mut target = Target::from_triple(&triple); + config.apply_gcc_config(toml.gcc); - if let Some(ref s) = cfg.llvm_config { - if config.download_rustc_commit.is_some() && triple == *config.build.triple { - panic!( - "setting llvm_config for the host is incompatible with download-rustc" - ); - } - target.llvm_config = Some(config.src.join(s)); - } - if let Some(patches) = cfg.llvm_has_rust_patches { - assert!( - config.submodules == Some(false) || cfg.llvm_config.is_some(), - "use of `llvm-has-rust-patches` is restricted to cases where either submodules are disabled or llvm-config been provided" - ); - target.llvm_has_rust_patches = Some(patches); - } - if let Some(ref s) = cfg.llvm_filecheck { - target.llvm_filecheck = Some(config.src.join(s)); - } - target.llvm_libunwind = cfg.llvm_libunwind.as_ref().map(|v| { - v.parse().unwrap_or_else(|_| { - panic!("failed to parse target.{triple}.llvm-libunwind") - }) - }); - if let Some(s) = cfg.no_std { - target.no_std = s; - } - target.cc = cfg.cc.map(PathBuf::from); - target.cxx = cfg.cxx.map(PathBuf::from); - target.ar = cfg.ar.map(PathBuf::from); - target.ranlib = cfg.ranlib.map(PathBuf::from); - target.linker = cfg.linker.map(PathBuf::from); - target.crt_static = cfg.crt_static; - target.musl_root = cfg.musl_root.map(PathBuf::from); - target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); - target.wasi_root = cfg.wasi_root.map(PathBuf::from); - target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); - target.runner = cfg.runner; - target.sanitizers = cfg.sanitizers; - target.profiler = cfg.profiler; - target.rpath = cfg.rpath; - target.optimized_compiler_builtins = cfg.optimized_compiler_builtins; - target.jemalloc = cfg.jemalloc; - - if let Some(ref backends) = cfg.codegen_backends { - let available_backends = ["llvm", "cranelift", "gcc"]; - - target.codegen_backends = Some(backends.iter().map(|s| { - if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { - if available_backends.contains(&backend) { - panic!("Invalid value '{s}' for 'target.{triple}.codegen-backends'. Instead, please use '{backend}'."); - } else { - println!("HELP: '{s}' for 'target.{triple}.codegen-backends' might fail. \ - Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ - In this case, it would be referred to as '{backend}'."); - } - } - - s.clone() - }).collect()); - } - - target.split_debuginfo = cfg.split_debuginfo.as_ref().map(|v| { - v.parse().unwrap_or_else(|_| { - panic!("invalid value for target.{triple}.split-debuginfo") - }) - }); - - config.target_config.insert(TargetSelection::from_user(&triple), target); - } - } + config.apply_target_config(toml.target); match ccache { Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), @@ -2412,69 +939,11 @@ impl Config { build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); } - if let Some(dist) = toml.dist { - let Dist { - sign_folder, - upload_addr, - src_tarball, - compression_formats, - compression_profile, - include_mingw_linker, - vendor, - } = dist; - config.dist_sign_folder = sign_folder.map(PathBuf::from); - config.dist_upload_addr = upload_addr; - config.dist_compression_formats = compression_formats; - set(&mut config.dist_compression_profile, compression_profile); - set(&mut config.rust_dist_src, src_tarball); - set(&mut config.dist_include_mingw_linker, include_mingw_linker); - config.dist_vendor = vendor.unwrap_or_else(|| { - // If we're building from git or tarball sources, enable it by default. - config.rust_info.is_managed_git_subrepository() - || config.rust_info.is_from_tarball() - }); - } + config.apply_dist_config(toml.dist); config.initial_rustfmt = if let Some(r) = rustfmt { Some(r) } else { config.maybe_download_rustfmt() }; - // Now that we've reached the end of our configuration, infer the - // default values for all options that we haven't otherwise stored yet. - - config.llvm_tests = llvm_tests.unwrap_or(false); - config.llvm_enzyme = llvm_enzyme.unwrap_or(false); - config.llvm_offload = llvm_offload.unwrap_or(false); - config.llvm_plugins = llvm_plugins.unwrap_or(false); - config.rust_optimize = optimize.unwrap_or(RustOptimize::Bool(true)); - - // We make `x86_64-unknown-linux-gnu` use the self-contained linker by default, so we will - // build our internal lld and use it as the default linker, by setting the `rust.lld` config - // to true by default: - // - on the `x86_64-unknown-linux-gnu` target - // - on the `dev` and `nightly` channels - // - when building our in-tree llvm (i.e. the target has not set an `llvm-config`), so that - // we're also able to build the corresponding lld - // - or when using an external llvm that's downloaded from CI, which also contains our prebuilt - // lld - // - otherwise, we'd be using an external llvm, and lld would not necessarily available and - // thus, disabled - // - similarly, lld will not be built nor used by default when explicitly asked not to, e.g. - // when the config sets `rust.lld = false` - if config.build.triple == "x86_64-unknown-linux-gnu" - && config.hosts == [config.build] - && (config.channel == "dev" || config.channel == "nightly") - { - let no_llvm_config = config - .target_config - .get(&config.build) - .is_some_and(|target_config| target_config.llvm_config.is_none()); - let enable_lld = config.llvm_from_ci || no_llvm_config; - // Prefer the config setting in case an explicit opt-out is needed. - config.lld_enabled = lld_enabled.unwrap_or(enable_lld); - } else { - set(&mut config.lld_enabled, lld_enabled); - } - if matches!(config.lld_mode, LldMode::SelfContained) && !config.lld_enabled && flags.stage.unwrap_or(0) > 0 @@ -2490,31 +959,6 @@ impl Config { ); } - let default_std_features = BTreeSet::from([String::from("panic-unwind")]); - config.rust_std_features = std_features.unwrap_or(default_std_features); - - let default = debug == Some(true); - config.rustc_debug_assertions = rustc_debug_assertions.unwrap_or(default); - config.std_debug_assertions = std_debug_assertions.unwrap_or(config.rustc_debug_assertions); - config.tools_debug_assertions = - tools_debug_assertions.unwrap_or(config.rustc_debug_assertions); - config.rust_overflow_checks = overflow_checks.unwrap_or(default); - config.rust_overflow_checks_std = - overflow_checks_std.unwrap_or(config.rust_overflow_checks); - - config.rust_debug_logging = debug_logging.unwrap_or(config.rustc_debug_assertions); - - let with_defaults = |debuginfo_level_specific: Option<_>| { - debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { - DebuginfoLevel::Limited - } else { - DebuginfoLevel::None - }) - }; - config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); - config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); - config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); - config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(DebuginfoLevel::None); config.optimized_compiler_builtins = optimized_compiler_builtins.unwrap_or(config.channel != "dev"); config.compiletest_diff_tool = compiletest_diff_tool; @@ -2531,9 +975,8 @@ impl Config { || bench_stage.is_some(); // See https://github.com/rust-lang/compiler-team/issues/326 config.stage = match config.cmd { - Subcommand::Check { .. } | Subcommand::Clippy { .. } | Subcommand::Fix => { - flags.stage.or(check_stage).unwrap_or(1) - } + Subcommand::Check { .. } => flags.stage.or(check_stage).unwrap_or(0), + Subcommand::Clippy { .. } | Subcommand::Fix => flags.stage.or(check_stage).unwrap_or(1), // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. Subcommand::Doc { .. } => { flags.stage.or(doc_stage).unwrap_or(if download_rustc { 2 } else { 1 }) @@ -2592,28 +1035,13 @@ impl Config { } pub fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } + self.exec_ctx.dry_run() } pub fn is_explicit_stage(&self) -> bool { self.explicit_stage_from_cli || self.explicit_stage_from_config } - /// Runs a command, printing out nice contextual information if it fails. - /// Exits if the command failed to execute at all, otherwise returns its - /// `status.success()`. - #[deprecated = "use `Builder::try_run` instead where possible"] - pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> { - if self.dry_run() { - return Ok(()); - } - self.verbose(|| println!("running: {cmd:?}")); - build_helper::util::try_run(cmd, self.is_verbose()) - } - pub(crate) fn test_args(&self) -> Vec<&str> { let mut test_args = match self.cmd { Subcommand::Test { ref test_args, .. } @@ -2647,7 +1075,7 @@ impl Config { let mut git = helpers::git(Some(&self.src)); git.arg("show").arg(format!("{commit}:{}", file.to_str().unwrap())); - output(git.as_command_mut()) + git.run_capture_stdout(self).stdout() } /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. @@ -2835,18 +1263,7 @@ impl Config { /// Runs a function if verbosity is greater than 0 pub fn verbose(&self, f: impl Fn()) { - if self.is_verbose() { - f() - } - } - - pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).and_then(|t| t.sanitizers).unwrap_or(self.sanitizers) - } - - pub fn needs_sanitizer_runtime_built(&self, target: TargetSelection) -> bool { - // MSVC uses the Microsoft-provided sanitizer runtime, but all other runtimes we build. - !target.is_msvc() && self.sanitizers_enabled(target) + self.exec_ctx.verbose(f); } pub fn any_sanitizers_to_build(&self) -> bool { @@ -2855,60 +1272,11 @@ impl Config { .any(|(ts, t)| !ts.is_msvc() && t.sanitizers.unwrap_or(self.sanitizers)) } - pub fn profiler_path(&self, target: TargetSelection) -> Option<&str> { - match self.target_config.get(&target)?.profiler.as_ref()? { - StringOrBool::String(s) => Some(s), - StringOrBool::Bool(_) => None, - } - } - - pub fn profiler_enabled(&self, target: TargetSelection) -> bool { - self.target_config - .get(&target) - .and_then(|t| t.profiler.as_ref()) - .map(StringOrBool::is_string_or_true) - .unwrap_or(self.profiler) - } - pub fn any_profiler_enabled(&self) -> bool { self.target_config.values().any(|t| matches!(&t.profiler, Some(p) if p.is_string_or_true())) || self.profiler } - pub fn rpath_enabled(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).and_then(|t| t.rpath).unwrap_or(self.rust_rpath) - } - - pub fn optimized_compiler_builtins(&self, target: TargetSelection) -> bool { - self.target_config - .get(&target) - .and_then(|t| t.optimized_compiler_builtins) - .unwrap_or(self.optimized_compiler_builtins) - } - - pub fn llvm_enabled(&self, target: TargetSelection) -> bool { - self.codegen_backends(target).contains(&"llvm".to_owned()) - } - - pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { - self.target_config - .get(&target) - .and_then(|t| t.llvm_libunwind) - .or(self.llvm_libunwind_default) - .unwrap_or(if target.contains("fuchsia") { - LlvmLibunwind::InTree - } else { - LlvmLibunwind::No - }) - } - - pub fn split_debuginfo(&self, target: TargetSelection) -> SplitDebuginfo { - self.target_config - .get(&target) - .and_then(|t| t.split_debuginfo) - .unwrap_or_else(|| SplitDebuginfo::default_for_platform(target)) - } - /// Returns whether or not submodules should be managed by bootstrap. pub fn submodules(&self) -> bool { // If not specified in config, the default is to only manage @@ -2916,21 +1284,6 @@ impl Config { self.submodules.unwrap_or(self.rust_info.is_managed_git_subrepository()) } - pub fn codegen_backends(&self, target: TargetSelection) -> &[String] { - self.target_config - .get(&target) - .and_then(|cfg| cfg.codegen_backends.as_deref()) - .unwrap_or(&self.rust_codegen_backends) - } - - pub fn jemalloc(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).and_then(|cfg| cfg.jemalloc).unwrap_or(self.jemalloc) - } - - pub fn default_codegen_backend(&self, target: TargetSelection) -> Option<String> { - self.codegen_backends(target).first().cloned() - } - pub fn git_config(&self) -> GitConfig<'_> { GitConfig { nightly_branch: &self.stage0_metadata.config.nightly_branch, @@ -2972,7 +1325,7 @@ impl Config { // NOTE: The check for the empty directory is here because when running x.py the first time, // the submodule won't be checked out. Check it out now so we can build it. - if !GitInfo::new(false, &absolute_path).is_managed_git_subrepository() + if !self.git_info(false, &absolute_path).is_managed_git_subrepository() && !helpers::dir_is_empty(&absolute_path) { return; @@ -2991,16 +1344,16 @@ impl Config { }; // Determine commit checked out in submodule. - let checked_out_hash = output(submodule_git().args(["rev-parse", "HEAD"]).as_command_mut()); + let checked_out_hash = + submodule_git().args(["rev-parse", "HEAD"]).run_capture_stdout(self).stdout(); let checked_out_hash = checked_out_hash.trim_end(); // Determine commit that the submodule *should* have. - let recorded = output( - helpers::git(Some(&self.src)) - .run_always() - .args(["ls-tree", "HEAD"]) - .arg(relative_path) - .as_command_mut(), - ); + let recorded = helpers::git(Some(&self.src)) + .run_always() + .args(["ls-tree", "HEAD"]) + .arg(relative_path) + .run_capture_stdout(self) + .stdout(); let actual_hash = recorded .split_whitespace() @@ -3024,21 +1377,20 @@ impl Config { let update = |progress: bool| { // Git is buggy and will try to fetch submodules from the tracking branch for *this* repository, // even though that has no relation to the upstream for the submodule. - let current_branch = output_result( - helpers::git(Some(&self.src)) - .allow_failure() - .run_always() - .args(["symbolic-ref", "--short", "HEAD"]) - .as_command_mut(), - ) - .map(|b| b.trim().to_owned()); + let current_branch = helpers::git(Some(&self.src)) + .allow_failure() + .run_always() + .args(["symbolic-ref", "--short", "HEAD"]) + .run_capture(self); let mut git = helpers::git(Some(&self.src)).allow_failure(); git.run_always(); - if let Ok(branch) = current_branch { + if current_branch.is_success() { // If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name. // This syntax isn't accepted by `branch.{branch}`. Strip it. - let branch = branch.strip_prefix("heads/").unwrap_or(&branch); + let branch = current_branch.stdout(); + let branch = branch.trim(); + let branch = branch.strip_prefix("heads/").unwrap_or(branch); git.arg("-c").arg(format!("branch.{branch}.remote=origin")); } git.args(["submodule", "update", "--init", "--recursive", "--depth=1"]); @@ -3083,7 +1435,8 @@ impl Config { return; } - let stage0_output = output(Command::new(program_path).arg("--version")); + let stage0_output = + command(program_path).arg("--version").run_capture_stdout(self).stdout(); let mut stage0_output = stage0_output.lines().next().unwrap().split(' '); let stage0_name = stage0_output.next().unwrap(); @@ -3114,7 +1467,7 @@ impl Config { } /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. - fn download_ci_rustc_commit( + pub fn download_ci_rustc_commit( &self, download_rustc: Option<StringOrBool>, debug_assertions_requested: bool, @@ -3194,7 +1547,7 @@ impl Config { Some(commit) } - fn parse_download_ci_llvm( + pub fn parse_download_ci_llvm( &self, download_ci_llvm: Option<StringOrBool>, asserts: bool, @@ -3277,6 +1630,83 @@ impl Config { .clone() } + pub fn ci_env(&self) -> CiEnv { + if self.is_running_on_ci { CiEnv::GitHubActions } else { CiEnv::None } + } + + pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).and_then(|t| t.sanitizers).unwrap_or(self.sanitizers) + } + + pub fn needs_sanitizer_runtime_built(&self, target: TargetSelection) -> bool { + // MSVC uses the Microsoft-provided sanitizer runtime, but all other runtimes we build. + !target.is_msvc() && self.sanitizers_enabled(target) + } + + pub fn profiler_path(&self, target: TargetSelection) -> Option<&str> { + match self.target_config.get(&target)?.profiler.as_ref()? { + StringOrBool::String(s) => Some(s), + StringOrBool::Bool(_) => None, + } + } + + pub fn profiler_enabled(&self, target: TargetSelection) -> bool { + self.target_config + .get(&target) + .and_then(|t| t.profiler.as_ref()) + .map(StringOrBool::is_string_or_true) + .unwrap_or(self.profiler) + } + + pub fn codegen_backends(&self, target: TargetSelection) -> &[String] { + self.target_config + .get(&target) + .and_then(|cfg| cfg.codegen_backends.as_deref()) + .unwrap_or(&self.rust_codegen_backends) + } + + pub fn jemalloc(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).and_then(|cfg| cfg.jemalloc).unwrap_or(self.jemalloc) + } + + pub fn default_codegen_backend(&self, target: TargetSelection) -> Option<String> { + self.codegen_backends(target).first().cloned() + } + + pub fn rpath_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).and_then(|t| t.rpath).unwrap_or(self.rust_rpath) + } + + pub fn optimized_compiler_builtins(&self, target: TargetSelection) -> bool { + self.target_config + .get(&target) + .and_then(|t| t.optimized_compiler_builtins) + .unwrap_or(self.optimized_compiler_builtins) + } + + pub fn llvm_enabled(&self, target: TargetSelection) -> bool { + self.codegen_backends(target).contains(&"llvm".to_owned()) + } + + pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { + self.target_config + .get(&target) + .and_then(|t| t.llvm_libunwind) + .or(self.llvm_libunwind_default) + .unwrap_or(if target.contains("fuchsia") { + LlvmLibunwind::InTree + } else { + LlvmLibunwind::No + }) + } + + pub fn split_debuginfo(&self, target: TargetSelection) -> SplitDebuginfo { + self.target_config + .get(&target) + .and_then(|t| t.split_debuginfo) + .unwrap_or_else(|| SplitDebuginfo::default_for_platform(target)) + } + /// Checks if the given target is the same as the host target. pub fn is_host_target(&self, target: TargetSelection) -> bool { self.build == target @@ -3313,289 +1743,17 @@ impl Config { } } - pub fn ci_env(&self) -> CiEnv { - if self.is_running_on_ci { CiEnv::GitHubActions } else { CiEnv::None } - } -} - -/// Compares the current `Llvm` options against those in the CI LLVM builder and detects any incompatible options. -/// It does this by destructuring the `Llvm` instance to make sure every `Llvm` field is covered and not missing. -#[cfg(not(test))] -pub(crate) fn check_incompatible_options_for_ci_llvm( - current_config_toml: TomlConfig, - ci_config_toml: TomlConfig, -) -> Result<(), String> { - macro_rules! err { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - return Err(format!( - "ERROR: Setting `llvm.{}` is incompatible with `llvm.download-ci-llvm`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - )); - }; - }; - }; - } - - macro_rules! warn { - ($current:expr, $expected:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - println!( - "WARNING: `llvm.{}` has no effect with `llvm.download-ci-llvm`. \ - Current value: {:?}, Expected value(s): {}{:?}", - stringify!($expected).replace("_", "-"), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - ); - }; - }; - }; - } - - let (Some(current_llvm_config), Some(ci_llvm_config)) = - (current_config_toml.llvm, ci_config_toml.llvm) - else { - return Ok(()); - }; - - let Llvm { - optimize, - thin_lto, - release_debuginfo, - assertions: _, - tests: _, - plugins, - ccache: _, - static_libstdcpp: _, - libzstd, - ninja: _, - targets, - experimental_targets, - link_jobs: _, - link_shared: _, - version_suffix, - clang_cl, - cflags, - cxxflags, - ldflags, - use_libcxx, - use_linker, - allow_old_toolchain, - offload, - polly, - clang, - enable_warnings, - download_ci_llvm: _, - build_config, - enzyme, - } = ci_llvm_config; - - err!(current_llvm_config.optimize, optimize); - err!(current_llvm_config.thin_lto, thin_lto); - err!(current_llvm_config.release_debuginfo, release_debuginfo); - err!(current_llvm_config.libzstd, libzstd); - err!(current_llvm_config.targets, targets); - err!(current_llvm_config.experimental_targets, experimental_targets); - err!(current_llvm_config.clang_cl, clang_cl); - err!(current_llvm_config.version_suffix, version_suffix); - err!(current_llvm_config.cflags, cflags); - err!(current_llvm_config.cxxflags, cxxflags); - err!(current_llvm_config.ldflags, ldflags); - err!(current_llvm_config.use_libcxx, use_libcxx); - err!(current_llvm_config.use_linker, use_linker); - err!(current_llvm_config.allow_old_toolchain, allow_old_toolchain); - err!(current_llvm_config.offload, offload); - err!(current_llvm_config.polly, polly); - err!(current_llvm_config.clang, clang); - err!(current_llvm_config.build_config, build_config); - err!(current_llvm_config.plugins, plugins); - err!(current_llvm_config.enzyme, enzyme); - - warn!(current_llvm_config.enable_warnings, enable_warnings); - - Ok(()) -} - -/// Compares the current Rust options against those in the CI rustc builder and detects any incompatible options. -/// It does this by destructuring the `Rust` instance to make sure every `Rust` field is covered and not missing. -fn check_incompatible_options_for_ci_rustc( - host: TargetSelection, - current_config_toml: TomlConfig, - ci_config_toml: TomlConfig, -) -> Result<(), String> { - macro_rules! err { - ($current:expr, $expected:expr, $config_section:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - return Err(format!( - "ERROR: Setting `{}` is incompatible with `rust.download-rustc`. \ - Current value: {:?}, Expected value(s): {}{:?}", - format!("{}.{}", $config_section, stringify!($expected).replace("_", "-")), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - )); - }; - }; - }; + pub fn exec_ctx(&self) -> &ExecutionContext { + &self.exec_ctx } - macro_rules! warn { - ($current:expr, $expected:expr, $config_section:expr) => { - if let Some(current) = &$current { - if Some(current) != $expected.as_ref() { - println!( - "WARNING: `{}` has no effect with `rust.download-rustc`. \ - Current value: {:?}, Expected value(s): {}{:?}", - format!("{}.{}", $config_section, stringify!($expected).replace("_", "-")), - $current, - if $expected.is_some() { "None/" } else { "" }, - $expected, - ); - }; - }; - }; - } - - let current_profiler = current_config_toml.build.as_ref().and_then(|b| b.profiler); - let profiler = ci_config_toml.build.as_ref().and_then(|b| b.profiler); - err!(current_profiler, profiler, "build"); - - let current_optimized_compiler_builtins = - current_config_toml.build.as_ref().and_then(|b| b.optimized_compiler_builtins); - let optimized_compiler_builtins = - ci_config_toml.build.as_ref().and_then(|b| b.optimized_compiler_builtins); - err!(current_optimized_compiler_builtins, optimized_compiler_builtins, "build"); - - // We always build the in-tree compiler on cross targets, so we only care - // about the host target here. - let host_str = host.to_string(); - if let Some(current_cfg) = current_config_toml.target.as_ref().and_then(|c| c.get(&host_str)) - && current_cfg.profiler.is_some() - { - let ci_target_toml = ci_config_toml.target.as_ref().and_then(|c| c.get(&host_str)); - let ci_cfg = ci_target_toml.ok_or(format!( - "Target specific config for '{host_str}' is not present for CI-rustc" - ))?; - - let profiler = &ci_cfg.profiler; - err!(current_cfg.profiler, profiler, "build"); - - let optimized_compiler_builtins = &ci_cfg.optimized_compiler_builtins; - err!(current_cfg.optimized_compiler_builtins, optimized_compiler_builtins, "build"); - } - - let (Some(current_rust_config), Some(ci_rust_config)) = - (current_config_toml.rust, ci_config_toml.rust) - else { - return Ok(()); - }; - - let Rust { - // Following options are the CI rustc incompatible ones. - optimize, - randomize_layout, - debug_logging, - debuginfo_level_rustc, - llvm_tools, - llvm_bitcode_linker, - lto, - stack_protector, - strip, - lld_mode, - jemalloc, - rpath, - channel, - description, - incremental, - default_linker, - std_features, - - // Rest of the options can simply be ignored. - debug: _, - codegen_units: _, - codegen_units_std: _, - rustc_debug_assertions: _, - std_debug_assertions: _, - tools_debug_assertions: _, - overflow_checks: _, - overflow_checks_std: _, - debuginfo_level: _, - debuginfo_level_std: _, - debuginfo_level_tools: _, - debuginfo_level_tests: _, - backtrace: _, - musl_root: _, - verbose_tests: _, - optimize_tests: _, - codegen_tests: _, - omit_git_hash: _, - dist_src: _, - save_toolstates: _, - codegen_backends: _, - lld: _, - deny_warnings: _, - backtrace_on_ice: _, - verify_llvm_ir: _, - thin_lto_import_instr_limit: _, - remap_debuginfo: _, - test_compare_mode: _, - llvm_libunwind: _, - control_flow_guard: _, - ehcont_guard: _, - new_symbol_mangling: _, - profile_generate: _, - profile_use: _, - download_rustc: _, - validate_mir_opts: _, - frame_pointers: _, - } = ci_rust_config; - - // There are two kinds of checks for CI rustc incompatible options: - // 1. Checking an option that may change the compiler behaviour/output. - // 2. Checking an option that have no effect on the compiler behaviour/output. - // - // If the option belongs to the first category, we call `err` macro for a hard error; - // otherwise, we just print a warning with `warn` macro. - - err!(current_rust_config.optimize, optimize, "rust"); - err!(current_rust_config.randomize_layout, randomize_layout, "rust"); - err!(current_rust_config.debug_logging, debug_logging, "rust"); - err!(current_rust_config.debuginfo_level_rustc, debuginfo_level_rustc, "rust"); - err!(current_rust_config.rpath, rpath, "rust"); - err!(current_rust_config.strip, strip, "rust"); - err!(current_rust_config.lld_mode, lld_mode, "rust"); - err!(current_rust_config.llvm_tools, llvm_tools, "rust"); - err!(current_rust_config.llvm_bitcode_linker, llvm_bitcode_linker, "rust"); - err!(current_rust_config.jemalloc, jemalloc, "rust"); - err!(current_rust_config.default_linker, default_linker, "rust"); - err!(current_rust_config.stack_protector, stack_protector, "rust"); - err!(current_rust_config.lto, lto, "rust"); - err!(current_rust_config.std_features, std_features, "rust"); - - warn!(current_rust_config.channel, channel, "rust"); - warn!(current_rust_config.description, description, "rust"); - warn!(current_rust_config.incremental, incremental, "rust"); - - Ok(()) -} - -fn set<T>(field: &mut T, val: Option<T>) { - if let Some(v) = val { - *field = v; + pub fn git_info(&self, omit_git_hash: bool, dir: &Path) -> GitInfo { + GitInfo::new(omit_git_hash, dir, self) } } -fn threads_from_config(v: u32) -> u32 { - match v { - 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, - n => n, +impl AsRef<ExecutionContext> for Config { + fn as_ref(&self) -> &ExecutionContext { + &self.exec_ctx } } diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs index 08bd87e03a1..bc4fa73a362 100644 --- a/src/bootstrap/src/core/config/flags.rs +++ b/src/bootstrap/src/core/config/flags.rs @@ -12,7 +12,8 @@ use tracing::instrument; use crate::core::build_steps::perf::PerfArgs; use crate::core::build_steps::setup::Profile; use crate::core::builder::{Builder, Kind}; -use crate::core::config::{Config, TargetSelectionList, target_selection_list}; +use crate::core::config::Config; +use crate::core::config::target_selection::{TargetSelectionList, target_selection_list}; use crate::{Build, DocTests}; #[derive(Copy, Clone, Default, Debug, ValueEnum)] @@ -182,6 +183,11 @@ pub struct Flags { /// Make bootstrap to behave as it's running on the CI environment or not. #[arg(global = true, long, value_name = "bool")] pub ci: Option<bool>, + /// Skip checking the standard library if `rust.download-rustc` isn't available. + /// This is mostly for RA as building the stage1 compiler to check the library tree + /// on each code change might be too much for some computers. + #[arg(global = true, long)] + pub skip_std_check_if_no_download_rustc: bool, } impl Flags { @@ -203,7 +209,8 @@ impl Flags { HelpVerboseOnly::try_parse_from(normalize_args(args)) { println!("NOTE: updating submodules before printing available paths"); - let config = Config::parse(Self::parse(&[String::from("build")])); + let flags = Self::parse(&[String::from("build")]); + let config = Config::parse(flags); let build = Build::new(config); let paths = Builder::get_help(&build, subcommand); if let Some(s) = paths { diff --git a/src/bootstrap/src/core/config/mod.rs b/src/bootstrap/src/core/config/mod.rs index 179e15e778b..285d20917e7 100644 --- a/src/bootstrap/src/core/config/mod.rs +++ b/src/bootstrap/src/core/config/mod.rs @@ -1,7 +1,416 @@ +//! Entry point for the `config` module. +//! +//! This module defines two macros: +//! +//! - `define_config!`: A declarative macro used instead of `#[derive(Deserialize)]` to reduce +//! compile time and binary size, especially for the bootstrap binary. +//! +//! - `check_ci_llvm!`: A compile-time assertion macro that ensures certain settings are +//! not enabled when `download-ci-llvm` is active. +//! +//! A declarative macro is used here in place of a procedural derive macro to minimize +//! the compile time of the bootstrap process. +//! +//! Additionally, this module defines common types, enums, and helper functions used across +//! various TOML configuration sections in `bootstrap.toml`. +//! +//! It provides shared definitions for: +//! - Data types deserialized from TOML. +//! - Utility enums for specific configuration options. +//! - Helper functions for managing configuration values. + #[expect(clippy::module_inception)] mod config; pub mod flags; +pub mod target_selection; #[cfg(test)] mod tests; +pub mod toml; + +use std::collections::HashSet; +use std::path::PathBuf; +use build_helper::exit; pub use config::*; +use serde::{Deserialize, Deserializer}; +use serde_derive::Deserialize; +pub use target_selection::TargetSelection; +pub use toml::BUILDER_CONFIG_FILENAME; +pub use toml::change_id::ChangeId; +pub use toml::rust::LldMode; +pub use toml::target::Target; + +use crate::Display; +use crate::str::FromStr; + +// We are using a decl macro instead of a derive proc macro here to reduce the compile time of bootstrap. +#[macro_export] +macro_rules! define_config { + ($(#[$attr:meta])* struct $name:ident { + $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* + }) => { + $(#[$attr])* + pub struct $name { + $(pub $field: Option<$field_ty>,)* + } + + impl Merge for $name { + fn merge( + &mut self, + _parent_config_path: Option<PathBuf>, + _included_extensions: &mut HashSet<PathBuf>, + other: Self, + replace: ReplaceOpt + ) { + $( + match replace { + ReplaceOpt::IgnoreDuplicate => { + if self.$field.is_none() { + self.$field = other.$field; + } + }, + ReplaceOpt::Override => { + if other.$field.is_some() { + self.$field = other.$field; + } + } + ReplaceOpt::ErrorOnDuplicate => { + if other.$field.is_some() { + if self.$field.is_some() { + if cfg!(test) { + panic!("overriding existing option") + } else { + eprintln!("overriding existing option: `{}`", stringify!($field)); + exit!(2); + } + } else { + self.$field = other.$field; + } + } + } + } + )* + } + } + + // The following is a trimmed version of what serde_derive generates. All parts not relevant + // for toml deserialization have been removed. This reduces the binary size and improves + // compile time of bootstrap. + impl<'de> Deserialize<'de> for $name { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: Deserializer<'de>, + { + struct Field; + impl<'de> serde::de::Visitor<'de> for Field { + type Value = $name; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(concat!("struct ", stringify!($name))) + } + + #[inline] + fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error> + where + A: serde::de::MapAccess<'de>, + { + $(let mut $field: Option<$field_ty> = None;)* + while let Some(key) = + match serde::de::MapAccess::next_key::<String>(&mut map) { + Ok(val) => val, + Err(err) => { + return Err(err); + } + } + { + match &*key { + $($field_key => { + if $field.is_some() { + return Err(<A::Error as serde::de::Error>::duplicate_field( + $field_key, + )); + } + $field = match serde::de::MapAccess::next_value::<$field_ty>( + &mut map, + ) { + Ok(val) => Some(val), + Err(err) => { + return Err(err); + } + }; + })* + key => { + return Err(serde::de::Error::unknown_field(key, FIELDS)); + } + } + } + Ok($name { $($field),* }) + } + } + const FIELDS: &'static [&'static str] = &[ + $($field_key,)* + ]; + Deserializer::deserialize_struct( + deserializer, + stringify!($name), + FIELDS, + Field, + ) + } + } + } +} + +#[macro_export] +macro_rules! check_ci_llvm { + ($name:expr) => { + assert!( + $name.is_none(), + "setting {} is incompatible with download-ci-llvm.", + stringify!($name).replace("_", "-") + ); + }; +} + +pub(crate) trait Merge { + fn merge( + &mut self, + parent_config_path: Option<PathBuf>, + included_extensions: &mut HashSet<PathBuf>, + other: Self, + replace: ReplaceOpt, + ); +} + +impl<T> Merge for Option<T> { + fn merge( + &mut self, + _parent_config_path: Option<PathBuf>, + _included_extensions: &mut HashSet<PathBuf>, + other: Self, + replace: ReplaceOpt, + ) { + match replace { + ReplaceOpt::IgnoreDuplicate => { + if self.is_none() { + *self = other; + } + } + ReplaceOpt::Override => { + if other.is_some() { + *self = other; + } + } + ReplaceOpt::ErrorOnDuplicate => { + if other.is_some() { + if self.is_some() { + if cfg!(test) { + panic!("overriding existing option") + } else { + eprintln!("overriding existing option"); + exit!(2); + } + } else { + *self = other; + } + } + } + } + } +} + +#[derive(Copy, Clone, Default, Debug, Eq, PartialEq)] +pub enum DebuginfoLevel { + #[default] + None, + LineDirectivesOnly, + LineTablesOnly, + Limited, + Full, +} + +// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only +// deserializes i64, and derive() only generates visit_u64 +impl<'de> Deserialize<'de> for DebuginfoLevel { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: Deserializer<'de>, + { + use serde::de::Error; + + Ok(match Deserialize::deserialize(deserializer)? { + StringOrInt::String(s) if s == "none" => DebuginfoLevel::None, + StringOrInt::Int(0) => DebuginfoLevel::None, + StringOrInt::String(s) if s == "line-directives-only" => { + DebuginfoLevel::LineDirectivesOnly + } + StringOrInt::String(s) if s == "line-tables-only" => DebuginfoLevel::LineTablesOnly, + StringOrInt::String(s) if s == "limited" => DebuginfoLevel::Limited, + StringOrInt::Int(1) => DebuginfoLevel::Limited, + StringOrInt::String(s) if s == "full" => DebuginfoLevel::Full, + StringOrInt::Int(2) => DebuginfoLevel::Full, + StringOrInt::Int(n) => { + let other = serde::de::Unexpected::Signed(n); + return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); + } + StringOrInt::String(s) => { + let other = serde::de::Unexpected::Str(&s); + return Err(D::Error::invalid_value( + other, + &"expected none, line-tables-only, limited, or full", + )); + } + }) + } +} + +/// Suitable for passing to `-C debuginfo` +impl Display for DebuginfoLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use DebuginfoLevel::*; + f.write_str(match self { + None => "0", + LineDirectivesOnly => "line-directives-only", + LineTablesOnly => "line-tables-only", + Limited => "1", + Full => "2", + }) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum StringOrBool { + String(String), + Bool(bool), +} + +impl Default for StringOrBool { + fn default() -> StringOrBool { + StringOrBool::Bool(false) + } +} + +impl StringOrBool { + pub fn is_string_or_true(&self) -> bool { + matches!(self, Self::String(_) | Self::Bool(true)) + } +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum StringOrInt { + String(String), + Int(i64), +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum LlvmLibunwind { + #[default] + No, + InTree, + System, +} + +impl FromStr for LlvmLibunwind { + type Err = String; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + match value { + "no" => Ok(Self::No), + "in-tree" => Ok(Self::InTree), + "system" => Ok(Self::System), + invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), + } + } +} + +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SplitDebuginfo { + Packed, + Unpacked, + #[default] + Off, +} + +impl std::str::FromStr for SplitDebuginfo { + type Err = (); + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "packed" => Ok(SplitDebuginfo::Packed), + "unpacked" => Ok(SplitDebuginfo::Unpacked), + "off" => Ok(SplitDebuginfo::Off), + _ => Err(()), + } + } +} + +/// Describes how to handle conflicts in merging two `TomlConfig` +#[derive(Copy, Clone, Debug)] +pub enum ReplaceOpt { + /// Silently ignore a duplicated value + IgnoreDuplicate, + /// Override the current value, even if it's `Some` + Override, + /// Exit with an error on duplicate values + ErrorOnDuplicate, +} + +#[derive(Clone, Default)] +pub enum DryRun { + /// This isn't a dry run. + #[default] + Disabled, + /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. + SelfCheck, + /// This is a dry run enabled by the `--dry-run` flag. + UserSelected, +} + +/// LTO mode used for compiling rustc itself. +#[derive(Default, Clone, PartialEq, Debug)] +pub enum RustcLto { + Off, + #[default] + ThinLocal, + Thin, + Fat, +} + +impl std::str::FromStr for RustcLto { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "thin-local" => Ok(RustcLto::ThinLocal), + "thin" => Ok(RustcLto::Thin), + "fat" => Ok(RustcLto::Fat), + "off" => Ok(RustcLto::Off), + _ => Err(format!("Invalid value for rustc LTO: {s}")), + } + } +} + +/// Determines how will GCC be provided. +#[derive(Default, Clone)] +pub enum GccCiMode { + /// Build GCC from the local `src/gcc` submodule. + #[default] + BuildLocally, + /// Try to download GCC from CI. + /// If it is not available on CI, it will be built locally instead. + DownloadFromCi, +} + +pub fn set<T>(field: &mut T, val: Option<T>) { + if let Some(v) = val { + *field = v; + } +} + +pub fn threads_from_config(v: u32) -> u32 { + match v { + 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, + n => n, + } +} diff --git a/src/bootstrap/src/core/config/target_selection.rs b/src/bootstrap/src/core/config/target_selection.rs new file mode 100644 index 00000000000..ebd3fe7a8c6 --- /dev/null +++ b/src/bootstrap/src/core/config/target_selection.rs @@ -0,0 +1,147 @@ +use std::fmt; + +use crate::core::config::SplitDebuginfo; +use crate::utils::cache::{INTERNER, Interned}; +use crate::{Path, env}; + +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +// N.B.: This type is used everywhere, and the entire codebase relies on it being Copy. +// Making !Copy is highly nontrivial! +pub struct TargetSelection { + pub triple: Interned<String>, + pub file: Option<Interned<String>>, + pub synthetic: bool, +} + +/// Newtype over `Vec<TargetSelection>` so we can implement custom parsing logic +#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct TargetSelectionList(pub Vec<TargetSelection>); + +pub fn target_selection_list(s: &str) -> Result<TargetSelectionList, String> { + Ok(TargetSelectionList( + s.split(',').filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), + )) +} + +impl TargetSelection { + pub fn from_user(selection: &str) -> Self { + let path = Path::new(selection); + + let (triple, file) = if path.exists() { + let triple = path + .file_stem() + .expect("Target specification file has no file stem") + .to_str() + .expect("Target specification file stem is not UTF-8"); + + (triple, Some(selection)) + } else { + (selection, None) + }; + + let triple = INTERNER.intern_str(triple); + let file = file.map(|f| INTERNER.intern_str(f)); + + Self { triple, file, synthetic: false } + } + + pub fn create_synthetic(triple: &str, file: &str) -> Self { + Self { + triple: INTERNER.intern_str(triple), + file: Some(INTERNER.intern_str(file)), + synthetic: true, + } + } + + pub fn rustc_target_arg(&self) -> &str { + self.file.as_ref().unwrap_or(&self.triple) + } + + pub fn contains(&self, needle: &str) -> bool { + self.triple.contains(needle) + } + + pub fn starts_with(&self, needle: &str) -> bool { + self.triple.starts_with(needle) + } + + pub fn ends_with(&self, needle: &str) -> bool { + self.triple.ends_with(needle) + } + + // See src/bootstrap/synthetic_targets.rs + pub fn is_synthetic(&self) -> bool { + self.synthetic + } + + pub fn is_msvc(&self) -> bool { + self.contains("msvc") + } + + pub fn is_windows(&self) -> bool { + self.contains("windows") + } + + pub fn is_windows_gnu(&self) -> bool { + self.ends_with("windows-gnu") + } + + pub fn is_cygwin(&self) -> bool { + self.is_windows() && + // ref. https://cygwin.com/pipermail/cygwin/2022-February/250802.html + env::var("OSTYPE").is_ok_and(|v| v.to_lowercase().contains("cygwin")) + } + + pub fn needs_crt_begin_end(&self) -> bool { + self.contains("musl") && !self.contains("unikraft") + } + + /// Path to the file defining the custom target, if any. + pub fn filepath(&self) -> Option<&Path> { + self.file.as_ref().map(Path::new) + } +} + +impl fmt::Display for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.triple)?; + if let Some(file) = self.file { + write!(f, "({file})")?; + } + Ok(()) + } +} + +impl fmt::Debug for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl PartialEq<&str> for TargetSelection { + fn eq(&self, other: &&str) -> bool { + self.triple == *other + } +} + +// Targets are often used as directory names throughout bootstrap. +// This impl makes it more ergonomics to use them as such. +impl AsRef<Path> for TargetSelection { + fn as_ref(&self) -> &Path { + self.triple.as_ref() + } +} + +impl SplitDebuginfo { + /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for + /// `rust.split-debuginfo` in `bootstrap.example.toml`. + pub fn default_for_platform(target: TargetSelection) -> Self { + if target.contains("apple") { + SplitDebuginfo::Unpacked + } else if target.is_windows() { + SplitDebuginfo::Packed + } else { + SplitDebuginfo::Off + } + } +} diff --git a/src/bootstrap/src/core/config/tests.rs b/src/bootstrap/src/core/config/tests.rs index 96ac8a6d52f..50eba12aba7 100644 --- a/src/bootstrap/src/core/config/tests.rs +++ b/src/bootstrap/src/core/config/tests.rs @@ -10,12 +10,14 @@ use clap::CommandFactory; use serde::Deserialize; use super::flags::Flags; -use super::{ChangeIdWrapper, Config, RUSTC_IF_UNCHANGED_ALLOWED_PATHS}; +use super::toml::change_id::ChangeIdWrapper; +use super::{Config, RUSTC_IF_UNCHANGED_ALLOWED_PATHS}; use crate::ChangeId; use crate::core::build_steps::clippy::{LintConfig, get_clippy_rules_in_order}; use crate::core::build_steps::llvm; use crate::core::build_steps::llvm::LLVM_INVALIDATION_PATHS; -use crate::core::config::{LldMode, Target, TargetSelection, TomlConfig}; +use crate::core::config::toml::TomlConfig; +use crate::core::config::{LldMode, Target, TargetSelection}; use crate::utils::tests::git::git_test; pub(crate) fn parse(config: &str) -> Config { diff --git a/src/bootstrap/src/core/config/toml/build.rs b/src/bootstrap/src/core/config/toml/build.rs new file mode 100644 index 00000000000..85ded3c87d9 --- /dev/null +++ b/src/bootstrap/src/core/config/toml/build.rs @@ -0,0 +1,72 @@ +//! This module defines the `Build` struct, which represents the `[build]` table +//! in the `bootstrap.toml` configuration file. +//! +//! The `[build]` table contains global options that influence the overall build process, +//! such as default host and target triples, paths to tools, build directories, and +//! various feature flags. These options apply across different stages and components +//! unless specifically overridden by other configuration sections or command-line flags. + +use serde::{Deserialize, Deserializer}; + +use crate::core::config::toml::ReplaceOpt; +use crate::core::config::{Merge, StringOrBool}; +use crate::{HashSet, PathBuf, define_config, exit}; + +define_config! { + /// TOML representation of various global build decisions. + #[derive(Default)] + struct Build { + build: Option<String> = "build", + description: Option<String> = "description", + host: Option<Vec<String>> = "host", + target: Option<Vec<String>> = "target", + build_dir: Option<String> = "build-dir", + cargo: Option<PathBuf> = "cargo", + rustc: Option<PathBuf> = "rustc", + rustfmt: Option<PathBuf> = "rustfmt", + cargo_clippy: Option<PathBuf> = "cargo-clippy", + docs: Option<bool> = "docs", + compiler_docs: Option<bool> = "compiler-docs", + library_docs_private_items: Option<bool> = "library-docs-private-items", + docs_minification: Option<bool> = "docs-minification", + submodules: Option<bool> = "submodules", + gdb: Option<String> = "gdb", + lldb: Option<String> = "lldb", + nodejs: Option<String> = "nodejs", + npm: Option<String> = "npm", + python: Option<String> = "python", + reuse: Option<String> = "reuse", + locked_deps: Option<bool> = "locked-deps", + vendor: Option<bool> = "vendor", + full_bootstrap: Option<bool> = "full-bootstrap", + bootstrap_cache_path: Option<PathBuf> = "bootstrap-cache-path", + extended: Option<bool> = "extended", + tools: Option<HashSet<String>> = "tools", + verbose: Option<usize> = "verbose", + sanitizers: Option<bool> = "sanitizers", + profiler: Option<bool> = "profiler", + cargo_native_static: Option<bool> = "cargo-native-static", + low_priority: Option<bool> = "low-priority", + configure_args: Option<Vec<String>> = "configure-args", + local_rebuild: Option<bool> = "local-rebuild", + print_step_timings: Option<bool> = "print-step-timings", + print_step_rusage: Option<bool> = "print-step-rusage", + check_stage: Option<u32> = "check-stage", + doc_stage: Option<u32> = "doc-stage", + build_stage: Option<u32> = "build-stage", + test_stage: Option<u32> = "test-stage", + install_stage: Option<u32> = "install-stage", + dist_stage: Option<u32> = "dist-stage", + bench_stage: Option<u32> = "bench-stage", + patch_binaries_for_nix: Option<bool> = "patch-binaries-for-nix", + // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally + metrics: Option<bool> = "metrics", + android_ndk: Option<PathBuf> = "android-ndk", + optimized_compiler_builtins: Option<bool> = "optimized-compiler-builtins", + jobs: Option<u32> = "jobs", + compiletest_diff_tool: Option<String> = "compiletest-diff-tool", + compiletest_use_stage0_libtest: Option<bool> = "compiletest-use-stage0-libtest", + ccache: Option<StringOrBool> = "ccache", + exclude: Option<Vec<PathBuf>> = "exclude", + } +} diff --git a/src/bootstrap/src/core/config/toml/change_id.rs b/src/bootstrap/src/core/config/toml/change_id.rs new file mode 100644 index 00000000000..41dd2531d43 --- /dev/null +++ b/src/bootstrap/src/core/config/toml/change_id.rs @@ -0,0 +1,34 @@ +use serde::{Deserialize, Deserializer}; +use serde_derive::Deserialize; + +/// This enum is used for deserializing change IDs from TOML, allowing both numeric values and the string `"ignore"`. +#[derive(Clone, Debug, PartialEq)] +pub enum ChangeId { + Ignore, + Id(usize), +} + +/// Since we use `#[serde(deny_unknown_fields)]` on `TomlConfig`, we need a wrapper type +/// for the "change-id" field to parse it even if other fields are invalid. This ensures +/// that if deserialization fails due to other fields, we can still provide the changelogs +/// to allow developers to potentially find the reason for the failure in the logs.. +#[derive(Deserialize, Default)] +pub(crate) struct ChangeIdWrapper { + #[serde(alias = "change-id", default, deserialize_with = "deserialize_change_id")] + pub(crate) inner: Option<ChangeId>, +} + +fn deserialize_change_id<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result<Option<ChangeId>, D::Error> { + let value = toml::Value::deserialize(deserializer)?; + Ok(match value { + toml::Value::String(s) if s == "ignore" => Some(ChangeId::Ignore), + toml::Value::Integer(i) => Some(ChangeId::Id(i as usize)), + _ => { + return Err(serde::de::Error::custom( + "expected \"ignore\" or an integer for change-id", + )); + } + }) +} diff --git a/src/bootstrap/src/core/config/toml/dist.rs b/src/bootstrap/src/core/config/toml/dist.rs new file mode 100644 index 00000000000..b1429ef1861 --- /dev/null +++ b/src/bootstrap/src/core/config/toml/dist.rs @@ -0,0 +1,52 @@ +//! This module defines the `Dist` struct, which represents the `[dist]` table +//! in the `bootstrap.toml` configuration file. +//! +//! The `[dist]` table contains options related to the distribution process, +//! including signing, uploading artifacts, source tarballs, compression settings, +//! and inclusion of specific tools. + +use serde::{Deserialize, Deserializer}; + +use crate::core::config::toml::ReplaceOpt; +use crate::core::config::{Merge, set}; +use crate::{Config, HashSet, PathBuf, define_config, exit}; + +define_config! { + struct Dist { + sign_folder: Option<String> = "sign-folder", + upload_addr: Option<String> = "upload-addr", + src_tarball: Option<bool> = "src-tarball", + compression_formats: Option<Vec<String>> = "compression-formats", + compression_profile: Option<String> = "compression-profile", + include_mingw_linker: Option<bool> = "include-mingw-linker", + vendor: Option<bool> = "vendor", + } +} + +impl Config { + /// Applies distribution-related configuration from the `Dist` struct + /// to the global `Config` structure. + pub fn apply_dist_config(&mut self, toml_dist: Option<Dist>) { + if let Some(dist) = toml_dist { + let Dist { + sign_folder, + upload_addr, + src_tarball, + compression_formats, + compression_profile, + include_mingw_linker, + vendor, + } = dist; + self.dist_sign_folder = sign_folder.map(PathBuf::from); + self.dist_upload_addr = upload_addr; + self.dist_compression_formats = compression_formats; + set(&mut self.dist_compression_profile, compression_profile); + set(&mut self.rust_dist_src, src_tarball); + set(&mut self.dist_include_mingw_linker, include_mingw_linker); + self.dist_vendor = vendor.unwrap_or_else(|| { + // If we're building from git or tarball sources, enable it by default. + self.rust_info.is_managed_git_subrepository() || self.rust_info.is_from_tarball() + }); + } + } +} diff --git a/src/bootstrap/src/core/config/toml/gcc.rs b/src/bootstrap/src/core/config/toml/gcc.rs new file mode 100644 index 00000000000..bb817c2aaef --- /dev/null +++ b/src/bootstrap/src/core/config/toml/gcc.rs @@ -0,0 +1,34 @@ +//! This module defines the `Gcc` struct, which represents the `[gcc]` table +//! in the `bootstrap.toml` configuration file. +//! +//! The `[gcc]` table contains options specifically related to building or +//! acquiring the GCC compiler for use within the Rust build process. + +use serde::{Deserialize, Deserializer}; + +use crate::core::config::toml::ReplaceOpt; +use crate::core::config::{GccCiMode, Merge}; +use crate::{Config, HashSet, PathBuf, define_config, exit}; + +define_config! { + /// TOML representation of how the GCC build is configured. + struct Gcc { + download_ci_gcc: Option<bool> = "download-ci-gcc", + } +} + +impl Config { + /// Applies GCC-related configuration from the `TomlGcc` struct to the + /// global `Config` structure. + pub fn apply_gcc_config(&mut self, toml_gcc: Option<Gcc>) { + if let Some(gcc) = toml_gcc { + self.gcc_ci_mode = match gcc.download_ci_gcc { + Some(value) => match value { + true => GccCiMode::DownloadFromCi, + false => GccCiMode::BuildLocally, + }, + None => GccCiMode::default(), + }; + } + } +} diff --git a/src/bootstrap/src/core/config/toml/install.rs b/src/bootstrap/src/core/config/toml/install.rs new file mode 100644 index 00000000000..6b9ab87a0b6 --- /dev/null +++ b/src/bootstrap/src/core/config/toml/install.rs @@ -0,0 +1,43 @@ +//! This module defines the `Install` struct, which represents the `[install]` table +//! in the `bootstrap.toml` configuration file. +//! +//! The `[install]` table contains options that specify the installation paths +//! for various components of the Rust toolchain. These paths determine where +//! executables, libraries, documentation, and other files will be placed +//! during the `install` stage of the build. + +use serde::{Deserialize, Deserializer}; + +use crate::core::config::toml::ReplaceOpt; +use crate::core::config::{Merge, set}; +use crate::{Config, HashSet, PathBuf, define_config, exit}; + +define_config! { + /// TOML representation of various global install decisions. + struct Install { + prefix: Option<String> = "prefix", + sysconfdir: Option<String> = "sysconfdir", + docdir: Option<String> = "docdir", + bindir: Option<String> = "bindir", + libdir: Option<String> = "libdir", + mandir: Option<String> = "mandir", + datadir: Option<String> = "datadir", + } +} + +impl Config { + /// Applies installation-related configuration from the `Install` struct + /// to the global `Config` structure. + pub fn apply_install_config(&mut self, toml_install: Option<Install>) { + if let Some(install) = toml_install { + let Install { prefix, sysconfdir, docdir, bindir, libdir, mandir, datadir } = install; + self.prefix = prefix.map(PathBuf::from); + self.sysconfdir = sysconfdir.map(PathBuf::from); + self.datadir = datadir.map(PathBuf::from); + self.docdir = docdir.map(PathBuf::from); + set(&mut self.bindir, bindir.map(PathBuf::from)); + self.libdir = libdir.map(PathBuf::from); + self.mandir = mandir.map(PathBuf::from); + } + } +} diff --git a/src/bootstrap/src/core/config/toml/llvm.rs b/src/bootstrap/src/core/config/toml/llvm.rs new file mode 100644 index 00000000000..4774e202bd8 --- /dev/null +++ b/src/bootstrap/src/core/config/toml/llvm.rs @@ -0,0 +1,284 @@ +//! This module defines the `Llvm` struct, which represents the `[llvm]` table +//! in the `bootstrap.toml` configuration file. + +use serde::{Deserialize, Deserializer}; + +use crate::core::config::toml::{Merge, ReplaceOpt, TomlConfig}; +use crate::core::config::{StringOrBool, set}; +use crate::{Config, HashMap, HashSet, PathBuf, define_config, exit}; + +define_config! { + /// TOML representation of how the LLVM build is configured. + struct Llvm { + optimize: Option<bool> = "optimize", + thin_lto: Option<bool> = "thin-lto", + release_debuginfo: Option<bool> = "release-debuginfo", + assertions: Option<bool> = "assertions", + tests: Option<bool> = "tests", + enzyme: Option<bool> = "enzyme", + plugins: Option<bool> = "plugins", + // FIXME: Remove this field at Q2 2025, it has been replaced by build.ccache + ccache: Option<StringOrBool> = "ccache", + static_libstdcpp: Option<bool> = "static-libstdcpp", + libzstd: Option<bool> = "libzstd", + ninja: Option<bool> = "ninja", + targets: Option<String> = "targets", + experimental_targets: Option<String> = "experimental-targets", + link_jobs: Option<u32> = "link-jobs", + link_shared: Option<bool> = "link-shared", + version_suffix: Option<String> = "version-suffix", + clang_cl: Option<String> = "clang-cl", + cflags: Option<String> = "cflags", + cxxflags: Option<String> = "cxxflags", + ldflags: Option<String> = "ldflags", + use_libcxx: Option<bool> = "use-libcxx", + use_linker: Option<String> = "use-linker", + allow_old_toolchain: Option<bool> = "allow-old-toolchain", + offload: Option<bool> = "offload", + polly: Option<bool> = "polly", + clang: Option<bool> = "clang", + enable_warnings: Option<bool> = "enable-warnings", + download_ci_llvm: Option<StringOrBool> = "download-ci-llvm", + build_config: Option<HashMap<String, String>> = "build-config", + } +} + +/// Compares the current `Llvm` options against those in the CI LLVM builder and detects any incompatible options. +/// It does this by destructuring the `Llvm` instance to make sure every `Llvm` field is covered and not missing. +#[cfg(not(test))] +pub fn check_incompatible_options_for_ci_llvm( + current_config_toml: TomlConfig, + ci_config_toml: TomlConfig, +) -> Result<(), String> { + macro_rules! err { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + return Err(format!( + "ERROR: Setting `llvm.{}` is incompatible with `llvm.download-ci-llvm`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + )); + }; + }; + }; + } + + macro_rules! warn { + ($current:expr, $expected:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + println!( + "WARNING: `llvm.{}` has no effect with `llvm.download-ci-llvm`. \ + Current value: {:?}, Expected value(s): {}{:?}", + stringify!($expected).replace("_", "-"), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + ); + }; + }; + }; + } + + let (Some(current_llvm_config), Some(ci_llvm_config)) = + (current_config_toml.llvm, ci_config_toml.llvm) + else { + return Ok(()); + }; + + let Llvm { + optimize, + thin_lto, + release_debuginfo, + assertions: _, + tests: _, + plugins, + ccache: _, + static_libstdcpp: _, + libzstd, + ninja: _, + targets, + experimental_targets, + link_jobs: _, + link_shared: _, + version_suffix, + clang_cl, + cflags, + cxxflags, + ldflags, + use_libcxx, + use_linker, + allow_old_toolchain, + offload, + polly, + clang, + enable_warnings, + download_ci_llvm: _, + build_config, + enzyme, + } = ci_llvm_config; + + err!(current_llvm_config.optimize, optimize); + err!(current_llvm_config.thin_lto, thin_lto); + err!(current_llvm_config.release_debuginfo, release_debuginfo); + err!(current_llvm_config.libzstd, libzstd); + err!(current_llvm_config.targets, targets); + err!(current_llvm_config.experimental_targets, experimental_targets); + err!(current_llvm_config.clang_cl, clang_cl); + err!(current_llvm_config.version_suffix, version_suffix); + err!(current_llvm_config.cflags, cflags); + err!(current_llvm_config.cxxflags, cxxflags); + err!(current_llvm_config.ldflags, ldflags); + err!(current_llvm_config.use_libcxx, use_libcxx); + err!(current_llvm_config.use_linker, use_linker); + err!(current_llvm_config.allow_old_toolchain, allow_old_toolchain); + err!(current_llvm_config.offload, offload); + err!(current_llvm_config.polly, polly); + err!(current_llvm_config.clang, clang); + err!(current_llvm_config.build_config, build_config); + err!(current_llvm_config.plugins, plugins); + err!(current_llvm_config.enzyme, enzyme); + + warn!(current_llvm_config.enable_warnings, enable_warnings); + + Ok(()) +} + +impl Config { + pub fn apply_llvm_config( + &mut self, + toml_llvm: Option<Llvm>, + ccache: &mut Option<StringOrBool>, + ) { + let mut llvm_tests = None; + let mut llvm_enzyme = None; + let mut llvm_offload = None; + let mut llvm_plugins = None; + + if let Some(llvm) = toml_llvm { + let Llvm { + optimize: optimize_toml, + thin_lto, + release_debuginfo, + assertions: _, + tests, + enzyme, + plugins, + ccache: llvm_ccache, + static_libstdcpp, + libzstd, + ninja, + targets, + experimental_targets, + link_jobs, + link_shared, + version_suffix, + clang_cl, + cflags, + cxxflags, + ldflags, + use_libcxx, + use_linker, + allow_old_toolchain, + offload, + polly, + clang, + enable_warnings, + download_ci_llvm, + build_config, + } = llvm; + if llvm_ccache.is_some() { + eprintln!("Warning: llvm.ccache is deprecated. Use build.ccache instead."); + } + + if ccache.is_none() { + *ccache = llvm_ccache; + } + set(&mut self.ninja_in_file, ninja); + llvm_tests = tests; + llvm_enzyme = enzyme; + llvm_offload = offload; + llvm_plugins = plugins; + set(&mut self.llvm_optimize, optimize_toml); + set(&mut self.llvm_thin_lto, thin_lto); + set(&mut self.llvm_release_debuginfo, release_debuginfo); + set(&mut self.llvm_static_stdcpp, static_libstdcpp); + set(&mut self.llvm_libzstd, libzstd); + if let Some(v) = link_shared { + self.llvm_link_shared.set(Some(v)); + } + self.llvm_targets.clone_from(&targets); + self.llvm_experimental_targets.clone_from(&experimental_targets); + self.llvm_link_jobs = link_jobs; + self.llvm_version_suffix.clone_from(&version_suffix); + self.llvm_clang_cl.clone_from(&clang_cl); + + self.llvm_cflags.clone_from(&cflags); + self.llvm_cxxflags.clone_from(&cxxflags); + self.llvm_ldflags.clone_from(&ldflags); + set(&mut self.llvm_use_libcxx, use_libcxx); + self.llvm_use_linker.clone_from(&use_linker); + self.llvm_allow_old_toolchain = allow_old_toolchain.unwrap_or(false); + self.llvm_offload = offload.unwrap_or(false); + self.llvm_polly = polly.unwrap_or(false); + self.llvm_clang = clang.unwrap_or(false); + self.llvm_enable_warnings = enable_warnings.unwrap_or(false); + self.llvm_build_config = build_config.clone().unwrap_or(Default::default()); + + self.llvm_from_ci = self.parse_download_ci_llvm(download_ci_llvm, self.llvm_assertions); + + if self.llvm_from_ci { + let warn = |option: &str| { + println!( + "WARNING: `{option}` will only be used on `compiler/rustc_llvm` build, not for the LLVM build." + ); + println!( + "HELP: To use `{option}` for LLVM builds, set `download-ci-llvm` option to false." + ); + }; + + if static_libstdcpp.is_some() { + warn("static-libstdcpp"); + } + + if link_shared.is_some() { + warn("link-shared"); + } + + // FIXME(#129153): instead of all the ad-hoc `download-ci-llvm` checks that follow, + // use the `builder-config` present in tarballs since #128822 to compare the local + // config to the ones used to build the LLVM artifacts on CI, and only notify users + // if they've chosen a different value. + + if libzstd.is_some() { + println!( + "WARNING: when using `download-ci-llvm`, the local `llvm.libzstd` option, \ + like almost all `llvm.*` options, will be ignored and set by the LLVM CI \ + artifacts builder config." + ); + println!( + "HELP: To use `llvm.libzstd` for LLVM/LLD builds, set `download-ci-llvm` option to false." + ); + } + } + + if !self.llvm_from_ci && self.llvm_thin_lto && link_shared.is_none() { + // If we're building with ThinLTO on, by default we want to link + // to LLVM shared, to avoid re-doing ThinLTO (which happens in + // the link step) with each stage. + self.llvm_link_shared.set(Some(true)); + } + } else { + self.llvm_from_ci = self.parse_download_ci_llvm(None, false); + } + + self.llvm_tests = llvm_tests.unwrap_or(false); + self.llvm_enzyme = llvm_enzyme.unwrap_or(false); + self.llvm_offload = llvm_offload.unwrap_or(false); + self.llvm_plugins = llvm_plugins.unwrap_or(false); + } +} diff --git a/src/bootstrap/src/core/config/toml/mod.rs b/src/bootstrap/src/core/config/toml/mod.rs new file mode 100644 index 00000000000..ac4e249e580 --- /dev/null +++ b/src/bootstrap/src/core/config/toml/mod.rs @@ -0,0 +1,184 @@ +//! This module defines the structures that directly mirror the `bootstrap.toml` +//! file's format. These types are used for `serde` deserialization. +//! +//! Crucially, this module also houses the core logic for loading, parsing, and merging +//! these raw TOML configurations from various sources (the main `bootstrap.toml`, +//! included files, profile defaults, and command-line overrides). This processed +//! TOML data then serves as an intermediate representation, which is further +//! transformed and applied to the final [`Config`] struct. + +use serde::Deserialize; +use serde_derive::Deserialize; +pub mod build; +pub mod change_id; +pub mod dist; +pub mod gcc; +pub mod install; +pub mod llvm; +pub mod rust; +pub mod target; + +use build::Build; +use change_id::{ChangeId, ChangeIdWrapper}; +use dist::Dist; +use gcc::Gcc; +use install::Install; +use llvm::Llvm; +use rust::Rust; +use target::TomlTarget; + +use crate::core::config::{Merge, ReplaceOpt}; +use crate::{Config, HashMap, HashSet, Path, PathBuf, exit, fs, t}; + +/// Structure of the `bootstrap.toml` file that configuration is read from. +/// +/// This structure uses `Decodable` to automatically decode a TOML configuration +/// file into this format, and then this is traversed and written into the above +/// `Config` structure. +#[derive(Deserialize, Default)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub(crate) struct TomlConfig { + #[serde(flatten)] + pub(crate) change_id: ChangeIdWrapper, + pub(super) build: Option<Build>, + pub(super) install: Option<Install>, + pub(super) llvm: Option<Llvm>, + pub(super) gcc: Option<Gcc>, + pub(super) rust: Option<Rust>, + pub(super) target: Option<HashMap<String, TomlTarget>>, + pub(super) dist: Option<Dist>, + pub(super) profile: Option<String>, + pub(super) include: Option<Vec<PathBuf>>, +} + +impl Merge for TomlConfig { + fn merge( + &mut self, + parent_config_path: Option<PathBuf>, + included_extensions: &mut HashSet<PathBuf>, + TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id, include }: Self, + replace: ReplaceOpt, + ) { + fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) { + if let Some(new) = y { + if let Some(original) = x { + original.merge(None, &mut Default::default(), new, replace); + } else { + *x = Some(new); + } + } + } + + self.change_id.inner.merge(None, &mut Default::default(), change_id.inner, replace); + self.profile.merge(None, &mut Default::default(), profile, replace); + + do_merge(&mut self.build, build, replace); + do_merge(&mut self.install, install, replace); + do_merge(&mut self.llvm, llvm, replace); + do_merge(&mut self.gcc, gcc, replace); + do_merge(&mut self.rust, rust, replace); + do_merge(&mut self.dist, dist, replace); + + match (self.target.as_mut(), target) { + (_, None) => {} + (None, Some(target)) => self.target = Some(target), + (Some(original_target), Some(new_target)) => { + for (triple, new) in new_target { + if let Some(original) = original_target.get_mut(&triple) { + original.merge(None, &mut Default::default(), new, replace); + } else { + original_target.insert(triple, new); + } + } + } + } + + let parent_dir = parent_config_path + .as_ref() + .and_then(|p| p.parent().map(ToOwned::to_owned)) + .unwrap_or_default(); + + // `include` handled later since we ignore duplicates using `ReplaceOpt::IgnoreDuplicate` to + // keep the upper-level configuration to take precedence. + for include_path in include.clone().unwrap_or_default().iter().rev() { + let include_path = parent_dir.join(include_path); + let include_path = include_path.canonicalize().unwrap_or_else(|e| { + eprintln!("ERROR: Failed to canonicalize '{}' path: {e}", include_path.display()); + exit!(2); + }); + + let included_toml = Config::get_toml_inner(&include_path).unwrap_or_else(|e| { + eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display()); + exit!(2); + }); + + assert!( + included_extensions.insert(include_path.clone()), + "Cyclic inclusion detected: '{}' is being included again before its previous inclusion was fully processed.", + include_path.display() + ); + + self.merge( + Some(include_path.clone()), + included_extensions, + included_toml, + // Ensures that parent configuration always takes precedence + // over child configurations. + ReplaceOpt::IgnoreDuplicate, + ); + + included_extensions.remove(&include_path); + } + } +} + +/// This file is embedded in the overlay directory of the tarball sources. It is +/// useful in scenarios where developers want to see how the tarball sources were +/// generated. +/// +/// We also use this file to compare the host's bootstrap.toml against the CI rustc builder +/// configuration to detect any incompatible options. +pub const BUILDER_CONFIG_FILENAME: &str = "builder-config"; + +impl Config { + pub(crate) fn get_builder_toml(&self, build_name: &str) -> Result<TomlConfig, toml::de::Error> { + if self.dry_run() { + return Ok(TomlConfig::default()); + } + + let builder_config_path = + self.out.join(self.build.triple).join(build_name).join(BUILDER_CONFIG_FILENAME); + Self::get_toml(&builder_config_path) + } + + pub(crate) fn get_toml(file: &Path) -> Result<TomlConfig, toml::de::Error> { + #[cfg(test)] + return Ok(TomlConfig::default()); + + #[cfg(not(test))] + Self::get_toml_inner(file) + } + + pub(crate) fn get_toml_inner(file: &Path) -> Result<TomlConfig, toml::de::Error> { + let contents = + t!(fs::read_to_string(file), format!("config file {} not found", file.display())); + // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of + // TomlConfig and sub types to be monomorphized 5x by toml. + toml::from_str(&contents) + .and_then(|table: toml::Value| TomlConfig::deserialize(table)) + .inspect_err(|_| { + if let Ok(ChangeIdWrapper { inner: Some(ChangeId::Id(id)) }) = + toml::from_str::<toml::Value>(&contents) + .and_then(|table: toml::Value| ChangeIdWrapper::deserialize(table)) + { + let changes = crate::find_recent_config_change_ids(id); + if !changes.is_empty() { + println!( + "WARNING: There have been changes to x.py since you last updated:\n{}", + crate::human_readable_changes(changes) + ); + } + } + }) + } +} diff --git a/src/bootstrap/src/core/config/toml/rust.rs b/src/bootstrap/src/core/config/toml/rust.rs new file mode 100644 index 00000000000..bb18fa802af --- /dev/null +++ b/src/bootstrap/src/core/config/toml/rust.rs @@ -0,0 +1,663 @@ +//! This module defines the `Rust` struct, which represents the `[rust]` table +//! in the `bootstrap.toml` configuration file. + +use std::str::FromStr; + +use serde::{Deserialize, Deserializer}; + +use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; +use crate::core::config::toml::TomlConfig; +use crate::core::config::{ + DebuginfoLevel, Merge, ReplaceOpt, RustcLto, StringOrBool, set, threads_from_config, +}; +use crate::flags::Warnings; +use crate::{BTreeSet, Config, HashSet, PathBuf, TargetSelection, define_config, exit}; + +define_config! { + /// TOML representation of how the Rust build is configured. + struct Rust { + optimize: Option<RustOptimize> = "optimize", + debug: Option<bool> = "debug", + codegen_units: Option<u32> = "codegen-units", + codegen_units_std: Option<u32> = "codegen-units-std", + rustc_debug_assertions: Option<bool> = "debug-assertions", + randomize_layout: Option<bool> = "randomize-layout", + std_debug_assertions: Option<bool> = "debug-assertions-std", + tools_debug_assertions: Option<bool> = "debug-assertions-tools", + overflow_checks: Option<bool> = "overflow-checks", + overflow_checks_std: Option<bool> = "overflow-checks-std", + debug_logging: Option<bool> = "debug-logging", + debuginfo_level: Option<DebuginfoLevel> = "debuginfo-level", + debuginfo_level_rustc: Option<DebuginfoLevel> = "debuginfo-level-rustc", + debuginfo_level_std: Option<DebuginfoLevel> = "debuginfo-level-std", + debuginfo_level_tools: Option<DebuginfoLevel> = "debuginfo-level-tools", + debuginfo_level_tests: Option<DebuginfoLevel> = "debuginfo-level-tests", + backtrace: Option<bool> = "backtrace", + incremental: Option<bool> = "incremental", + default_linker: Option<String> = "default-linker", + channel: Option<String> = "channel", + // FIXME: Remove this field at Q2 2025, it has been replaced by build.description + description: Option<String> = "description", + musl_root: Option<String> = "musl-root", + rpath: Option<bool> = "rpath", + strip: Option<bool> = "strip", + frame_pointers: Option<bool> = "frame-pointers", + stack_protector: Option<String> = "stack-protector", + verbose_tests: Option<bool> = "verbose-tests", + optimize_tests: Option<bool> = "optimize-tests", + codegen_tests: Option<bool> = "codegen-tests", + omit_git_hash: Option<bool> = "omit-git-hash", + dist_src: Option<bool> = "dist-src", + save_toolstates: Option<String> = "save-toolstates", + codegen_backends: Option<Vec<String>> = "codegen-backends", + llvm_bitcode_linker: Option<bool> = "llvm-bitcode-linker", + lld: Option<bool> = "lld", + lld_mode: Option<LldMode> = "use-lld", + llvm_tools: Option<bool> = "llvm-tools", + deny_warnings: Option<bool> = "deny-warnings", + backtrace_on_ice: Option<bool> = "backtrace-on-ice", + verify_llvm_ir: Option<bool> = "verify-llvm-ir", + thin_lto_import_instr_limit: Option<u32> = "thin-lto-import-instr-limit", + remap_debuginfo: Option<bool> = "remap-debuginfo", + jemalloc: Option<bool> = "jemalloc", + test_compare_mode: Option<bool> = "test-compare-mode", + llvm_libunwind: Option<String> = "llvm-libunwind", + control_flow_guard: Option<bool> = "control-flow-guard", + ehcont_guard: Option<bool> = "ehcont-guard", + new_symbol_mangling: Option<bool> = "new-symbol-mangling", + profile_generate: Option<String> = "profile-generate", + profile_use: Option<String> = "profile-use", + // ignored; this is set from an env var set by bootstrap.py + download_rustc: Option<StringOrBool> = "download-rustc", + lto: Option<String> = "lto", + validate_mir_opts: Option<u32> = "validate-mir-opts", + std_features: Option<BTreeSet<String>> = "std-features", + } +} + +/// LLD in bootstrap works like this: +/// - Self-contained lld: use `rust-lld` from the compiler's sysroot +/// - External: use an external `lld` binary +/// +/// It is configured depending on the target: +/// 1) Everything except MSVC +/// - Self-contained: `-Clinker-flavor=gnu-lld-cc -Clink-self-contained=+linker` +/// - External: `-Clinker-flavor=gnu-lld-cc` +/// 2) MSVC +/// - Self-contained: `-Clinker=<path to rust-lld>` +/// - External: `-Clinker=lld` +#[derive(Copy, Clone, Default, Debug, PartialEq)] +pub enum LldMode { + /// Do not use LLD + #[default] + Unused, + /// Use `rust-lld` from the compiler's sysroot + SelfContained, + /// Use an externally provided `lld` binary. + /// Note that the linker name cannot be overridden, the binary has to be named `lld` and it has + /// to be in $PATH. + External, +} + +impl LldMode { + pub fn is_used(&self) -> bool { + match self { + LldMode::SelfContained | LldMode::External => true, + LldMode::Unused => false, + } + } +} + +impl<'de> Deserialize<'de> for LldMode { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: Deserializer<'de>, + { + struct LldModeVisitor; + + impl serde::de::Visitor<'_> for LldModeVisitor { + type Value = LldMode; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("one of true, 'self-contained' or 'external'") + } + + fn visit_bool<E>(self, v: bool) -> Result<Self::Value, E> + where + E: serde::de::Error, + { + Ok(if v { LldMode::External } else { LldMode::Unused }) + } + + fn visit_str<E>(self, v: &str) -> Result<Self::Value, E> + where + E: serde::de::Error, + { + match v { + "external" => Ok(LldMode::External), + "self-contained" => Ok(LldMode::SelfContained), + _ => Err(E::custom(format!("unknown mode {v}"))), + } + } + } + + deserializer.deserialize_any(LldModeVisitor) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RustOptimize { + String(String), + Int(u8), + Bool(bool), +} + +impl Default for RustOptimize { + fn default() -> RustOptimize { + RustOptimize::Bool(false) + } +} + +impl<'de> Deserialize<'de> for RustOptimize { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(OptimizeVisitor) + } +} + +struct OptimizeVisitor; + +impl serde::de::Visitor<'_> for OptimizeVisitor { + type Value = RustOptimize; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) + } + + fn visit_str<E>(self, value: &str) -> Result<Self::Value, E> + where + E: serde::de::Error, + { + if matches!(value, "s" | "z") { + Ok(RustOptimize::String(value.to_string())) + } else { + Err(serde::de::Error::custom(format_optimize_error_msg(value))) + } + } + + fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E> + where + E: serde::de::Error, + { + if matches!(value, 0..=3) { + Ok(RustOptimize::Int(value as u8)) + } else { + Err(serde::de::Error::custom(format_optimize_error_msg(value))) + } + } + + fn visit_bool<E>(self, value: bool) -> Result<Self::Value, E> + where + E: serde::de::Error, + { + Ok(RustOptimize::Bool(value)) + } +} + +fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { + format!( + r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# + ) +} + +impl RustOptimize { + pub(crate) fn is_release(&self) -> bool { + match &self { + RustOptimize::Bool(true) | RustOptimize::String(_) => true, + RustOptimize::Int(i) => *i > 0, + RustOptimize::Bool(false) => false, + } + } + + pub(crate) fn get_opt_level(&self) -> Option<String> { + match &self { + RustOptimize::String(s) => Some(s.clone()), + RustOptimize::Int(i) => Some(i.to_string()), + RustOptimize::Bool(_) => None, + } + } +} + +/// Compares the current Rust options against those in the CI rustc builder and detects any incompatible options. +/// It does this by destructuring the `Rust` instance to make sure every `Rust` field is covered and not missing. +pub fn check_incompatible_options_for_ci_rustc( + host: TargetSelection, + current_config_toml: TomlConfig, + ci_config_toml: TomlConfig, +) -> Result<(), String> { + macro_rules! err { + ($current:expr, $expected:expr, $config_section:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + return Err(format!( + "ERROR: Setting `{}` is incompatible with `rust.download-rustc`. \ + Current value: {:?}, Expected value(s): {}{:?}", + format!("{}.{}", $config_section, stringify!($expected).replace("_", "-")), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + )); + }; + }; + }; + } + + macro_rules! warn { + ($current:expr, $expected:expr, $config_section:expr) => { + if let Some(current) = &$current { + if Some(current) != $expected.as_ref() { + println!( + "WARNING: `{}` has no effect with `rust.download-rustc`. \ + Current value: {:?}, Expected value(s): {}{:?}", + format!("{}.{}", $config_section, stringify!($expected).replace("_", "-")), + $current, + if $expected.is_some() { "None/" } else { "" }, + $expected, + ); + }; + }; + }; + } + + let current_profiler = current_config_toml.build.as_ref().and_then(|b| b.profiler); + let profiler = ci_config_toml.build.as_ref().and_then(|b| b.profiler); + err!(current_profiler, profiler, "build"); + + let current_optimized_compiler_builtins = + current_config_toml.build.as_ref().and_then(|b| b.optimized_compiler_builtins); + let optimized_compiler_builtins = + ci_config_toml.build.as_ref().and_then(|b| b.optimized_compiler_builtins); + err!(current_optimized_compiler_builtins, optimized_compiler_builtins, "build"); + + // We always build the in-tree compiler on cross targets, so we only care + // about the host target here. + let host_str = host.to_string(); + if let Some(current_cfg) = current_config_toml.target.as_ref().and_then(|c| c.get(&host_str)) + && current_cfg.profiler.is_some() + { + let ci_target_toml = ci_config_toml.target.as_ref().and_then(|c| c.get(&host_str)); + let ci_cfg = ci_target_toml.ok_or(format!( + "Target specific config for '{host_str}' is not present for CI-rustc" + ))?; + + let profiler = &ci_cfg.profiler; + err!(current_cfg.profiler, profiler, "build"); + + let optimized_compiler_builtins = &ci_cfg.optimized_compiler_builtins; + err!(current_cfg.optimized_compiler_builtins, optimized_compiler_builtins, "build"); + } + + let (Some(current_rust_config), Some(ci_rust_config)) = + (current_config_toml.rust, ci_config_toml.rust) + else { + return Ok(()); + }; + + let Rust { + // Following options are the CI rustc incompatible ones. + optimize, + randomize_layout, + debug_logging, + debuginfo_level_rustc, + llvm_tools, + llvm_bitcode_linker, + lto, + stack_protector, + strip, + lld_mode, + jemalloc, + rpath, + channel, + description, + default_linker, + std_features, + + // Rest of the options can simply be ignored. + incremental: _, + debug: _, + codegen_units: _, + codegen_units_std: _, + rustc_debug_assertions: _, + std_debug_assertions: _, + tools_debug_assertions: _, + overflow_checks: _, + overflow_checks_std: _, + debuginfo_level: _, + debuginfo_level_std: _, + debuginfo_level_tools: _, + debuginfo_level_tests: _, + backtrace: _, + musl_root: _, + verbose_tests: _, + optimize_tests: _, + codegen_tests: _, + omit_git_hash: _, + dist_src: _, + save_toolstates: _, + codegen_backends: _, + lld: _, + deny_warnings: _, + backtrace_on_ice: _, + verify_llvm_ir: _, + thin_lto_import_instr_limit: _, + remap_debuginfo: _, + test_compare_mode: _, + llvm_libunwind: _, + control_flow_guard: _, + ehcont_guard: _, + new_symbol_mangling: _, + profile_generate: _, + profile_use: _, + download_rustc: _, + validate_mir_opts: _, + frame_pointers: _, + } = ci_rust_config; + + // There are two kinds of checks for CI rustc incompatible options: + // 1. Checking an option that may change the compiler behaviour/output. + // 2. Checking an option that have no effect on the compiler behaviour/output. + // + // If the option belongs to the first category, we call `err` macro for a hard error; + // otherwise, we just print a warning with `warn` macro. + + err!(current_rust_config.optimize, optimize, "rust"); + err!(current_rust_config.randomize_layout, randomize_layout, "rust"); + err!(current_rust_config.debug_logging, debug_logging, "rust"); + err!(current_rust_config.debuginfo_level_rustc, debuginfo_level_rustc, "rust"); + err!(current_rust_config.rpath, rpath, "rust"); + err!(current_rust_config.strip, strip, "rust"); + err!(current_rust_config.lld_mode, lld_mode, "rust"); + err!(current_rust_config.llvm_tools, llvm_tools, "rust"); + err!(current_rust_config.llvm_bitcode_linker, llvm_bitcode_linker, "rust"); + err!(current_rust_config.jemalloc, jemalloc, "rust"); + err!(current_rust_config.default_linker, default_linker, "rust"); + err!(current_rust_config.stack_protector, stack_protector, "rust"); + err!(current_rust_config.lto, lto, "rust"); + err!(current_rust_config.std_features, std_features, "rust"); + + warn!(current_rust_config.channel, channel, "rust"); + warn!(current_rust_config.description, description, "rust"); + + Ok(()) +} + +impl Config { + pub fn apply_rust_config( + &mut self, + toml_rust: Option<Rust>, + warnings: Warnings, + description: &mut Option<String>, + ) { + let mut debug = None; + let mut rustc_debug_assertions = None; + let mut std_debug_assertions = None; + let mut tools_debug_assertions = None; + let mut overflow_checks = None; + let mut overflow_checks_std = None; + let mut debug_logging = None; + let mut debuginfo_level = None; + let mut debuginfo_level_rustc = None; + let mut debuginfo_level_std = None; + let mut debuginfo_level_tools = None; + let mut debuginfo_level_tests = None; + let mut optimize = None; + let mut lld_enabled = None; + let mut std_features = None; + + if let Some(rust) = toml_rust { + let Rust { + optimize: optimize_toml, + debug: debug_toml, + codegen_units, + codegen_units_std, + rustc_debug_assertions: rustc_debug_assertions_toml, + std_debug_assertions: std_debug_assertions_toml, + tools_debug_assertions: tools_debug_assertions_toml, + overflow_checks: overflow_checks_toml, + overflow_checks_std: overflow_checks_std_toml, + debug_logging: debug_logging_toml, + debuginfo_level: debuginfo_level_toml, + debuginfo_level_rustc: debuginfo_level_rustc_toml, + debuginfo_level_std: debuginfo_level_std_toml, + debuginfo_level_tools: debuginfo_level_tools_toml, + debuginfo_level_tests: debuginfo_level_tests_toml, + backtrace, + incremental, + randomize_layout, + default_linker, + channel: _, // already handled above + description: rust_description, + musl_root, + rpath, + verbose_tests, + optimize_tests, + codegen_tests, + omit_git_hash: _, // already handled above + dist_src, + save_toolstates, + codegen_backends, + lld: lld_enabled_toml, + llvm_tools, + llvm_bitcode_linker, + deny_warnings, + backtrace_on_ice, + verify_llvm_ir, + thin_lto_import_instr_limit, + remap_debuginfo, + jemalloc, + test_compare_mode, + llvm_libunwind, + control_flow_guard, + ehcont_guard, + new_symbol_mangling, + profile_generate, + profile_use, + download_rustc, + lto, + validate_mir_opts, + frame_pointers, + stack_protector, + strip, + lld_mode, + std_features: std_features_toml, + } = rust; + + // FIXME(#133381): alt rustc builds currently do *not* have rustc debug assertions + // enabled. We should not download a CI alt rustc if we need rustc to have debug + // assertions (e.g. for crashes test suite). This can be changed once something like + // [Enable debug assertions on alt + // builds](https://github.com/rust-lang/rust/pull/131077) lands. + // + // Note that `rust.debug = true` currently implies `rust.debug-assertions = true`! + // + // This relies also on the fact that the global default for `download-rustc` will be + // `false` if it's not explicitly set. + let debug_assertions_requested = matches!(rustc_debug_assertions_toml, Some(true)) + || (matches!(debug_toml, Some(true)) + && !matches!(rustc_debug_assertions_toml, Some(false))); + + if debug_assertions_requested + && let Some(ref opt) = download_rustc + && opt.is_string_or_true() + { + eprintln!( + "WARN: currently no CI rustc builds have rustc debug assertions \ + enabled. Please either set `rust.debug-assertions` to `false` if you \ + want to use download CI rustc or set `rust.download-rustc` to `false`." + ); + } + + self.download_rustc_commit = self.download_ci_rustc_commit( + download_rustc, + debug_assertions_requested, + self.llvm_assertions, + ); + + debug = debug_toml; + rustc_debug_assertions = rustc_debug_assertions_toml; + std_debug_assertions = std_debug_assertions_toml; + tools_debug_assertions = tools_debug_assertions_toml; + overflow_checks = overflow_checks_toml; + overflow_checks_std = overflow_checks_std_toml; + debug_logging = debug_logging_toml; + debuginfo_level = debuginfo_level_toml; + debuginfo_level_rustc = debuginfo_level_rustc_toml; + debuginfo_level_std = debuginfo_level_std_toml; + debuginfo_level_tools = debuginfo_level_tools_toml; + debuginfo_level_tests = debuginfo_level_tests_toml; + lld_enabled = lld_enabled_toml; + std_features = std_features_toml; + + optimize = optimize_toml; + self.rust_new_symbol_mangling = new_symbol_mangling; + set(&mut self.rust_optimize_tests, optimize_tests); + set(&mut self.codegen_tests, codegen_tests); + set(&mut self.rust_rpath, rpath); + set(&mut self.rust_strip, strip); + set(&mut self.rust_frame_pointers, frame_pointers); + self.rust_stack_protector = stack_protector; + set(&mut self.jemalloc, jemalloc); + set(&mut self.test_compare_mode, test_compare_mode); + set(&mut self.backtrace, backtrace); + if rust_description.is_some() { + eprintln!( + "Warning: rust.description is deprecated. Use build.description instead." + ); + } + if description.is_none() { + *description = rust_description; + } + set(&mut self.rust_dist_src, dist_src); + set(&mut self.verbose_tests, verbose_tests); + // in the case "false" is set explicitly, do not overwrite the command line args + if let Some(true) = incremental { + self.incremental = true; + } + set(&mut self.lld_mode, lld_mode); + set(&mut self.llvm_bitcode_linker_enabled, llvm_bitcode_linker); + + self.rust_randomize_layout = randomize_layout.unwrap_or_default(); + self.llvm_tools_enabled = llvm_tools.unwrap_or(true); + + self.llvm_enzyme = self.channel == "dev" || self.channel == "nightly"; + self.rustc_default_linker = default_linker; + self.musl_root = musl_root.map(PathBuf::from); + self.save_toolstates = save_toolstates.map(PathBuf::from); + set( + &mut self.deny_warnings, + match warnings { + Warnings::Deny => Some(true), + Warnings::Warn => Some(false), + Warnings::Default => deny_warnings, + }, + ); + set(&mut self.backtrace_on_ice, backtrace_on_ice); + set(&mut self.rust_verify_llvm_ir, verify_llvm_ir); + self.rust_thin_lto_import_instr_limit = thin_lto_import_instr_limit; + set(&mut self.rust_remap_debuginfo, remap_debuginfo); + set(&mut self.control_flow_guard, control_flow_guard); + set(&mut self.ehcont_guard, ehcont_guard); + self.llvm_libunwind_default = + llvm_libunwind.map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); + + if let Some(ref backends) = codegen_backends { + let available_backends = ["llvm", "cranelift", "gcc"]; + + self.rust_codegen_backends = backends.iter().map(|s| { + if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { + if available_backends.contains(&backend) { + panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); + } else { + println!("HELP: '{s}' for 'rust.codegen-backends' might fail. \ + Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + In this case, it would be referred to as '{backend}'."); + } + } + + s.clone() + }).collect(); + } + + self.rust_codegen_units = codegen_units.map(threads_from_config); + self.rust_codegen_units_std = codegen_units_std.map(threads_from_config); + + if self.rust_profile_use.is_none() { + self.rust_profile_use = profile_use; + } + + if self.rust_profile_generate.is_none() { + self.rust_profile_generate = profile_generate; + } + + self.rust_lto = + lto.as_deref().map(|value| RustcLto::from_str(value).unwrap()).unwrap_or_default(); + self.rust_validate_mir_opts = validate_mir_opts; + } + + self.rust_optimize = optimize.unwrap_or(RustOptimize::Bool(true)); + + // We make `x86_64-unknown-linux-gnu` use the self-contained linker by default, so we will + // build our internal lld and use it as the default linker, by setting the `rust.lld` config + // to true by default: + // - on the `x86_64-unknown-linux-gnu` target + // - on the `dev` and `nightly` channels + // - when building our in-tree llvm (i.e. the target has not set an `llvm-config`), so that + // we're also able to build the corresponding lld + // - or when using an external llvm that's downloaded from CI, which also contains our prebuilt + // lld + // - otherwise, we'd be using an external llvm, and lld would not necessarily available and + // thus, disabled + // - similarly, lld will not be built nor used by default when explicitly asked not to, e.g. + // when the config sets `rust.lld = false` + if self.build.triple == "x86_64-unknown-linux-gnu" + && self.hosts == [self.build] + && (self.channel == "dev" || self.channel == "nightly") + { + let no_llvm_config = self + .target_config + .get(&self.build) + .is_some_and(|target_config| target_config.llvm_config.is_none()); + let enable_lld = self.llvm_from_ci || no_llvm_config; + // Prefer the config setting in case an explicit opt-out is needed. + self.lld_enabled = lld_enabled.unwrap_or(enable_lld); + } else { + set(&mut self.lld_enabled, lld_enabled); + } + + let default_std_features = BTreeSet::from([String::from("panic-unwind")]); + self.rust_std_features = std_features.unwrap_or(default_std_features); + + let default = debug == Some(true); + self.rustc_debug_assertions = rustc_debug_assertions.unwrap_or(default); + self.std_debug_assertions = std_debug_assertions.unwrap_or(self.rustc_debug_assertions); + self.tools_debug_assertions = tools_debug_assertions.unwrap_or(self.rustc_debug_assertions); + self.rust_overflow_checks = overflow_checks.unwrap_or(default); + self.rust_overflow_checks_std = overflow_checks_std.unwrap_or(self.rust_overflow_checks); + + self.rust_debug_logging = debug_logging.unwrap_or(self.rustc_debug_assertions); + + let with_defaults = |debuginfo_level_specific: Option<_>| { + debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { + DebuginfoLevel::Limited + } else { + DebuginfoLevel::None + }) + }; + self.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); + self.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); + self.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); + self.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(DebuginfoLevel::None); + } +} diff --git a/src/bootstrap/src/core/config/toml/target.rs b/src/bootstrap/src/core/config/toml/target.rs new file mode 100644 index 00000000000..b8fc3e74c35 --- /dev/null +++ b/src/bootstrap/src/core/config/toml/target.rs @@ -0,0 +1,174 @@ +//! This module defines the structures and logic for handling target-specific configuration +//! within the `bootstrap.toml` file. This allows you to customize build settings, tools, +//! and flags for individual compilation targets. +//! +//! It includes: +//! +//! * [`TomlTarget`]: This struct directly mirrors the `[target.<triple>]` sections in your +//! `bootstrap.toml`. It's used for deserializing raw TOML data for a specific target. +//! * [`Target`]: This struct represents the processed and validated configuration for a +//! build target, which is is stored in the main [`Config`] structure. +//! * [`Config::apply_target_config`]: This method processes the `TomlTarget` data and +//! applies it to the global [`Config`], ensuring proper path resolution, validation, +//! and integration with other build settings. + +use std::collections::HashMap; + +use serde::{Deserialize, Deserializer}; + +use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; +use crate::core::config::{LlvmLibunwind, Merge, ReplaceOpt, SplitDebuginfo, StringOrBool}; +use crate::{Config, HashSet, PathBuf, TargetSelection, define_config, exit}; + +define_config! { + /// TOML representation of how each build target is configured. + struct TomlTarget { + cc: Option<String> = "cc", + cxx: Option<String> = "cxx", + ar: Option<String> = "ar", + ranlib: Option<String> = "ranlib", + default_linker: Option<PathBuf> = "default-linker", + linker: Option<String> = "linker", + split_debuginfo: Option<String> = "split-debuginfo", + llvm_config: Option<String> = "llvm-config", + llvm_has_rust_patches: Option<bool> = "llvm-has-rust-patches", + llvm_filecheck: Option<String> = "llvm-filecheck", + llvm_libunwind: Option<String> = "llvm-libunwind", + sanitizers: Option<bool> = "sanitizers", + profiler: Option<StringOrBool> = "profiler", + rpath: Option<bool> = "rpath", + crt_static: Option<bool> = "crt-static", + musl_root: Option<String> = "musl-root", + musl_libdir: Option<String> = "musl-libdir", + wasi_root: Option<String> = "wasi-root", + qemu_rootfs: Option<String> = "qemu-rootfs", + no_std: Option<bool> = "no-std", + codegen_backends: Option<Vec<String>> = "codegen-backends", + runner: Option<String> = "runner", + optimized_compiler_builtins: Option<bool> = "optimized-compiler-builtins", + jemalloc: Option<bool> = "jemalloc", + } +} + +/// Per-target configuration stored in the global configuration structure. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Target { + /// Some(path to llvm-config) if using an external LLVM. + pub llvm_config: Option<PathBuf>, + pub llvm_has_rust_patches: Option<bool>, + /// Some(path to FileCheck) if one was specified. + pub llvm_filecheck: Option<PathBuf>, + pub llvm_libunwind: Option<LlvmLibunwind>, + pub cc: Option<PathBuf>, + pub cxx: Option<PathBuf>, + pub ar: Option<PathBuf>, + pub ranlib: Option<PathBuf>, + pub default_linker: Option<PathBuf>, + pub linker: Option<PathBuf>, + pub split_debuginfo: Option<SplitDebuginfo>, + pub sanitizers: Option<bool>, + pub profiler: Option<StringOrBool>, + pub rpath: Option<bool>, + pub crt_static: Option<bool>, + pub musl_root: Option<PathBuf>, + pub musl_libdir: Option<PathBuf>, + pub wasi_root: Option<PathBuf>, + pub qemu_rootfs: Option<PathBuf>, + pub runner: Option<String>, + pub no_std: bool, + pub codegen_backends: Option<Vec<String>>, + pub optimized_compiler_builtins: Option<bool>, + pub jemalloc: Option<bool>, +} + +impl Target { + pub fn from_triple(triple: &str) -> Self { + let mut target: Self = Default::default(); + if !build_helper::targets::target_supports_std(triple) { + target.no_std = true; + } + if triple.contains("emscripten") { + target.runner = Some("node".into()); + } + target + } +} + +impl Config { + pub fn apply_target_config(&mut self, toml_target: Option<HashMap<String, TomlTarget>>) { + if let Some(t) = toml_target { + for (triple, cfg) in t { + let mut target = Target::from_triple(&triple); + + if let Some(ref s) = cfg.llvm_config { + if self.download_rustc_commit.is_some() && triple == *self.build.triple { + panic!( + "setting llvm_config for the host is incompatible with download-rustc" + ); + } + target.llvm_config = Some(self.src.join(s)); + } + if let Some(patches) = cfg.llvm_has_rust_patches { + assert!( + self.submodules == Some(false) || cfg.llvm_config.is_some(), + "use of `llvm-has-rust-patches` is restricted to cases where either submodules are disabled or llvm-config been provided" + ); + target.llvm_has_rust_patches = Some(patches); + } + if let Some(ref s) = cfg.llvm_filecheck { + target.llvm_filecheck = Some(self.src.join(s)); + } + target.llvm_libunwind = cfg.llvm_libunwind.as_ref().map(|v| { + v.parse().unwrap_or_else(|_| { + panic!("failed to parse target.{triple}.llvm-libunwind") + }) + }); + if let Some(s) = cfg.no_std { + target.no_std = s; + } + target.cc = cfg.cc.map(PathBuf::from); + target.cxx = cfg.cxx.map(PathBuf::from); + target.ar = cfg.ar.map(PathBuf::from); + target.ranlib = cfg.ranlib.map(PathBuf::from); + target.linker = cfg.linker.map(PathBuf::from); + target.crt_static = cfg.crt_static; + target.musl_root = cfg.musl_root.map(PathBuf::from); + target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); + target.wasi_root = cfg.wasi_root.map(PathBuf::from); + target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); + target.runner = cfg.runner; + target.sanitizers = cfg.sanitizers; + target.profiler = cfg.profiler; + target.rpath = cfg.rpath; + target.optimized_compiler_builtins = cfg.optimized_compiler_builtins; + target.jemalloc = cfg.jemalloc; + + if let Some(ref backends) = cfg.codegen_backends { + let available_backends = ["llvm", "cranelift", "gcc"]; + + target.codegen_backends = Some(backends.iter().map(|s| { + if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { + if available_backends.contains(&backend) { + panic!("Invalid value '{s}' for 'target.{triple}.codegen-backends'. Instead, please use '{backend}'."); + } else { + println!("HELP: '{s}' for 'target.{triple}.codegen-backends' might fail. \ + Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + In this case, it would be referred to as '{backend}'."); + } + } + + s.clone() + }).collect()); + } + + target.split_debuginfo = cfg.split_debuginfo.as_ref().map(|v| { + v.parse().unwrap_or_else(|_| { + panic!("invalid value for target.{triple}.split-debuginfo") + }) + }); + + self.target_config.insert(TargetSelection::from_user(&triple), target); + } + } + } +} diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index d942334d1c3..f349b9a87ed 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -3,7 +3,6 @@ use std::ffi::OsString; use std::fs::{self, File}; use std::io::{BufRead, BufReader, BufWriter, ErrorKind, Write}; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; use std::sync::OnceLock; use xz2::bufread::XzDecoder; @@ -16,14 +15,7 @@ use crate::{Config, t}; static SHOULD_FIX_BINS_AND_DYLIBS: OnceLock<bool> = OnceLock::new(); -/// `Config::try_run` wrapper for this module to avoid warnings on `try_run`, since we don't have access to a `builder` yet. -fn try_run(config: &Config, cmd: &mut Command) -> Result<(), ()> { - #[expect(deprecated)] - config.try_run(cmd) -} - -fn extract_curl_version(out: &[u8]) -> semver::Version { - let out = String::from_utf8_lossy(out); +fn extract_curl_version(out: String) -> semver::Version { // The output should look like this: "curl <major>.<minor>.<patch> ..." out.lines() .next() @@ -32,18 +24,21 @@ fn extract_curl_version(out: &[u8]) -> semver::Version { .unwrap_or(semver::Version::new(1, 0, 0)) } -fn curl_version() -> semver::Version { - let mut curl = Command::new("curl"); +fn curl_version(config: &Config) -> semver::Version { + let mut curl = command("curl"); curl.arg("-V"); - let Ok(out) = curl.output() else { return semver::Version::new(1, 0, 0) }; - let out = out.stdout; - extract_curl_version(&out) + let curl = curl.run_capture_stdout(config); + if curl.is_failure() { + return semver::Version::new(1, 0, 0); + } + let output = curl.stdout(); + extract_curl_version(output) } /// Generic helpers that are useful anywhere in bootstrap. impl Config { pub fn is_verbose(&self) -> bool { - self.verbose > 0 + self.exec_ctx.is_verbose() } pub(crate) fn create<P: AsRef<Path>>(&self, path: P, s: &str) { @@ -85,20 +80,14 @@ impl Config { /// on NixOS fn should_fix_bins_and_dylibs(&self) -> bool { let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| { - match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { - Err(_) => return false, - Ok(output) if !output.status.success() => return false, - Ok(output) => { - let mut os_name = output.stdout; - if os_name.last() == Some(&b'\n') { - os_name.pop(); - } - if os_name != b"Linux" { - return false; - } - } + let uname = command("uname").allow_failure().arg("-s").run_capture_stdout(self); + if uname.is_failure() { + return false; + } + let output = uname.stdout(); + if !output.starts_with("Linux") { + return false; } - // If the user has asked binaries to be patched for Nix, then // don't check for NixOS or `/lib`. // NOTE: this intentionally comes after the Linux check: @@ -173,23 +162,18 @@ impl Config { ]; } "; - nix_build_succeeded = try_run( - self, - Command::new("nix-build").args([ - Path::new("-E"), - Path::new(NIX_EXPR), - Path::new("-o"), - &nix_deps_dir, - ]), - ) - .is_ok(); + nix_build_succeeded = command("nix-build") + .allow_failure() + .args([Path::new("-E"), Path::new(NIX_EXPR), Path::new("-o"), &nix_deps_dir]) + .run_capture_stdout(self) + .is_success(); nix_deps_dir }); if !nix_build_succeeded { return; } - let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); + let mut patchelf = command(nix_deps_dir.join("bin/patchelf")); patchelf.args(&[ OsString::from("--add-rpath"), OsString::from(t!(fs::canonicalize(nix_deps_dir)).join("lib")), @@ -200,8 +184,8 @@ impl Config { let dynamic_linker = t!(fs::read_to_string(dynamic_linker_path)); patchelf.args(["--set-interpreter", dynamic_linker.trim_end()]); } - - let _ = try_run(self, patchelf.arg(fname)); + patchelf.arg(fname); + let _ = patchelf.allow_failure().run_capture_stdout(self); } fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) { @@ -267,7 +251,7 @@ impl Config { curl.arg("--progress-bar"); } // --retry-all-errors was added in 7.71.0, don't use it if curl is old. - if curl_version() >= semver::Version::new(7, 71, 0) { + if curl_version(self) >= semver::Version::new(7, 71, 0) { curl.arg("--retry-all-errors"); } curl.arg(url); @@ -275,7 +259,7 @@ impl Config { if self.build.contains("windows-msvc") { eprintln!("Fallback to PowerShell"); for _ in 0..3 { - if try_run(self, Command::new("PowerShell.exe").args([ + let powershell = command("PowerShell.exe").allow_failure().args([ "/nologo", "-Command", "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", @@ -283,9 +267,12 @@ impl Config { "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), ), - ])).is_err() { + ]).run_capture_stdout(self); + + if powershell.is_failure() { return; } + eprintln!("\nspurious failure, trying again"); } } @@ -727,7 +714,7 @@ download-rustc = false use build_helper::git::PathFreshness; use crate::core::build_steps::llvm::detect_llvm_freshness; - use crate::core::config::check_incompatible_options_for_ci_llvm; + use crate::core::config::toml::llvm::check_incompatible_options_for_ci_llvm; if !self.llvm_from_ci { return; diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs index af4ec679d08..b7bbd294f0e 100644 --- a/src/bootstrap/src/core/sanity.rs +++ b/src/bootstrap/src/core/sanity.rs @@ -34,6 +34,8 @@ pub struct Finder { // Targets can be removed from this list once they are present in the stage0 compiler (usually by updating the beta compiler of the bootstrap). const STAGE0_MISSING_TARGETS: &[&str] = &[ // just a dummy comment so the list doesn't get onelined + "loongarch32-unknown-none", + "loongarch32-unknown-none-softfloat", ]; /// Minimum version threshold for libstdc++ required when using prebuilt LLVM @@ -362,7 +364,7 @@ than building it. // Cygwin. The Cygwin build does not have generators for Visual // Studio, so detect that here and error. let out = - command("cmake").arg("--help").run_always().run_capture_stdout(build).stdout(); + command("cmake").arg("--help").run_always().run_capture_stdout(&build).stdout(); if !out.contains("Visual Studio") { panic!( " diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index e4643653d97..88d5d026820 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -32,6 +32,7 @@ use cc::Tool; use termcolor::{ColorChoice, StandardStream, WriteColor}; use utils::build_stamp::BuildStamp; use utils::channel::GitInfo; +use utils::execution_context::ExecutionContext; use crate::core::builder; use crate::core::builder::Kind; @@ -81,7 +82,10 @@ const LLD_FILE_NAMES: &[&str] = &["ld.lld", "ld64.lld", "lld-link", "wasm-ld"]; /// (Mode restriction, config name, config values (if any)) #[expect(clippy::type_complexity)] // It's fine for hard-coded list and type is explained above. const EXTRA_CHECK_CFGS: &[(Option<Mode>, &str, Option<&[&'static str]>)] = &[ - (None, "bootstrap", None), + (Some(Mode::Rustc), "bootstrap", None), + (Some(Mode::Codegen), "bootstrap", None), + (Some(Mode::ToolRustc), "bootstrap", None), + (Some(Mode::ToolStd), "bootstrap", None), (Some(Mode::Rustc), "llvm_enzyme", None), (Some(Mode::Codegen), "llvm_enzyme", None), (Some(Mode::ToolRustc), "llvm_enzyme", None), @@ -195,7 +199,6 @@ pub struct Build { crates: HashMap<String, Crate>, crate_paths: HashMap<PathBuf, String>, is_sudo: bool, - delayed_failures: RefCell<Vec<String>>, prerelease_version: Cell<Option<u32>>, #[cfg(feature = "build-metrics")] @@ -272,6 +275,16 @@ impl Mode { } } +/// When `rust.rust_remap_debuginfo` is requested, the compiler needs to know how to +/// opportunistically unremap compiler vs non-compiler sources. We use two schemes, +/// [`RemapScheme::Compiler`] and [`RemapScheme::NonCompiler`]. +pub enum RemapScheme { + /// The [`RemapScheme::Compiler`] scheme will remap to `/rustc-dev/{hash}`. + Compiler, + /// The [`RemapScheme::NonCompiler`] scheme will remap to `/rustc/{hash}`. + NonCompiler, +} + #[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum CLang { C, @@ -456,7 +469,6 @@ impl Build { crates: HashMap::new(), crate_paths: HashMap::new(), is_sudo, - delayed_failures: RefCell::new(Vec::new()), prerelease_version: Cell::new(None), #[cfg(feature = "build-metrics")] @@ -616,7 +628,7 @@ impl Build { return; } - if GitInfo::new(false, Path::new(submodule)).is_managed_git_subrepository() { + if config.git_info(false, Path::new(submodule)).is_managed_git_subrepository() { config.update_submodule(submodule); } } @@ -671,7 +683,7 @@ impl Build { #[cfg(feature = "tracing")] let _sanity_check_span = span!(tracing::Level::DEBUG, "(1) executing dry-run sanity-check").entered(); - self.config.dry_run = DryRun::SelfCheck; + self.config.set_dry_run(DryRun::SelfCheck); let builder = builder::Builder::new(self); builder.execute_cli(); } @@ -681,7 +693,7 @@ impl Build { #[cfg(feature = "tracing")] let _actual_run_span = span!(tracing::Level::DEBUG, "(2) executing actual run").entered(); - self.config.dry_run = DryRun::Disabled; + self.config.set_dry_run(DryRun::Disabled); let builder = builder::Builder::new(self); builder.execute_cli(); } @@ -697,14 +709,7 @@ impl Build { debug!("checking for postponed test failures from `test --no-fail-fast`"); // Check for postponed failures from `test --no-fail-fast`. - let failures = self.delayed_failures.borrow(); - if !failures.is_empty() { - eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); - for failure in failures.iter() { - eprintln!(" - {failure}\n"); - } - exit!(1); - } + self.config.exec_ctx().report_failures_and_exit(); #[cfg(feature = "build-metrics")] self.metrics.persist(self); @@ -940,133 +945,6 @@ impl Build { }) } - /// Execute a command and return its output. - /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to - /// execute commands. They internally call this method. - #[track_caller] - fn run( - &self, - command: &mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - ) -> CommandOutput { - command.mark_as_executed(); - if self.config.dry_run() && !command.run_always { - return CommandOutput::default(); - } - - #[cfg(feature = "tracing")] - let _run_span = trace_cmd!(command); - - let created_at = command.get_created_location(); - let executed_at = std::panic::Location::caller(); - - self.verbose(|| { - println!("running: {command:?} (created at {created_at}, executed at {executed_at})") - }); - - let cmd = command.as_command_mut(); - cmd.stdout(stdout.stdio()); - cmd.stderr(stderr.stdio()); - - let output = cmd.output(); - - use std::fmt::Write; - - let mut message = String::new(); - let output: CommandOutput = match output { - // Command has succeeded - Ok(output) if output.status.success() => { - CommandOutput::from_output(output, stdout, stderr) - } - // Command has started, but then it failed - Ok(output) => { - writeln!( - message, - r#" -Command {command:?} did not execute successfully. -Expected success, got {} -Created at: {created_at} -Executed at: {executed_at}"#, - output.status, - ) - .unwrap(); - - let output: CommandOutput = CommandOutput::from_output(output, stdout, stderr); - - // If the output mode is OutputMode::Capture, we can now print the output. - // If it is OutputMode::Print, then the output has already been printed to - // stdout/stderr, and we thus don't have anything captured to print anyway. - if stdout.captures() { - writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); - } - if stderr.captures() { - writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); - } - output - } - // The command did not even start - Err(e) => { - writeln!( - message, - "\n\nCommand {command:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}" - ) - .unwrap(); - CommandOutput::did_not_start(stdout, stderr) - } - }; - - let fail = |message: &str, output: CommandOutput| -> ! { - if self.is_verbose() { - println!("{message}"); - } else { - let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); - // If the command captures output, the user would not see any indication that - // it has failed. In this case, print a more verbose error, since to provide more - // context. - if stdout.is_some() || stderr.is_some() { - if let Some(stdout) = - output.stdout_if_present().take_if(|s| !s.trim().is_empty()) - { - println!("STDOUT:\n{stdout}\n"); - } - if let Some(stderr) = - output.stderr_if_present().take_if(|s| !s.trim().is_empty()) - { - println!("STDERR:\n{stderr}\n"); - } - println!("Command {command:?} has failed. Rerun with -v to see more details."); - } else { - println!("Command has failed. Rerun with -v to see more details."); - } - } - exit!(1); - }; - - if !output.is_success() { - match command.failure_behavior { - BehaviorOnFailure::DelayFail => { - if self.fail_fast { - fail(&message, output); - } - - let mut failures = self.delayed_failures.borrow_mut(); - failures.push(message); - } - BehaviorOnFailure::Exit => { - fail(&message, output); - } - BehaviorOnFailure::Ignore => { - // If failures are allowed, either the error has been printed already - // (OutputMode::Print) or the user used a capture output mode and wants to - // handle the error output on their own. - } - } - } - output - } - /// Check if verbosity is greater than the `level` pub fn is_verbose_than(&self, level: usize) -> bool { self.verbosity > level @@ -1080,7 +958,7 @@ Executed at: {executed_at}"#, } fn info(&self, msg: &str) { - match self.config.dry_run { + match self.config.get_dry_run() { DryRun::SelfCheck => (), DryRun::Disabled | DryRun::UserSelected => { println!("{msg}"); @@ -1104,8 +982,15 @@ Executed at: {executed_at}"#, &self, what: impl Display, target: impl Into<Option<TargetSelection>>, + custom_stage: Option<u32>, ) -> Option<gha::Group> { - self.msg(Kind::Check, self.config.stage, what, self.config.build, target) + self.msg( + Kind::Check, + custom_stage.unwrap_or(self.config.stage), + what, + self.config.build, + target, + ) } #[must_use = "Groups should not be dropped until the Step finishes running"] @@ -1196,7 +1081,7 @@ Executed at: {executed_at}"#, #[track_caller] fn group(&self, msg: &str) -> Option<gha::Group> { - match self.config.dry_run { + match self.config.get_dry_run() { DryRun::SelfCheck => None, DryRun::Disabled | DryRun::UserSelected => Some(gha::group(msg)), } @@ -1210,7 +1095,7 @@ Executed at: {executed_at}"#, }) } - fn debuginfo_map_to(&self, which: GitRepo) -> Option<String> { + fn debuginfo_map_to(&self, which: GitRepo, remap_scheme: RemapScheme) -> Option<String> { if !self.config.rust_remap_debuginfo { return None; } @@ -1218,7 +1103,24 @@ Executed at: {executed_at}"#, match which { GitRepo::Rustc => { let sha = self.rust_sha().unwrap_or(&self.version); - Some(format!("/rustc/{sha}")) + + match remap_scheme { + RemapScheme::Compiler => { + // For compiler sources, remap via `/rustc-dev/{sha}` to allow + // distinguishing between compiler sources vs library sources, since + // `rustc-dev` dist component places them under + // `$sysroot/lib/rustlib/rustc-src/rust` as opposed to `rust-src`'s + // `$sysroot/lib/rustlib/src/rust`. + // + // Keep this scheme in sync with `rustc_metadata::rmeta::decoder`'s + // `try_to_translate_virtual_to_real`. + Some(format!("/rustc-dev/{sha}")) + } + RemapScheme::NonCompiler => { + // For non-compiler sources, use `/rustc/{sha}` remapping scheme. + Some(format!("/rustc/{sha}")) + } + } } GitRepo::Llvm => Some(String::from("/rustc/llvm")), } @@ -1285,7 +1187,7 @@ Executed at: {executed_at}"#, base.push("-fno-omit-frame-pointer".into()); } - if let Some(map_to) = self.debuginfo_map_to(which) { + if let Some(map_to) = self.debuginfo_map_to(which, RemapScheme::NonCompiler) { let map = format!("{}={}", self.src.display(), map_to); let cc = self.cc(target); if cc.ends_with("clang") || cc.ends_with("gcc") { @@ -1788,11 +1690,12 @@ Executed at: {executed_at}"#, let now = t!(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)); let _ = fs::rename(dst, format!("{}-{}", dst.display(), now.as_nanos())); } - let metadata = t!(src.symlink_metadata(), format!("src = {}", src.display())); + let mut metadata = t!(src.symlink_metadata(), format!("src = {}", src.display())); let mut src = src.to_path_buf(); if metadata.file_type().is_symlink() { if dereference_symlinks { src = t!(fs::canonicalize(src)); + metadata = t!(fs::metadata(&src), format!("target = {}", src.display())); } else { let link = t!(fs::read_link(src)); t!(self.symlink_file(link, dst)); @@ -2007,6 +1910,16 @@ to download LLVM rather than building it. stream.reset().unwrap(); result } + + pub fn exec_ctx(&self) -> &ExecutionContext { + &self.config.exec_ctx + } +} + +impl AsRef<ExecutionContext> for Build { + fn as_ref(&self) -> &ExecutionContext { + &self.config.exec_ctx + } } #[cfg(unix)] diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 459a34d14cc..e939a8362ad 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -416,4 +416,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Warning, summary: "Stage0 library no longer matches the in-tree library, which means stage1 compiler now uses the beta library.", }, + ChangeInfo { + change_id: 141970, + severity: ChangeSeverity::Info, + summary: "Added new bootstrap flag `--skip-std-check-if-no-download-rustc` that skips std checks when download-rustc is unavailable. Mainly intended for developers to reduce RA overhead.", + }, ]; diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs index 4a9ecc7a4f8..38f250af42f 100644 --- a/src/bootstrap/src/utils/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -8,6 +8,7 @@ use std::fs; use std::path::Path; +use super::execution_context::ExecutionContext; use super::helpers; use crate::Build; use crate::utils::helpers::{start_process, t}; @@ -34,7 +35,7 @@ pub struct Info { } impl GitInfo { - pub fn new(omit_git_hash: bool, dir: &Path) -> GitInfo { + pub fn new(omit_git_hash: bool, dir: &Path, exec_ctx: impl AsRef<ExecutionContext>) -> GitInfo { // See if this even begins to look like a git dir if !dir.join(".git").exists() { match read_commit_info_file(dir) { @@ -43,10 +44,12 @@ impl GitInfo { } } - // Make sure git commands work - match helpers::git(Some(dir)).arg("rev-parse").as_command_mut().output() { - Ok(ref out) if out.status.success() => {} - _ => return GitInfo::Absent, + let mut git_command = helpers::git(Some(dir)); + git_command.arg("rev-parse"); + let output = git_command.allow_failure().run_capture(exec_ctx); + + if output.is_failure() { + return GitInfo::Absent; } // If we're ignoring the git info, we don't actually need to collect it, just make sure this diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 64e46f10563..f297300e34a 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -11,7 +11,7 @@ use std::process::{Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio} use build_helper::ci::CiEnv; use build_helper::drop_bomb::DropBomb; -use crate::Build; +use super::execution_context::ExecutionContext; /// What should be done when the command fails. #[derive(Debug, Copy, Clone)] @@ -125,7 +125,6 @@ impl BootstrapCommand { Self { failure_behavior: BehaviorOnFailure::DelayFail, ..self } } - #[expect(dead_code)] pub fn fail_fast(self) -> Self { Self { failure_behavior: BehaviorOnFailure::Exit, ..self } } @@ -143,20 +142,20 @@ impl BootstrapCommand { /// Run the command, while printing stdout and stderr. /// Returns true if the command has succeeded. #[track_caller] - pub fn run(&mut self, builder: &Build) -> bool { - builder.run(self, OutputMode::Print, OutputMode::Print).is_success() + pub fn run(&mut self, exec_ctx: impl AsRef<ExecutionContext>) -> bool { + exec_ctx.as_ref().run(self, OutputMode::Print, OutputMode::Print).is_success() } /// Run the command, while capturing and returning all its output. #[track_caller] - pub fn run_capture(&mut self, builder: &Build) -> CommandOutput { - builder.run(self, OutputMode::Capture, OutputMode::Capture) + pub fn run_capture(&mut self, exec_ctx: impl AsRef<ExecutionContext>) -> CommandOutput { + exec_ctx.as_ref().run(self, OutputMode::Capture, OutputMode::Capture) } /// Run the command, while capturing and returning stdout, and printing stderr. #[track_caller] - pub fn run_capture_stdout(&mut self, builder: &Build) -> CommandOutput { - builder.run(self, OutputMode::Capture, OutputMode::Print) + pub fn run_capture_stdout(&mut self, exec_ctx: impl AsRef<ExecutionContext>) -> CommandOutput { + exec_ctx.as_ref().run(self, OutputMode::Capture, OutputMode::Print) } /// Provides access to the stdlib Command inside. @@ -280,7 +279,6 @@ impl CommandOutput { !self.is_success() } - #[expect(dead_code)] pub fn status(&self) -> Option<ExitStatus> { match self.status { CommandStatus::Finished(status) => Some(status), diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs new file mode 100644 index 00000000000..a5e1e9bcc07 --- /dev/null +++ b/src/bootstrap/src/utils/execution_context.rs @@ -0,0 +1,204 @@ +//! Shared execution context for running bootstrap commands. +//! +//! This module provides the [`ExecutionContext`] type, which holds global configuration +//! relevant during the execution of commands in bootstrap. This includes dry-run +//! mode, verbosity level, and behavior on failure. +use std::sync::{Arc, Mutex}; + +use crate::core::config::DryRun; +#[cfg(feature = "tracing")] +use crate::trace_cmd; +use crate::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, exit}; + +#[derive(Clone, Default)] +pub struct ExecutionContext { + dry_run: DryRun, + verbose: u8, + pub fail_fast: bool, + delayed_failures: Arc<Mutex<Vec<String>>>, +} + +impl ExecutionContext { + pub fn new() -> Self { + ExecutionContext::default() + } + + pub fn dry_run(&self) -> bool { + match self.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } + } + + pub fn get_dry_run(&self) -> &DryRun { + &self.dry_run + } + + pub fn verbose(&self, f: impl Fn()) { + if self.is_verbose() { + f() + } + } + + pub fn is_verbose(&self) -> bool { + self.verbose > 0 + } + + pub fn fail_fast(&self) -> bool { + self.fail_fast + } + + pub fn set_dry_run(&mut self, value: DryRun) { + self.dry_run = value; + } + + pub fn set_verbose(&mut self, value: u8) { + self.verbose = value; + } + + pub fn set_fail_fast(&mut self, value: bool) { + self.fail_fast = value; + } + + pub fn add_to_delay_failure(&self, message: String) { + self.delayed_failures.lock().unwrap().push(message); + } + + pub fn report_failures_and_exit(&self) { + let failures = self.delayed_failures.lock().unwrap(); + if failures.is_empty() { + return; + } + eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); + for failure in &*failures { + eprintln!(" - {failure}"); + } + exit!(1); + } + + /// Execute a command and return its output. + /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to + /// execute commands. They internally call this method. + #[track_caller] + pub fn run( + &self, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + ) -> CommandOutput { + command.mark_as_executed(); + if self.dry_run() && !command.run_always { + return CommandOutput::default(); + } + + #[cfg(feature = "tracing")] + let _run_span = trace_cmd!(command); + + let created_at = command.get_created_location(); + let executed_at = std::panic::Location::caller(); + + self.verbose(|| { + println!("running: {command:?} (created at {created_at}, executed at {executed_at})") + }); + + let cmd = command.as_command_mut(); + cmd.stdout(stdout.stdio()); + cmd.stderr(stderr.stdio()); + + let output = cmd.output(); + + use std::fmt::Write; + + let mut message = String::new(); + let output: CommandOutput = match output { + // Command has succeeded + Ok(output) if output.status.success() => { + CommandOutput::from_output(output, stdout, stderr) + } + // Command has started, but then it failed + Ok(output) => { + writeln!( + message, + r#" +Command {command:?} did not execute successfully. +Expected success, got {} +Created at: {created_at} +Executed at: {executed_at}"#, + output.status, + ) + .unwrap(); + + let output: CommandOutput = CommandOutput::from_output(output, stdout, stderr); + + // If the output mode is OutputMode::Capture, we can now print the output. + // If it is OutputMode::Print, then the output has already been printed to + // stdout/stderr, and we thus don't have anything captured to print anyway. + if stdout.captures() { + writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); + } + if stderr.captures() { + writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); + } + output + } + // The command did not even start + Err(e) => { + writeln!( + message, + "\n\nCommand {command:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}" + ) + .unwrap(); + CommandOutput::did_not_start(stdout, stderr) + } + }; + + let fail = |message: &str, output: CommandOutput| -> ! { + if self.is_verbose() { + println!("{message}"); + } else { + let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); + // If the command captures output, the user would not see any indication that + // it has failed. In this case, print a more verbose error, since to provide more + // context. + if stdout.is_some() || stderr.is_some() { + if let Some(stdout) = + output.stdout_if_present().take_if(|s| !s.trim().is_empty()) + { + println!("STDOUT:\n{stdout}\n"); + } + if let Some(stderr) = + output.stderr_if_present().take_if(|s| !s.trim().is_empty()) + { + println!("STDERR:\n{stderr}\n"); + } + println!("Command {command:?} has failed. Rerun with -v to see more details."); + } else { + println!("Command has failed. Rerun with -v to see more details."); + } + } + exit!(1); + }; + + if !output.is_success() { + match command.failure_behavior { + BehaviorOnFailure::DelayFail => { + if self.fail_fast { + fail(&message, output); + } + + self.add_to_delay_failure(message); + } + BehaviorOnFailure::Exit => { + fail(&message, output); + } + BehaviorOnFailure::Ignore => { + // If failures are allowed, either the error has been printed already + // (OutputMode::Print) or the user used a capture output mode and wants to + // handle the error output on their own. + } + } + } + output + } +} diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs index 169fcec303e..5a0b90801e7 100644 --- a/src/bootstrap/src/utils/mod.rs +++ b/src/bootstrap/src/utils/mod.rs @@ -8,6 +8,7 @@ pub(crate) mod cc_detect; pub(crate) mod change_tracker; pub(crate) mod channel; pub(crate) mod exec; +pub(crate) mod execution_context; pub(crate) mod helpers; pub(crate) mod job; pub(crate) mod render_tests; diff --git a/src/bootstrap/src/utils/render_tests.rs b/src/bootstrap/src/utils/render_tests.rs index 418f3ff975d..77e645a9e3c 100644 --- a/src/bootstrap/src/utils/render_tests.rs +++ b/src/bootstrap/src/utils/render_tests.rs @@ -43,8 +43,7 @@ pub(crate) fn try_run_tests( if builder.fail_fast { crate::exit!(1); } else { - let mut failures = builder.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); + builder.config.exec_ctx().add_to_delay_failure(format!("{cmd:?}")); false } } else { |