diff options
Diffstat (limited to 'src/bootstrap')
| -rw-r--r-- | src/bootstrap/src/bin/rustc.rs | 10 | ||||
| -rw-r--r-- | src/bootstrap/src/core/build_steps/check.rs | 352 | ||||
| -rw-r--r-- | src/bootstrap/src/core/build_steps/clippy.rs | 33 | ||||
| -rw-r--r-- | src/bootstrap/src/core/build_steps/compile.rs | 132 | ||||
| -rw-r--r-- | src/bootstrap/src/core/build_steps/dist.rs | 105 | ||||
| -rw-r--r-- | src/bootstrap/src/core/build_steps/doc.rs | 8 | ||||
| -rw-r--r-- | src/bootstrap/src/core/build_steps/run.rs | 18 | ||||
| -rw-r--r-- | src/bootstrap/src/core/build_steps/test.rs | 4 | ||||
| -rw-r--r-- | src/bootstrap/src/core/builder/cargo.rs | 30 | ||||
| -rw-r--r-- | src/bootstrap/src/core/builder/tests.rs | 2 | ||||
| -rw-r--r-- | src/bootstrap/src/core/config/mod.rs | 2 | ||||
| -rw-r--r-- | src/bootstrap/src/core/config/target_selection.rs | 2 | ||||
| -rw-r--r-- | src/bootstrap/src/lib.rs | 24 | ||||
| -rw-r--r-- | src/bootstrap/src/utils/tracing.rs | 93 |
14 files changed, 591 insertions, 224 deletions
diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs index 5865df67b66..f15b76fa85c 100644 --- a/src/bootstrap/src/bin/rustc.rs +++ b/src/bootstrap/src/bin/rustc.rs @@ -179,6 +179,16 @@ fn main() { } } + // Here we pass additional paths that essentially act as a sysroot. + // These are used to load rustc crates (e.g. `extern crate rustc_ast;`) + // for rustc_private tools, so that we do not have to copy them into the + // actual sysroot of the compiler that builds the tool. + if let Ok(dirs) = env::var("RUSTC_ADDITIONAL_SYSROOT_PATHS") { + for dir in dirs.split(",") { + cmd.arg(format!("-L{dir}")); + } + } + // Force all crates compiled by this compiler to (a) be unstable and (b) // allow the `rustc_private` feature to link to other unstable crates // also in the sysroot. We also do this for host crates, since those diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs index 4a110b733e1..bebae893ee7 100644 --- a/src/bootstrap/src/core/build_steps/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -1,5 +1,8 @@ //! Implementation of compiling the compiler and standard library, in "check"-based modes. +use std::fs; +use std::path::{Path, PathBuf}; + use crate::core::build_steps::compile::{ add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make, }; @@ -9,11 +12,11 @@ use crate::core::build_steps::tool::{ prepare_tool_cargo, }; use crate::core::builder::{ - self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, StepMetadata, crate_description, + self, Alias, Builder, Cargo, Kind, RunConfig, ShouldRun, Step, StepMetadata, crate_description, }; use crate::core::config::TargetSelection; use crate::utils::build_stamp::{self, BuildStamp}; -use crate::{CodegenBackendKind, Compiler, Mode, Subcommand}; +use crate::{CodegenBackendKind, Compiler, Mode, Subcommand, t}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Std { @@ -33,7 +36,7 @@ impl Std { } impl Step for Std { - type Output = (); + type Output = BuildStamp; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -60,13 +63,14 @@ impl Step for Std { let crates = std_crates_for_run_make(&run); run.builder.ensure(Std { - build_compiler: prepare_compiler_for_check(run.builder, run.target, Mode::Std), + build_compiler: prepare_compiler_for_check(run.builder, run.target, Mode::Std) + .build_compiler(), target: run.target, crates, }); } - fn run(self, builder: &Builder<'_>) { + fn run(self, builder: &Builder<'_>) -> Self::Output { let build_compiler = self.build_compiler; let target = self.target; @@ -93,18 +97,27 @@ impl Step for Std { Kind::Check, format_args!("library artifacts{}", crate_description(&self.crates)), Mode::Std, - self.build_compiler, + build_compiler, target, ); - let stamp = build_stamp::libstd_stamp(builder, build_compiler, target).with_prefix("check"); - run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); + let check_stamp = + build_stamp::libstd_stamp(builder, build_compiler, target).with_prefix("check"); + run_cargo( + builder, + cargo, + builder.config.free_args.clone(), + &check_stamp, + vec![], + true, + false, + ); drop(_guard); // don't check test dependencies if we haven't built libtest if !self.crates.iter().any(|krate| krate == "test") { - return; + return check_stamp; } // Then run cargo again, once we've put the rmeta files for the library @@ -137,10 +150,11 @@ impl Step for Std { Kind::Check, "library test/bench/example targets", Mode::Std, - self.build_compiler, + build_compiler, target, ); run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); + check_stamp } fn metadata(&self) -> Option<StepMetadata> { @@ -148,12 +162,135 @@ impl Step for Std { } } -/// Checks rustc using `build_compiler` and copies the built -/// .rmeta files into the sysroot of `build_compiler`. +/// Represents a proof that rustc was **checked**. +/// Contains directories with .rmeta files generated by checking rustc for a specific +/// target. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct RmetaSysroot { + host_dir: PathBuf, + target_dir: PathBuf, +} + +impl RmetaSysroot { + /// Copy rmeta artifacts from the given `stamp` into a sysroot located at `directory`. + fn from_stamp( + builder: &Builder<'_>, + stamp: BuildStamp, + target: TargetSelection, + directory: &Path, + ) -> Self { + let host_dir = directory.join("host"); + let target_dir = directory.join(target); + let _ = fs::remove_dir_all(directory); + t!(fs::create_dir_all(directory)); + add_to_sysroot(builder, &target_dir, &host_dir, &stamp); + + Self { host_dir, target_dir } + } + + /// Configure the given cargo invocation so that the compiled crate will be able to use + /// rustc .rmeta artifacts that were previously generated. + fn configure_cargo(&self, cargo: &mut Cargo) { + cargo.append_to_env( + "RUSTC_ADDITIONAL_SYSROOT_PATHS", + format!("{},{}", self.host_dir.to_str().unwrap(), self.target_dir.to_str().unwrap()), + ",", + ); + } +} + +/// Checks rustc using the given `build_compiler` for the given `target`, and produces +/// a sysroot in the build directory that stores the generated .rmeta files. +/// +/// This step exists so that we can store the generated .rmeta artifacts into a separate +/// directory, instead of copying them into the sysroot of `build_compiler`, which would +/// "pollute" it (that is especially problematic for the external stage0 rustc). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct PrepareRustcRmetaSysroot { + build_compiler: CompilerForCheck, + target: TargetSelection, +} + +impl PrepareRustcRmetaSysroot { + fn new(build_compiler: CompilerForCheck, target: TargetSelection) -> Self { + Self { build_compiler, target } + } +} + +impl Step for PrepareRustcRmetaSysroot { + type Output = RmetaSysroot; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + // Check rustc + let stamp = builder.ensure(Rustc::from_build_compiler( + self.build_compiler.clone(), + self.target, + vec![], + )); + + let build_compiler = self.build_compiler.build_compiler(); + + // Copy the generated rmeta artifacts to a separate directory + let dir = builder + .out + .join(build_compiler.host) + .join(format!("stage{}-rustc-rmeta-artifacts", build_compiler.stage + 1)); + RmetaSysroot::from_stamp(builder, stamp, self.target, &dir) + } +} + +/// Checks std using the given `build_compiler` for the given `target`, and produces +/// a sysroot in the build directory that stores the generated .rmeta files. +/// +/// This step exists so that we can store the generated .rmeta artifacts into a separate +/// directory, instead of copying them into the sysroot of `build_compiler`, which would +/// "pollute" it (that is especially problematic for the external stage0 rustc). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct PrepareStdRmetaSysroot { + build_compiler: Compiler, + target: TargetSelection, +} + +impl PrepareStdRmetaSysroot { + fn new(build_compiler: Compiler, target: TargetSelection) -> Self { + Self { build_compiler, target } + } +} + +impl Step for PrepareStdRmetaSysroot { + type Output = RmetaSysroot; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + // Check std + let stamp = builder.ensure(Std { + build_compiler: self.build_compiler, + target: self.target, + crates: vec![], + }); + + // Copy the generated rmeta artifacts to a separate directory + let dir = builder + .out + .join(self.build_compiler.host) + .join(format!("stage{}-std-rmeta-artifacts", self.build_compiler.stage)); + + RmetaSysroot::from_stamp(builder, stamp, self.target, &dir) + } +} + +/// Checks rustc using `build_compiler`. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Rustc { /// Compiler that will check this rustc. - pub build_compiler: Compiler, + pub build_compiler: CompilerForCheck, pub target: TargetSelection, /// Whether to build only a subset of crates. /// @@ -166,12 +303,20 @@ pub struct Rustc { impl Rustc { pub fn new(builder: &Builder<'_>, target: TargetSelection, crates: Vec<String>) -> Self { let build_compiler = prepare_compiler_for_check(builder, target, Mode::Rustc); + Self::from_build_compiler(build_compiler, target, crates) + } + + fn from_build_compiler( + build_compiler: CompilerForCheck, + target: TargetSelection, + crates: Vec<String>, + ) -> Self { Self { build_compiler, target, crates } } } impl Step for Rustc { - type Output = (); + type Output = BuildStamp; const IS_HOST: bool = true; const DEFAULT: bool = true; @@ -191,8 +336,8 @@ impl Step for Rustc { /// created will also be linked into the sysroot directory. /// /// If we check a stage 2 compiler, we will have to first build a stage 1 compiler to check it. - fn run(self, builder: &Builder<'_>) { - let build_compiler = self.build_compiler; + fn run(self, builder: &Builder<'_>) -> Self::Output { + let build_compiler = self.build_compiler.build_compiler; let target = self.target; let mut cargo = builder::Cargo::new( @@ -205,6 +350,7 @@ impl Step for Rustc { ); rustc_cargo(builder, &mut cargo, target, &build_compiler, &self.crates); + self.build_compiler.configure_cargo(&mut cargo); // Explicitly pass -p for all compiler crates -- this will force cargo // to also check the tests/benches/examples for these crates, rather @@ -217,7 +363,7 @@ impl Step for Rustc { Kind::Check, format_args!("compiler artifacts{}", crate_description(&self.crates)), Mode::Rustc, - self.build_compiler, + self.build_compiler.build_compiler(), target, ); @@ -226,13 +372,12 @@ impl Step for Rustc { run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); - let libdir = builder.sysroot_target_libdir(build_compiler, target); - let hostdir = builder.sysroot_target_libdir(build_compiler, build_compiler.host); - add_to_sysroot(builder, &libdir, &hostdir, &stamp); + stamp } fn metadata(&self) -> Option<StepMetadata> { - let metadata = StepMetadata::check("rustc", self.target).built_by(self.build_compiler); + let metadata = StepMetadata::check("rustc", self.target) + .built_by(self.build_compiler.build_compiler()); let metadata = if self.crates.is_empty() { metadata } else { @@ -242,45 +387,101 @@ impl Step for Rustc { } } +/// Represents a compiler that can check something. +/// +/// If the compiler was created for `Mode::ToolRustc` or `Mode::Codegen`, it will also contain +/// .rmeta artifacts from rustc that was already checked using `build_compiler`. +/// +/// All steps that use this struct in a "general way" (i.e. they don't know exactly what kind of +/// thing is being built) should call `configure_cargo` to ensure that the rmeta artifacts are +/// properly linked, if present. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CompilerForCheck { + build_compiler: Compiler, + rustc_rmeta_sysroot: Option<RmetaSysroot>, + std_rmeta_sysroot: Option<RmetaSysroot>, +} + +impl CompilerForCheck { + pub fn build_compiler(&self) -> Compiler { + self.build_compiler + } + + /// If there are any rustc rmeta artifacts available, configure the Cargo invocation + /// so that the artifact being built can find them. + pub fn configure_cargo(&self, cargo: &mut Cargo) { + if let Some(sysroot) = &self.rustc_rmeta_sysroot { + sysroot.configure_cargo(cargo); + } + if let Some(sysroot) = &self.std_rmeta_sysroot { + sysroot.configure_cargo(cargo); + } + } +} + +/// Prepare the standard library for checking something (that requires stdlib) using +/// `build_compiler`. +fn prepare_std( + builder: &Builder<'_>, + build_compiler: Compiler, + target: TargetSelection, +) -> Option<RmetaSysroot> { + // We need to build the host stdlib even if we only check, to compile build scripts and proc + // macros + builder.std(build_compiler, builder.host_target); + + // If we're cross-compiling, we generate the rmeta files for the given target + // This check has to be here, because if we generate both .so and .rmeta files, rustc will fail, + // as it will have multiple candidates for linking. + if builder.host_target != target { + Some(builder.ensure(PrepareStdRmetaSysroot::new(build_compiler, target))) + } else { + None + } +} + /// Prepares a compiler that will check something with the given `mode`. pub fn prepare_compiler_for_check( builder: &Builder<'_>, target: TargetSelection, mode: Mode, -) -> Compiler { +) -> CompilerForCheck { let host = builder.host_target; - match mode { + let mut rustc_rmeta_sysroot = None; + let mut std_rmeta_sysroot = None; + let build_compiler = match mode { Mode::ToolBootstrap => builder.compiler(0, host), + // We could also only check std here and use `prepare_std`, but `ToolTarget` is currently + // only used for running in-tree Clippy on bootstrap tools, so it does not seem worth it to + // optimize it. Therefore, here we build std for the target, instead of just checking it. Mode::ToolTarget => get_tool_target_compiler(builder, ToolTargetBuildMode::Build(target)), Mode::ToolStd => { if builder.config.compile_time_deps { // When --compile-time-deps is passed, we can't use any rustc // other than the bootstrap compiler. Luckily build scripts and // proc macros for tools are unlikely to need nightly. - return builder.compiler(0, host); + builder.compiler(0, host) + } else { + // These tools require the local standard library to be checked + let build_compiler = builder.compiler(builder.top_stage, host); + std_rmeta_sysroot = prepare_std(builder, build_compiler, target); + build_compiler } - - // These tools require the local standard library to be checked - let build_compiler = builder.compiler(builder.top_stage, host); - - // We need to build the host stdlib to check the tool itself. - // We need to build the target stdlib so that the tool can link to it. - builder.std(build_compiler, host); - // We could only check this library in theory, but `check::Std` doesn't copy rmetas - // into `build_compiler`'s sysroot to avoid clashes with `.rlibs`, so we build it - // instead. - builder.std(build_compiler, target); - build_compiler } Mode::ToolRustc | Mode::Codegen => { // Check Rustc to produce the required rmeta artifacts for rustc_private, and then // return the build compiler that was used to check rustc. // We do not need to check examples/tests/etc. of Rustc for rustc_private, so we pass // an empty set of crates, which will avoid using `cargo -p`. - let check = Rustc::new(builder, target, vec![]); - let build_compiler = check.build_compiler; - builder.ensure(check); + let compiler_for_rustc = prepare_compiler_for_check(builder, target, Mode::Rustc); + rustc_rmeta_sysroot = Some( + builder.ensure(PrepareRustcRmetaSysroot::new(compiler_for_rustc.clone(), target)), + ); + let build_compiler = compiler_for_rustc.build_compiler(); + + // To check a rustc_private tool, we also need to check std that it will link to + std_rmeta_sysroot = prepare_std(builder, build_compiler, target); build_compiler } Mode::Rustc => { @@ -294,15 +495,8 @@ pub fn prepare_compiler_for_check( let stage = if host == target { builder.top_stage - 1 } else { builder.top_stage }; let build_compiler = builder.compiler(stage, host); - // Build host std for compiling build scripts - builder.std(build_compiler, build_compiler.host); - - // Build target std so that the checked rustc can link to it during the check - // FIXME: maybe we can a way to only do a check of std here? - // But for that we would have to copy the stdlib rmetas to the sysroot of the build - // compiler, which conflicts with std rlibs, if we also build std. - builder.std(build_compiler, target); - + // To check rustc, we need to check std that it will link to + std_rmeta_sysroot = prepare_std(builder, build_compiler, target); build_compiler } Mode::Std => { @@ -311,13 +505,14 @@ pub fn prepare_compiler_for_check( // stage 0 stdlib is used to compile build scripts and proc macros. builder.compiler(builder.top_stage, host) } - } + }; + CompilerForCheck { build_compiler, rustc_rmeta_sysroot, std_rmeta_sysroot } } /// Check the Cranelift codegen backend. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CraneliftCodegenBackend { - build_compiler: Compiler, + build_compiler: CompilerForCheck, target: TargetSelection, } @@ -332,12 +527,14 @@ impl Step for CraneliftCodegenBackend { } fn make_run(run: RunConfig<'_>) { - let build_compiler = prepare_compiler_for_check(run.builder, run.target, Mode::Codegen); - run.builder.ensure(CraneliftCodegenBackend { build_compiler, target: run.target }); + run.builder.ensure(CraneliftCodegenBackend { + build_compiler: prepare_compiler_for_check(run.builder, run.target, Mode::Codegen), + target: run.target, + }); } fn run(self, builder: &Builder<'_>) { - let build_compiler = self.build_compiler; + let build_compiler = self.build_compiler.build_compiler(); let target = self.target; let mut cargo = builder::Cargo::new( @@ -353,12 +550,13 @@ impl Step for CraneliftCodegenBackend { .arg("--manifest-path") .arg(builder.src.join("compiler/rustc_codegen_cranelift/Cargo.toml")); rustc_cargo_env(builder, &mut cargo, target); + self.build_compiler.configure_cargo(&mut cargo); let _guard = builder.msg( Kind::Check, "rustc_codegen_cranelift", Mode::Codegen, - self.build_compiler, + build_compiler, target, ); @@ -376,7 +574,7 @@ impl Step for CraneliftCodegenBackend { fn metadata(&self) -> Option<StepMetadata> { Some( StepMetadata::check("rustc_codegen_cranelift", self.target) - .built_by(self.build_compiler), + .built_by(self.build_compiler.build_compiler()), ) } } @@ -384,7 +582,7 @@ impl Step for CraneliftCodegenBackend { /// Check the GCC codegen backend. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct GccCodegenBackend { - build_compiler: Compiler, + build_compiler: CompilerForCheck, target: TargetSelection, } @@ -399,8 +597,10 @@ impl Step for GccCodegenBackend { } fn make_run(run: RunConfig<'_>) { - let build_compiler = prepare_compiler_for_check(run.builder, run.target, Mode::Codegen); - run.builder.ensure(GccCodegenBackend { build_compiler, target: run.target }); + run.builder.ensure(GccCodegenBackend { + build_compiler: prepare_compiler_for_check(run.builder, run.target, Mode::Codegen), + target: run.target, + }); } fn run(self, builder: &Builder<'_>) { @@ -410,7 +610,7 @@ impl Step for GccCodegenBackend { return; } - let build_compiler = self.build_compiler; + let build_compiler = self.build_compiler.build_compiler(); let target = self.target; let mut cargo = builder::Cargo::new( @@ -424,14 +624,10 @@ impl Step for GccCodegenBackend { cargo.arg("--manifest-path").arg(builder.src.join("compiler/rustc_codegen_gcc/Cargo.toml")); rustc_cargo_env(builder, &mut cargo, target); + self.build_compiler.configure_cargo(&mut cargo); - let _guard = builder.msg( - Kind::Check, - "rustc_codegen_gcc", - Mode::Codegen, - self.build_compiler, - target, - ); + let _guard = + builder.msg(Kind::Check, "rustc_codegen_gcc", Mode::Codegen, build_compiler, target); let stamp = build_stamp::codegen_backend_stamp( builder, @@ -445,7 +641,10 @@ impl Step for GccCodegenBackend { } fn metadata(&self) -> Option<StepMetadata> { - Some(StepMetadata::check("rustc_codegen_gcc", self.target).built_by(self.build_compiler)) + Some( + StepMetadata::check("rustc_codegen_gcc", self.target) + .built_by(self.build_compiler.build_compiler()), + ) } } @@ -467,8 +666,8 @@ macro_rules! tool_check_step { ) => { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct $name { - pub build_compiler: Compiler, - pub target: TargetSelection, + compiler: CompilerForCheck, + target: TargetSelection, } impl Step for $name { @@ -486,7 +685,7 @@ macro_rules! tool_check_step { let builder = run.builder; let mode = $mode(builder); - let build_compiler = prepare_compiler_for_check(run.builder, target, mode); + let compiler = prepare_compiler_for_check(run.builder, target, mode); // It doesn't make sense to cross-check bootstrap tools if mode == Mode::ToolBootstrap && target != run.builder.host_target { @@ -494,11 +693,11 @@ macro_rules! tool_check_step { return; }; - run.builder.ensure($name { target, build_compiler }); + run.builder.ensure($name { target, compiler }); } fn run(self, builder: &Builder<'_>) { - let Self { target, build_compiler } = self; + let Self { target, compiler } = self; let allow_features = { let mut _value = ""; $( _value = $allow_features; )? @@ -506,11 +705,11 @@ macro_rules! tool_check_step { }; let extra_features: &[&str] = &[$($($enable_features),*)?]; let mode = $mode(builder); - run_tool_check_step(builder, build_compiler, target, $path, mode, allow_features, extra_features); + run_tool_check_step(builder, compiler, target, $path, mode, allow_features, extra_features); } fn metadata(&self) -> Option<StepMetadata> { - Some(StepMetadata::check(stringify!($name), self.target).built_by(self.build_compiler)) + Some(StepMetadata::check(stringify!($name), self.target).built_by(self.compiler.build_compiler)) } } } @@ -519,7 +718,7 @@ macro_rules! tool_check_step { /// Used by the implementation of `Step::run` in `tool_check_step!`. fn run_tool_check_step( builder: &Builder<'_>, - build_compiler: Compiler, + compiler: CompilerForCheck, target: TargetSelection, path: &str, mode: Mode, @@ -528,6 +727,8 @@ fn run_tool_check_step( ) { let display_name = path.rsplit('/').next().unwrap(); + let build_compiler = compiler.build_compiler(); + let extra_features = extra_features.iter().map(|f| f.to_string()).collect::<Vec<String>>(); let mut cargo = prepare_tool_cargo( builder, @@ -544,6 +745,7 @@ fn run_tool_check_step( &extra_features, ); cargo.allow_features(allow_features); + compiler.configure_cargo(&mut cargo); // FIXME: check bootstrap doesn't currently work when multiple targets are checked // FIXME: rust-analyzer does not work with --all-targets diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs index 3c4aa0886c2..05f8b240291 100644 --- a/src/bootstrap/src/core/build_steps/clippy.rs +++ b/src/bootstrap/src/core/build_steps/clippy.rs @@ -18,7 +18,7 @@ use build_helper::exit; use super::compile::{run_cargo, rustc_cargo, std_cargo}; use super::tool::{SourceType, prepare_tool_cargo}; use crate::builder::{Builder, ShouldRun}; -use crate::core::build_steps::check::prepare_compiler_for_check; +use crate::core::build_steps::check::{CompilerForCheck, prepare_compiler_for_check}; use crate::core::build_steps::compile::std_crates_for_run_make; use crate::core::builder; use crate::core::builder::{Alias, Kind, RunConfig, Step, StepMetadata, crate_description}; @@ -231,7 +231,7 @@ impl Step for Std { /// in-tree rustc. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Rustc { - build_compiler: Compiler, + build_compiler: CompilerForCheck, target: TargetSelection, config: LintConfig, /// Whether to lint only a subset of crates. @@ -271,7 +271,7 @@ impl Step for Rustc { } fn run(self, builder: &Builder<'_>) { - let build_compiler = self.build_compiler; + let build_compiler = self.build_compiler.build_compiler(); let target = self.target; let mut cargo = builder::Cargo::new( @@ -284,6 +284,7 @@ impl Step for Rustc { ); rustc_cargo(builder, &mut cargo, target, &build_compiler, &self.crates); + self.build_compiler.configure_cargo(&mut cargo); // Explicitly pass -p for all compiler crates -- this will force cargo // to also lint the tests/benches/examples for these crates, rather @@ -312,13 +313,16 @@ impl Step for Rustc { } fn metadata(&self) -> Option<StepMetadata> { - Some(StepMetadata::clippy("rustc", self.target).built_by(self.build_compiler)) + Some( + StepMetadata::clippy("rustc", self.target) + .built_by(self.build_compiler.build_compiler()), + ) } } #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct CodegenGcc { - build_compiler: Compiler, + build_compiler: CompilerForCheck, target: TargetSelection, config: LintConfig, } @@ -347,10 +351,10 @@ impl Step for CodegenGcc { } fn run(self, builder: &Builder<'_>) -> Self::Output { - let build_compiler = self.build_compiler; + let build_compiler = self.build_compiler.build_compiler(); let target = self.target; - let cargo = prepare_tool_cargo( + let mut cargo = prepare_tool_cargo( builder, build_compiler, Mode::Codegen, @@ -360,6 +364,7 @@ impl Step for CodegenGcc { SourceType::InTree, &[], ); + self.build_compiler.configure_cargo(&mut cargo); let _guard = builder.msg(Kind::Clippy, "rustc_codegen_gcc", Mode::ToolRustc, build_compiler, target); @@ -379,7 +384,10 @@ impl Step for CodegenGcc { } fn metadata(&self) -> Option<StepMetadata> { - Some(StepMetadata::clippy("rustc_codegen_gcc", self.target).built_by(self.build_compiler)) + Some( + StepMetadata::clippy("rustc_codegen_gcc", self.target) + .built_by(self.build_compiler.build_compiler()), + ) } } @@ -396,7 +404,7 @@ macro_rules! lint_any { #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct $name { - build_compiler: Compiler, + build_compiler: CompilerForCheck, target: TargetSelection, config: LintConfig, } @@ -419,9 +427,9 @@ macro_rules! lint_any { } fn run(self, builder: &Builder<'_>) -> Self::Output { - let build_compiler = self.build_compiler; + let build_compiler = self.build_compiler.build_compiler(); let target = self.target; - let cargo = prepare_tool_cargo( + let mut cargo = prepare_tool_cargo( builder, build_compiler, $mode, @@ -431,6 +439,7 @@ macro_rules! lint_any { SourceType::InTree, &[], ); + self.build_compiler.configure_cargo(&mut cargo); let _guard = builder.msg( Kind::Clippy, @@ -456,7 +465,7 @@ macro_rules! lint_any { } fn metadata(&self) -> Option<StepMetadata> { - Some(StepMetadata::clippy($readable_name, self.target).built_by(self.build_compiler)) + Some(StepMetadata::clippy($readable_name, self.target).built_by(self.build_compiler.build_compiler())) } } )+ diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 997a152a31f..004b132f28f 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -12,6 +12,7 @@ use std::ffi::OsStr; use std::io::BufReader; use std::io::prelude::*; use std::path::{Path, PathBuf}; +use std::time::SystemTime; use std::{env, fs, str}; use serde_derive::Deserialize; @@ -38,7 +39,7 @@ use crate::{ }; /// Build a standard library for the given `target` using the given `build_compiler`. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Std { pub target: TargetSelection, /// Compiler that builds the standard library. @@ -933,13 +934,22 @@ fn cp_rustc_component_to_ci_sysroot(builder: &Builder<'_>, sysroot: &Path, conte } } +/// Represents information about a built rustc. +#[derive(Clone, Debug)] +pub struct BuiltRustc { + /// The compiler that actually built this *rustc*. + /// This can be different from the *build_compiler* passed to the `Rustc` step because of + /// uplifting. + pub build_compiler: Compiler, +} + /// Build rustc using the passed `build_compiler`. /// /// - Makes sure that `build_compiler` has a standard library prepared for its host target, /// so that it can compile build scripts and proc macros when building this `rustc`. /// - Makes sure that `build_compiler` has a standard library prepared for `target`, /// so that the built `rustc` can *link to it* and use it at runtime. -#[derive(Debug, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Rustc { /// The target on which rustc will run (its host). pub target: TargetSelection, @@ -960,7 +970,7 @@ impl Rustc { } impl Step for Rustc { - type Output = (); + type Output = BuiltRustc; const IS_HOST: bool = true; const DEFAULT: bool = false; @@ -1000,7 +1010,7 @@ impl Step for Rustc { /// This will build the compiler for a particular stage of the build using /// the `build_compiler` targeting the `target` architecture. The artifacts /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { + fn run(self, builder: &Builder<'_>) -> Self::Output { let build_compiler = self.build_compiler; let target = self.target; @@ -1016,7 +1026,7 @@ impl Step for Rustc { &sysroot, builder.config.ci_rustc_dev_contents(), ); - return; + return BuiltRustc { build_compiler }; } // Build a standard library for `target` using the `build_compiler`. @@ -1028,9 +1038,9 @@ impl Step for Rustc { builder.info("WARNING: Using a potentially old librustc. This may not behave well."); builder.info("WARNING: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); - builder.ensure(RustcLink::from_rustc(self, build_compiler)); + builder.ensure(RustcLink::from_rustc(self)); - return; + return BuiltRustc { build_compiler }; } // The stage of the compiler that we're building @@ -1042,21 +1052,35 @@ impl Step for Rustc { && !builder.config.full_bootstrap && (target == builder.host_target || builder.hosts.contains(&target)) { - // If we're cross-compiling, the earliest rustc that we could have is stage 2. - // If we're not cross-compiling, then we should have rustc stage 1. - let stage_to_uplift = if target == builder.host_target { 1 } else { 2 }; - let rustc_to_uplift = builder.compiler(stage_to_uplift, target); - let msg = if rustc_to_uplift.host == target { - format!("Uplifting rustc (stage{} -> stage{stage})", rustc_to_uplift.stage,) + // Here we need to determine the **build compiler** that built the stage that we will + // be uplifting. We cannot uplift stage 1, as it has a different ABI than stage 2+, + // so we always uplift the stage2 compiler (compiled with stage 1). + let uplift_build_compiler = builder.compiler(1, build_compiler.host); + let msg = if uplift_build_compiler.host == target { + format!("Uplifting rustc (stage2 -> stage{stage})") } else { format!( - "Uplifting rustc (stage{}:{} -> stage{stage}:{target})", - rustc_to_uplift.stage, rustc_to_uplift.host, + "Uplifting rustc (stage2:{} -> stage{stage}:{target})", + uplift_build_compiler.host ) }; builder.info(&msg); - builder.ensure(RustcLink::from_rustc(self, rustc_to_uplift)); - return; + + // Here the compiler that built the rlibs (`uplift_build_compiler`) can be different + // from the compiler whose sysroot should be modified in this step. So we need to copy + // the (previously built) rlibs into the correct sysroot. + builder.ensure(RustcLink::from_build_compiler_and_sysroot( + // This is the compiler that actually built the rustc rlibs + uplift_build_compiler, + // We copy the rlibs into the sysroot of `build_compiler` + build_compiler, + target, + self.crates, + )); + + // Here we have performed an uplift, so we return the actual build compiler that "built" + // this rustc. + return BuiltRustc { build_compiler: uplift_build_compiler }; } // Build a standard library for the current host target using the `build_compiler`. @@ -1129,10 +1153,8 @@ impl Step for Rustc { strip_debug(builder, target, &target_root_dir.join("rustc-main")); } - builder.ensure(RustcLink::from_rustc( - self, - builder.compiler(build_compiler.stage, builder.config.host_target), - )); + builder.ensure(RustcLink::from_rustc(self)); + BuiltRustc { build_compiler } } fn metadata(&self) -> Option<StepMetadata> { @@ -1441,31 +1463,51 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect } } -/// `RustcLink` copies all of the rlibs from the rustc build into the previous stage's sysroot. +/// `RustcLink` copies compiler rlibs from a rustc build into a compiler sysroot. +/// It works with (potentially up to) three compilers: +/// - `build_compiler` is a compiler that built rustc rlibs +/// - `sysroot_compiler` is a compiler into whose sysroot we will copy the rlibs +/// - In most situations, `build_compiler` == `sysroot_compiler` +/// - `target_compiler` is the compiler whose rlibs were built. It is not represented explicitly +/// in this step, rather we just read the rlibs from a rustc build stamp of `build_compiler`. +/// /// This is necessary for tools using `rustc_private`, where the previous compiler will build /// a tool against the next compiler. /// To build a tool against a compiler, the rlibs of that compiler that it links against /// must be in the sysroot of the compiler that's doing the compiling. #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct RustcLink { - /// The compiler whose rlibs we are copying around. - pub compiler: Compiler, - /// This is the compiler into whose sysroot we want to copy the rlibs into. - pub previous_stage_compiler: Compiler, - pub target: TargetSelection, + /// This compiler **built** some rustc, whose rlibs we will copy into a sysroot. + build_compiler: Compiler, + /// This is the compiler into whose sysroot we want to copy the built rlibs. + /// In most cases, it will correspond to `build_compiler`. + sysroot_compiler: Compiler, + target: TargetSelection, /// Not actually used; only present to make sure the cache invalidation is correct. crates: Vec<String>, } impl RustcLink { - fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self { + /// Copy rlibs from the build compiler that build this `rustc` into the sysroot of that + /// build compiler. + fn from_rustc(rustc: Rustc) -> Self { Self { - compiler: host_compiler, - previous_stage_compiler: rustc.build_compiler, + build_compiler: rustc.build_compiler, + sysroot_compiler: rustc.build_compiler, target: rustc.target, crates: rustc.crates, } } + + /// Copy rlibs **built** by `build_compiler` into the sysroot of `sysroot_compiler`. + fn from_build_compiler_and_sysroot( + build_compiler: Compiler, + sysroot_compiler: Compiler, + target: TargetSelection, + crates: Vec<String>, + ) -> Self { + Self { build_compiler, sysroot_compiler, target, crates } + } } impl Step for RustcLink { @@ -1477,14 +1519,14 @@ impl Step for RustcLink { /// Same as `std_link`, only for librustc fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let previous_stage_compiler = self.previous_stage_compiler; + let build_compiler = self.build_compiler; + let sysroot_compiler = self.sysroot_compiler; let target = self.target; add_to_sysroot( builder, - &builder.sysroot_target_libdir(previous_stage_compiler, target), - &builder.sysroot_target_libdir(previous_stage_compiler, compiler.host), - &build_stamp::librustc_stamp(builder, compiler, target), + &builder.sysroot_target_libdir(sysroot_compiler, target), + &builder.sysroot_target_libdir(sysroot_compiler, sysroot_compiler.host), + &build_stamp::librustc_stamp(builder, build_compiler, target), ); } } @@ -1918,7 +1960,7 @@ impl Step for Sysroot { /// linker wrappers (LLD, LLVM bitcode linker, etc.). /// /// This will assemble a compiler in `build/$target/stage$stage`. -#[derive(Debug, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Assemble { /// The compiler which we will produce in this step. Assemble itself will /// take care of ensuring that the necessary prerequisites to do so exist, @@ -2099,7 +2141,10 @@ impl Step for Assemble { "target_compiler.host" = ?target_compiler.host, "building compiler libraries to link to" ); - builder.ensure(Rustc::new(build_compiler, target_compiler.host)); + + // It is possible that an uplift has happened, so we override build_compiler here. + let BuiltRustc { build_compiler } = + builder.ensure(Rustc::new(build_compiler, target_compiler.host)); let stage = target_compiler.stage; let host = target_compiler.host; @@ -2286,6 +2331,7 @@ impl Step for Assemble { /// /// For a particular stage this will link the file listed in `stamp` into the /// `sysroot_dst` provided. +#[track_caller] pub fn add_to_sysroot( builder: &Builder<'_>, sysroot_dst: &Path, @@ -2568,7 +2614,17 @@ pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) } let previous_mtime = t!(t!(path.metadata()).modified()); - command("strip").arg("--strip-debug").arg(path).run_capture(builder); + let stamp = BuildStamp::new(path.parent().unwrap()) + .with_prefix(path.file_name().unwrap().to_str().unwrap()) + .with_prefix("strip") + .add_stamp(previous_mtime.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_nanos()); + + // Running strip can be relatively expensive (~1s on librustc_driver.so), so we don't rerun it + // if the file is unchanged. + if !stamp.is_up_to_date() { + command("strip").arg("--strip-debug").arg(path).run_capture(builder); + } + t!(stamp.write()); let file = t!(fs::File::open(path)); diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 414f4464d1e..beb71e70035 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -54,7 +54,7 @@ fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { builder.config.tools.as_ref().is_none_or(|tools| tools.contains(tool)) } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Docs { pub host: TargetSelection, } @@ -91,7 +91,7 @@ impl Step for Docs { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct JsonDocs { build_compiler: Compiler, target: TargetSelection, @@ -354,7 +354,7 @@ fn get_cc_search_dirs( (bin_path, lib_path) } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Mingw { pub host: TargetSelection, } @@ -394,7 +394,7 @@ impl Step for Mingw { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Rustc { pub compiler: Compiler, } @@ -730,7 +730,7 @@ fn copy_target_libs( } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Std { pub compiler: Compiler, pub target: TargetSelection, @@ -785,7 +785,7 @@ impl Step for Std { /// `rust.download-rustc`. /// /// (Don't confuse this with [`RustDev`], without the `c`!) -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustcDev { pub compiler: Compiler, pub target: TargetSelection, @@ -916,6 +916,12 @@ fn copy_src_dirs( exclude_dirs: &[&str], dst_dir: &Path, ) { + // The src directories should be relative to `base`, we depend on them not being absolute + // paths below. + for src_dir in src_dirs { + assert!(Path::new(src_dir).is_relative()); + } + // Iterating, filtering and copying a large number of directories can be quite slow. // Avoid doing it in dry run (and thus also tests). if builder.config.dry_run() { @@ -923,6 +929,7 @@ fn copy_src_dirs( } fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool { + // The paths are relative, e.g. `llvm-project/...`. let spath = match path.to_str() { Some(path) => path, None => return false, @@ -930,65 +937,53 @@ fn copy_src_dirs( if spath.ends_with('~') || spath.ends_with(".pyc") { return false; } + // Normalize slashes + let spath = spath.replace("\\", "/"); - const LLVM_PROJECTS: &[&str] = &[ + static LLVM_PROJECTS: &[&str] = &[ "llvm-project/clang", - "llvm-project\\clang", "llvm-project/libunwind", - "llvm-project\\libunwind", "llvm-project/lld", - "llvm-project\\lld", "llvm-project/lldb", - "llvm-project\\lldb", "llvm-project/llvm", - "llvm-project\\llvm", "llvm-project/compiler-rt", - "llvm-project\\compiler-rt", "llvm-project/cmake", - "llvm-project\\cmake", "llvm-project/runtimes", - "llvm-project\\runtimes", "llvm-project/third-party", - "llvm-project\\third-party", ]; - if spath.contains("llvm-project") - && !spath.ends_with("llvm-project") - && !LLVM_PROJECTS.iter().any(|path| spath.contains(path)) - { - return false; - } + if spath.starts_with("llvm-project") && spath != "llvm-project" { + if !LLVM_PROJECTS.iter().any(|path| spath.starts_with(path)) { + return false; + } - // Keep only these third party libraries - const LLVM_THIRD_PARTY: &[&str] = - &["llvm-project/third-party/siphash", "llvm-project\\third-party\\siphash"]; - if (spath.starts_with("llvm-project/third-party") - || spath.starts_with("llvm-project\\third-party")) - && !(spath.ends_with("llvm-project/third-party") - || spath.ends_with("llvm-project\\third-party")) - && !LLVM_THIRD_PARTY.iter().any(|path| spath.contains(path)) - { - return false; - } + // Keep siphash third-party dependency + if spath.starts_with("llvm-project/third-party") + && spath != "llvm-project/third-party" + && !spath.starts_with("llvm-project/third-party/siphash") + { + return false; + } - const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"]; - if LLVM_TEST.iter().any(|path| spath.contains(path)) - && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s")) - { - return false; + if spath.starts_with("llvm-project/llvm/test") + && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s")) + { + return false; + } } // Cargo tests use some files like `.gitignore` that we would otherwise exclude. - const CARGO_TESTS: &[&str] = &["tools/cargo/tests", "tools\\cargo\\tests"]; - if CARGO_TESTS.iter().any(|path| spath.contains(path)) { + if spath.starts_with("tools/cargo/tests") { return true; } - let full_path = Path::new(dir).join(path); - if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) { - return false; + if !exclude_dirs.is_empty() { + let full_path = Path::new(dir).join(path); + if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) { + return false; + } } - let excludes = [ + static EXCLUDES: &[&str] = &[ "CVS", "RCS", "SCCS", @@ -1011,7 +1006,15 @@ fn copy_src_dirs( ".hgrags", "_darcs", ]; - !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s)) + + // We want to check if any component of `path` doesn't contain the strings in `EXCLUDES`. + // However, since we traverse directories top-down in `Builder::cp_link_filtered`, + // it is enough to always check only the last component: + // - If the path is a file, we will iterate to it and then check it's filename + // - If the path is a dir, if it's dir name contains an excluded string, we will not even + // recurse into it. + let last_component = path.iter().next_back().map(|s| s.to_str().unwrap()).unwrap(); + !EXCLUDES.contains(&last_component) } // Copy the directories using our filter @@ -1023,7 +1026,7 @@ fn copy_src_dirs( } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Src; impl Step for Src { @@ -1084,7 +1087,7 @@ impl Step for Src { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct PlainSourceTarball; impl Step for PlainSourceTarball { @@ -1230,7 +1233,7 @@ impl Step for PlainSourceTarball { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Cargo { pub build_compiler: Compiler, pub target: TargetSelection, @@ -1284,7 +1287,7 @@ impl Step for Cargo { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustAnalyzer { pub build_compiler: Compiler, pub target: TargetSelection, @@ -1560,7 +1563,7 @@ impl Step for Rustfmt { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Extended { stage: u32, host: TargetSelection, @@ -2401,7 +2404,7 @@ impl Step for LlvmTools { /// Distributes the `llvm-bitcode-linker` tool so that it can be used by a compiler whose host /// is `target`. -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct LlvmBitcodeLinker { /// The linker will be compiled by this compiler. pub build_compiler: Compiler, diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index f6b27d83120..7fe19c00ef5 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -580,7 +580,7 @@ impl Step for SharedAssets { } /// Document the standard library using `build_compiler`. -#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Std { build_compiler: Compiler, target: TargetSelection, @@ -715,7 +715,7 @@ impl Step for Std { /// or remote link. const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"]; -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub enum DocumentationFormat { Html, Json, @@ -1230,7 +1230,7 @@ fn symlink_dir_force(config: &Config, original: &Path, link: &Path) { } /// Builds the Rust compiler book. -#[derive(Ord, PartialOrd, Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustcBook { build_compiler: Compiler, target: TargetSelection, @@ -1334,7 +1334,7 @@ impl Step for RustcBook { /// Documents the reference. /// It has to always be done using a stage 1+ compiler, because it references in-tree /// compiler/stdlib concepts. -#[derive(Ord, PartialOrd, Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct Reference { build_compiler: Compiler, target: TargetSelection, diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 7f1a7d00257..c6288f63847 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -17,7 +17,7 @@ use crate::core::config::flags::get_completion; use crate::utils::exec::command; use crate::{Mode, t}; -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct BuildManifest; impl Step for BuildManifest { @@ -56,7 +56,7 @@ impl Step for BuildManifest { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct BumpStage0; impl Step for BumpStage0 { @@ -78,7 +78,7 @@ impl Step for BumpStage0 { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct ReplaceVersionPlaceholder; impl Step for ReplaceVersionPlaceholder { @@ -169,7 +169,7 @@ impl Step for Miri { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct CollectLicenseMetadata; impl Step for CollectLicenseMetadata { @@ -200,7 +200,7 @@ impl Step for CollectLicenseMetadata { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct GenerateCopyright; impl Step for GenerateCopyright { @@ -264,7 +264,7 @@ impl Step for GenerateCopyright { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct GenerateWindowsSys; impl Step for GenerateWindowsSys { @@ -326,7 +326,7 @@ impl Step for GenerateCompletions { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct UnicodeTableGenerator; impl Step for UnicodeTableGenerator { @@ -348,7 +348,7 @@ impl Step for UnicodeTableGenerator { } } -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct FeaturesStatusDump; impl Step for FeaturesStatusDump { @@ -408,7 +408,7 @@ impl Step for CyclicStep { /// /// The coverage-dump tool is an internal detail of coverage tests, so this run /// step is only needed when testing coverage-dump manually. -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct CoverageDump; impl Step for CoverageDump { diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 4006bed4ac5..e7a57a7f375 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -2732,7 +2732,7 @@ fn prepare_cargo_test( /// FIXME(Zalathar): Try to split this into two separate steps: a user-visible /// step for testing standard library crates, and an internal step used for both /// library crates and compiler crates. -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Crate { pub compiler: Compiler, pub target: TargetSelection, @@ -3747,7 +3747,7 @@ impl Step for TestFloatParse { /// Runs the tool `src/tools/collect-license-metadata` in `ONLY_CHECK=1` mode, /// which verifies that `license-metadata.json` is up-to-date and therefore /// running the tool normally would not update anything. -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct CollectLicenseMetadata; impl Step for CollectLicenseMetadata { diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index 3ce21eb151c..69a744a86cb 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -101,6 +101,7 @@ pub struct Cargo { impl Cargo { /// Calls [`Builder::cargo`] and [`Cargo::configure_linker`] to prepare an invocation of `cargo` /// to be run. + #[track_caller] pub fn new( builder: &Builder<'_>, compiler: Compiler, @@ -139,6 +140,7 @@ impl Cargo { /// Same as [`Cargo::new`] except this one doesn't configure the linker with /// [`Cargo::configure_linker`]. + #[track_caller] pub fn new_for_mir_opt_tests( builder: &Builder<'_>, compiler: Compiler, @@ -186,6 +188,32 @@ impl Cargo { self } + /// Append a value to an env var of the cargo command instance. + /// If the variable was unset previously, this is equivalent to [`Cargo::env`]. + /// If the variable was already set, this will append `delimiter` and then `value` to it. + /// + /// Note that this only considers the existence of the env. var. configured on this `Cargo` + /// instance. It does not look at the environment of this process. + pub fn append_to_env( + &mut self, + key: impl AsRef<OsStr>, + value: impl AsRef<OsStr>, + delimiter: impl AsRef<OsStr>, + ) -> &mut Cargo { + assert_ne!(key.as_ref(), "RUSTFLAGS"); + assert_ne!(key.as_ref(), "RUSTDOCFLAGS"); + + let key = key.as_ref(); + if let Some((_, Some(previous_value))) = self.command.get_envs().find(|(k, _)| *k == key) { + let mut combined: OsString = previous_value.to_os_string(); + combined.push(delimiter.as_ref()); + combined.push(value.as_ref()); + self.env(key, combined) + } else { + self.env(key, value) + } + } + pub fn add_rustc_lib_path(&mut self, builder: &Builder<'_>) { builder.add_rustc_lib_path(self.compiler, &mut self.command); } @@ -396,6 +424,7 @@ impl From<Cargo> for BootstrapCommand { impl Builder<'_> { /// Like [`Builder::cargo`], but only passes flags that are valid for all commands. + #[track_caller] pub fn bare_cargo( &self, compiler: Compiler, @@ -480,6 +509,7 @@ impl Builder<'_> { /// scoped by `mode`'s output directory, it will pass the `--target` flag for the specified /// `target`, and will be executing the Cargo command `cmd`. `cmd` can be `miri-cmd` for /// commands to be run with Miri. + #[track_caller] fn cargo( &self, compiler: Compiler, diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index a9398a654e9..f4266a6085b 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -1569,7 +1569,7 @@ mod snapshot { [build] llvm <host> [build] rustc 0 <host> -> rustc 1 <host> [build] rustc 1 <host> -> std 1 <host> - [build] rustc 1 <host> -> std 1 <target1> + [check] rustc 1 <host> -> std 1 <target1> [check] rustc 1 <host> -> rustc 2 <target1> (73 crates) [check] rustc 1 <host> -> rustc 2 <target1> [check] rustc 1 <host> -> Rustdoc 2 <target1> diff --git a/src/bootstrap/src/core/config/mod.rs b/src/bootstrap/src/core/config/mod.rs index 285d20917e7..8c5f9037251 100644 --- a/src/bootstrap/src/core/config/mod.rs +++ b/src/bootstrap/src/core/config/mod.rs @@ -324,7 +324,7 @@ impl FromStr for LlvmLibunwind { } } -#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Hash)] pub enum SplitDebuginfo { Packed, Unpacked, diff --git a/src/bootstrap/src/core/config/target_selection.rs b/src/bootstrap/src/core/config/target_selection.rs index ebd3fe7a8c6..40b63a7f9c7 100644 --- a/src/bootstrap/src/core/config/target_selection.rs +++ b/src/bootstrap/src/core/config/target_selection.rs @@ -14,7 +14,7 @@ pub struct TargetSelection { } /// Newtype over `Vec<TargetSelection>` so we can implement custom parsing logic -#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +#[derive(Clone, Default, PartialEq, Eq, Hash, Debug)] pub struct TargetSelectionList(pub Vec<TargetSelection>); pub fn target_selection_list(s: &str) -> Result<TargetSelectionList, String> { diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 706a3cbb210..8f766ed00a5 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -279,7 +279,7 @@ pub enum DependencyType { /// /// These entries currently correspond to the various output directories of the /// build system, with each mod generating output in a different directory. -#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq)] pub enum Mode { /// Build the standard library, placing output in the "stageN-std" directory. Std, @@ -357,7 +357,7 @@ pub enum RemapScheme { NonCompiler, } -#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq)] pub enum CLang { C, Cxx, @@ -1743,6 +1743,7 @@ impl Build { /// /// If `src` is a symlink, `src` will be resolved to the actual path /// and copied to `dst` instead of the symlink itself. + #[track_caller] pub fn resolve_symlink_and_copy(&self, src: &Path, dst: &Path) { self.copy_link_internal(src, dst, true); } @@ -1751,6 +1752,7 @@ impl Build { /// Attempts to use hard links if possible, falling back to copying. /// You can neither rely on this being a copy nor it being a link, /// so do not write to dst. + #[track_caller] pub fn copy_link(&self, src: &Path, dst: &Path, file_type: FileType) { self.copy_link_internal(src, dst, false); @@ -1765,6 +1767,7 @@ impl Build { } } + #[track_caller] fn copy_link_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { if self.config.dry_run() { return; @@ -1773,6 +1776,10 @@ impl Build { if src == dst { return; } + + #[cfg(feature = "tracing")] + let _span = trace_io!("file-copy-link", ?src, ?dst); + if let Err(e) = fs::remove_file(dst) && cfg!(windows) && e.kind() != io::ErrorKind::NotFound @@ -1815,6 +1822,7 @@ impl Build { /// Links the `src` directory recursively to `dst`. Both are assumed to exist /// when this function is called. /// Will attempt to use hard links if possible and fall back to copying. + #[track_caller] pub fn cp_link_r(&self, src: &Path, dst: &Path) { if self.config.dry_run() { return; @@ -1837,12 +1845,14 @@ impl Build { /// Will attempt to use hard links if possible and fall back to copying. /// Unwanted files or directories can be skipped /// by returning `false` from the filter function. + #[track_caller] pub fn cp_link_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { // Immediately recurse with an empty relative path self.cp_link_filtered_recurse(src, dst, Path::new(""), filter) } // Inner function does the actual work + #[track_caller] fn cp_link_filtered_recurse( &self, src: &Path, @@ -1862,7 +1872,6 @@ impl Build { self.create_dir(&dst); self.cp_link_filtered_recurse(&path, &dst, &relative, filter); } else { - let _ = fs::remove_file(&dst); self.copy_link(&path, &dst, FileType::Regular); } } @@ -1904,10 +1913,15 @@ impl Build { t!(fs::read_to_string(path)) } + #[track_caller] fn create_dir(&self, dir: &Path) { if self.config.dry_run() { return; } + + #[cfg(feature = "tracing")] + let _span = trace_io!("dir-create", ?dir); + t!(fs::create_dir_all(dir)) } @@ -1915,6 +1929,10 @@ impl Build { if self.config.dry_run() { return; } + + #[cfg(feature = "tracing")] + let _span = trace_io!("dir-remove", ?dir); + t!(fs::remove_dir_all(dir)) } diff --git a/src/bootstrap/src/utils/tracing.rs b/src/bootstrap/src/utils/tracing.rs index 428ba013c98..472781ffa73 100644 --- a/src/bootstrap/src/utils/tracing.rs +++ b/src/bootstrap/src/utils/tracing.rs @@ -49,13 +49,36 @@ macro_rules! error { } #[cfg(feature = "tracing")] +pub const IO_SPAN_TARGET: &str = "IO"; + +/// Create a tracing span around an I/O operation, if tracing is enabled. +/// Note that at least one tracing value field has to be passed to this macro, otherwise it will not +/// compile. +#[macro_export] +macro_rules! trace_io { + ($name:expr, $($args:tt)*) => { + ::tracing::trace_span!( + target: $crate::utils::tracing::IO_SPAN_TARGET, + $name, + $($args)*, + location = $crate::utils::tracing::format_location(*::std::panic::Location::caller()) + ).entered() + } +} + +#[cfg(feature = "tracing")] +pub fn format_location(location: std::panic::Location<'static>) -> String { + format!("{}:{}", location.file(), location.line()) +} + +#[cfg(feature = "tracing")] const COMMAND_SPAN_TARGET: &str = "COMMAND"; #[cfg(feature = "tracing")] pub fn trace_cmd(command: &crate::BootstrapCommand) -> tracing::span::EnteredSpan { let fingerprint = command.fingerprint(); let location = command.get_created_location(); - let location = format!("{}:{}", location.file(), location.line()); + let location = format_location(location); tracing::span!( target: COMMAND_SPAN_TARGET, @@ -84,6 +107,7 @@ mod inner { use std::fmt::Debug; use std::fs::File; use std::io::Write; + use std::path::{Path, PathBuf}; use std::sync::atomic::Ordering; use chrono::{DateTime, Utc}; @@ -93,8 +117,8 @@ mod inner { use tracing_subscriber::registry::{LookupSpan, SpanRef}; use tracing_subscriber::{EnvFilter, Layer}; + use super::{COMMAND_SPAN_TARGET, IO_SPAN_TARGET}; use crate::STEP_SPAN_TARGET; - use crate::utils::tracing::COMMAND_SPAN_TARGET; pub fn setup_tracing(env_name: &str) -> TracingGuard { let filter = EnvFilter::from_env(env_name); @@ -291,6 +315,23 @@ mod inner { Ok(()) } + // Write fields while treating the "location" field specially, and assuming that it + // contains the source file location relevant to the span. + let write_with_location = |writer: &mut W| -> std::io::Result<()> { + if let Some(values) = field_values { + write_fields( + writer, + values.fields.iter().filter(|(name, _)| *name != "location"), + )?; + let location = + &values.fields.iter().find(|(name, _)| *name == "location").unwrap().1; + let (filename, line) = location.rsplit_once(':').unwrap(); + let filename = shorten_filename(filename); + write!(writer, " ({filename}:{line})",)?; + } + Ok(()) + }; + // We handle steps specially. We instrument them dynamically in `Builder::ensure`, // and we want to have custom name for each step span. But tracing doesn't allow setting // dynamic span names. So we detect step spans here and override their name. @@ -311,17 +352,11 @@ mod inner { // Executed command COMMAND_SPAN_TARGET => { write!(writer, "{}", span.name())?; - if let Some(values) = field_values { - write_fields( - writer, - values.fields.iter().filter(|(name, _)| *name != "location"), - )?; - write!( - writer, - " ({})", - values.fields.iter().find(|(name, _)| *name == "location").unwrap().1 - )?; - } + write_with_location(writer)?; + } + IO_SPAN_TARGET => { + write!(writer, "{}", span.name())?; + write_with_location(writer)?; } // Other span _ => { @@ -342,21 +377,10 @@ mod inner { writer: &mut W, metadata: &'static tracing::Metadata<'static>, ) -> std::io::Result<()> { - use std::path::{Path, PathBuf}; - if let Some(filename) = metadata.file() { - // Keep only the module name and file name to make it shorter - let filename: PathBuf = Path::new(filename) - .components() - // Take last two path components - .rev() - .take(2) - .collect::<Vec<_>>() - .into_iter() - .rev() - .collect(); - - write!(writer, " ({}", filename.display())?; + let filename = shorten_filename(filename); + + write!(writer, " ({filename}")?; if let Some(line) = metadata.line() { write!(writer, ":{line}")?; } @@ -365,6 +389,21 @@ mod inner { Ok(()) } + /// Keep only the module name and file name to make it shorter + fn shorten_filename(filename: &str) -> String { + Path::new(filename) + .components() + // Take last two path components + .rev() + .take(2) + .collect::<Vec<_>>() + .into_iter() + .rev() + .collect::<PathBuf>() + .display() + .to_string() + } + impl<S> Layer<S> for TracingPrinter where S: Subscriber, |