diff options
Diffstat (limited to 'src/bootstrap')
| -rw-r--r-- | src/bootstrap/Cargo.toml | 2 | ||||
| -rw-r--r-- | src/bootstrap/README.md | 3 | ||||
| -rw-r--r-- | src/bootstrap/bin/rustdoc.rs | 5 | ||||
| -rw-r--r-- | src/bootstrap/bootstrap.py | 4 | ||||
| -rw-r--r-- | src/bootstrap/builder.rs | 53 | ||||
| -rw-r--r-- | src/bootstrap/channel.rs | 2 | ||||
| -rw-r--r-- | src/bootstrap/compile.rs | 72 | ||||
| -rw-r--r-- | src/bootstrap/config.rs | 2 | ||||
| -rw-r--r-- | src/bootstrap/dist.rs | 2 | ||||
| -rw-r--r-- | src/bootstrap/doc.rs | 7 | ||||
| -rw-r--r-- | src/bootstrap/flags.rs | 35 | ||||
| -rw-r--r-- | src/bootstrap/format.rs | 57 | ||||
| -rw-r--r-- | src/bootstrap/lib.rs | 15 | ||||
| -rw-r--r-- | src/bootstrap/native.rs | 98 | ||||
| -rw-r--r-- | src/bootstrap/run.rs | 43 | ||||
| -rw-r--r-- | src/bootstrap/test.rs | 53 | ||||
| -rw-r--r-- | src/bootstrap/tool.rs | 7 | ||||
| -rw-r--r-- | src/bootstrap/toolstate.rs | 55 | ||||
| -rw-r--r-- | src/bootstrap/util.rs | 27 |
19 files changed, 384 insertions, 158 deletions
diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index c09f58cc591..f7856f6a7fc 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -54,4 +54,4 @@ version = "0.3" features = ["fileapi", "ioapiset", "jobapi2", "handleapi", "winioctl"] [dev-dependencies] -pretty_assertions = "0.5" +pretty_assertions = "0.6" diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index c501378bff5..87da7327fe6 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -55,6 +55,9 @@ The script accepts commands, flags, and arguments to determine what to do: # run all unit tests ./x.py test + # execute tool tests + ./x.py test tidy + # execute the UI test suite ./x.py test src/test/ui diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs index 04345867bf5..ba644e61118 100644 --- a/src/bootstrap/bin/rustdoc.rs +++ b/src/bootstrap/bin/rustdoc.rs @@ -52,12 +52,7 @@ fn main() { // Bootstrap's Cargo-command builder sets this variable to the current Rust version; let's pick // it up so we can make rustdoc print this into the docs if let Some(version) = env::var_os("RUSTDOC_CRATE_VERSION") { - // This "unstable-options" can be removed when `--crate-version` is stabilized - if !has_unstable { - cmd.arg("-Z").arg("unstable-options"); - } cmd.arg("--crate-version").arg(version); - has_unstable = true; } // Needed to be able to run all rustdoc tests. diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 50e1726240f..d5efed61b54 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -664,6 +664,10 @@ class RustBuild(object): if self.clean and os.path.exists(build_dir): shutil.rmtree(build_dir) env = os.environ.copy() + # `CARGO_BUILD_TARGET` breaks bootstrap build. + # See also: <https://github.com/rust-lang/rust/issues/70208>. + if "CARGO_BUILD_TARGET" in env: + del env["CARGO_BUILD_TARGET"] env["RUSTC_BOOTSTRAP"] = '1' env["CARGO_TARGET_DIR"] = build_dir env["RUSTC"] = self.rustc() diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index e4b57cddfb8..b14352d7f4b 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -11,7 +11,7 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{Duration, Instant}; -use build_helper::t; +use build_helper::{output, t}; use crate::cache::{Cache, Interned, INTERNER}; use crate::check; @@ -21,9 +21,10 @@ use crate::doc; use crate::flags::Subcommand; use crate::install; use crate::native; +use crate::run; use crate::test; use crate::tool; -use crate::util::{self, add_lib_path, exe, libdir}; +use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir}; use crate::{Build, DocTests, GitRepo, Mode}; pub use crate::Compiler; @@ -313,6 +314,7 @@ pub enum Kind { Dist, Doc, Install, + Run, } impl<'a> Builder<'a> { @@ -353,6 +355,7 @@ impl<'a> Builder<'a> { } Kind::Test => describe!( crate::toolstate::ToolStateCheck, + test::ExpandYamlAnchors, test::Tidy, test::Ui, test::CompileFail, @@ -454,6 +457,7 @@ impl<'a> Builder<'a> { install::Src, install::Rustc ), + Kind::Run => describe!(run::ExpandYamlAnchors,), } } @@ -507,6 +511,7 @@ impl<'a> Builder<'a> { Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]), Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]), Subcommand::Install { ref paths } => (Kind::Install, &paths[..]), + Subcommand::Run { ref paths } => (Kind::Run, &paths[..]), Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(), }; @@ -660,7 +665,7 @@ impl<'a> Builder<'a> { return; } - add_lib_path(vec![self.rustc_libdir(compiler)], &mut cmd.command); + add_dylib_path(vec![self.rustc_libdir(compiler)], &mut cmd.command); } /// Gets a path to the compiler specified. @@ -698,6 +703,20 @@ impl<'a> Builder<'a> { cmd } + /// Return the path to `llvm-config` for the target, if it exists. + /// + /// Note that this returns `None` if LLVM is disabled, or if we're in a + /// check build or dry-run, where there's no need to build all of LLVM. + fn llvm_config(&self, target: Interned<String>) -> Option<PathBuf> { + if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run { + let llvm_config = self.ensure(native::Llvm { target }); + if llvm_config.is_file() { + return Some(llvm_config); + } + } + None + } + /// Prepares an invocation of `cargo` to be run. /// /// This will create a `Command` that represents a pending execution of @@ -725,7 +744,7 @@ impl<'a> Builder<'a> { self.clear_if_dirty(&my_out, &rustdoc); } - cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd).arg("-Zconfig-profile"); + cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd); let profile_var = |name: &str| { let profile = if self.config.rust_optimize { "RELEASE" } else { "DEV" }; @@ -847,13 +866,7 @@ impl<'a> Builder<'a> { rustflags.arg("-Zforce-unstable-if-unmarked"); } - // cfg(bootstrap): the flag was renamed from `-Zexternal-macro-backtrace` - // to `-Zmacro-backtrace`, keep only the latter after beta promotion. - if stage == 0 { - rustflags.arg("-Zexternal-macro-backtrace"); - } else { - rustflags.arg("-Zmacro-backtrace"); - } + rustflags.arg("-Zmacro-backtrace"); let want_rustdoc = self.doc_tests != DocTests::No; @@ -1009,8 +1022,13 @@ impl<'a> Builder<'a> { cargo.env("RUSTC_HOST_CRT_STATIC", x.to_string()); } - if let Some(map) = self.build.debuginfo_map(GitRepo::Rustc) { + if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) { + let map = format!("{}={}", self.build.src.display(), map_to); cargo.env("RUSTC_DEBUGINFO_MAP", map); + + // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to, + // in order to opportunistically reverse it later. + cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); } // Enable usage of unstable features @@ -1040,6 +1058,17 @@ impl<'a> Builder<'a> { .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler)); } + // Tools that use compiler libraries may inherit the `-lLLVM` link + // requirement, but the `-L` library path is not propagated across + // separate Cargo projects. We can add LLVM's library path to the + // platform-specific environment variable as a workaround. + if mode == Mode::ToolRustc { + if let Some(llvm_config) = self.llvm_config(target) { + let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); + add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cargo); + } + } + if self.config.incremental { cargo.env("CARGO_INCREMENTAL", "1"); } else { diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs index 504cba45570..be2b0f36d14 100644 --- a/src/bootstrap/channel.rs +++ b/src/bootstrap/channel.rs @@ -13,7 +13,7 @@ use build_helper::output; use crate::Build; // The version number -pub const CFG_RELEASE_NUM: &str = "1.43.0"; +pub const CFG_RELEASE_NUM: &str = "1.44.0"; pub struct GitInfo { inner: Option<Info>, diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 7dded96e18e..32ce170a5a1 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -22,7 +22,7 @@ use serde::Deserialize; use crate::builder::Cargo; use crate::dist; use crate::native; -use crate::util::{exe, is_dylib}; +use crate::util::{exe, is_dylib, symlink_dir}; use crate::{Compiler, GitRepo, Mode}; use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; @@ -141,7 +141,7 @@ fn copy_third_party_objects( copy_and_stamp(&srcdir, "crt1.o"); } - // Copies libunwind.a compiled to be linked wit x86_64-fortanix-unknown-sgx. + // Copies libunwind.a compiled to be linked with x86_64-fortanix-unknown-sgx. // // This target needs to be linked to Fortanix's port of llvm's libunwind. // libunwind requires support for rwlock and printing to stderr, @@ -451,44 +451,6 @@ impl Step for Rustc { false, ); - // We used to build librustc_codegen_llvm as a separate step, - // which produced a dylib that the compiler would dlopen() at runtime. - // This meant that we only needed to make sure that libLLVM.so was - // installed by the time we went to run a tool using it - since - // librustc_codegen_llvm was effectively a standalone artifact, - // other crates were completely oblivious to its dependency - // on `libLLVM.so` during build time. - // - // However, librustc_codegen_llvm is now built as an ordinary - // crate during the same step as the rest of the compiler crates. - // This means that any crates depending on it will see the fact - // that it uses `libLLVM.so` as a native library, and will - // cause us to pass `-llibLLVM.so` to the linker when we link - // a binary. - // - // For `rustc` itself, this works out fine. - // During the `Assemble` step, we call `dist::maybe_install_llvm_dylib` - // to copy libLLVM.so into the `stage` directory. We then link - // the compiler binary, which will find `libLLVM.so` in the correct place. - // - // However, this is insufficient for tools that are build against stage0 - // (e.g. stage1 rustdoc). Since `Assemble` for stage0 doesn't actually do anything, - // we won't have `libLLVM.so` in the stage0 sysroot. In the past, this wasn't - // a problem - we would copy the tool binary into its correct stage directory - // (e.g. stage1 for a stage1 rustdoc built against a stage0 compiler). - // Since libLLVM.so wasn't resolved until runtime, it was fine for it to - // not exist while we were building it. - // - // To ensure that we can still build stage1 tools against a stage0 compiler, - // we explicitly copy libLLVM.so into the stage0 sysroot when building - // the stage0 compiler. This ensures that tools built against stage0 - // will see libLLVM.so at build time, making the linker happy. - if compiler.stage == 0 { - builder.info(&format!("Installing libLLVM.so to stage 0 ({})", compiler.host)); - let sysroot = builder.sysroot(compiler); - dist::maybe_install_llvm_dylib(builder, compiler.host, &sysroot); - } - builder.ensure(RustcLink { compiler: builder.compiler(compiler.stage, builder.config.build), target_compiler: compiler, @@ -671,6 +633,30 @@ impl Step for Sysroot { }; let _ = fs::remove_dir_all(&sysroot); t!(fs::create_dir_all(&sysroot)); + + // Symlink the source root into the same location inside the sysroot, + // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), + // so that any tools relying on `rust-src` also work for local builds, + // and also for translating the virtual `/rustc/$hash` back to the real + // directory (for running tests with `rust.remap-debuginfo = true`). + let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); + t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); + let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); + if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { + eprintln!( + "warning: creating symbolic link `{}` to `{}` failed with {}", + sysroot_lib_rustlib_src_rust.display(), + builder.src.display(), + e, + ); + if builder.config.rust_remap_debuginfo { + eprintln!( + "warning: some `src/test/ui` tests will fail when lacking `{}`", + sysroot_lib_rustlib_src_rust.display(), + ); + } + } + INTERNER.intern_path(sysroot) } } @@ -949,7 +935,11 @@ pub fn stream_cargo( } // Instruct Cargo to give us json messages on stdout, critically leaving // stderr as piped so we can get those pretty colors. - let mut message_format = String::from("json-render-diagnostics"); + let mut message_format = if builder.config.json_output { + String::from("json") + } else { + String::from("json-render-diagnostics") + }; if let Some(s) = &builder.config.rustc_error_format { message_format.push_str(",json-diagnostic-"); message_format.push_str(s); diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 56164b74f30..133709421a5 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -48,6 +48,7 @@ pub struct Config { pub ignore_git: bool, pub exclude: Vec<PathBuf>, pub rustc_error_format: Option<String>, + pub json_output: bool, pub test_compare_mode: bool, pub llvm_libunwind: bool, @@ -415,6 +416,7 @@ impl Config { let mut config = Config::default_opts(); config.exclude = flags.exclude; config.rustc_error_format = flags.rustc_error_format; + config.json_output = flags.json_output; config.on_fail = flags.on_fail; config.stage = flags.stage; config.jobs = flags.jobs.map(threads_from_config); diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 76478abd0de..8215211ea1c 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1002,8 +1002,6 @@ impl Step for Src { "src/tools/rustc-std-workspace-core", "src/tools/rustc-std-workspace-alloc", "src/tools/rustc-std-workspace-std", - "src/librustc", - "src/librustc_ast", ]; copy_src_dirs(builder, &std_src_dirs[..], &[], &dst_src); diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index b0d9a5b9464..04da3cc1015 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -313,6 +313,9 @@ impl Step for Standalone { } let mut cmd = builder.rustdoc_cmd(compiler); + // Needed for --index-page flag + cmd.arg("-Z").arg("unstable-options"); + cmd.arg("--html-after-content") .arg(&footer) .arg("--html-before-content") @@ -395,7 +398,7 @@ impl Step for Std { // Keep a whitelist so we do not build internal stdlib crates, these will be // build by the rustc step later if enabled. - cargo.arg("-Z").arg("unstable-options").arg("-p").arg(package); + cargo.arg("-p").arg(package); // Create all crate output directories first to make sure rustdoc uses // relative links. // FIXME: Cargo should probably do this itself. @@ -406,6 +409,8 @@ impl Step for Std { .arg("rust.css") .arg("--markdown-no-toc") .arg("--generate-redirect-pages") + .arg("-Z") + .arg("unstable-options") .arg("--resource-suffix") .arg(crate::channel::CFG_RELEASE_NUM) .arg("--index-page") diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 516be6a30c2..5d6e401d5b3 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -31,9 +31,10 @@ pub struct Flags { pub incremental: bool, pub exclude: Vec<PathBuf>, pub rustc_error_format: Option<String>, + pub json_output: bool, pub dry_run: bool, - // This overrides the deny-warnings configuation option, + // This overrides the deny-warnings configuration option, // which passes -Dwarnings to the compiler invocations. // // true => deny, false => warn @@ -86,6 +87,9 @@ pub enum Subcommand { Install { paths: Vec<PathBuf>, }, + Run { + paths: Vec<PathBuf>, + }, } impl Default for Subcommand { @@ -113,6 +117,7 @@ Subcommands: clean Clean out build directories dist Build distribution artifacts install Install distribution artifacts + run Run tools contained in this repository To learn more about a subcommand, run `./x.py <subcommand> -h`", ); @@ -152,6 +157,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", "VALUE", ); opts.optopt("", "error-format", "rustc error format", "FORMAT"); + opts.optflag("", "json-output", "use message-format=json"); opts.optopt( "", "llvm-skip-rebuild", @@ -188,6 +194,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", || (s == "clean") || (s == "dist") || (s == "install") + || (s == "run") }); let subcommand = match subcommand { Some(s) => s, @@ -359,7 +366,7 @@ Arguments: subcommand_help.push_str( "\n Arguments: - This subcommand accepts a number of paths to directories to tests that + This subcommand accepts a number of paths to test directories that should be compiled and run. For example: ./x.py test src/test/ui @@ -372,6 +379,10 @@ Arguments: just like `build src/libstd --stage N` it tests the compiler produced by the previous stage. + Execute tool tests with a tool name argument: + + ./x.py test tidy + If no arguments are passed then the complete artifacts for that stage are compiled and tested. @@ -396,6 +407,18 @@ Arguments: ./x.py doc --stage 1", ); } + "run" => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a number of paths to tools to build and run. For + example: + + ./x.py run src/tool/expand-yaml-anchors + + At least a tool needs to be called.", + ); + } _ => {} }; // Get any optional paths which occur after the subcommand @@ -464,6 +487,13 @@ Arguments: "fmt" => Subcommand::Format { check: matches.opt_present("check") }, "dist" => Subcommand::Dist { paths }, "install" => Subcommand::Install { paths }, + "run" => { + if paths.is_empty() { + println!("\nrun requires at least a path!\n"); + usage(1, &opts, &subcommand_help, &extra_help); + } + Subcommand::Run { paths } + } _ => { usage(1, &opts, &subcommand_help, &extra_help); } @@ -475,6 +505,7 @@ Arguments: dry_run: matches.opt_present("dry-run"), on_fail: matches.opt_str("on-fail"), rustc_error_format: matches.opt_str("error-format"), + json_output: matches.opt_present("json-output"), keep_stage: matches .opt_strs("keep-stage") .into_iter() diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs index a4acb14ee4b..6653c505bf5 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/format.rs @@ -4,7 +4,7 @@ use crate::Build; use build_helper::{output, t}; use ignore::WalkBuilder; use std::path::Path; -use std::process::Command; +use std::process::{Command, Stdio}; fn rustfmt(src: &Path, rustfmt: &Path, path: &Path, check: bool) { let mut cmd = Command::new(&rustfmt); @@ -37,6 +37,9 @@ struct RustfmtConfig { } pub fn format(build: &Build, check: bool) { + if build.config.dry_run { + return; + } let mut builder = ignore::types::TypesBuilder::new(); builder.add_defaults(); builder.select("rust"); @@ -53,16 +56,48 @@ pub fn format(build: &Build, check: bool) { for ignore in rustfmt_config.ignore { ignore_fmt.add(&format!("!{}", ignore)).expect(&ignore); } - let untracked_paths_output = output( - Command::new("git").arg("status").arg("--porcelain").arg("--untracked-files=normal"), - ); - let untracked_paths = untracked_paths_output - .lines() - .filter(|entry| entry.starts_with("??")) - .map(|entry| entry.split(" ").nth(1).expect("every git status entry should list a path")); - for untracked_path in untracked_paths { - eprintln!("skip untracked path {} during rustfmt invocations", untracked_path); - ignore_fmt.add(&format!("!{}", untracked_path)).expect(&untracked_path); + let git_available = match Command::new("git") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + }; + if git_available { + let in_working_tree = match Command::new("git") + .arg("rev-parse") + .arg("--is-inside-work-tree") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + }; + if in_working_tree { + let untracked_paths_output = output( + Command::new("git") + .arg("status") + .arg("--porcelain") + .arg("--untracked-files=normal"), + ); + let untracked_paths = untracked_paths_output + .lines() + .filter(|entry| entry.starts_with("??")) + .map(|entry| { + entry.split(" ").nth(1).expect("every git status entry should list a path") + }); + for untracked_path in untracked_paths { + eprintln!("skip untracked path {} during rustfmt invocations", untracked_path); + ignore_fmt.add(&format!("!{}", untracked_path)).expect(&untracked_path); + } + } else { + eprintln!("Not in git tree. Skipping git-aware format checks"); + } + } else { + eprintln!("Could not find usable git. Skipping git-aware format checks"); } let ignore_fmt = ignore_fmt.build().unwrap(); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index a476d25f102..31bbd92cd62 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -140,6 +140,7 @@ mod format; mod install; mod metadata; mod native; +mod run; mod sanity; mod test; mod tool; @@ -739,19 +740,18 @@ impl Build { self.config.jobs.unwrap_or_else(|| num_cpus::get() as u32) } - fn debuginfo_map(&self, which: GitRepo) -> Option<String> { + fn debuginfo_map_to(&self, which: GitRepo) -> Option<String> { if !self.config.rust_remap_debuginfo { return None; } - let path = match which { + match which { GitRepo::Rustc => { let sha = self.rust_sha().unwrap_or(channel::CFG_RELEASE_NUM); - format!("/rustc/{}", sha) + Some(format!("/rustc/{}", sha)) } - GitRepo::Llvm => String::from("/rustc/llvm"), - }; - Some(format!("{}={}", self.src.display(), path)) + GitRepo::Llvm => Some(String::from("/rustc/llvm")), + } } /// Returns the path to the C compiler for the target specified. @@ -786,7 +786,8 @@ impl Build { base.push("-fno-omit-frame-pointer".into()); } - if let Some(map) = self.debuginfo_map(which) { + if let Some(map_to) = self.debuginfo_map_to(which) { + let map = format!("{}={}", self.src.display(), map_to); let cc = self.cc(target); if cc.ends_with("clang") || cc.ends_with("gcc") { base.push(format!("-fdebug-prefix-map={}", map)); diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index c22c2a336f1..d4d66abd520 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -11,6 +11,7 @@ use std::env; use std::ffi::OsString; use std::fs::{self, File}; +use std::io; use std::path::{Path, PathBuf}; use std::process::Command; @@ -54,7 +55,6 @@ impl Step for Llvm { } } - let llvm_info = &builder.in_tree_llvm_info; let root = "src/llvm-project/llvm"; let out_dir = builder.llvm_out(target); let mut llvm_config_ret_dir = builder.llvm_out(builder.config.build); @@ -65,40 +65,35 @@ impl Step for Llvm { let build_llvm_config = llvm_config_ret_dir.join(exe("llvm-config", &*builder.config.build)); - let done_stamp = out_dir.join("llvm-finished-building"); - if done_stamp.exists() { - if builder.config.llvm_skip_rebuild { - builder.info( - "Warning: \ - Using a potentially stale build of LLVM; \ - This may not behave well.", - ); - return build_llvm_config; - } + let stamp = out_dir.join("llvm-finished-building"); + let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); - if let Some(llvm_commit) = llvm_info.sha() { - let done_contents = t!(fs::read(&done_stamp)); + if builder.config.llvm_skip_rebuild && stamp.path.exists() { + builder.info( + "Warning: \ + Using a potentially stale build of LLVM; \ + This may not behave well.", + ); + return build_llvm_config; + } - // If LLVM was already built previously and the submodule's commit didn't change - // from the previous build, then no action is required. - if done_contents == llvm_commit.as_bytes() { - return build_llvm_config; - } - } else { + if stamp.is_done() { + if stamp.hash.is_none() { builder.info( "Could not determine the LLVM submodule commit hash. \ Assuming that an LLVM rebuild is not necessary.", ); builder.info(&format!( "To force LLVM to rebuild, remove the file `{}`", - done_stamp.display() + stamp.path.display() )); - return build_llvm_config; } + return build_llvm_config; } builder.info(&format!("Building LLVM for {}", target)); + t!(stamp.remove()); let _time = util::timeit(&builder); t!(fs::create_dir_all(&out_dir)); @@ -271,7 +266,7 @@ impl Step for Llvm { cfg.build(); - t!(fs::write(&done_stamp, llvm_info.sha().unwrap_or(""))); + t!(stamp.write()); build_llvm_config } @@ -584,17 +579,21 @@ impl Step for Sanitizers { return runtimes; } - let done_stamp = out_dir.join("sanitizers-finished-building"); - if done_stamp.exists() { - builder.info(&format!( - "Assuming that sanitizers rebuild is not necessary. \ - To force a rebuild, remove the file `{}`", - done_stamp.display() - )); + let stamp = out_dir.join("sanitizers-finished-building"); + let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); + + if stamp.is_done() { + if stamp.hash.is_none() { + builder.info(&format!( + "Rebuild sanitizers by removing the file `{}`", + stamp.path.display() + )); + } return runtimes; } builder.info(&format!("Building sanitizers for {}", self.target)); + t!(stamp.remove()); let _time = util::timeit(&builder); let mut cfg = cmake::Config::new(&compiler_rt_dir); @@ -623,8 +622,7 @@ impl Step for Sanitizers { cfg.build_target(&runtime.cmake_target); cfg.build(); } - - t!(fs::write(&done_stamp, b"")); + t!(stamp.write()); runtimes } @@ -689,3 +687,41 @@ fn supported_sanitizers( } result } + +struct HashStamp { + path: PathBuf, + hash: Option<Vec<u8>>, +} + +impl HashStamp { + fn new(path: PathBuf, hash: Option<&str>) -> Self { + HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) } + } + + fn is_done(&self) -> bool { + match fs::read(&self.path) { + Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(), + Err(e) if e.kind() == io::ErrorKind::NotFound => false, + Err(e) => { + panic!("failed to read stamp file `{}`: {}", self.path.display(), e); + } + } + } + + fn remove(&self) -> io::Result<()> { + match fs::remove_file(&self.path) { + Ok(()) => Ok(()), + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + Ok(()) + } else { + Err(e) + } + } + } + } + + fn write(&self) -> io::Result<()> { + fs::write(&self.path, self.hash.as_deref().unwrap_or(b"")) + } +} diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs new file mode 100644 index 00000000000..90053471427 --- /dev/null +++ b/src/bootstrap/run.rs @@ -0,0 +1,43 @@ +use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::tool::Tool; +use std::process::Command; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + + /// Runs the `expand-yaml_anchors` tool. + /// + /// This tool in `src/tools` read the CI configuration files written in YAML and expands the + /// anchors in them, since GitHub Actions doesn't support them. + fn run(self, builder: &Builder<'_>) { + builder.info("Expanding YAML anchors in the GitHub Actions configuration"); + try_run( + builder, + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + +fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if !builder.fail_fast { + if !builder.try_run(cmd) { + let mut failures = builder.delayed_failures.borrow_mut(); + failures.push(format!("{:?}", cmd)); + return false; + } + } else { + builder.run(cmd); + } + true +} diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index f041555c932..0bf507f9ebb 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -21,7 +21,7 @@ use crate::flags::Subcommand; use crate::native; use crate::tool::{self, SourceType, Tool}; use crate::toolstate::ToolState; -use crate::util::{self, dylib_path, dylib_path_var}; +use crate::util::{self, add_link_lib_path, dylib_path, dylib_path_var}; use crate::Crate as CargoCrate; use crate::{envify, DocTests, GitRepo, Mode}; @@ -607,7 +607,6 @@ impl Step for RustdocTheme { #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct RustdocJSStd { - pub host: Interned<String>, pub target: Interned<String>, } @@ -621,13 +620,16 @@ impl Step for RustdocJSStd { } fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustdocJSStd { host: run.host, target: run.target }); + run.builder.ensure(RustdocJSStd { target: run.target }); } fn run(self, builder: &Builder<'_>) { if let Some(ref nodejs) = builder.config.nodejs { let mut command = Command::new(nodejs); - command.args(&["src/tools/rustdoc-js-std/tester.js", &*self.host]); + command + .arg(builder.src.join("src/tools/rustdoc-js-std/tester.js")) + .arg(builder.doc_out(self.target)) + .arg(builder.src.join("src/test/rustdoc-js-std")); builder.ensure(crate::doc::Std { target: self.target, stage: builder.top_stage }); builder.run(&mut command); } else { @@ -726,9 +728,6 @@ impl Step for Tidy { let mut cmd = builder.tool_cmd(Tool::Tidy); cmd.arg(builder.src.join("src")); cmd.arg(&builder.initial_cargo); - if !builder.config.vendor { - cmd.arg("--no-vendor"); - } if builder.is_verbose() { cmd.arg("--verbose"); } @@ -751,6 +750,35 @@ impl Step for Tidy { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + const ONLY_HOSTS: bool = true; + + /// Ensure the `generate-ci-config` tool was run locally. + /// + /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the + /// appropriate configuration for all our CI providers. This step ensures the tool was called + /// by the user before committing CI changes. + fn run(self, builder: &Builder<'_>) { + builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded"); + try_run( + builder, + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + fn testdir(builder: &Builder<'_>, host: Interned<String>) -> PathBuf { builder.out.join(host).join("test") } @@ -1142,12 +1170,23 @@ impl Step for Compiletest { let llvm_config = builder.ensure(native::Llvm { target: builder.config.build }); if !builder.config.dry_run { let llvm_version = output(Command::new(&llvm_config).arg("--version")); + // Remove trailing newline from llvm-config output. + let llvm_version = llvm_version.trim_end(); cmd.arg("--llvm-version").arg(llvm_version); } if !builder.is_rust_llvm(target) { cmd.arg("--system-llvm"); } + // Tests that use compiler libraries may inherit the `-lLLVM` link + // requirement, but the `-L` library path is not propagated across + // separate compilations. We can add LLVM's library path to the + // platform-specific environment variable as a workaround. + if !builder.config.dry_run && suite.ends_with("fulldeps") { + let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); + add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd); + } + // Only pass correct values for these flags for the `run-make` suite as it // requires that a C++ compiler was configured which isn't always the case. if !builder.config.dry_run && suite == "run-make-fulldeps" { diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 67e0ed5c580..52f750f448e 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -12,7 +12,7 @@ use crate::channel; use crate::channel::GitInfo; use crate::compile; use crate::toolstate::ToolState; -use crate::util::{add_lib_path, exe, CiEnv}; +use crate::util::{add_dylib_path, exe, CiEnv}; use crate::Compiler; use crate::Mode; @@ -378,6 +378,7 @@ bootstrap_tool!( RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true; RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; + ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; ); #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -388,7 +389,7 @@ pub struct ErrorIndex { impl ErrorIndex { pub fn command(builder: &Builder<'_>, compiler: Compiler) -> Command { let mut cmd = Command::new(builder.ensure(ErrorIndex { compiler })); - add_lib_path( + add_dylib_path( vec![PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))], &mut cmd, ); @@ -689,7 +690,7 @@ impl<'a> Builder<'a> { } } - add_lib_path(lib_paths, &mut cmd); + add_dylib_path(lib_paths, &mut cmd); cmd } } diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs index 31ff01d5be6..095c3c03c30 100644 --- a/src/bootstrap/toolstate.rs +++ b/src/bootstrap/toolstate.rs @@ -215,6 +215,9 @@ impl Step for ToolStateCheck { tool, old_state, state ); } else { + // This warning only appears in the logs, which most + // people won't read. It's mostly here for testing and + // debugging. eprintln!( "warning: Tool `{}` is not test-pass (is `{}`), \ this should be fixed before beta is branched.", @@ -222,6 +225,11 @@ impl Step for ToolStateCheck { ); } } + // `publish_toolstate.py` is responsible for updating + // `latest.json` and creating comments/issues warning people + // if there is a regression. That all happens in a separate CI + // job on the master branch once the PR has passed all tests + // on the `auto` branch. } } @@ -230,7 +238,7 @@ impl Step for ToolStateCheck { } if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { - commit_toolstate_change(&toolstates, in_beta_week); + commit_toolstate_change(&toolstates); } } @@ -325,11 +333,11 @@ fn prepare_toolstate_config(token: &str) { Err(_) => false, }; if !success { - panic!("git config key={} value={} successful (status: {:?})", key, value, status); + panic!("git config key={} value={} failed (status: {:?})", key, value, status); } } - // If changing anything here, then please check that src/ci/publish_toolstate.sh is up to date + // If changing anything here, then please check that `src/ci/publish_toolstate.sh` is up to date // as well. git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); git_config("user.name", "Rust Toolstate Update"); @@ -373,14 +381,14 @@ fn read_old_toolstate() -> Vec<RepoState> { /// /// * See <https://help.github.com/articles/about-commit-email-addresses/> /// if a private email by GitHub is wanted. -fn commit_toolstate_change(current_toolstate: &ToolstateData, in_beta_week: bool) { - let old_toolstate = read_old_toolstate(); - +fn commit_toolstate_change(current_toolstate: &ToolstateData) { let message = format!("({} CI update)", OS.expect("linux/windows only")); let mut success = false; for _ in 1..=5 { - // Update the toolstate results (the new commit-to-toolstate mapping) in the toolstate repo. - change_toolstate(¤t_toolstate, &old_toolstate, in_beta_week); + // Upload the test results (the new commit-to-toolstate mapping) to the toolstate repo. + // This does *not* change the "current toolstate"; that only happens post-landing + // via `src/ci/docker/publish_toolstate.sh`. + publish_test_results(¤t_toolstate); // `git commit` failing means nothing to commit. let status = t!(Command::new("git") @@ -429,31 +437,12 @@ fn commit_toolstate_change(current_toolstate: &ToolstateData, in_beta_week: bool } } -fn change_toolstate( - current_toolstate: &ToolstateData, - old_toolstate: &[RepoState], - in_beta_week: bool, -) { - let mut regressed = false; - for repo_state in old_toolstate { - let tool = &repo_state.tool; - let state = repo_state.state(); - let new_state = current_toolstate[tool.as_str()]; - - if new_state != state { - eprintln!("The state of `{}` has changed from `{}` to `{}`", tool, state, new_state); - if new_state < state { - if !NIGHTLY_TOOLS.iter().any(|(name, _path)| name == tool) { - regressed = true; - } - } - } - } - - if regressed && in_beta_week { - std::process::exit(1); - } - +/// Updates the "history" files with the latest results. +/// +/// These results will later be promoted to `latest.json` by the +/// `publish_toolstate.py` script if the PR passes all tests and is merged to +/// master. +fn publish_test_results(current_toolstate: &ToolstateData) { let commit = t!(std::process::Command::new("git").arg("rev-parse").arg("HEAD").output()); let commit = t!(String::from_utf8(commit.stdout)); diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index eac790fe504..2bc6f1939d9 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -40,7 +40,7 @@ pub fn libdir(target: &str) -> &'static str { } /// Adds a list of lookup paths to `cmd`'s dynamic library lookup path. -pub fn add_lib_path(path: Vec<PathBuf>, cmd: &mut Command) { +pub fn add_dylib_path(path: Vec<PathBuf>, cmd: &mut Command) { let mut list = dylib_path(); for path in path { list.insert(0, path); @@ -72,6 +72,31 @@ pub fn dylib_path() -> Vec<PathBuf> { env::split_paths(&var).collect() } +/// Adds a list of lookup paths to `cmd`'s link library lookup path. +pub fn add_link_lib_path(path: Vec<PathBuf>, cmd: &mut Command) { + let mut list = link_lib_path(); + for path in path { + list.insert(0, path); + } + cmd.env(link_lib_path_var(), t!(env::join_paths(list))); +} + +/// Returns the environment variable which the link library lookup path +/// resides in for this platform. +fn link_lib_path_var() -> &'static str { + if cfg!(target_env = "msvc") { "LIB" } else { "LIBRARY_PATH" } +} + +/// Parses the `link_lib_path_var()` environment variable, returning a list of +/// paths that are members of this lookup path. +fn link_lib_path() -> Vec<PathBuf> { + let var = match env::var_os(link_lib_path_var()) { + Some(v) => v, + None => return vec![], + }; + env::split_paths(&var).collect() +} + /// `push` all components to `buf`. On windows, append `.exe` to the last component. pub fn push_exe_path(mut buf: PathBuf, components: &[&str]) -> PathBuf { let (&file, components) = components.split_last().expect("at least one component required"); |