From baa95efa7a8045d7174bf5aa57508d4ac82899f8 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Wed, 16 Oct 2024 19:42:25 +0300 Subject: use allowed "if-unchanged" logic for compiler builds Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 82 ++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 17 deletions(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index f977c285a74..bb1c7219eda 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -28,6 +28,53 @@ use crate::utils::cache::{INTERNER, Interned}; use crate::utils::channel::{self, GitInfo}; use crate::utils::helpers::{self, exe, output, t}; +/// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic. +/// This means they can be modified and changes to these paths should never trigger a compiler build +/// when "if-unchanged" is set. +/// +/// NOTE: Paths must have the ":!" prefix to tell git to ignore changes in those paths during +/// the diff check. +/// +/// WARNING: Be cautious when adding paths to this list. If a path that influences the compiler build +/// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results. +const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ + ":!.clang-format", + ":!.editorconfig", + ":!.git-blame-ignore-revs", + ":!.gitattributes", + ":!.gitignore", + ":!.gitmodules", + ":!.ignore", + ":!.mailmap", + ":!CODE_OF_CONDUCT.md", + ":!CONTRIBUTING.md", + ":!COPYRIGHT", + ":!INSTALL.md", + ":!LICENSE-APACHE", + ":!LICENSE-MIT", + ":!LICENSES", + ":!README.md", + ":!RELEASES.md", + ":!REUSE.toml", + ":!config.example.toml", + ":!configure", + ":!rust-bors.toml", + ":!rustfmt.toml", + ":!tests", + ":!triagebot.toml", + ":!x", + ":!x.ps1", + ":!x.py", + ":!src/ci/cpu-usage-over-time.py", + ":!src/ci/publish_toolstate.sh", + ":!src/doc", + ":!src/etc", + ":!src/librustdoc", + ":!src/rustdoc-json-types", + ":!src/tools", + ":!src/README.md", +]; + macro_rules! check_ci_llvm { ($name:expr) => { assert!( @@ -2768,32 +2815,33 @@ impl Config { } }; - let mut files_to_track = vec!["compiler", "src/version", "src/stage0", "src/ci/channel"]; + // RUSTC_IF_UNCHANGED_ALLOWED_PATHS + let mut allowed_paths = RUSTC_IF_UNCHANGED_ALLOWED_PATHS.to_vec(); - // In CI, disable ci-rustc if there are changes in the library tree. But for non-CI, ignore + // In CI, disable ci-rustc if there are changes in the library tree. But for non-CI, allow // these changes to speed up the build process for library developers. This provides consistent // functionality for library developers between `download-rustc=true` and `download-rustc="if-unchanged"` // options. - if CiEnv::is_ci() { - files_to_track.push("library"); + if !CiEnv::is_ci() { + allowed_paths.push(":!library"); } // Look for a version to compare to based on the current commit. // Only commits merged by bors will have CI artifacts. - let commit = - match self.last_modified_commit(&files_to_track, "download-rustc", if_unchanged) { - Some(commit) => commit, - None => { - if if_unchanged { - return None; - } - println!("ERROR: could not find commit hash for downloading rustc"); - println!("HELP: maybe your repository history is too shallow?"); - println!("HELP: consider disabling `download-rustc`"); - println!("HELP: or fetch enough history to include one upstream commit"); - crate::exit!(1); + let commit = match self.last_modified_commit(&allowed_paths, "download-rustc", if_unchanged) + { + Some(commit) => commit, + None => { + if if_unchanged { + return None; } - }; + println!("ERROR: could not find commit hash for downloading rustc"); + println!("HELP: maybe your repository history is too shallow?"); + println!("HELP: consider setting `rust.download-rustc=false` in config.toml"); + println!("HELP: or fetch enough history to include one upstream commit"); + crate::exit!(1); + } + }; if CiEnv::is_ci() && { let head_sha = -- cgit 1.4.1-3-g733a5 From 72e63e3ad58347ffe8a5e00e47738c1fd6295fa2 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 17 Oct 2024 15:15:34 +0300 Subject: add test coverage for `RUSTC_IF_UNCHANGED_ALLOWED_PATHS` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 2 +- src/bootstrap/src/core/config/tests.rs | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index bb1c7219eda..b4bea7a68de 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -37,7 +37,7 @@ use crate::utils::helpers::{self, exe, output, t}; /// /// WARNING: Be cautious when adding paths to this list. If a path that influences the compiler build /// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results. -const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ +pub(crate) const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ ":!.clang-format", ":!.editorconfig", ":!.git-blame-ignore-revs", diff --git a/src/bootstrap/src/core/config/tests.rs b/src/bootstrap/src/core/config/tests.rs index 1f02757682c..7e4eca3c5b0 100644 --- a/src/bootstrap/src/core/config/tests.rs +++ b/src/bootstrap/src/core/config/tests.rs @@ -8,7 +8,7 @@ use clap::CommandFactory; use serde::Deserialize; use super::flags::Flags; -use super::{ChangeIdWrapper, Config}; +use super::{ChangeIdWrapper, Config, RUSTC_IF_UNCHANGED_ALLOWED_PATHS}; use crate::core::build_steps::clippy::get_clippy_rules_in_order; use crate::core::build_steps::llvm; use crate::core::config::{LldMode, Target, TargetSelection, TomlConfig}; @@ -410,3 +410,18 @@ fn jobs_precedence() { ); assert_eq!(config.jobs, Some(123)); } + +#[test] +fn check_rustc_if_unchanged_paths() { + let config = parse(""); + let normalised_allowed_paths: Vec<_> = RUSTC_IF_UNCHANGED_ALLOWED_PATHS + .iter() + .map(|t| { + t.strip_prefix(":!").expect(&format!("{t} doesn't have ':!' prefix, but it should.")) + }) + .collect(); + + for p in normalised_allowed_paths { + assert!(config.src.join(p).exists(), "{p} doesn't exist."); + } +} -- cgit 1.4.1-3-g733a5 From a00fd7a965c455d137c887ebbb7ed580719f080c Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 10 Nov 2024 23:14:39 +0300 Subject: warn about "src/bootstrap" on `RUSTC_IF_UNCHANGED_ALLOWED_PATHS` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/bootstrap') diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index b4bea7a68de..adfd7c42f23 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -37,6 +37,8 @@ use crate::utils::helpers::{self, exe, output, t}; /// /// WARNING: Be cautious when adding paths to this list. If a path that influences the compiler build /// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results. +/// For example, "src/bootstrap" should never be included in this list as it plays a crucial role in the +/// final output/compiler, which can be significantly affected by changes made to the bootstrap sources. pub(crate) const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ ":!.clang-format", ":!.editorconfig", -- cgit 1.4.1-3-g733a5 From 6b38a159e5d1c3c154c24ff38786e40b71dd4710 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 10 Nov 2024 23:36:44 +0300 Subject: reduce `RUSTC_IF_UNCHANGED_ALLOWED_PATHS` significantly Signed-off-by: onur-ozkan --- src/bootstrap/src/core/config/config.rs | 35 ++------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index adfd7c42f23..8afabda1403 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -39,42 +39,11 @@ use crate::utils::helpers::{self, exe, output, t}; /// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results. /// For example, "src/bootstrap" should never be included in this list as it plays a crucial role in the /// final output/compiler, which can be significantly affected by changes made to the bootstrap sources. +#[rustfmt::skip] // We don't want rustfmt to oneline this list pub(crate) const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[ - ":!.clang-format", - ":!.editorconfig", - ":!.git-blame-ignore-revs", - ":!.gitattributes", - ":!.gitignore", - ":!.gitmodules", - ":!.ignore", - ":!.mailmap", - ":!CODE_OF_CONDUCT.md", - ":!CONTRIBUTING.md", - ":!COPYRIGHT", - ":!INSTALL.md", - ":!LICENSE-APACHE", - ":!LICENSE-MIT", - ":!LICENSES", - ":!README.md", - ":!RELEASES.md", - ":!REUSE.toml", - ":!config.example.toml", - ":!configure", - ":!rust-bors.toml", - ":!rustfmt.toml", + ":!src/tools", ":!tests", ":!triagebot.toml", - ":!x", - ":!x.ps1", - ":!x.py", - ":!src/ci/cpu-usage-over-time.py", - ":!src/ci/publish_toolstate.sh", - ":!src/doc", - ":!src/etc", - ":!src/librustdoc", - ":!src/rustdoc-json-types", - ":!src/tools", - ":!src/README.md", ]; macro_rules! check_ci_llvm { -- cgit 1.4.1-3-g733a5 From bc7531089efdbb868711ccd66dd83f2b18cb560c Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 10 Nov 2024 23:52:56 +0300 Subject: move `src/tools/build_helper` into `src/build_helper` Signed-off-by: onur-ozkan --- Cargo.toml | 2 +- src/bootstrap/Cargo.toml | 2 +- src/bootstrap/src/core/build_steps/clippy.rs | 2 +- src/bootstrap/src/core/build_steps/doc.rs | 2 +- src/bootstrap/src/core/build_steps/test.rs | 4 +- src/build_helper/Cargo.toml | 10 ++ src/build_helper/README.md | 1 + src/build_helper/src/ci.rs | 95 ++++++++++++ src/build_helper/src/drop_bomb/mod.rs | 54 +++++++ src/build_helper/src/drop_bomb/tests.rs | 15 ++ src/build_helper/src/git.rs | 208 ++++++++++++++++++++++++++ src/build_helper/src/lib.rs | 31 ++++ src/build_helper/src/metrics.rs | 92 ++++++++++++ src/build_helper/src/stage0_parser.rs | 76 ++++++++++ src/build_helper/src/util.rs | 75 ++++++++++ src/tools/build_helper/Cargo.toml | 10 -- src/tools/build_helper/README.md | 1 - src/tools/build_helper/src/ci.rs | 95 ------------ src/tools/build_helper/src/drop_bomb/mod.rs | 54 ------- src/tools/build_helper/src/drop_bomb/tests.rs | 15 -- src/tools/build_helper/src/git.rs | 208 -------------------------- src/tools/build_helper/src/lib.rs | 31 ---- src/tools/build_helper/src/metrics.rs | 92 ------------ src/tools/build_helper/src/stage0_parser.rs | 76 ---------- src/tools/build_helper/src/util.rs | 75 ---------- src/tools/bump-stage0/Cargo.toml | 2 +- src/tools/compiletest/Cargo.toml | 2 +- src/tools/opt-dist/Cargo.toml | 2 +- src/tools/run-make-support/Cargo.toml | 2 +- src/tools/rustdoc-gui-test/Cargo.toml | 2 +- src/tools/suggest-tests/Cargo.toml | 2 +- src/tools/tidy/Cargo.toml | 2 +- 32 files changed, 670 insertions(+), 670 deletions(-) create mode 100644 src/build_helper/Cargo.toml create mode 100644 src/build_helper/README.md create mode 100644 src/build_helper/src/ci.rs create mode 100644 src/build_helper/src/drop_bomb/mod.rs create mode 100644 src/build_helper/src/drop_bomb/tests.rs create mode 100644 src/build_helper/src/git.rs create mode 100644 src/build_helper/src/lib.rs create mode 100644 src/build_helper/src/metrics.rs create mode 100644 src/build_helper/src/stage0_parser.rs create mode 100644 src/build_helper/src/util.rs delete mode 100644 src/tools/build_helper/Cargo.toml delete mode 100644 src/tools/build_helper/README.md delete mode 100644 src/tools/build_helper/src/ci.rs delete mode 100644 src/tools/build_helper/src/drop_bomb/mod.rs delete mode 100644 src/tools/build_helper/src/drop_bomb/tests.rs delete mode 100644 src/tools/build_helper/src/git.rs delete mode 100644 src/tools/build_helper/src/lib.rs delete mode 100644 src/tools/build_helper/src/metrics.rs delete mode 100644 src/tools/build_helper/src/stage0_parser.rs delete mode 100644 src/tools/build_helper/src/util.rs (limited to 'src/bootstrap') diff --git a/Cargo.toml b/Cargo.toml index e1d667bf015..b773030b4ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,12 +2,12 @@ resolver = "2" members = [ "compiler/rustc", + "src/build_helper", "src/etc/test-float-parse", "src/rustc-std-workspace/rustc-std-workspace-core", "src/rustc-std-workspace/rustc-std-workspace-alloc", "src/rustc-std-workspace/rustc-std-workspace-std", "src/rustdoc-json-types", - "src/tools/build_helper", "src/tools/cargotest", "src/tools/clippy", "src/tools/clippy/clippy_dev", diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index ba505089a00..7950f1004a2 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -40,7 +40,7 @@ test = false cc = "=1.1.22" cmake = "=0.1.48" -build_helper = { path = "../tools/build_helper" } +build_helper = { path = "../build_helper" } clap = { version = "4.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } clap_complete = "4.4" fd-lock = "4.0" diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs index cd198c425c0..fb030b9b781 100644 --- a/src/bootstrap/src/core/build_steps/clippy.rs +++ b/src/bootstrap/src/core/build_steps/clippy.rs @@ -295,7 +295,7 @@ macro_rules! lint_any { lint_any!( Bootstrap, "src/bootstrap", "bootstrap"; - BuildHelper, "src/tools/build_helper", "build_helper"; + BuildHelper, "src/build_helper", "build_helper"; BuildManifest, "src/tools/build-manifest", "build-manifest"; CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri"; Clippy, "src/tools/clippy", "clippy"; diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 69ec832a44a..8a9321f8e79 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -1030,7 +1030,7 @@ macro_rules! tool_doc { // NOTE: make sure to register these in `Builder::get_step_description`. tool_doc!( BuildHelper, - "src/tools/build_helper", + "src/build_helper", rustc_tool = false, is_library = true, crates = ["build_helper"] diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 2c36d8bab82..532c8f767eb 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1354,7 +1354,7 @@ impl Step for CrateBuildHelper { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/build_helper") + run.path("src/build_helper") } fn make_run(run: RunConfig<'_>) { @@ -1372,7 +1372,7 @@ impl Step for CrateBuildHelper { Mode::ToolBootstrap, host, Kind::Test, - "src/tools/build_helper", + "src/build_helper", SourceType::InTree, &[], ); diff --git a/src/build_helper/Cargo.toml b/src/build_helper/Cargo.toml new file mode 100644 index 00000000000..66894e1abc4 --- /dev/null +++ b/src/build_helper/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "build_helper" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = "1" +serde_derive = "1" diff --git a/src/build_helper/README.md b/src/build_helper/README.md new file mode 100644 index 00000000000..f81b631c3fd --- /dev/null +++ b/src/build_helper/README.md @@ -0,0 +1 @@ +Types and functions shared across tools in this workspace. diff --git a/src/build_helper/src/ci.rs b/src/build_helper/src/ci.rs new file mode 100644 index 00000000000..60f319129a0 --- /dev/null +++ b/src/build_helper/src/ci.rs @@ -0,0 +1,95 @@ +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum CiEnv { + /// Not a CI environment. + None, + /// The GitHub Actions environment, for Linux (including Docker), Windows and macOS builds. + GitHubActions, +} + +impl CiEnv { + /// Obtains the current CI environment. + pub fn current() -> CiEnv { + if std::env::var("GITHUB_ACTIONS").map_or(false, |e| e == "true") { + CiEnv::GitHubActions + } else { + CiEnv::None + } + } + + pub fn is_ci() -> bool { + Self::current() != CiEnv::None + } + + /// Checks if running in rust-lang/rust managed CI job. + pub fn is_rust_lang_managed_ci_job() -> bool { + Self::is_ci() + // If both are present, we can assume it's an upstream CI job + // as they are always set unconditionally. + && std::env::var_os("CI_JOB_NAME").is_some() + && std::env::var_os("TOOLSTATE_REPO").is_some() + } +} + +pub mod gha { + use std::sync::Mutex; + + static ACTIVE_GROUPS: Mutex> = Mutex::new(Vec::new()); + + /// All github actions log messages from this call to the Drop of the return value + /// will be grouped and hidden by default in logs. Note that since github actions doesn't + /// support group nesting, any active group will be first finished when a subgroup is started, + /// and then re-started when the subgroup finishes. + #[track_caller] + pub fn group(name: impl std::fmt::Display) -> Group { + let mut groups = ACTIVE_GROUPS.lock().unwrap(); + + // A group is currently active. End it first to avoid nesting. + if !groups.is_empty() { + end_group(); + } + + let name = name.to_string(); + start_group(&name); + groups.push(name); + Group(()) + } + + /// A guard that closes the current github actions log group on drop. + #[must_use] + pub struct Group(()); + + impl Drop for Group { + fn drop(&mut self) { + end_group(); + + let mut groups = ACTIVE_GROUPS.lock().unwrap(); + // Remove the current group + groups.pop(); + + // If there was some previous group, restart it + if is_in_gha() { + if let Some(name) = groups.last() { + start_group(format!("{name} (continued)")); + } + } + } + } + + fn start_group(name: impl std::fmt::Display) { + if is_in_gha() { + println!("::group::{name}"); + } else { + println!("{name}") + } + } + + fn end_group() { + if is_in_gha() { + println!("::endgroup::"); + } + } + + fn is_in_gha() -> bool { + std::env::var_os("GITHUB_ACTIONS").is_some() + } +} diff --git a/src/build_helper/src/drop_bomb/mod.rs b/src/build_helper/src/drop_bomb/mod.rs new file mode 100644 index 00000000000..0a5bb04b55b --- /dev/null +++ b/src/build_helper/src/drop_bomb/mod.rs @@ -0,0 +1,54 @@ +//! This module implements "drop bombs" intended for use by command wrappers to ensure that the +//! constructed commands are *eventually* executed. This is exactly like `rustc_errors::Diag` where +//! we force every `Diag` to be consumed or we emit a bug, but we panic instead. +//! +//! This is adapted from and simplified for our +//! purposes. + +use std::ffi::{OsStr, OsString}; +use std::panic; + +#[cfg(test)] +mod tests; + +#[derive(Debug)] +pub struct DropBomb { + command: OsString, + defused: bool, + armed_location: panic::Location<'static>, +} + +impl DropBomb { + /// Arm a [`DropBomb`]. If the value is dropped without being [`defused`][Self::defused], then + /// it will panic. It is expected that the command wrapper uses `#[track_caller]` to help + /// propagate the caller location. + #[track_caller] + pub fn arm>(command: S) -> DropBomb { + DropBomb { + command: command.as_ref().into(), + defused: false, + armed_location: *panic::Location::caller(), + } + } + + pub fn get_created_location(&self) -> panic::Location<'static> { + self.armed_location + } + + /// Defuse the [`DropBomb`]. This will prevent the drop bomb from panicking when dropped. + pub fn defuse(&mut self) { + self.defused = true; + } +} + +impl Drop for DropBomb { + fn drop(&mut self) { + if !self.defused && !std::thread::panicking() { + panic!( + "command constructed at `{}` was dropped without being executed: `{}`", + self.armed_location, + self.command.to_string_lossy() + ) + } + } +} diff --git a/src/build_helper/src/drop_bomb/tests.rs b/src/build_helper/src/drop_bomb/tests.rs new file mode 100644 index 00000000000..4a488c0f670 --- /dev/null +++ b/src/build_helper/src/drop_bomb/tests.rs @@ -0,0 +1,15 @@ +use super::DropBomb; + +#[test] +#[should_panic] +fn test_arm() { + let bomb = DropBomb::arm("hi :3"); + drop(bomb); // <- armed bomb should explode when not defused +} + +#[test] +fn test_defuse() { + let mut bomb = DropBomb::arm("hi :3"); + bomb.defuse(); + drop(bomb); // <- defused bomb should not explode +} diff --git a/src/build_helper/src/git.rs b/src/build_helper/src/git.rs new file mode 100644 index 00000000000..2aad5650fa8 --- /dev/null +++ b/src/build_helper/src/git.rs @@ -0,0 +1,208 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use crate::ci::CiEnv; + +pub struct GitConfig<'a> { + pub git_repository: &'a str, + pub nightly_branch: &'a str, + pub git_merge_commit_email: &'a str, +} + +/// Runs a command and returns the output +pub fn output_result(cmd: &mut Command) -> Result { + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => return Err(format!("failed to run command: {:?}: {}", cmd, e)), + }; + if !output.status.success() { + return Err(format!( + "command did not execute successfully: {:?}\n\ + expected success, got: {}\n{}", + cmd, + output.status, + String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? + )); + } + String::from_utf8(output.stdout).map_err(|err| format!("{err:?}")) +} + +/// Finds the remote for rust-lang/rust. +/// For example for these remotes it will return `upstream`. +/// ```text +/// origin https://github.com/Nilstrieb/rust.git (fetch) +/// origin https://github.com/Nilstrieb/rust.git (push) +/// upstream https://github.com/rust-lang/rust (fetch) +/// upstream https://github.com/rust-lang/rust (push) +/// ``` +pub fn get_rust_lang_rust_remote( + config: &GitConfig<'_>, + git_dir: Option<&Path>, +) -> Result { + let mut git = Command::new("git"); + if let Some(git_dir) = git_dir { + git.current_dir(git_dir); + } + git.args(["config", "--local", "--get-regex", "remote\\..*\\.url"]); + let stdout = output_result(&mut git)?; + + let rust_lang_remote = stdout + .lines() + .find(|remote| remote.contains(config.git_repository)) + .ok_or_else(|| format!("{} remote not found", config.git_repository))?; + + let remote_name = + rust_lang_remote.split('.').nth(1).ok_or_else(|| "remote name not found".to_owned())?; + Ok(remote_name.into()) +} + +pub fn rev_exists(rev: &str, git_dir: Option<&Path>) -> Result { + let mut git = Command::new("git"); + if let Some(git_dir) = git_dir { + git.current_dir(git_dir); + } + git.args(["rev-parse", rev]); + let output = git.output().map_err(|err| format!("{err:?}"))?; + + match output.status.code() { + Some(0) => Ok(true), + Some(128) => Ok(false), + None => Err(format!( + "git didn't exit properly: {}", + String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? + )), + Some(code) => Err(format!( + "git command exited with status code: {code}: {}", + String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? + )), + } +} + +/// Returns the master branch from which we can take diffs to see changes. +/// This will usually be rust-lang/rust master, but sometimes this might not exist. +/// This could be because the user is updating their forked master branch using the GitHub UI +/// and therefore doesn't need an upstream master branch checked out. +/// We will then fall back to origin/master in the hope that at least this exists. +pub fn updated_master_branch( + config: &GitConfig<'_>, + git_dir: Option<&Path>, +) -> Result { + let upstream_remote = get_rust_lang_rust_remote(config, git_dir)?; + let branch = config.nightly_branch; + for upstream_master in [format!("{upstream_remote}/{branch}"), format!("origin/{branch}")] { + if rev_exists(&upstream_master, git_dir)? { + return Ok(upstream_master); + } + } + + Err("Cannot find any suitable upstream master branch".to_owned()) +} + +/// Finds the nearest merge commit by comparing the local `HEAD` with the upstream branch's state. +/// To work correctly, the upstream remote must be properly configured using `git remote add `. +/// In most cases `get_closest_merge_commit` is the function you are looking for as it doesn't require remote +/// to be configured. +fn git_upstream_merge_base( + config: &GitConfig<'_>, + git_dir: Option<&Path>, +) -> Result { + let updated_master = updated_master_branch(config, git_dir)?; + let mut git = Command::new("git"); + if let Some(git_dir) = git_dir { + git.current_dir(git_dir); + } + Ok(output_result(git.arg("merge-base").arg(&updated_master).arg("HEAD"))?.trim().to_owned()) +} + +/// Searches for the nearest merge commit in the repository that also exists upstream. +/// +/// It looks for the most recent commit made by the merge bot by matching the author's email +/// address with the merge bot's email. +pub fn get_closest_merge_commit( + git_dir: Option<&Path>, + config: &GitConfig<'_>, + target_paths: &[PathBuf], +) -> Result { + let mut git = Command::new("git"); + + if let Some(git_dir) = git_dir { + git.current_dir(git_dir); + } + + let merge_base = { + if CiEnv::is_ci() { + git_upstream_merge_base(config, git_dir).unwrap() + } else { + // For non-CI environments, ignore rust-lang/rust upstream as it usually gets + // outdated very quickly. + "HEAD".to_string() + } + }; + + git.args([ + "rev-list", + &format!("--author={}", config.git_merge_commit_email), + "-n1", + "--first-parent", + &merge_base, + ]); + + if !target_paths.is_empty() { + git.arg("--").args(target_paths); + } + + Ok(output_result(&mut git)?.trim().to_owned()) +} + +/// Returns the files that have been modified in the current branch compared to the master branch. +/// The `extensions` parameter can be used to filter the files by their extension. +/// Does not include removed files. +/// If `extensions` is empty, all files will be returned. +pub fn get_git_modified_files( + config: &GitConfig<'_>, + git_dir: Option<&Path>, + extensions: &[&str], +) -> Result>, String> { + let merge_base = get_closest_merge_commit(git_dir, config, &[])?; + + let mut git = Command::new("git"); + if let Some(git_dir) = git_dir { + git.current_dir(git_dir); + } + let files = output_result(git.args(["diff-index", "--name-status", merge_base.trim()]))? + .lines() + .filter_map(|f| { + let (status, name) = f.trim().split_once(char::is_whitespace).unwrap(); + if status == "D" { + None + } else if Path::new(name).extension().map_or(false, |ext| { + extensions.is_empty() || extensions.contains(&ext.to_str().unwrap()) + }) { + Some(name.to_owned()) + } else { + None + } + }) + .collect(); + Ok(Some(files)) +} + +/// Returns the files that haven't been added to git yet. +pub fn get_git_untracked_files( + config: &GitConfig<'_>, + git_dir: Option<&Path>, +) -> Result>, String> { + let Ok(_updated_master) = updated_master_branch(config, git_dir) else { + return Ok(None); + }; + let mut git = Command::new("git"); + if let Some(git_dir) = git_dir { + git.current_dir(git_dir); + } + + let files = output_result(git.arg("ls-files").arg("--others").arg("--exclude-standard"))? + .lines() + .map(|s| s.trim().to_owned()) + .collect(); + Ok(Some(files)) +} diff --git a/src/build_helper/src/lib.rs b/src/build_helper/src/lib.rs new file mode 100644 index 00000000000..4a4f0ca2a9d --- /dev/null +++ b/src/build_helper/src/lib.rs @@ -0,0 +1,31 @@ +//! Types and functions shared across tools in this workspace. + +pub mod ci; +pub mod drop_bomb; +pub mod git; +pub mod metrics; +pub mod stage0_parser; +pub mod util; + +/// The default set of crates for opt-dist to collect LLVM profiles. +pub const LLVM_PGO_CRATES: &[&str] = &[ + "syn-1.0.89", + "cargo-0.60.0", + "serde-1.0.136", + "ripgrep-13.0.0", + "regex-1.5.5", + "clap-3.1.6", + "hyper-0.14.18", +]; + +/// The default set of crates for opt-dist to collect rustc profiles. +pub const RUSTC_PGO_CRATES: &[&str] = &[ + "externs", + "ctfe-stress-5", + "cargo-0.60.0", + "token-stream-stress", + "match-stress", + "tuple-stress", + "diesel-1.4.8", + "bitmaps-3.1.0", +]; diff --git a/src/build_helper/src/metrics.rs b/src/build_helper/src/metrics.rs new file mode 100644 index 00000000000..2d0c66a8f33 --- /dev/null +++ b/src/build_helper/src/metrics.rs @@ -0,0 +1,92 @@ +use serde_derive::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct JsonRoot { + #[serde(default)] // For version 0 the field was not present. + pub format_version: usize, + pub system_stats: JsonInvocationSystemStats, + pub invocations: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct JsonInvocation { + // Unix timestamp in seconds + // + // This is necessary to easily correlate this invocation with logs or other data. + pub start_time: u64, + pub duration_including_children_sec: f64, + pub children: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum JsonNode { + RustbuildStep { + #[serde(rename = "type")] + type_: String, + debug_repr: String, + + duration_excluding_children_sec: f64, + system_stats: JsonStepSystemStats, + + children: Vec, + }, + TestSuite(TestSuite), +} + +#[derive(Serialize, Deserialize)] +pub struct TestSuite { + pub metadata: TestSuiteMetadata, + pub tests: Vec, +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum TestSuiteMetadata { + CargoPackage { + crates: Vec, + target: String, + host: String, + stage: u32, + }, + Compiletest { + suite: String, + mode: String, + compare_mode: Option, + target: String, + host: String, + stage: u32, + }, +} + +#[derive(Serialize, Deserialize)] +pub struct Test { + pub name: String, + #[serde(flatten)] + pub outcome: TestOutcome, +} + +#[derive(Serialize, Deserialize)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum TestOutcome { + Passed, + Failed, + Ignored { ignore_reason: Option }, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct JsonInvocationSystemStats { + pub cpu_threads_count: usize, + pub cpu_model: String, + + pub memory_total_bytes: u64, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct JsonStepSystemStats { + pub cpu_utilization_percent: f64, +} diff --git a/src/build_helper/src/stage0_parser.rs b/src/build_helper/src/stage0_parser.rs new file mode 100644 index 00000000000..2a0c12a1c91 --- /dev/null +++ b/src/build_helper/src/stage0_parser.rs @@ -0,0 +1,76 @@ +use std::collections::BTreeMap; + +#[derive(Default, Clone)] +pub struct Stage0 { + pub compiler: VersionMetadata, + pub rustfmt: Option, + pub config: Stage0Config, + pub checksums_sha256: BTreeMap, +} + +#[derive(Default, Clone)] +pub struct VersionMetadata { + pub date: String, + pub version: String, +} + +#[derive(Default, Clone)] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, + pub git_repository: String, + pub nightly_branch: String, +} + +pub fn parse_stage0_file() -> Stage0 { + let stage0_content = include_str!("../../stage0"); + + let mut stage0 = Stage0::default(); + for line in stage0_content.lines() { + let line = line.trim(); + + if line.is_empty() { + continue; + } + + // Ignore comments + if line.starts_with('#') { + continue; + } + + let (key, value) = line.split_once('=').unwrap(); + + match key { + "dist_server" => stage0.config.dist_server = value.to_owned(), + "artifacts_server" => stage0.config.artifacts_server = value.to_owned(), + "artifacts_with_llvm_assertions_server" => { + stage0.config.artifacts_with_llvm_assertions_server = value.to_owned() + } + "git_merge_commit_email" => stage0.config.git_merge_commit_email = value.to_owned(), + "git_repository" => stage0.config.git_repository = value.to_owned(), + "nightly_branch" => stage0.config.nightly_branch = value.to_owned(), + + "compiler_date" => stage0.compiler.date = value.to_owned(), + "compiler_version" => stage0.compiler.version = value.to_owned(), + + "rustfmt_date" => { + stage0.rustfmt.get_or_insert(VersionMetadata::default()).date = value.to_owned(); + } + "rustfmt_version" => { + stage0.rustfmt.get_or_insert(VersionMetadata::default()).version = value.to_owned(); + } + + dist if dist.starts_with("dist") => { + stage0.checksums_sha256.insert(key.to_owned(), value.to_owned()); + } + + unsupported => { + println!("'{unsupported}' field is not supported."); + } + } + } + + stage0 +} diff --git a/src/build_helper/src/util.rs b/src/build_helper/src/util.rs new file mode 100644 index 00000000000..72c05c4c48a --- /dev/null +++ b/src/build_helper/src/util.rs @@ -0,0 +1,75 @@ +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::Command; +use std::sync::OnceLock; + +/// Invokes `build_helper::util::detail_exit` with `cfg!(test)` +/// +/// This is a macro instead of a function so that it uses `cfg(test)` in the *calling* crate, not in build helper. +#[macro_export] +macro_rules! exit { + ($code:expr) => { + $crate::util::detail_exit($code, cfg!(test)); + }; +} + +/// If code is not 0 (successful exit status), exit status is 101 (rust's default error code.) +/// If `is_test` true and code is an error code, it will cause a panic. +pub fn detail_exit(code: i32, is_test: bool) -> ! { + // if in test and code is an error code, panic with status code provided + if is_test { + panic!("status code: {}", code); + } else { + // otherwise,exit with provided status code + std::process::exit(code); + } +} + +pub fn fail(s: &str) -> ! { + eprintln!("\n\n{}\n\n", s); + detail_exit(1, cfg!(test)); +} + +pub fn try_run(cmd: &mut Command, print_cmd_on_fail: bool) -> Result<(), ()> { + let status = match cmd.status() { + Ok(status) => status, + Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)), + }; + if !status.success() { + if print_cmd_on_fail { + println!( + "\n\ncommand did not execute successfully: {:?}\n\ + expected success, got: {}\n\n", + cmd, status + ); + } + Err(()) + } else { + Ok(()) + } +} + +/// Returns the submodule paths from the `.gitmodules` file in the given directory. +pub fn parse_gitmodules(target_dir: &Path) -> &[String] { + static SUBMODULES_PATHS: OnceLock> = OnceLock::new(); + let gitmodules = target_dir.join(".gitmodules"); + assert!(gitmodules.exists(), "'{}' file is missing.", gitmodules.display()); + + let init_submodules_paths = || { + let file = File::open(gitmodules).unwrap(); + + let mut submodules_paths = vec![]; + for line in BufReader::new(file).lines().map_while(Result::ok) { + let line = line.trim(); + if line.starts_with("path") { + let actual_path = line.split(' ').last().expect("Couldn't get value of path"); + submodules_paths.push(actual_path.to_owned()); + } + } + + submodules_paths + }; + + SUBMODULES_PATHS.get_or_init(|| init_submodules_paths()) +} diff --git a/src/tools/build_helper/Cargo.toml b/src/tools/build_helper/Cargo.toml deleted file mode 100644 index 66894e1abc4..00000000000 --- a/src/tools/build_helper/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "build_helper" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -serde = "1" -serde_derive = "1" diff --git a/src/tools/build_helper/README.md b/src/tools/build_helper/README.md deleted file mode 100644 index f81b631c3fd..00000000000 --- a/src/tools/build_helper/README.md +++ /dev/null @@ -1 +0,0 @@ -Types and functions shared across tools in this workspace. diff --git a/src/tools/build_helper/src/ci.rs b/src/tools/build_helper/src/ci.rs deleted file mode 100644 index 60f319129a0..00000000000 --- a/src/tools/build_helper/src/ci.rs +++ /dev/null @@ -1,95 +0,0 @@ -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub enum CiEnv { - /// Not a CI environment. - None, - /// The GitHub Actions environment, for Linux (including Docker), Windows and macOS builds. - GitHubActions, -} - -impl CiEnv { - /// Obtains the current CI environment. - pub fn current() -> CiEnv { - if std::env::var("GITHUB_ACTIONS").map_or(false, |e| e == "true") { - CiEnv::GitHubActions - } else { - CiEnv::None - } - } - - pub fn is_ci() -> bool { - Self::current() != CiEnv::None - } - - /// Checks if running in rust-lang/rust managed CI job. - pub fn is_rust_lang_managed_ci_job() -> bool { - Self::is_ci() - // If both are present, we can assume it's an upstream CI job - // as they are always set unconditionally. - && std::env::var_os("CI_JOB_NAME").is_some() - && std::env::var_os("TOOLSTATE_REPO").is_some() - } -} - -pub mod gha { - use std::sync::Mutex; - - static ACTIVE_GROUPS: Mutex> = Mutex::new(Vec::new()); - - /// All github actions log messages from this call to the Drop of the return value - /// will be grouped and hidden by default in logs. Note that since github actions doesn't - /// support group nesting, any active group will be first finished when a subgroup is started, - /// and then re-started when the subgroup finishes. - #[track_caller] - pub fn group(name: impl std::fmt::Display) -> Group { - let mut groups = ACTIVE_GROUPS.lock().unwrap(); - - // A group is currently active. End it first to avoid nesting. - if !groups.is_empty() { - end_group(); - } - - let name = name.to_string(); - start_group(&name); - groups.push(name); - Group(()) - } - - /// A guard that closes the current github actions log group on drop. - #[must_use] - pub struct Group(()); - - impl Drop for Group { - fn drop(&mut self) { - end_group(); - - let mut groups = ACTIVE_GROUPS.lock().unwrap(); - // Remove the current group - groups.pop(); - - // If there was some previous group, restart it - if is_in_gha() { - if let Some(name) = groups.last() { - start_group(format!("{name} (continued)")); - } - } - } - } - - fn start_group(name: impl std::fmt::Display) { - if is_in_gha() { - println!("::group::{name}"); - } else { - println!("{name}") - } - } - - fn end_group() { - if is_in_gha() { - println!("::endgroup::"); - } - } - - fn is_in_gha() -> bool { - std::env::var_os("GITHUB_ACTIONS").is_some() - } -} diff --git a/src/tools/build_helper/src/drop_bomb/mod.rs b/src/tools/build_helper/src/drop_bomb/mod.rs deleted file mode 100644 index 0a5bb04b55b..00000000000 --- a/src/tools/build_helper/src/drop_bomb/mod.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! This module implements "drop bombs" intended for use by command wrappers to ensure that the -//! constructed commands are *eventually* executed. This is exactly like `rustc_errors::Diag` where -//! we force every `Diag` to be consumed or we emit a bug, but we panic instead. -//! -//! This is adapted from and simplified for our -//! purposes. - -use std::ffi::{OsStr, OsString}; -use std::panic; - -#[cfg(test)] -mod tests; - -#[derive(Debug)] -pub struct DropBomb { - command: OsString, - defused: bool, - armed_location: panic::Location<'static>, -} - -impl DropBomb { - /// Arm a [`DropBomb`]. If the value is dropped without being [`defused`][Self::defused], then - /// it will panic. It is expected that the command wrapper uses `#[track_caller]` to help - /// propagate the caller location. - #[track_caller] - pub fn arm>(command: S) -> DropBomb { - DropBomb { - command: command.as_ref().into(), - defused: false, - armed_location: *panic::Location::caller(), - } - } - - pub fn get_created_location(&self) -> panic::Location<'static> { - self.armed_location - } - - /// Defuse the [`DropBomb`]. This will prevent the drop bomb from panicking when dropped. - pub fn defuse(&mut self) { - self.defused = true; - } -} - -impl Drop for DropBomb { - fn drop(&mut self) { - if !self.defused && !std::thread::panicking() { - panic!( - "command constructed at `{}` was dropped without being executed: `{}`", - self.armed_location, - self.command.to_string_lossy() - ) - } - } -} diff --git a/src/tools/build_helper/src/drop_bomb/tests.rs b/src/tools/build_helper/src/drop_bomb/tests.rs deleted file mode 100644 index 4a488c0f670..00000000000 --- a/src/tools/build_helper/src/drop_bomb/tests.rs +++ /dev/null @@ -1,15 +0,0 @@ -use super::DropBomb; - -#[test] -#[should_panic] -fn test_arm() { - let bomb = DropBomb::arm("hi :3"); - drop(bomb); // <- armed bomb should explode when not defused -} - -#[test] -fn test_defuse() { - let mut bomb = DropBomb::arm("hi :3"); - bomb.defuse(); - drop(bomb); // <- defused bomb should not explode -} diff --git a/src/tools/build_helper/src/git.rs b/src/tools/build_helper/src/git.rs deleted file mode 100644 index 2aad5650fa8..00000000000 --- a/src/tools/build_helper/src/git.rs +++ /dev/null @@ -1,208 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; - -use crate::ci::CiEnv; - -pub struct GitConfig<'a> { - pub git_repository: &'a str, - pub nightly_branch: &'a str, - pub git_merge_commit_email: &'a str, -} - -/// Runs a command and returns the output -pub fn output_result(cmd: &mut Command) -> Result { - let output = match cmd.stderr(Stdio::inherit()).output() { - Ok(status) => status, - Err(e) => return Err(format!("failed to run command: {:?}: {}", cmd, e)), - }; - if !output.status.success() { - return Err(format!( - "command did not execute successfully: {:?}\n\ - expected success, got: {}\n{}", - cmd, - output.status, - String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? - )); - } - String::from_utf8(output.stdout).map_err(|err| format!("{err:?}")) -} - -/// Finds the remote for rust-lang/rust. -/// For example for these remotes it will return `upstream`. -/// ```text -/// origin https://github.com/Nilstrieb/rust.git (fetch) -/// origin https://github.com/Nilstrieb/rust.git (push) -/// upstream https://github.com/rust-lang/rust (fetch) -/// upstream https://github.com/rust-lang/rust (push) -/// ``` -pub fn get_rust_lang_rust_remote( - config: &GitConfig<'_>, - git_dir: Option<&Path>, -) -> Result { - let mut git = Command::new("git"); - if let Some(git_dir) = git_dir { - git.current_dir(git_dir); - } - git.args(["config", "--local", "--get-regex", "remote\\..*\\.url"]); - let stdout = output_result(&mut git)?; - - let rust_lang_remote = stdout - .lines() - .find(|remote| remote.contains(config.git_repository)) - .ok_or_else(|| format!("{} remote not found", config.git_repository))?; - - let remote_name = - rust_lang_remote.split('.').nth(1).ok_or_else(|| "remote name not found".to_owned())?; - Ok(remote_name.into()) -} - -pub fn rev_exists(rev: &str, git_dir: Option<&Path>) -> Result { - let mut git = Command::new("git"); - if let Some(git_dir) = git_dir { - git.current_dir(git_dir); - } - git.args(["rev-parse", rev]); - let output = git.output().map_err(|err| format!("{err:?}"))?; - - match output.status.code() { - Some(0) => Ok(true), - Some(128) => Ok(false), - None => Err(format!( - "git didn't exit properly: {}", - String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? - )), - Some(code) => Err(format!( - "git command exited with status code: {code}: {}", - String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? - )), - } -} - -/// Returns the master branch from which we can take diffs to see changes. -/// This will usually be rust-lang/rust master, but sometimes this might not exist. -/// This could be because the user is updating their forked master branch using the GitHub UI -/// and therefore doesn't need an upstream master branch checked out. -/// We will then fall back to origin/master in the hope that at least this exists. -pub fn updated_master_branch( - config: &GitConfig<'_>, - git_dir: Option<&Path>, -) -> Result { - let upstream_remote = get_rust_lang_rust_remote(config, git_dir)?; - let branch = config.nightly_branch; - for upstream_master in [format!("{upstream_remote}/{branch}"), format!("origin/{branch}")] { - if rev_exists(&upstream_master, git_dir)? { - return Ok(upstream_master); - } - } - - Err("Cannot find any suitable upstream master branch".to_owned()) -} - -/// Finds the nearest merge commit by comparing the local `HEAD` with the upstream branch's state. -/// To work correctly, the upstream remote must be properly configured using `git remote add `. -/// In most cases `get_closest_merge_commit` is the function you are looking for as it doesn't require remote -/// to be configured. -fn git_upstream_merge_base( - config: &GitConfig<'_>, - git_dir: Option<&Path>, -) -> Result { - let updated_master = updated_master_branch(config, git_dir)?; - let mut git = Command::new("git"); - if let Some(git_dir) = git_dir { - git.current_dir(git_dir); - } - Ok(output_result(git.arg("merge-base").arg(&updated_master).arg("HEAD"))?.trim().to_owned()) -} - -/// Searches for the nearest merge commit in the repository that also exists upstream. -/// -/// It looks for the most recent commit made by the merge bot by matching the author's email -/// address with the merge bot's email. -pub fn get_closest_merge_commit( - git_dir: Option<&Path>, - config: &GitConfig<'_>, - target_paths: &[PathBuf], -) -> Result { - let mut git = Command::new("git"); - - if let Some(git_dir) = git_dir { - git.current_dir(git_dir); - } - - let merge_base = { - if CiEnv::is_ci() { - git_upstream_merge_base(config, git_dir).unwrap() - } else { - // For non-CI environments, ignore rust-lang/rust upstream as it usually gets - // outdated very quickly. - "HEAD".to_string() - } - }; - - git.args([ - "rev-list", - &format!("--author={}", config.git_merge_commit_email), - "-n1", - "--first-parent", - &merge_base, - ]); - - if !target_paths.is_empty() { - git.arg("--").args(target_paths); - } - - Ok(output_result(&mut git)?.trim().to_owned()) -} - -/// Returns the files that have been modified in the current branch compared to the master branch. -/// The `extensions` parameter can be used to filter the files by their extension. -/// Does not include removed files. -/// If `extensions` is empty, all files will be returned. -pub fn get_git_modified_files( - config: &GitConfig<'_>, - git_dir: Option<&Path>, - extensions: &[&str], -) -> Result>, String> { - let merge_base = get_closest_merge_commit(git_dir, config, &[])?; - - let mut git = Command::new("git"); - if let Some(git_dir) = git_dir { - git.current_dir(git_dir); - } - let files = output_result(git.args(["diff-index", "--name-status", merge_base.trim()]))? - .lines() - .filter_map(|f| { - let (status, name) = f.trim().split_once(char::is_whitespace).unwrap(); - if status == "D" { - None - } else if Path::new(name).extension().map_or(false, |ext| { - extensions.is_empty() || extensions.contains(&ext.to_str().unwrap()) - }) { - Some(name.to_owned()) - } else { - None - } - }) - .collect(); - Ok(Some(files)) -} - -/// Returns the files that haven't been added to git yet. -pub fn get_git_untracked_files( - config: &GitConfig<'_>, - git_dir: Option<&Path>, -) -> Result>, String> { - let Ok(_updated_master) = updated_master_branch(config, git_dir) else { - return Ok(None); - }; - let mut git = Command::new("git"); - if let Some(git_dir) = git_dir { - git.current_dir(git_dir); - } - - let files = output_result(git.arg("ls-files").arg("--others").arg("--exclude-standard"))? - .lines() - .map(|s| s.trim().to_owned()) - .collect(); - Ok(Some(files)) -} diff --git a/src/tools/build_helper/src/lib.rs b/src/tools/build_helper/src/lib.rs deleted file mode 100644 index 4a4f0ca2a9d..00000000000 --- a/src/tools/build_helper/src/lib.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! Types and functions shared across tools in this workspace. - -pub mod ci; -pub mod drop_bomb; -pub mod git; -pub mod metrics; -pub mod stage0_parser; -pub mod util; - -/// The default set of crates for opt-dist to collect LLVM profiles. -pub const LLVM_PGO_CRATES: &[&str] = &[ - "syn-1.0.89", - "cargo-0.60.0", - "serde-1.0.136", - "ripgrep-13.0.0", - "regex-1.5.5", - "clap-3.1.6", - "hyper-0.14.18", -]; - -/// The default set of crates for opt-dist to collect rustc profiles. -pub const RUSTC_PGO_CRATES: &[&str] = &[ - "externs", - "ctfe-stress-5", - "cargo-0.60.0", - "token-stream-stress", - "match-stress", - "tuple-stress", - "diesel-1.4.8", - "bitmaps-3.1.0", -]; diff --git a/src/tools/build_helper/src/metrics.rs b/src/tools/build_helper/src/metrics.rs deleted file mode 100644 index 2d0c66a8f33..00000000000 --- a/src/tools/build_helper/src/metrics.rs +++ /dev/null @@ -1,92 +0,0 @@ -use serde_derive::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub struct JsonRoot { - #[serde(default)] // For version 0 the field was not present. - pub format_version: usize, - pub system_stats: JsonInvocationSystemStats, - pub invocations: Vec, -} - -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub struct JsonInvocation { - // Unix timestamp in seconds - // - // This is necessary to easily correlate this invocation with logs or other data. - pub start_time: u64, - pub duration_including_children_sec: f64, - pub children: Vec, -} - -#[derive(Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum JsonNode { - RustbuildStep { - #[serde(rename = "type")] - type_: String, - debug_repr: String, - - duration_excluding_children_sec: f64, - system_stats: JsonStepSystemStats, - - children: Vec, - }, - TestSuite(TestSuite), -} - -#[derive(Serialize, Deserialize)] -pub struct TestSuite { - pub metadata: TestSuiteMetadata, - pub tests: Vec, -} - -#[derive(Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum TestSuiteMetadata { - CargoPackage { - crates: Vec, - target: String, - host: String, - stage: u32, - }, - Compiletest { - suite: String, - mode: String, - compare_mode: Option, - target: String, - host: String, - stage: u32, - }, -} - -#[derive(Serialize, Deserialize)] -pub struct Test { - pub name: String, - #[serde(flatten)] - pub outcome: TestOutcome, -} - -#[derive(Serialize, Deserialize)] -#[serde(tag = "outcome", rename_all = "snake_case")] -pub enum TestOutcome { - Passed, - Failed, - Ignored { ignore_reason: Option }, -} - -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub struct JsonInvocationSystemStats { - pub cpu_threads_count: usize, - pub cpu_model: String, - - pub memory_total_bytes: u64, -} - -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub struct JsonStepSystemStats { - pub cpu_utilization_percent: f64, -} diff --git a/src/tools/build_helper/src/stage0_parser.rs b/src/tools/build_helper/src/stage0_parser.rs deleted file mode 100644 index ff05b116989..00000000000 --- a/src/tools/build_helper/src/stage0_parser.rs +++ /dev/null @@ -1,76 +0,0 @@ -use std::collections::BTreeMap; - -#[derive(Default, Clone)] -pub struct Stage0 { - pub compiler: VersionMetadata, - pub rustfmt: Option, - pub config: Stage0Config, - pub checksums_sha256: BTreeMap, -} - -#[derive(Default, Clone)] -pub struct VersionMetadata { - pub date: String, - pub version: String, -} - -#[derive(Default, Clone)] -pub struct Stage0Config { - pub dist_server: String, - pub artifacts_server: String, - pub artifacts_with_llvm_assertions_server: String, - pub git_merge_commit_email: String, - pub git_repository: String, - pub nightly_branch: String, -} - -pub fn parse_stage0_file() -> Stage0 { - let stage0_content = include_str!("../../../stage0"); - - let mut stage0 = Stage0::default(); - for line in stage0_content.lines() { - let line = line.trim(); - - if line.is_empty() { - continue; - } - - // Ignore comments - if line.starts_with('#') { - continue; - } - - let (key, value) = line.split_once('=').unwrap(); - - match key { - "dist_server" => stage0.config.dist_server = value.to_owned(), - "artifacts_server" => stage0.config.artifacts_server = value.to_owned(), - "artifacts_with_llvm_assertions_server" => { - stage0.config.artifacts_with_llvm_assertions_server = value.to_owned() - } - "git_merge_commit_email" => stage0.config.git_merge_commit_email = value.to_owned(), - "git_repository" => stage0.config.git_repository = value.to_owned(), - "nightly_branch" => stage0.config.nightly_branch = value.to_owned(), - - "compiler_date" => stage0.compiler.date = value.to_owned(), - "compiler_version" => stage0.compiler.version = value.to_owned(), - - "rustfmt_date" => { - stage0.rustfmt.get_or_insert(VersionMetadata::default()).date = value.to_owned(); - } - "rustfmt_version" => { - stage0.rustfmt.get_or_insert(VersionMetadata::default()).version = value.to_owned(); - } - - dist if dist.starts_with("dist") => { - stage0.checksums_sha256.insert(key.to_owned(), value.to_owned()); - } - - unsupported => { - println!("'{unsupported}' field is not supported."); - } - } - } - - stage0 -} diff --git a/src/tools/build_helper/src/util.rs b/src/tools/build_helper/src/util.rs deleted file mode 100644 index 72c05c4c48a..00000000000 --- a/src/tools/build_helper/src/util.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::fs::File; -use std::io::{BufRead, BufReader}; -use std::path::Path; -use std::process::Command; -use std::sync::OnceLock; - -/// Invokes `build_helper::util::detail_exit` with `cfg!(test)` -/// -/// This is a macro instead of a function so that it uses `cfg(test)` in the *calling* crate, not in build helper. -#[macro_export] -macro_rules! exit { - ($code:expr) => { - $crate::util::detail_exit($code, cfg!(test)); - }; -} - -/// If code is not 0 (successful exit status), exit status is 101 (rust's default error code.) -/// If `is_test` true and code is an error code, it will cause a panic. -pub fn detail_exit(code: i32, is_test: bool) -> ! { - // if in test and code is an error code, panic with status code provided - if is_test { - panic!("status code: {}", code); - } else { - // otherwise,exit with provided status code - std::process::exit(code); - } -} - -pub fn fail(s: &str) -> ! { - eprintln!("\n\n{}\n\n", s); - detail_exit(1, cfg!(test)); -} - -pub fn try_run(cmd: &mut Command, print_cmd_on_fail: bool) -> Result<(), ()> { - let status = match cmd.status() { - Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", cmd, e)), - }; - if !status.success() { - if print_cmd_on_fail { - println!( - "\n\ncommand did not execute successfully: {:?}\n\ - expected success, got: {}\n\n", - cmd, status - ); - } - Err(()) - } else { - Ok(()) - } -} - -/// Returns the submodule paths from the `.gitmodules` file in the given directory. -pub fn parse_gitmodules(target_dir: &Path) -> &[String] { - static SUBMODULES_PATHS: OnceLock> = OnceLock::new(); - let gitmodules = target_dir.join(".gitmodules"); - assert!(gitmodules.exists(), "'{}' file is missing.", gitmodules.display()); - - let init_submodules_paths = || { - let file = File::open(gitmodules).unwrap(); - - let mut submodules_paths = vec![]; - for line in BufReader::new(file).lines().map_while(Result::ok) { - let line = line.trim(); - if line.starts_with("path") { - let actual_path = line.split(' ').last().expect("Couldn't get value of path"); - submodules_paths.push(actual_path.to_owned()); - } - } - - submodules_paths - }; - - SUBMODULES_PATHS.get_or_init(|| init_submodules_paths()) -} diff --git a/src/tools/bump-stage0/Cargo.toml b/src/tools/bump-stage0/Cargo.toml index de5d821133d..6ee7a831839 100644 --- a/src/tools/bump-stage0/Cargo.toml +++ b/src/tools/bump-stage0/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] anyhow = "1.0.34" -build_helper = { path = "../build_helper" } +build_helper = { path = "../../build_helper" } curl = "0.4.38" indexmap = { version = "2.0.0", features = ["serde"] } serde = { version = "1.0.125", features = ["derive"] } diff --git a/src/tools/compiletest/Cargo.toml b/src/tools/compiletest/Cargo.toml index cef6b525a7b..b784bdb7139 100644 --- a/src/tools/compiletest/Cargo.toml +++ b/src/tools/compiletest/Cargo.toml @@ -14,7 +14,7 @@ unified-diff = "0.2.1" getopts = "0.2" indexmap = "2.0.0" miropt-test-tools = { path = "../miropt-test-tools" } -build_helper = { path = "../build_helper" } +build_helper = { path = "../../build_helper" } tracing = "0.1" tracing-subscriber = { version = "0.3.3", default-features = false, features = ["fmt", "env-filter", "smallvec", "parking_lot", "ansi"] } regex = "1.0" diff --git a/src/tools/opt-dist/Cargo.toml b/src/tools/opt-dist/Cargo.toml index d34f8ad0520..d0413911014 100644 --- a/src/tools/opt-dist/Cargo.toml +++ b/src/tools/opt-dist/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -build_helper = { path = "../build_helper" } +build_helper = { path = "../../build_helper" } env_logger = "0.11" log = "0.4" anyhow = { version = "1", features = ["backtrace"] } diff --git a/src/tools/run-make-support/Cargo.toml b/src/tools/run-make-support/Cargo.toml index 3affa199fa5..3c172b2d956 100644 --- a/src/tools/run-make-support/Cargo.toml +++ b/src/tools/run-make-support/Cargo.toml @@ -10,7 +10,7 @@ similar = "2.5.0" wasmparser = { version = "0.216", default-features = false, features = ["std"] } regex = "1.8" # 1.8 to avoid memchr 2.6.0, as 2.5.0 is pinned in the workspace gimli = "0.31.0" -build_helper = { path = "../build_helper" } +build_helper = { path = "../../build_helper" } serde_json = "1.0" libc = "0.2" diff --git a/src/tools/rustdoc-gui-test/Cargo.toml b/src/tools/rustdoc-gui-test/Cargo.toml index 4cb200ebc7c..f7384a98f85 100644 --- a/src/tools/rustdoc-gui-test/Cargo.toml +++ b/src/tools/rustdoc-gui-test/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -build_helper = { path = "../build_helper" } +build_helper = { path = "../../build_helper" } compiletest = { path = "../compiletest" } getopts = "0.2" walkdir = "2" diff --git a/src/tools/suggest-tests/Cargo.toml b/src/tools/suggest-tests/Cargo.toml index 7c048d53a50..d6f86078d7e 100644 --- a/src/tools/suggest-tests/Cargo.toml +++ b/src/tools/suggest-tests/Cargo.toml @@ -5,4 +5,4 @@ edition = "2021" [dependencies] glob = "0.3.0" -build_helper = { version = "0.1.0", path = "../build_helper" } +build_helper = { version = "0.1.0", path = "../../build_helper" } diff --git a/src/tools/tidy/Cargo.toml b/src/tools/tidy/Cargo.toml index 42e608ff5ce..bc75787fb1a 100644 --- a/src/tools/tidy/Cargo.toml +++ b/src/tools/tidy/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" autobins = false [dependencies] -build_helper = { path = "../build_helper" } +build_helper = { path = "../../build_helper" } cargo_metadata = "0.18" regex = "1" miropt-test-tools = { path = "../miropt-test-tools" } -- cgit 1.4.1-3-g733a5