diff options
| -rw-r--r-- | src/ci/citool/src/jobs.rs | 116 | ||||
| -rw-r--r-- | src/ci/citool/src/jobs/tests.rs | 220 | ||||
| -rw-r--r-- | src/ci/citool/tests/jobs.rs | 2 |
3 files changed, 333 insertions, 5 deletions
diff --git a/src/ci/citool/src/jobs.rs b/src/ci/citool/src/jobs.rs index 410274227e4..47516cbc1f4 100644 --- a/src/ci/citool/src/jobs.rs +++ b/src/ci/citool/src/jobs.rs @@ -1,9 +1,9 @@ #[cfg(test)] mod tests; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; -use anyhow::Context as _; +use anyhow::{Context as _, anyhow}; use serde_yaml::Value; use crate::GitHubContext; @@ -85,6 +85,10 @@ impl JobDatabase { .cloned() .collect() } + + fn find_auto_job_by_name(&self, job_name: &str) -> Option<&Job> { + self.auto_jobs.iter().find(|job| job.name == job_name) + } } pub fn load_job_db(db: &str) -> anyhow::Result<JobDatabase> { @@ -97,14 +101,118 @@ pub fn load_job_db(db: &str) -> anyhow::Result<JobDatabase> { db.apply_merge().context("failed to apply merge keys") }; - // Apply merge twice to handle nested merges + // Apply merge twice to handle nested merges up to depth 2. apply_merge(&mut db)?; apply_merge(&mut db)?; - let db: JobDatabase = serde_yaml::from_value(db).context("failed to parse job database")?; + let mut db: JobDatabase = serde_yaml::from_value(db).context("failed to parse job database")?; + + register_pr_jobs_as_auto_jobs(&mut db)?; + + validate_job_database(&db)?; + Ok(db) } +/// Maintain invariant that PR CI jobs must be a subset of Auto CI jobs modulo carve-outs. +/// +/// When PR jobs are auto-registered as Auto jobs, they will have `continue_on_error` overridden to +/// be `false` to avoid wasting Auto CI resources. +/// +/// When a job is already both a PR job and a auto job, we will post-validate their "equivalence +/// modulo certain carve-outs" in [`validate_job_database`]. +/// +/// This invariant is important to make sure that it's not easily possible (without modifying +/// `citool`) to have PRs with red PR-only CI jobs merged into `master`, causing all subsequent PR +/// CI runs to be red until the cause is fixed. +fn register_pr_jobs_as_auto_jobs(db: &mut JobDatabase) -> anyhow::Result<()> { + for pr_job in &db.pr_jobs { + // It's acceptable to "override" a PR job in Auto job, for instance, `x86_64-gnu-tools` will + // receive an additional `DEPLOY_TOOLSTATES_JSON: toolstates-linux.json` env when under Auto + // environment versus PR environment. + if db.find_auto_job_by_name(&pr_job.name).is_some() { + continue; + } + + let auto_registered_job = Job { continue_on_error: Some(false), ..pr_job.clone() }; + db.auto_jobs.push(auto_registered_job); + } + + Ok(()) +} + +fn validate_job_database(db: &JobDatabase) -> anyhow::Result<()> { + fn ensure_no_duplicate_job_names(section: &str, jobs: &Vec<Job>) -> anyhow::Result<()> { + let mut job_names = HashSet::new(); + for job in jobs { + let job_name = job.name.as_str(); + if !job_names.insert(job_name) { + return Err(anyhow::anyhow!( + "duplicate job name `{job_name}` in section `{section}`" + )); + } + } + Ok(()) + } + + ensure_no_duplicate_job_names("pr", &db.pr_jobs)?; + ensure_no_duplicate_job_names("auto", &db.auto_jobs)?; + ensure_no_duplicate_job_names("try", &db.try_jobs)?; + ensure_no_duplicate_job_names("optional", &db.optional_jobs)?; + + fn equivalent_modulo_carve_out(pr_job: &Job, auto_job: &Job) -> anyhow::Result<()> { + let Job { + name, + os, + only_on_channel, + free_disk, + doc_url, + codebuild, + + // Carve-out configs allowed to be different. + env: _, + continue_on_error: _, + } = pr_job; + + if *name == auto_job.name + && *os == auto_job.os + && *only_on_channel == auto_job.only_on_channel + && *free_disk == auto_job.free_disk + && *doc_url == auto_job.doc_url + && *codebuild == auto_job.codebuild + { + Ok(()) + } else { + Err(anyhow!( + "PR job `{}` differs from corresponding Auto job `{}` in configuration other than `continue_on_error` and `env`", + pr_job.name, + auto_job.name + )) + } + } + + for pr_job in &db.pr_jobs { + // At this point, any PR job must also be an Auto job, auto-registered or overridden. + let auto_job = db + .find_auto_job_by_name(&pr_job.name) + .expect("PR job must either be auto-registered as Auto job or overridden"); + + equivalent_modulo_carve_out(pr_job, auto_job)?; + } + + // Auto CI jobs must all "fail-fast" to avoid wasting Auto CI resources. For instance, `tidy`. + for auto_job in &db.auto_jobs { + if auto_job.continue_on_error == Some(true) { + return Err(anyhow!( + "Auto job `{}` cannot have `continue_on_error: true`", + auto_job.name + )); + } + } + + Ok(()) +} + /// Representation of a job outputted to a GitHub Actions workflow. #[derive(serde::Serialize, Debug)] struct GithubActionsJob { diff --git a/src/ci/citool/src/jobs/tests.rs b/src/ci/citool/src/jobs/tests.rs index 63ac508b632..f1f6274e1ed 100644 --- a/src/ci/citool/src/jobs/tests.rs +++ b/src/ci/citool/src/jobs/tests.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::path::Path; use super::Job; @@ -146,3 +147,222 @@ fn validate_jobs() { panic!("Job validation failed:\n{error_messages}"); } } + +#[test] +fn pr_job_implies_auto_job() { + let db = load_job_db( + r#" +envs: + pr: + try: + auto: + optional: + +pr: + - name: pr-ci-a + os: ubuntu + env: {} +try: +auto: +optional: +"#, + ) + .unwrap(); + + assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["pr-ci-a"]) +} + +#[test] +fn implied_auto_job_keeps_env_and_fails_fast() { + let db = load_job_db( + r#" +envs: + pr: + try: + auto: + optional: + +pr: + - name: tidy + env: + DEPLOY_TOOLSTATES_JSON: toolstates-linux.json + continue_on_error: true + os: ubuntu +try: +auto: +optional: +"#, + ) + .unwrap(); + + assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["tidy"]); + assert_eq!(db.auto_jobs[0].continue_on_error, Some(false)); + assert_eq!( + db.auto_jobs[0].env, + BTreeMap::from([( + "DEPLOY_TOOLSTATES_JSON".to_string(), + serde_yaml::Value::String("toolstates-linux.json".to_string()) + )]) + ); +} + +#[test] +#[should_panic = "duplicate"] +fn duplicate_job_name() { + let _ = load_job_db( + r#" +envs: + pr: + try: + auto: + + +pr: + - name: pr-ci-a + os: ubuntu + env: {} + - name: pr-ci-a + os: ubuntu + env: {} +try: +auto: +optional: +"#, + ) + .unwrap(); +} + +#[test] +fn auto_job_can_override_pr_job_spec() { + let db = load_job_db( + r#" +envs: + pr: + try: + auto: + optional: + +pr: + - name: tidy + os: ubuntu + env: {} +try: +auto: + - name: tidy + env: + DEPLOY_TOOLSTATES_JSON: toolstates-linux.json + continue_on_error: false + os: ubuntu +optional: +"#, + ) + .unwrap(); + + assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["tidy"]); + assert_eq!(db.auto_jobs[0].continue_on_error, Some(false)); + assert_eq!( + db.auto_jobs[0].env, + BTreeMap::from([( + "DEPLOY_TOOLSTATES_JSON".to_string(), + serde_yaml::Value::String("toolstates-linux.json".to_string()) + )]) + ); +} + +#[test] +fn compatible_divergence_pr_auto_job() { + let db = load_job_db( + r#" +envs: + pr: + try: + auto: + optional: + +pr: + - name: tidy + continue_on_error: true + env: + ENV_ALLOWED_TO_DIFFER: "hello world" + os: ubuntu +try: +auto: + - name: tidy + continue_on_error: false + env: + ENV_ALLOWED_TO_DIFFER: "goodbye world" + os: ubuntu +optional: +"#, + ) + .unwrap(); + + // `continue_on_error` and `env` are carve-outs *allowed* to diverge between PR and Auto job of + // the same name. Should load successfully. + + assert_eq!(db.auto_jobs.iter().map(|j| j.name.as_str()).collect::<Vec<_>>(), vec!["tidy"]); + assert_eq!(db.auto_jobs[0].continue_on_error, Some(false)); + assert_eq!( + db.auto_jobs[0].env, + BTreeMap::from([( + "ENV_ALLOWED_TO_DIFFER".to_string(), + serde_yaml::Value::String("goodbye world".to_string()) + )]) + ); +} + +#[test] +#[should_panic = "differs"] +fn incompatible_divergence_pr_auto_job() { + // `os` is not one of the carve-out options allowed to diverge. This should fail. + let _ = load_job_db( + r#" +envs: + pr: + try: + auto: + optional: + +pr: + - name: tidy + continue_on_error: true + env: + ENV_ALLOWED_TO_DIFFER: "hello world" + os: ubuntu +try: +auto: + - name: tidy + continue_on_error: false + env: + ENV_ALLOWED_TO_DIFFER: "goodbye world" + os: windows +optional: +"#, + ) + .unwrap(); +} + +#[test] +#[should_panic = "cannot have `continue_on_error: true`"] +fn auto_job_continue_on_error() { + // Auto CI jobs must fail-fast. + let _ = load_job_db( + r#" +envs: + pr: + try: + auto: + optional: + +pr: +try: +auto: + - name: tidy + continue_on_error: true + os: windows + env: {} +optional: +"#, + ) + .unwrap(); +} diff --git a/src/ci/citool/tests/jobs.rs b/src/ci/citool/tests/jobs.rs index dbaf13d4f42..24e0b85cab2 100644 --- a/src/ci/citool/tests/jobs.rs +++ b/src/ci/citool/tests/jobs.rs @@ -6,7 +6,7 @@ const TEST_JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/tes fn auto_jobs() { let stdout = get_matrix("push", "commit", "refs/heads/auto"); insta::assert_snapshot!(stdout, @r#" - jobs=[{"name":"aarch64-gnu","full_name":"auto - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"x86_64-gnu-llvm-18-1","full_name":"auto - x86_64-gnu-llvm-18-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DOCKER_SCRIPT":"stage_2_test_set1.sh","IMAGE":"x86_64-gnu-llvm-18","READ_ONLY_SRC":"0","RUST_BACKTRACE":1,"TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"aarch64-apple","full_name":"auto - aarch64-apple","os":"macos-14","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","MACOSX_DEPLOYMENT_TARGET":11.0,"MACOSX_STD_DEPLOYMENT_TARGET":11.0,"NO_DEBUG_ASSERTIONS":1,"NO_LLVM_ASSERTIONS":1,"NO_OVERFLOW_CHECKS":1,"RUSTC_RETRY_LINKER_ON_SEGFAULT":1,"RUST_CONFIGURE_ARGS":"--enable-sanitizers --enable-profiler --set rust.jemalloc","SCRIPT":"./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin","SELECT_XCODE":"/Applications/Xcode_15.4.app","TOOLSTATE_PUBLISH":1,"USE_XCODE_CLANG":1}},{"name":"dist-i686-msvc","full_name":"auto - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}}] + jobs=[{"name":"aarch64-gnu","full_name":"auto - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"x86_64-gnu-llvm-18-1","full_name":"auto - x86_64-gnu-llvm-18-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DOCKER_SCRIPT":"stage_2_test_set1.sh","IMAGE":"x86_64-gnu-llvm-18","READ_ONLY_SRC":"0","RUST_BACKTRACE":1,"TOOLSTATE_PUBLISH":1},"free_disk":true},{"name":"aarch64-apple","full_name":"auto - aarch64-apple","os":"macos-14","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","MACOSX_DEPLOYMENT_TARGET":11.0,"MACOSX_STD_DEPLOYMENT_TARGET":11.0,"NO_DEBUG_ASSERTIONS":1,"NO_LLVM_ASSERTIONS":1,"NO_OVERFLOW_CHECKS":1,"RUSTC_RETRY_LINKER_ON_SEGFAULT":1,"RUST_CONFIGURE_ARGS":"--enable-sanitizers --enable-profiler --set rust.jemalloc","SCRIPT":"./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin","SELECT_XCODE":"/Applications/Xcode_15.4.app","TOOLSTATE_PUBLISH":1,"USE_XCODE_CLANG":1}},{"name":"dist-i686-msvc","full_name":"auto - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":1,"RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":1}},{"name":"pr-check-1","full_name":"auto - pr-check-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"continue_on_error":false,"free_disk":true},{"name":"pr-check-2","full_name":"auto - pr-check-2","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"continue_on_error":false,"free_disk":true},{"name":"tidy","full_name":"auto - tidy","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":1},"continue_on_error":false,"free_disk":true,"doc_url":"https://foo.bar"}] run_type=auto "#); } |