Implement CI tooling in Rust

4 files changed, 653 insertions, 0 deletions

diff --git a/src/ci/citool/Cargo.lock b/src/ci/citool/Cargo.lock
new file mode 100644
index 00000000000..d7931af464e
--- /dev/null
+++ b/src/ci/citool/Cargo.lock
@@ -0,0 +1,345 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "anstream"
+version = "0.6.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
+dependencies = [
+ "anstyle",
+ "once_cell",
+ "windows-sys",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.95"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
+
+[[package]]
+name = "citool"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "serde",
+ "serde_json",
+ "serde_yaml",
+]
+
+[[package]]
+name = "clap"
+version = "4.5.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.5.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.5.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
+
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
+[[package]]
+name = "hashbrown"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "indexmap"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
+
+[[package]]
+name = "itoa"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
+
+[[package]]
+name = "memchr"
+version = "2.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+
+[[package]]
+name = "once_cell"
+version = "1.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd"
+
+[[package]]
+name = "serde"
+version = "1.0.217"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.217"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.138"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949"
+dependencies = [
+ "itoa",
+ "memchr",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_yaml"
+version = "0.9.34+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
+dependencies = [
+ "indexmap",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
+[[package]]
+name = "syn"
+version = "2.0.98"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
+
+[[package]]
+name = "unsafe-libyaml"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml
new file mode 100644
index 00000000000..40c6e5b4dc2
--- /dev/null
+++ b/src/ci/citool/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "citool"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1"
+clap = { version = "4.5", features = ["derive"] }
+serde = { version = "1", features = ["derive"] }
+serde_yaml = "0.9"
+serde_json = "1"
+
+[workspace]
diff --git a/src/ci/citool/README.md b/src/ci/citool/README.md
new file mode 100644
index 00000000000..685a255f152
--- /dev/null
+++ b/src/ci/citool/README.md
@@ -0,0 +1,2 @@
+# CI tooling
+This is a simple Rust script that determines which jobs should be executed on CI based on the situation (pull request, try job, merge attempt). It also provides a simple way of executing (some) CI jobs locally.
diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs
new file mode 100644
index 00000000000..5f0202e9c34
--- /dev/null
+++ b/src/ci/citool/src/main.rs
@@ -0,0 +1,293 @@
+use std::collections::HashMap;
+use std::path::Path;
+
+use anyhow::Context;
+use clap::Parser;
+use serde_yaml::Value;
+
+const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/..");
+const JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../github-actions/jobs.yml");
+
+/// Representation of a job loaded from the jobs.yml file.
+#[derive(serde::Deserialize, Debug, Clone)]
+struct Job {
+    /// Name of the job, e.g. mingw-check
+    name: String,
+    /// GitHub runner on which the job should be executed
+    os: String,
+    env: HashMap<String, Value>,
+    /// Should the job be only executed on a specific channel?
+    #[serde(default)]
+    only_on_channel: Option<String>,
+    /// Rest of attributes that will be passed through to GitHub actions
+    #[serde(flatten)]
+    extra_keys: HashMap<String, Value>,
+}
+
+#[derive(serde::Deserialize, Debug)]
+struct JobEnvironments {
+    #[serde(rename = "pr")]
+    pr_env: HashMap<String, Value>,
+    #[serde(rename = "try")]
+    try_env: HashMap<String, Value>,
+    #[serde(rename = "auto")]
+    auto_env: HashMap<String, Value>,
+}
+
+#[derive(serde::Deserialize, Debug)]
+struct JobDatabase {
+    #[serde(rename = "pr")]
+    pr_jobs: Vec<Job>,
+    #[serde(rename = "try")]
+    try_jobs: Vec<Job>,
+    #[serde(rename = "auto")]
+    auto_jobs: Vec<Job>,
+
+    /// Shared environments for the individual run types.
+    envs: JobEnvironments,
+}
+
+impl JobDatabase {
+    fn find_auto_job_by_name(&self, name: &str) -> Option<Job> {
+        self.auto_jobs.iter().find(|j| j.name == name).cloned()
+    }
+}
+
+fn load_job_db(path: &Path) -> anyhow::Result<JobDatabase> {
+    let db = std::fs::read_to_string(path)?;
+    let mut db: Value = serde_yaml::from_str(&db)?;
+
+    // We need to expand merge keys (<<), because serde_yaml can't deal with them
+    // `apply_merge` only applies the merge once, so do it a few times to unwrap nested merges.
+    db.apply_merge()?;
+    db.apply_merge()?;
+
+    let db: JobDatabase = serde_yaml::from_value(db)?;
+    Ok(db)
+}
+
+/// Representation of a job outputted to a GitHub Actions workflow.
+#[derive(serde::Serialize, Debug)]
+struct GithubActionsJob {
+    name: String,
+    full_name: String,
+    os: String,
+    env: HashMap<String, String>,
+    #[serde(flatten)]
+    extra_keys: HashMap<String, serde_json::Value>,
+}
+
+/// Type of workflow that is being executed on CI
+#[derive(Debug)]
+enum RunType {
+    /// Workflows that run after a push to a PR branch
+    PullRequest,
+    /// Try run started with @bors try
+    TryJob { custom_jobs: Option<Vec<String>> },
+    /// Merge attempt workflow
+    AutoJob,
+}
+
+struct GitHubContext {
+    event_name: String,
+    branch_ref: String,
+    commit_message: Option<String>,
+}
+
+impl GitHubContext {
+    fn get_run_type(&self) -> Option<RunType> {
+        if self.event_name == "pull_request" {
+            return Some(RunType::PullRequest);
+        } else if self.event_name == "push" {
+            let is_try_build =
+                ["refs/heads/try", "refs/heads/try-perf", "refs/heads/automation/bors/try"]
+                    .iter()
+                    .any(|r| **r == self.branch_ref);
+            // Unrolled branch from a rollup for testing perf
+            // This should **not** allow custom try jobs
+            let is_unrolled_perf_build = self.branch_ref == "refs/heads/try-perf";
+            if is_try_build {
+                let custom_jobs =
+                    if !is_unrolled_perf_build { Some(self.get_custom_jobs()) } else { None };
+                return Some(RunType::TryJob { custom_jobs });
+            }
+
+            if self.branch_ref == "refs/heads/auto" {
+                return Some(RunType::AutoJob);
+            }
+        }
+        None
+    }
+
+    /// Tries to parse names of specific CI jobs that should be executed in the form of
+    /// try-job: <job-name>
+    /// from the commit message of the passed GitHub context.
+    fn get_custom_jobs(&self) -> Vec<String> {
+        if let Some(ref msg) = self.commit_message {
+            msg.lines()
+                .filter_map(|line| line.trim().strip_prefix("try-job: "))
+                .map(|l| l.to_string())
+                .collect()
+        } else {
+            vec![]
+        }
+    }
+}
+
+fn load_env_var(name: &str) -> anyhow::Result<String> {
+    std::env::var(name).with_context(|| format!("Cannot find variable {name}"))
+}
+
+fn load_github_ctx() -> anyhow::Result<GitHubContext> {
+    let event_name = load_env_var("GITHUB_EVENT_NAME")?;
+    let commit_message =
+        if event_name == "push" { Some(load_env_var("COMMIT_MESSAGE")?) } else { None };
+
+    Ok(GitHubContext { event_name, branch_ref: load_env_var("GITHUB_REF")?, commit_message })
+}
+
+/// Skip CI jobs that are not supposed to be executed on the given `channel`.
+fn skip_jobs(jobs: Vec<Job>, channel: &str) -> Vec<Job> {
+    jobs.into_iter()
+        .filter(|job| {
+            job.only_on_channel.is_none() || job.only_on_channel.as_deref() == Some(channel)
+        })
+        .collect()
+}
+
+fn to_string_map(map: &HashMap<String, Value>) -> HashMap<String, String> {
+    map.iter()
+        .map(|(key, value)| {
+            (
+                key.clone(),
+                serde_yaml::to_string(value)
+                    .expect("Cannot serialize YAML value to string")
+                    .trim()
+                    .to_string(),
+            )
+        })
+        .collect()
+}
+
+fn calculate_jobs(
+    run_type: &RunType,
+    db: &JobDatabase,
+    channel: &str,
+) -> anyhow::Result<Vec<GithubActionsJob>> {
+    let (jobs, prefix, base_env) = match run_type {
+        RunType::PullRequest => (db.pr_jobs.clone(), "PR", &db.envs.pr_env),
+        RunType::TryJob { custom_jobs } => {
+            let jobs = if let Some(custom_jobs) = custom_jobs {
+                if custom_jobs.len() > 10 {
+                    return Err(anyhow::anyhow!(
+                        "It is only possible to schedule up to 10 custom jobs, received {} custom jobs",
+                        custom_jobs.len()
+                    ));
+                }
+
+                let mut jobs = vec![];
+                let mut unknown_jobs = vec![];
+                for custom_job in custom_jobs {
+                    if let Some(job) = db.find_auto_job_by_name(custom_job) {
+                        jobs.push(job);
+                    } else {
+                        unknown_jobs.push(custom_job.clone());
+                    }
+                }
+                if !unknown_jobs.is_empty() {
+                    return Err(anyhow::anyhow!(
+                        "Custom job(s) `{}` not found in auto jobs",
+                        unknown_jobs.join(", ")
+                    ));
+                }
+                jobs
+            } else {
+                db.try_jobs.clone()
+            };
+            (jobs, "try", &db.envs.try_env)
+        }
+        RunType::AutoJob => (db.auto_jobs.clone(), "auto", &db.envs.auto_env),
+    };
+    let jobs = skip_jobs(jobs, channel);
+    let jobs = jobs
+        .into_iter()
+        .map(|job| {
+            let mut env: HashMap<String, String> = to_string_map(base_env);
+            env.extend(to_string_map(&job.env));
+            let full_name = format!("{prefix} - {}", job.name);
+
+            GithubActionsJob {
+                name: job.name,
+                full_name,
+                os: job.os,
+                env,
+                extra_keys: job
+                    .extra_keys
+                    .into_iter()
+                    .map(|(key, value)| {
+                        (
+                            key,
+                            serde_json::to_value(&value)
+                                .expect("Cannot convert extra key value from YAML to JSON"),
+                        )
+                    })
+                    .collect(),
+            }
+        })
+        .collect();
+
+    Ok(jobs)
+}
+
+fn calculate_job_matrix(
+    db: JobDatabase,
+    gh_ctx: GitHubContext,
+    channel: &str,
+) -> anyhow::Result<()> {
+    let run_type = gh_ctx.get_run_type().ok_or_else(|| {
+        anyhow::anyhow!("Cannot determine the type of workflow that is being executed")
+    })?;
+    eprintln!("Run type: {run_type:?}");
+
+    let jobs = calculate_jobs(&run_type, &db, channel)?;
+    if jobs.is_empty() {
+        return Err(anyhow::anyhow!("Computed job list is empty"));
+    }
+
+    let run_type = match run_type {
+        RunType::PullRequest => "pr",
+        RunType::TryJob { .. } => "try",
+        RunType::AutoJob => "auto",
+    };
+
+    eprintln!("Output:\njobs={jobs:?}\nrun_type={run_type}");
+    println!("jobs={}", serde_json::to_string(&jobs)?);
+    println!("run_type={run_type}");
+
+    Ok(())
+}
+
+#[derive(clap::Parser)]
+enum Args {
+    /// Calculate a list of jobs that should be executed on CI.
+    /// Should only be used on CI inside GitHub actions.
+    CalculateJobMatrix,
+}
+
+fn main() -> anyhow::Result<()> {
+    let args = Args::parse();
+    let db = load_job_db(Path::new(JOBS_YML_PATH)).context("Cannot load jobs.yml")?;
+
+    match args {
+        Args::CalculateJobMatrix => {
+            let gh_ctx = load_github_ctx()
+                .context("Cannot load environment variables from GitHub Actions")?;
+            let channel = std::fs::read_to_string(Path::new(CI_DIRECTORY).join("channel"))
+                .context("Cannot read channel file")?;
+
+            calculate_job_matrix(db, gh_ctx, &channel).context("Failed to calculate job matrix")?;
+        }
+    }
+
+    Ok(())
+}