Auto merge of #124175 - Kobzol:ci-dynamic-job, r=pietroalbini

CI: dynamic jobs

This PR modifies our CI workflows to be dynamic. This means that when a GitHub event is generated, we will run a Python script (`calculate-job-matrix.py`), which decides which CI jobs should be generated. These jobs are defined in `src/ci/github-actions/jobs.yml`).

This should provide a few benefits:
- Once the migration to dynamic jobs is complete, we shouldn't need `expand-yaml-anchors` anymore.
- The job table on PRs (and also the left job column on auto/try builds) should be much cleaner and contain only the jobs that are actually relevant/executed.
- It should be much easier to support dynamic try builds, i.e. to run an arbitrary CI job on a try build.

See [this Zulip discussion](https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/job.20matrix.20re-ordered.20PR.20list) for more context.

r? `@ghost`

4 files changed, 562 insertions, 442 deletions

diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py
new file mode 100755
index 00000000000..c24cefa8d89
--- /dev/null
+++ b/src/ci/github-actions/calculate-job-matrix.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+"""
+This script serves for generating a matrix of jobs that should
+be executed on CI.
+
+It reads job definitions from `src/ci/github-actions/jobs.yml`
+and filters them based on the event that happened on CI.
+"""
+import dataclasses
+import enum
+import json
+import logging
+import os
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+import yaml
+
+JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
+
+
+def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
+    """
+    Add a `name` attribute to each job, based on its image and the given `prefix`.
+    """
+    for job in jobs:
+        job["name"] = f"{prefix} - {job['image']}"
+    return jobs
+
+
+def add_base_env(jobs: List[Dict], environment: Dict[str, str]) -> List[Dict]:
+    """
+    Prepends `environment` to the `env` attribute of each job.
+    The `env` of each job has higher precedence than `environment`.
+    """
+    for job in jobs:
+        env = environment.copy()
+        env.update(job.get("env", {}))
+        job["env"] = env
+    return jobs
+
+
+class JobType(enum.Enum):
+    PR = enum.auto()
+    Try = enum.auto()
+    Auto = enum.auto()
+
+
+@dataclasses.dataclass
+class GitHubCtx:
+    event_name: str
+    ref: str
+    repository: str
+
+
+def find_job_type(ctx: GitHubCtx) -> Optional[JobType]:
+    if ctx.event_name == "pull_request":
+        return JobType.PR
+    elif ctx.event_name == "push":
+        old_bors_try_build = (
+            ctx.ref in ("refs/heads/try", "refs/heads/try-perf") and
+            ctx.repository == "rust-lang-ci/rust"
+        )
+        new_bors_try_build = (
+            ctx.ref == "refs/heads/automation/bors/try" and
+            ctx.repository == "rust-lang/rust"
+        )
+        try_build = old_bors_try_build or new_bors_try_build
+
+        if try_build:
+            return JobType.Try
+
+        if ctx.ref == "refs/heads/auto" and ctx.repository == "rust-lang-ci/rust":
+            return JobType.Auto
+
+    return None
+
+
+def calculate_jobs(job_type: JobType, job_data: Dict[str, Any]) -> List[Dict[str, Any]]:
+    if job_type == JobType.PR:
+        return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"])
+    elif job_type == JobType.Try:
+        return add_base_env(name_jobs(job_data["try"], "try"), job_data["envs"]["try"])
+    elif job_type == JobType.Auto:
+        return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"])
+
+    return []
+
+
+def get_github_ctx() -> GitHubCtx:
+    return GitHubCtx(
+        event_name=os.environ["GITHUB_EVENT_NAME"],
+        ref=os.environ["GITHUB_REF"],
+        repository=os.environ["GITHUB_REPOSITORY"]
+    )
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+
+    with open(JOBS_YAML_PATH) as f:
+        data = yaml.safe_load(f)
+
+    github_ctx = get_github_ctx()
+
+    job_type = find_job_type(github_ctx)
+    logging.info(f"Job type: {job_type}")
+
+    jobs = []
+    if job_type is not None:
+        jobs = calculate_jobs(job_type, data)
+
+    logging.info(f"Output:\n{yaml.dump(jobs, indent=4)}")
+    print(f"jobs={json.dumps(jobs)}")
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index de71b9f874f..19d6b517552 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -342,6 +342,8 @@ concurrency:
 
 jobs:
   # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
+  # It calculates which jobs should be executed, based on the data of the ${{ github }} context.
+  # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
   calculate_matrix:
     name: Calculate job matrix
     runs-on: ubuntu-latest
@@ -351,422 +353,34 @@ jobs:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
-        run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
-  pr:
+  job:
     <<: *base-ci-job
-    name: PR - ${{ matrix.name }}
+    name: ${{ matrix.name }}
     needs: [ calculate_matrix ]
     env:
-      <<: [*shared-ci-variables, *public-variables]
-      PR_CI_JOB: 1
-    if: github.event_name == 'pull_request'
-    continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
+      CI_JOB_NAME: ${{ matrix.image }}
+      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
+      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      SCCACHE_BUCKET: rust-lang-ci-sccache2
+      TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
+      CACHE_DOMAIN: ci-caches.rust-lang.org
+    continue-on-error: ${{ matrix.continue_on_error || false }}
     strategy:
       matrix:
         # Check the `calculate_matrix` job to see how is the matrix defined.
         include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
-
-  auto:
-    <<: *base-ci-job
-    name: auto - ${{ matrix.name }}
-    env:
-      <<: [*shared-ci-variables, *prod-variables]
-    if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
-    strategy:
-      matrix:
-        include:
-          #############################
-          #   Linux/Docker builders   #
-          #############################
-
-          - name: aarch64-gnu
-            <<: *job-aarch64-linux
-
-          - name: arm-android
-            <<: *job-linux-8c
-
-          - name: armhf-gnu
-            <<: *job-linux-8c
-
-          - name: dist-aarch64-linux
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-8c
-
-          - name: dist-android
-            <<: *job-linux-8c
-
-          - name: dist-arm-linux
-            <<: *job-linux-16c
-
-          - name: dist-armhf-linux
-            <<: *job-linux-8c
-
-          - name: dist-armv7-linux
-            <<: *job-linux-8c
-
-          - name: dist-i586-gnu-i586-i686-musl
-            <<: *job-linux-8c
-
-          - name: dist-i686-linux
-            <<: *job-linux-8c
-
-          - name: dist-loongarch64-linux
-            <<: *job-linux-8c
-
-          - name: dist-ohos
-            <<: *job-linux-8c
-
-          - name: dist-powerpc-linux
-            <<: *job-linux-8c
-
-          - name: dist-powerpc64-linux
-            <<: *job-linux-8c
-
-          - name: dist-powerpc64le-linux
-            <<: *job-linux-8c
-
-          - name: dist-riscv64-linux
-            <<: *job-linux-8c
-
-          - name: dist-s390x-linux
-            <<: *job-linux-8c
-
-          - name: dist-various-1
-            <<: *job-linux-8c
-
-          - name: dist-various-2
-            <<: *job-linux-8c
-
-          - name: dist-x86_64-freebsd
-            <<: *job-linux-8c
-
-          - name: dist-x86_64-illumos
-            <<: *job-linux-8c
-
-          - &dist-x86_64-linux
-            name: dist-x86_64-linux
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-16c
-
-          - name: dist-x86_64-linux-alt
-            env:
-              IMAGE: dist-x86_64-linux
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-16c
-
-          - name: dist-x86_64-musl
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-8c
-
-          - name: dist-x86_64-netbsd
-            <<: *job-linux-8c
-
-          - name: i686-gnu
-            <<: *job-linux-8c
-
-          - name: i686-gnu-nopt
-            <<: *job-linux-8c
-
-          - name: mingw-check
-            <<: *job-linux-4c
-
-          - name: test-various
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu
-            <<: *job-linux-4c
-
-          # This job ensures commits landing on nightly still pass the full
-          # test suite on the stable channel. There are some UI tests that
-          # depend on the channel being built (for example if they include the
-          # channel name on the output), and this builder prevents landing
-          # changes that would result in broken builds after a promotion.
-          - name: x86_64-gnu-stable
-            env:
-              IMAGE: x86_64-gnu
-              RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
-              # Only run this job on the nightly channel. Running this on beta
-              # could cause failures when `dev: 1` in `stage0.txt`, and running
-              # this on stable is useless.
-              CI_ONLY_WHEN_CHANNEL: nightly
-            <<: *job-linux-4c
-
-          - name: x86_64-gnu-aux
-            <<: *job-linux-4c
-
-          - name: x86_64-gnu-integration
-            env:
-              # Only run this job on the nightly channel. Fuchsia requires
-              # nightly features to compile, and this job would fail if
-              # executed on beta and stable.
-              CI_ONLY_WHEN_CHANNEL: nightly
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-debug
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-distcheck
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-llvm-18
-            env:
-              RUST_BACKTRACE: 1
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-llvm-17
-            env:
-              RUST_BACKTRACE: 1
-            <<: *job-linux-8c
-
-          - name: x86_64-gnu-nopt
-            <<: *job-linux-4c
-
-          - name: x86_64-gnu-tools
-            env:
-              DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
-            <<: *job-linux-8c
-
-          ####################
-          #  macOS Builders  #
-          ####################
-
-          - name: dist-x86_64-apple
-            env:
-              SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
-              RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-macos-xl
-
-          - name: dist-apple-various
-            env:
-              SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim
-              RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            <<: *job-macos-xl
-
-          - name: x86_64-apple-1
-            env: &env-x86_64-apple-tests
-              SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps
-              RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              MACOSX_DEPLOYMENT_TARGET: 10.12
-              MACOSX_STD_DEPLOYMENT_TARGET: 10.12
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            <<: *job-macos-xl
-
-          - name: x86_64-apple-2
-            env:
-              SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps
-              <<: *env-x86_64-apple-tests
-            <<: *job-macos-xl
-
-          # This target only needs to support 11.0 and up as nothing else supports the hardware
-          - name: dist-aarch64-apple
-            env:
-              SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin
-              RUST_CONFIGURE_ARGS: >-
-                --enable-full-tools
-                --enable-sanitizers
-                --enable-profiler
-                --set rust.jemalloc
-                --set llvm.ninja=false
-                --set rust.lto=thin
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              USE_XCODE_CLANG: 1
-              MACOSX_DEPLOYMENT_TARGET: 11.0
-              MACOSX_STD_DEPLOYMENT_TARGET: 11.0
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-macos-m1
-
-          # This target only needs to support 11.0 and up as nothing else supports the hardware
-          - name: aarch64-apple
-            env:
-              SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin
-              RUST_CONFIGURE_ARGS: >-
-                --enable-sanitizers
-                --enable-profiler
-                --set rust.jemalloc
-              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-              SELECT_XCODE: /Applications/Xcode_14.3.1.app
-              USE_XCODE_CLANG: 1
-              MACOSX_DEPLOYMENT_TARGET: 11.0
-              MACOSX_STD_DEPLOYMENT_TARGET: 11.0
-              NO_LLVM_ASSERTIONS: 1
-              NO_DEBUG_ASSERTIONS: 1
-              NO_OVERFLOW_CHECKS: 1
-            <<: *job-macos-m1
-
-          ######################
-          #  Windows Builders  #
-          ######################
-
-          - name: x86_64-msvc
-            env:
-              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-              SCRIPT: make ci-msvc
-            <<: *job-windows-8c
-
-          - name: i686-msvc
-            env:
-              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-              SCRIPT: make ci-msvc
-            <<: *job-windows-8c
-
-          - name: x86_64-msvc-ext
-            env:
-              SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
-              HOST_TARGET: x86_64-pc-windows-msvc
-              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
-              DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
-            <<: *job-windows-8c
-
-          # 32/64-bit MinGW builds.
-          #
-          # We are using MinGW with POSIX threads since LLVM requires
-          # C++'s std::thread which is disabled in libstdc++ with win32 threads.
-          # FIXME: Libc++ doesn't have this limitation so we can avoid
-          # winpthreads if we switch to it.
-          #
-          # Instead of relying on the MinGW version installed on CI we download
-          # and install one ourselves so we won't be surprised by changes to CI's
-          # build image.
-          #
-          # Finally, note that the downloads below are all in the `rust-lang-ci` S3
-          # bucket, but they clearly didn't originate there! The downloads originally
-          # came from the mingw-w64 SourceForge download site. Unfortunately
-          # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
-
-          - name: i686-mingw
-            env:
-              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
-              SCRIPT: make ci-mingw
-              # We are intentionally allowing an old toolchain on this builder (and that's
-              # incompatible with LLVM downloads today).
-              NO_DOWNLOAD_CI_LLVM: 1
-              CUSTOM_MINGW: 1
-            <<: *job-windows-8c
-
-          - name: x86_64-mingw
-            env:
-              SCRIPT: make ci-mingw
-              RUST_CONFIGURE_ARGS: >-
-                --build=x86_64-pc-windows-gnu
-                --enable-profiler
-              # We are intentionally allowing an old toolchain on this builder (and that's
-              # incompatible with LLVM downloads today).
-              NO_DOWNLOAD_CI_LLVM: 1
-              CUSTOM_MINGW: 1
-            <<: *job-windows-8c
-
-          - name: dist-x86_64-msvc
-            env:
-              RUST_CONFIGURE_ARGS: >-
-                --build=x86_64-pc-windows-msvc
-                --host=x86_64-pc-windows-msvc
-                --target=x86_64-pc-windows-msvc
-                --enable-full-tools
-                --enable-profiler
-                --set rust.codegen-units=1
-              SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-i686-msvc
-            env:
-              RUST_CONFIGURE_ARGS: >-
-                --build=i686-pc-windows-msvc
-                --host=i686-pc-windows-msvc
-                --target=i686-pc-windows-msvc,i586-pc-windows-msvc
-                --enable-full-tools
-                --enable-profiler
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-aarch64-msvc
-            env:
-              RUST_CONFIGURE_ARGS: >-
-                --build=x86_64-pc-windows-msvc
-                --host=aarch64-pc-windows-msvc
-                --enable-full-tools
-                --enable-profiler
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-i686-mingw
-            env:
-              RUST_CONFIGURE_ARGS: >-
-                --build=i686-pc-windows-gnu
-                --enable-full-tools
-                --enable-profiler
-              # We are intentionally allowing an old toolchain on this builder (and that's
-              # incompatible with LLVM downloads today).
-              NO_DOWNLOAD_CI_LLVM: 1
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              CUSTOM_MINGW: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-x86_64-mingw
-            env:
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-              RUST_CONFIGURE_ARGS: >-
-                --build=x86_64-pc-windows-gnu
-                --enable-full-tools
-                --enable-profiler
-              # We are intentionally allowing an old toolchain on this builder (and that's
-              # incompatible with LLVM downloads today).
-              NO_DOWNLOAD_CI_LLVM: 1
-              CUSTOM_MINGW: 1
-              DIST_REQUIRE_ALL_TOOLS: 1
-            <<: *job-windows-8c
-
-          - name: dist-x86_64-msvc-alt
-            env:
-              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
-              SCRIPT: python x.py dist bootstrap --include-default-paths
-            <<: *job-windows-8c
-
-  try:
-    <<: *base-ci-job
-    name: try - ${{ matrix.name }}
-    env:
-      DIST_TRY_BUILD: 1
-      <<: [*shared-ci-variables, *prod-variables]
-    if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))
-    strategy:
-      matrix:
-        include:
-          - &dist-x86_64-linux
-            name: dist-x86_64-linux
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-16c
-
+    # GitHub Actions fails the workflow if an empty list of jobs is provided to
+    # the workflow, so we need to skip this job if nothing was produced by
+    # the Python script.
+    #
+    # Unfortunately checking whether a list is empty is not possible in a nice
+    # way due to GitHub Actions expressions limits.
+    # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82
+    if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null
 
   master:
     name: master
@@ -791,18 +405,18 @@ jobs:
   # build completed, as there is no practical way to detect when a workflow is
   # successful listening to webhooks only.
   try-success:
-    needs: [try]
+    needs: [ job ]
     if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
     <<: *base-success-job
   try-failure:
-    needs: [try]
+    needs: [ job ]
     if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
     <<: *base-failure-job
   auto-success:
-    needs: [auto]
+    needs: [ job ]
     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
     <<: *base-success-job
   auto-failure:
-    needs: [auto]
+    needs: [ job ]
     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
     <<: *base-failure-job
diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
index 7e89eef2670..ec58bd0924e 100644
--- a/src/ci/github-actions/jobs.yml
+++ b/src/ci/github-actions/jobs.yml
@@ -2,7 +2,7 @@
 # dynamically in CI from ci.yml.
 # You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
 # modify this file.
-shared_defs:
+runners:
   - &base-job
     env: { }
 
@@ -37,14 +37,430 @@ shared_defs:
   - &job-aarch64-linux
     os: [ self-hosted, ARM64, linux ]
 
+envs:
+  production:
+    &production
+    DEPLOY_BUCKET: rust-lang-ci2
+    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
+    TOOLSTATE_PUBLISH: 1
+    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
+    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
+    # rotate them in a single branch while keeping the old key in another
+    # branch, which wouldn't be possible if the key was named with the kind
+    # (caches, artifacts...).
+    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
+    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
+    AWS_REGION: us-west-1
+
+  try:
+    <<: *production
+    DIST_TRY_BUILD: 1
+
+  auto:
+    <<: *production
+
+  pr:
+    PR_CI_JOB: 1
+
+# Jobs that run on each push to a pull request (PR)
+# These jobs automatically inherit envs.pr, to avoid repeating
+# it in each job definition.
 pr:
-  - name: mingw-check
+  - image: mingw-check
     <<: *job-linux-4c
-  - name: mingw-check-tidy
+  - image: mingw-check-tidy
+    continue_on_error: true
     <<: *job-linux-4c
-  - name: x86_64-gnu-llvm-17
+  - image: x86_64-gnu-llvm-17
     env:
       ENABLE_GCC_CODEGEN: "1"
     <<: *job-linux-16c
-  - name: x86_64-gnu-tools
+  - image: x86_64-gnu-tools
+    <<: *job-linux-16c
+
+# Jobs that run when you perform a try build (@bors try)
+# These jobs automatically inherit envs.production, to avoid repeating
+# it in each job definition.
+try:
+  - image: dist-x86_64-linux
+    env:
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-linux-16c
+
+# Main CI jobs that have to be green to merge a commit into master
+# These jobs automatically inherit envs.production, to avoid repeating
+# it in each job definition.
+auto:
+  #############################
+  #   Linux/Docker builders   #
+  #############################
+
+  - image: aarch64-gnu
+    <<: *job-aarch64-linux
+
+  - image: arm-android
+    <<: *job-linux-8c
+
+  - image: armhf-gnu
+    <<: *job-linux-8c
+
+  - image: dist-aarch64-linux
+    env:
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-linux-8c
+
+  - image: dist-android
+    <<: *job-linux-8c
+
+  - image: dist-arm-linux
+    <<: *job-linux-16c
+
+  - image: dist-armhf-linux
+    <<: *job-linux-8c
+
+  - image: dist-armv7-linux
+    <<: *job-linux-8c
+
+  - image: dist-i586-gnu-i586-i686-musl
+    <<: *job-linux-8c
+
+  - image: dist-i686-linux
+    <<: *job-linux-8c
+
+  - image: dist-loongarch64-linux
+    <<: *job-linux-8c
+
+  - image: dist-ohos
+    <<: *job-linux-8c
+
+  - image: dist-powerpc-linux
+    <<: *job-linux-8c
+
+  - image: dist-powerpc64-linux
+    <<: *job-linux-8c
+
+  - image: dist-powerpc64le-linux
+    <<: *job-linux-8c
+
+  - image: dist-riscv64-linux
+    <<: *job-linux-8c
+
+  - image: dist-s390x-linux
+    <<: *job-linux-8c
+
+  - image: dist-various-1
+    <<: *job-linux-8c
+
+  - image: dist-various-2
+    <<: *job-linux-8c
+
+  - image: dist-x86_64-freebsd
+    <<: *job-linux-8c
+
+  - image: dist-x86_64-illumos
+    <<: *job-linux-8c
+
+  - image: dist-x86_64-linux
+    env:
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-linux-16c
+
+  - image: dist-x86_64-linux-alt
+    env:
+      IMAGE: dist-x86_64-linux
+      CODEGEN_BACKENDS: llvm,cranelift
     <<: *job-linux-16c
+
+  - image: dist-x86_64-musl
+    env:
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-linux-8c
+
+  - image: dist-x86_64-netbsd
+    <<: *job-linux-8c
+
+  - image: i686-gnu
+    <<: *job-linux-8c
+
+  - image: i686-gnu-nopt
+    <<: *job-linux-8c
+
+  - image: mingw-check
+    <<: *job-linux-4c
+
+  - image: test-various
+    <<: *job-linux-8c
+
+  - image: x86_64-gnu
+    <<: *job-linux-4c
+
+  # This job ensures commits landing on nightly still pass the full
+  # test suite on the stable channel. There are some UI tests that
+  # depend on the channel being built (for example if they include the
+  # channel name on the output), and this builder prevents landing
+  # changes that would result in broken builds after a promotion.
+  - image: x86_64-gnu-stable
+    env:
+      IMAGE: x86_64-gnu
+      RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable
+      # Only run this job on the nightly channel. Running this on beta
+      # could cause failures when `dev: 1` in `stage0.txt`, and running
+      # this on stable is useless.
+      CI_ONLY_WHEN_CHANNEL: nightly
+    <<: *job-linux-4c
+
+  - image: x86_64-gnu-aux
+    <<: *job-linux-4c
+
+  - image: x86_64-gnu-integration
+    env:
+      # Only run this job on the nightly channel. Fuchsia requires
+      # nightly features to compile, and this job would fail if
+      # executed on beta and stable.
+      CI_ONLY_WHEN_CHANNEL: nightly
+    <<: *job-linux-8c
+
+  - image: x86_64-gnu-debug
+    <<: *job-linux-8c
+
+  - image: x86_64-gnu-distcheck
+    <<: *job-linux-8c
+
+  - image: x86_64-gnu-llvm-18
+    env:
+      RUST_BACKTRACE: 1
+    <<: *job-linux-8c
+
+  - image: x86_64-gnu-llvm-17
+    env:
+      RUST_BACKTRACE: 1
+    <<: *job-linux-8c
+
+  - image: x86_64-gnu-nopt
+    <<: *job-linux-4c
+
+  - image: x86_64-gnu-tools
+    env:
+      DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
+    <<: *job-linux-8c
+
+  ####################
+  #  macOS Builders  #
+  ####################
+
+  - image: dist-x86_64-apple
+    env:
+      SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin
+      RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      MACOSX_DEPLOYMENT_TARGET: 10.12
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+      DIST_REQUIRE_ALL_TOOLS: 1
+      CODEGEN_BACKENDS: llvm,cranelift
+    <<: *job-macos-xl
+
+  - image: dist-apple-various
+    env:
+      SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim
+      RUST_CONFIGURE_ARGS: --enable-sanitizers --enable-profiler --set rust.jemalloc
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      MACOSX_DEPLOYMENT_TARGET: 10.12
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+    <<: *job-macos-xl
+
+  - image: x86_64-apple-1
+    env: &env-x86_64-apple-tests
+      SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc --skip tests/run-make-fulldeps
+      RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      MACOSX_DEPLOYMENT_TARGET: 10.12
+      MACOSX_STD_DEPLOYMENT_TARGET: 10.12
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+    <<: *job-macos-xl
+
+  - image: x86_64-apple-2
+    env:
+      SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc tests/run-make-fulldeps
+      <<: *env-x86_64-apple-tests
+    <<: *job-macos-xl
+
+  # This target only needs to support 11.0 and up as nothing else supports the hardware
+  - image: dist-aarch64-apple
+    env:
+      SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin
+      RUST_CONFIGURE_ARGS: >-
+        --enable-full-tools
+        --enable-sanitizers
+        --enable-profiler
+        --set rust.jemalloc
+        --set llvm.ninja=false
+        --set rust.lto=thin
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      USE_XCODE_CLANG: 1
+      MACOSX_DEPLOYMENT_TARGET: 11.0
+      MACOSX_STD_DEPLOYMENT_TARGET: 11.0
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-macos-m1
+
+  # This target only needs to support 11.0 and up as nothing else supports the hardware
+  - image: aarch64-apple
+    env:
+      SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin
+      RUST_CONFIGURE_ARGS: >-
+        --enable-sanitizers
+        --enable-profiler
+        --set rust.jemalloc
+      RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+      SELECT_XCODE: /Applications/Xcode_14.3.1.app
+      USE_XCODE_CLANG: 1
+      MACOSX_DEPLOYMENT_TARGET: 11.0
+      MACOSX_STD_DEPLOYMENT_TARGET: 11.0
+      NO_LLVM_ASSERTIONS: 1
+      NO_DEBUG_ASSERTIONS: 1
+      NO_OVERFLOW_CHECKS: 1
+    <<: *job-macos-m1
+
+  ######################
+  #  Windows Builders  #
+  ######################
+
+  - image: x86_64-msvc
+    env:
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+      SCRIPT: make ci-msvc
+    <<: *job-windows-8c
+
+  - image: i686-msvc
+    env:
+      RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
+      SCRIPT: make ci-msvc
+    <<: *job-windows-8c
+
+  - image: x86_64-msvc-ext
+    env:
+      SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
+      HOST_TARGET: x86_64-pc-windows-msvc
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
+      DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
+    <<: *job-windows-8c
+
+  # 32/64-bit MinGW builds.
+  #
+  # We are using MinGW with POSIX threads since LLVM requires
+  # C++'s std::thread which is disabled in libstdc++ with win32 threads.
+  # FIXME: Libc++ doesn't have this limitation so we can avoid
+  # winpthreads if we switch to it.
+  #
+  # Instead of relying on the MinGW version installed on CI we download
+  # and install one ourselves so we won't be surprised by changes to CI's
+  # build image.
+  #
+  # Finally, note that the downloads below are all in the `rust-lang-ci` S3
+  # bucket, but they clearly didn't originate there! The downloads originally
+  # came from the mingw-w64 SourceForge download site. Unfortunately
+  # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
+
+  - image: i686-mingw
+    env:
+      RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+      SCRIPT: make ci-mingw
+      # We are intentionally allowing an old toolchain on this builder (and that's
+      # incompatible with LLVM downloads today).
+      NO_DOWNLOAD_CI_LLVM: 1
+      CUSTOM_MINGW: 1
+    <<: *job-windows-8c
+
+  - image: x86_64-mingw
+    env:
+      SCRIPT: make ci-mingw
+      RUST_CONFIGURE_ARGS: >-
+        --build=x86_64-pc-windows-gnu
+        --enable-profiler
+      # We are intentionally allowing an old toolchain on this builder (and that's
+      # incompatible with LLVM downloads today).
+      NO_DOWNLOAD_CI_LLVM: 1
+      CUSTOM_MINGW: 1
+    <<: *job-windows-8c
+
+  - image: dist-x86_64-msvc
+    env:
+      RUST_CONFIGURE_ARGS: >-
+        --build=x86_64-pc-windows-msvc
+        --host=x86_64-pc-windows-msvc
+        --target=x86_64-pc-windows-msvc
+        --enable-full-tools
+        --enable-profiler
+        --set rust.codegen-units=1
+      SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - image: dist-i686-msvc
+    env:
+      RUST_CONFIGURE_ARGS: >-
+        --build=i686-pc-windows-msvc
+        --host=i686-pc-windows-msvc
+        --target=i686-pc-windows-msvc,i586-pc-windows-msvc
+        --enable-full-tools
+        --enable-profiler
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - image: dist-aarch64-msvc
+    env:
+      RUST_CONFIGURE_ARGS: >-
+        --build=x86_64-pc-windows-msvc
+        --host=aarch64-pc-windows-msvc
+        --enable-full-tools
+        --enable-profiler
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - image: dist-i686-mingw
+    env:
+      RUST_CONFIGURE_ARGS: >-
+        --build=i686-pc-windows-gnu
+        --enable-full-tools
+        --enable-profiler
+      # We are intentionally allowing an old toolchain on this builder (and that's
+      # incompatible with LLVM downloads today).
+      NO_DOWNLOAD_CI_LLVM: 1
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+      CUSTOM_MINGW: 1
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - image: dist-x86_64-mingw
+    env:
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+      RUST_CONFIGURE_ARGS: >-
+        --build=x86_64-pc-windows-gnu
+        --enable-full-tools
+        --enable-profiler
+      # We are intentionally allowing an old toolchain on this builder (and that's
+      # incompatible with LLVM downloads today).
+      NO_DOWNLOAD_CI_LLVM: 1
+      CUSTOM_MINGW: 1
+      DIST_REQUIRE_ALL_TOOLS: 1
+    <<: *job-windows-8c
+
+  - image: dist-x86_64-msvc-alt
+    env:
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
+      SCRIPT: python x.py dist bootstrap --include-default-paths
+    <<: *job-windows-8c
diff --git a/src/ci/scripts/calculate-job-matrix.py b/src/ci/scripts/calculate-job-matrix.py
deleted file mode 100755
index 9b1e74c23c3..00000000000
--- a/src/ci/scripts/calculate-job-matrix.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-This script serves for generating a matrix of jobs that should
-be executed on CI.
-
-It reads job definitions from `src/ci/github-actions/jobs.yml`
-and filters them based on the event that happened on CI.
-
-Currently, it only supports PR builds.
-"""
-
-import json
-from pathlib import Path
-
-import yaml
-
-JOBS_YAML_PATH = Path(__file__).absolute().parent.parent / "github-actions" / "jobs.yml"
-
-
-if __name__ == "__main__":
-    with open(JOBS_YAML_PATH) as f:
-        jobs = yaml.safe_load(f)
-    job_output = jobs["pr"]
-    print(f"jobs={json.dumps(job_output)}")