path: root/src/ci/github-actions/ci.yml

######################################################
#   WARNING! Action needed when changing this file   #
######################################################

# Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
# CI configuration stored on the repository. To work around that this file is
# expanded by a tool in the repository, and the expansion is committed as well.
#
# After you make any change to the file you'll need to run this command:
#
#   ./x.py run src/tools/expand-yaml-anchors
#
# ...and commit the file it updated in addition to this one. If you forget this
# step CI will fail.

---
###############################
#   YAML Anchors Definition   #
###############################

# This key contains most of the YAML anchors that will be used later in the
# document. YAML anchors allows us to greatly reduce duplication inside the CI
# configuration by reusing parts of the configuration.
#
# YAML anchors work by defining an anchor with `&anchor-name` and reusing its
# content in another place with `*anchor-name`. The special `<<` map key merges
# the content of the map with the content of the anchor (or list of anchors).
#
# The expand-yaml-anchors tool will automatically remove this block from the
# output YAML file.
x--expand-yaml-anchors--remove:
  - &shared-ci-variables
    CI_JOB_NAME: ${{ matrix.name }}
    CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
    # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
    HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
    DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  - &public-variables
    SCCACHE_BUCKET: rust-lang-ci-sccache2
    TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
    CACHE_DOMAIN: ci-caches.rust-lang.org

  - &prod-variables
    SCCACHE_BUCKET: rust-lang-ci-sccache2
    DEPLOY_BUCKET: rust-lang-ci2
    TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
    TOOLSTATE_PUBLISH: 1
    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
    # rotate them in a single branch while keeping the old key in another
    # branch, which wouldn't be possible if the key was named with the kind
    # (caches, artifacts...).
    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
    AWS_REGION: us-west-1
    CACHE_DOMAIN: ci-caches.rust-lang.org

  - &dummy-variables
    SCCACHE_BUCKET: rust-lang-gha-caches
    DEPLOY_BUCKET: rust-lang-gha
    TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
    TOOLSTATE_PUBLISH: 1
    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
    # rotate them in a single branch while keeping the old key in another
    # branch, which wouldn't be possible if the key was named with the kind
    # (caches, artifacts...).
    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
    AWS_REGION: us-west-1
    CACHE_DOMAIN: ci-caches-gha.rust-lang.org

  - &base-job
    env: {}

  - &job-linux-4c
    os: ubuntu-20.04-4core-16gb
    <<: *base-job

  - &job-linux-8c
    os: ubuntu-20.04-8core-32gb
    <<: *base-job

  - &job-linux-16c
    os: ubuntu-20.04-16core-64gb
    <<: *base-job

  - &job-macos-xl
    os: macos-13 # We use the standard runner for now
    <<: *base-job

  - &job-macos-m1
    os: macos-14
    <<: *base-job

  - &job-windows-8c
    os: windows-2019-8core-32gb
    <<: *base-job

  - &job-windows-16c
    os: windows-2019-16core-64gb
    <<: *base-job

  - &job-aarch64-linux
    os: [self-hosted, ARM64, linux]

  - &base-ci-job
    defaults:
      run:
        shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
    timeout-minutes: 600
    runs-on: "${{ matrix.os }}"
    env: *shared-ci-variables
    steps:
      - if: contains(matrix.os, 'windows')
        uses: msys2/setup-msys2@v2.22.0
        with:
          # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
          msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
          # don't try to download updates for already installed packages
          update: false
          # don't try to use the msys that comes built-in to the github runner,
          # so we can control what is installed (i.e. not python)
          release: true
          # Inherit the full path from the Windows environment, with MSYS2's */bin/
          # dirs placed in front. This lets us run Windows-native Python etc.
          path-type: inherit
          install: >
            make
            dos2unix
            diffutils

      - name: disable git crlf conversion
        run: git config --global core.autocrlf false

      - name: checkout the source code
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      # Rust Log Analyzer can't currently detect the PR number of a GitHub
      # Actions build on its own, so a hint in the log message is needed to
      # point it in the right direction.
      - name: configure the PR in which the error message will be posted
        run: echo "[CI_PR_NUMBER=$num]"
        env:
          num: ${{ github.event.number }}
        if: success() && github.event_name == 'pull_request'

      - name: add extra environment variables
        run: src/ci/scripts/setup-environment.sh
        env:
          # Since it's not possible to merge `${{ matrix.env }}` with the other
          # variables in `job.<name>.env`, the variables defined in the matrix
          # are passed to the `setup-environment.sh` script encoded in JSON,
          # which then uses log commands to actually set them.
          EXTRA_VARIABLES: ${{ toJson(matrix.env) }}

      - name: ensure the channel matches the target branch
        run: src/ci/scripts/verify-channel.sh

      - name: collect CPU statistics
        run: src/ci/scripts/collect-cpu-stats.sh

      - name: show the current environment
        run: src/ci/scripts/dump-environment.sh

      - name: install awscli
        run: src/ci/scripts/install-awscli.sh

      - name: install sccache
        run: src/ci/scripts/install-sccache.sh

      - name: select Xcode
        run: src/ci/scripts/select-xcode.sh

      - name: install clang
        run: src/ci/scripts/install-clang.sh

      - name: install tidy
        run: src/ci/scripts/install-tidy.sh

      - name: install WIX
        run: src/ci/scripts/install-wix.sh

      - name: disable git crlf conversion
        run: src/ci/scripts/disable-git-crlf-conversion.sh

      - name: checkout submodules
        run: src/ci/scripts/checkout-submodules.sh

      - name: install MSYS2
        run: src/ci/scripts/install-msys2.sh

      - name: install MinGW
        run: src/ci/scripts/install-mingw.sh

      - name: install ninja
        run: src/ci/scripts/install-ninja.sh

      - name: enable ipv6 on Docker
        run: src/ci/scripts/enable-docker-ipv6.sh

      # Disable automatic line ending conversion (again). On Windows, when we're
      # installing dependencies, something switches the git configuration directory or
      # re-enables autocrlf. We've not tracked down the exact cause -- and there may
      # be multiple -- but this should ensure submodules are checked out with the
      # appropriate line endings.
      - name: disable git crlf conversion
        run: src/ci/scripts/disable-git-crlf-conversion.sh

      - name: ensure line endings are correct
        run: src/ci/scripts/verify-line-endings.sh

      - name: ensure backported commits are in upstream branches
        run: src/ci/scripts/verify-backported-commits.sh

      - name: ensure the stable version number is correct
        run: src/ci/scripts/verify-stable-version-number.sh

      - name: run the build
        # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
        run: src/ci/scripts/run-build-from-ci.sh 2>&1
        env:
          AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}

      - name: create github artifacts
        run: src/ci/scripts/create-doc-artifacts.sh

      - name: upload artifacts to github
        uses: actions/upload-artifact@v4
        with:
          # name is set in previous step
          name: ${{ env.DOC_ARTIFACT_NAME }}
          path: obj/artifacts/doc
          if-no-files-found: ignore
          retention-days: 5

      - name: upload artifacts to S3
        run: src/ci/scripts/upload-artifacts.sh
        env:
          AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
        # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
        # builders *should* have the AWS credentials available. Still, explicitly
        # adding the condition is helpful as this way CI will not silently skip
        # deploying artifacts from a dist builder if the variables are misconfigured,
        # erroring about invalid credentials instead.
        if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')

  # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
  # Check out their documentation for more information on why they're needed.

  - &base-outcome-job
    name: bors build finished
    runs-on: ubuntu-latest

  - &base-success-job
    steps:
      - name: mark the job as a success
        run: exit 0
        shell: bash
    <<: *base-outcome-job

  - &base-failure-job
    steps:
      - name: mark the job as a failure
        run: exit 1
        shell: bash
    <<: *base-outcome-job

###########################
#   Builders definition   #
###########################

name: CI
on:
  push:
    branches:
      - auto
      - try
      - try-perf
      - automation/bors/try
      - master
  pull_request:
    branches:
      - "**"

permissions:
  contents: read
  packages: write

defaults:
  run:
    # On Linux, macOS, and Windows, use the system-provided bash as the default
    # shell. (This should only make a difference on Windows, where the default
    # shell is PowerShell.)
    shell: bash

concurrency:
  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
  # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
  # are all triggered on the same branch, but which should be able to run concurrently.
  group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
  cancel-in-progress: true

jobs:
  # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
  # It calculates which jobs should be executed, based on the data of the ${{ github }} context.
  # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
  calculate_matrix:
    name: Calculate job matrix
    runs-on: ubuntu-latest
    outputs:
      jobs: ${{ steps.jobs.outputs.jobs }}
    steps:
      - name: Checkout the source code
        uses: actions/checkout@v4
      - name: Calculate the CI job matrix
        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
        id: jobs
  job:
    <<: *base-ci-job
    name: ${{ matrix.name }}
    needs: [ calculate_matrix ]
    env:
      CI_JOB_NAME: ${{ matrix.image }}
      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      SCCACHE_BUCKET: rust-lang-ci-sccache2
      TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
      CACHE_DOMAIN: ci-caches.rust-lang.org
    continue-on-error: ${{ matrix.continue_on_error || false }}
    strategy:
      matrix:
        # Check the `calculate_matrix` job to see how is the matrix defined.
        include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
    # GitHub Actions fails the workflow if an empty list of jobs is provided to
    # the workflow, so we need to skip this job if nothing was produced by
    # the Python script.
    #
    # Unfortunately checking whether a list is empty is not possible in a nice
    # way due to GitHub Actions expressions limits.
    # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82
    if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null

  master:
    name: master
    runs-on: ubuntu-latest
    env:
      <<: [*prod-variables]
    if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
    steps:
      - name: checkout the source code
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: publish toolstate
        run: src/ci/publish_toolstate.sh
        shell: bash
        env:
          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}

  # These jobs don't actually test anything, but they're used to tell bors the
  # build completed, as there is no practical way to detect when a workflow is
  # successful listening to webhooks only.
  try-success:
    needs: [ job ]
    if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
    <<: *base-success-job
  try-failure:
    needs: [ job ]
    if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
    <<: *base-failure-job
  auto-success:
    needs: [ job ]
    if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
    <<: *base-success-job
  auto-failure:
    needs: [ job ]
    if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
    <<: *base-failure-job