path: root/.github/workflows/ci.yml

# This file defines our primary CI workflow that runs on pull requests
# and also on pushes to special branches (auto, try).
#
# The actual definition of the executed jobs is calculated by the
# `src/ci/citool` crate, which
# uses job definition data from src/ci/github-actions/jobs.yml.
# You should primarily modify the `jobs.yml` file if you want to modify
# what jobs are executed in CI.

name: CI
on:
  push:
    branches:
      - auto
      - try
      - try-perf
      - automation/bors/try
  pull_request:
    branches:
      - "**"

permissions:
  contents: read
  packages: write

defaults:
  run:
    # On Linux, macOS, and Windows, use the system-provided bash as the default
    # shell. (This should only make a difference on Windows, where the default
    # shell is PowerShell.)
    shell: bash

concurrency:
  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
  # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
  # are all triggered on the same branch, but which should be able to run concurrently.
  group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try') && github.sha) || github.ref }}
  cancel-in-progress: true
env:
  TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
  # This will be empty in PR jobs.
  TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
jobs:
  # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
  # It calculates which jobs should be executed, based on the data of the ${{ github }} context.
  # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
  calculate_matrix:
    name: Calculate job matrix
    runs-on: ubuntu-24.04-arm
    outputs:
      jobs: ${{ steps.jobs.outputs.jobs }}
      run_type: ${{ steps.jobs.outputs.run_type }}
    steps:
      - name: Checkout the source code
        uses: actions/checkout@v4
      - name: Test citool
        # Only test citool on the auto branch, to reduce latency of the calculate matrix job
        # on PR/try builds.
        if: ${{ github.ref == 'refs/heads/auto' }}
        run: |
          cd src/ci/citool
          CARGO_INCREMENTAL=0 cargo test
      - name: Calculate the CI job matrix
        env:
          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
        run: |
          cd src/ci/citool
          CARGO_INCREMENTAL=0 cargo run calculate-job-matrix >> $GITHUB_OUTPUT
        id: jobs
  job:
    name: ${{ matrix.full_name }}
    needs: [ calculate_matrix ]
    runs-on: "${{ matrix.os }}"
    timeout-minutes: 360
    # The bors environment contains secrets required for elevated workflows (try and auto builds),
    # which need to access e.g. S3 and upload artifacts. We want to provide access to that
    # environment only on the try/auto branches, which are only accessible to bors.
    # This also ensures that PR CI (which doesn't get write access to S3) works, as it cannot
    # access the environment.
    #
    # We only enable the environment for the rust-lang/rust repository, so that CI works on forks.
    environment: ${{ ((github.repository == 'rust-lang/rust' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try' || github.ref == 'refs/heads/auto')) && 'bors') || '' }}
    env:
      CI_JOB_NAME: ${{ matrix.name }}
      CI_JOB_DOC_URL: ${{ matrix.doc_url }}
      GITHUB_WORKFLOW_RUN_ID: ${{ github.run_id }}
      GITHUB_REPOSITORY: ${{ github.repository }}
      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      SCCACHE_BUCKET: rust-lang-ci-sccache2
      SCCACHE_REGION: us-west-1
      CACHE_DOMAIN: ci-caches.rust-lang.org
    continue-on-error: ${{ matrix.continue_on_error || false }}
    strategy:
      matrix:
        # Check the `calculate_matrix` job to see how is the matrix defined.
        include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
    steps:
      - name: Install cargo in AWS CodeBuild
        if: matrix.codebuild
        run: |
          # Check if cargo is installed
          if ! command -v cargo &> /dev/null; then
            echo "Cargo not found, installing Rust..."
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal
            # Make cargo available in PATH
            echo "$HOME/.cargo/bin" >> $GITHUB_PATH
          fi

      - name: disable git crlf conversion
        run: git config --global core.autocrlf false

      - name: checkout the source code
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      # Free up disk space on Linux by removing preinstalled components that
      # we do not need. We do this to enable some of the less resource
      # intensive jobs to run on free runners, which however also have
      # less disk space.
      - name: free up disk space
        run: src/ci/scripts/free-disk-space.sh
        if: matrix.free_disk

      # Rust Log Analyzer can't currently detect the PR number of a GitHub
      # Actions build on its own, so a hint in the log message is needed to
      # point it in the right direction.
      - name: configure the PR in which the error message will be posted
        run: echo "[CI_PR_NUMBER=$num]"
        env:
          num: ${{ github.event.number }}
        if: needs.calculate_matrix.outputs.run_type == 'pr'

      - name: add extra environment variables
        run: src/ci/scripts/setup-environment.sh
        env:
          # Since it's not possible to merge `${{ matrix.env }}` with the other
          # variables in `job.<name>.env`, the variables defined in the matrix
          # are passed to the `setup-environment.sh` script encoded in JSON,
          # which then uses log commands to actually set them.
          EXTRA_VARIABLES: ${{ toJson(matrix.env) }}

      - name: ensure the channel matches the target branch
        run: src/ci/scripts/verify-channel.sh

      - name: collect CPU statistics
        run: src/ci/scripts/collect-cpu-stats.sh

      - name: show the current environment
        run: src/ci/scripts/dump-environment.sh

      - name: install awscli
        run: src/ci/scripts/install-awscli.sh

      - name: install sccache
        run: src/ci/scripts/install-sccache.sh

      - name: select Xcode
        run: src/ci/scripts/select-xcode.sh

      - name: install clang
        run: src/ci/scripts/install-clang.sh

      - name: install tidy
        run: src/ci/scripts/install-tidy.sh

      - name: install WIX
        run: src/ci/scripts/install-wix.sh

      - name: disable git crlf conversion
        run: src/ci/scripts/disable-git-crlf-conversion.sh

      - name: checkout submodules
        run: src/ci/scripts/checkout-submodules.sh

      - name: install MinGW
        run: src/ci/scripts/install-mingw.sh

      - name: install ninja
        run: src/ci/scripts/install-ninja.sh

      - name: enable ipv6 on Docker
        # Don't run on codebuild because systemctl is not available
        if: ${{ !matrix.codebuild }}
        run: src/ci/scripts/enable-docker-ipv6.sh

      # Disable automatic line ending conversion (again). On Windows, when we're
      # installing dependencies, something switches the git configuration directory or
      # re-enables autocrlf. We've not tracked down the exact cause -- and there may
      # be multiple -- but this should ensure submodules are checked out with the
      # appropriate line endings.
      - name: disable git crlf conversion
        run: src/ci/scripts/disable-git-crlf-conversion.sh

      - name: ensure line endings are correct
        run: src/ci/scripts/verify-line-endings.sh

      - name: ensure backported commits are in upstream branches
        run: src/ci/scripts/verify-backported-commits.sh

      - name: ensure the stable version number is correct
        run: src/ci/scripts/verify-stable-version-number.sh

      # Show the environment just before we run the build
      # This makes it easier to diagnose problems with the above install scripts.
      - name: show the current environment
        run: src/ci/scripts/dump-environment.sh

      # Pre-build citool before the following step uninstalls rustup
      # Build it into the build directory, to avoid modifying sources
      - name: build citool
        run: |
          cd src/ci/citool
          CARGO_INCREMENTAL=0 CARGO_TARGET_DIR=../../../build/citool cargo build

      - name: run the build
        run: |
          set +e
          # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
          src/ci/scripts/run-build-from-ci.sh 2>&1
          STATUS=$?
          set -e

          if [[ "$STATUS" -ne 0 && -n "$CI_JOB_DOC_URL" ]]; then
            echo "****************************************************************************"
            echo "To find more information about this job, visit the following URL:"
            echo "$CI_JOB_DOC_URL"
            echo "****************************************************************************"
          fi
          exit ${STATUS}
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.CACHES_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.CACHES_AWS_SECRET_ACCESS_KEY }}

      - name: create github artifacts
        run: src/ci/scripts/create-doc-artifacts.sh

      - name: print disk usage
        run: |
          echo "disk usage:"
          df -h

      - name: upload artifacts to github
        uses: actions/upload-artifact@v4
        with:
          # name is set in previous step
          name: ${{ env.DOC_ARTIFACT_NAME }}
          path: obj/artifacts/doc
          if-no-files-found: ignore
          retention-days: 5

      - name: upload artifacts to S3
        run: src/ci/scripts/upload-artifacts.sh
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.ARTIFACTS_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.ARTIFACTS_AWS_SECRET_ACCESS_KEY }}
        # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
        # builders *should* have the AWS credentials available. Still, explicitly
        # adding the condition is helpful as this way CI will not silently skip
        # deploying artifacts from a dist builder if the variables are misconfigured,
        # erroring about invalid credentials instead.
        if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'

      - name: postprocess metrics into the summary
        # This step is not critical, and if some I/O problem happens, we don't want
        # to cancel the build.
        continue-on-error: true
        run: |
          if [ -f build/metrics.json ]; then
            METRICS=build/metrics.json
          elif [ -f obj/build/metrics.json ]; then
            METRICS=obj/build/metrics.json
          else
            echo "No metrics.json found"
            exit 0
          fi

          # Get closest bors merge commit
          PARENT_COMMIT=`git rev-list --author='bors <bors@rust-lang.org>' -n1 --first-parent HEAD^1`

          ./build/citool/debug/citool postprocess-metrics \
              --job-name ${CI_JOB_NAME} \
              --parent ${PARENT_COMMIT} \
              ${METRICS} >> ${GITHUB_STEP_SUMMARY}

      - name: upload job metrics to DataDog
        # This step is not critical, and if some I/O problem happens, we don't want
        # to cancel the build.
        continue-on-error: true
        if: needs.calculate_matrix.outputs.run_type != 'pr'
        env:
          DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
          DD_GITHUB_JOB_NAME: ${{ matrix.full_name }}
        run: ./build/citool/debug/citool upload-build-metrics build/cpu-usage.csv

  # This job isused to tell bors the final status of the build, as there is no practical way to detect
  # when a workflow is successful listening to webhooks only in our current bors implementation (homu).
  outcome:
    name: bors build finished
    runs-on: ubuntu-24.04
    needs: [ calculate_matrix, job ]
    # !cancelled() executes the job regardless of whether the previous jobs passed or failed
    if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }}
    steps:
      - name: checkout the source code
        uses: actions/checkout@v4
        with:
          fetch-depth: 2
      # Calculate the exit status of the whole CI workflow.
      # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
      # If a some dependent job has failed, this exits with 1.
      - name: calculate the correct exit status
        run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
      # Publish the toolstate if an auto build succeeds (just before push to master)
      - name: publish toolstate
        run: src/ci/publish_toolstate.sh
        shell: bash
        if: needs.calculate_matrix.outputs.run_type == 'auto'
        env:
          TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
          TOOLSTATE_PUBLISH: 1