about summary refs log tree commit diff
path: root/src/ci
diff options
context:
space:
mode:
authorLaurențiu Nicola <lnicola@dend.ro>2025-02-10 07:49:43 +0200
committerLaurențiu Nicola <lnicola@dend.ro>2025-02-10 07:49:43 +0200
commit15cd1f011c4f854f7a7cfbe2e727cc4b02b1d254 (patch)
treebdd9d79bd54ffe237db9c987d6c1e2ab12349e16 /src/ci
parent879dc387cde7ac4b85aa5a185158c9c3c488c5e7 (diff)
parentd9a4a47b8b3dc0bdff83360cea2013200d60d49c (diff)
Merge from rust-lang/rust
Diffstat (limited to 'src/ci')
-rw-r--r--src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile1
-rw-r--r--src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile3
-rw-r--r--src/ci/docker/host-x86_64/mingw-check/Dockerfile5
-rw-r--r--src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version2
-rwxr-xr-xsrc/ci/docker/run.sh9
-rwxr-xr-xsrc/ci/docker/scripts/build-zstd.sh6
-rwxr-xr-xsrc/ci/docker/scripts/rfl-build.sh4
-rwxr-xr-xsrc/ci/github-actions/ci.py2
-rw-r--r--src/ci/github-actions/jobs.yml57
-rwxr-xr-xsrc/ci/scripts/free-disk-space.sh176
10 files changed, 231 insertions, 34 deletions
diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile
index 6f33c632181..2b8a3f829c6 100644
--- a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile
+++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile
@@ -20,6 +20,7 @@ RUN yum upgrade -y && \
       gcc-c++ \
       git \
       glibc-devel \
+      glibc-static \
       libedit-devel \
       libstdc++-devel \
       make \
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
index dde6fe7f6d0..0b4682ac32b 100644
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
@@ -21,6 +21,8 @@ RUN yum upgrade -y && \
       git \
       glibc-devel.i686 \
       glibc-devel.x86_64 \
+      glibc-static.i686 \
+      glibc-static.x86_64 \
       libedit-devel \
       libstdc++-devel.i686 \
       libstdc++-devel.x86_64 \
@@ -34,6 +36,7 @@ RUN yum upgrade -y && \
       python3 \
       unzip \
       wget \
+      flex \
       xz \
       zlib-devel.i686 \
       zlib-devel.x86_64 \
diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile
index d408cd518a0..9234c6dc921 100644
--- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile
+++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile
@@ -29,7 +29,7 @@ ENV PATH="/node/bin:${PATH}"
 
 # Install es-check
 # Pin its version to prevent unrelated CI failures due to future es-check versions.
-RUN npm install es-check@6.1.1 eslint@8.6.0 -g
+RUN npm install es-check@6.1.1 eslint@8.6.0 typescript@5.7.3 -g
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
@@ -68,4 +68,5 @@ ENV SCRIPT \
            es-check es2019 ../src/librustdoc/html/static/js/*.js && \
            eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \
            eslint -c ../src/tools/rustdoc-js/.eslintrc.js ../src/tools/rustdoc-js/tester.js && \
-           eslint -c ../src/tools/rustdoc-gui/.eslintrc.js ../src/tools/rustdoc-gui/tester.js
+           eslint -c ../src/tools/rustdoc-gui/.eslintrc.js ../src/tools/rustdoc-gui/tester.js && \
+           tsc --project ../src/librustdoc/html/static/js/tsconfig.json
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version
index 7211b157c69..a158e5b6253 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version
@@ -1 +1 @@
-0.18.2
\ No newline at end of file
+0.20.2
\ No newline at end of file
diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index d1bc0519bc1..d2697ac27ab 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -123,6 +123,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       build_args+=("--build-arg" "SCRIPT_ARG=${DOCKER_SCRIPT}")
     fi
 
+    GHCR_BUILDKIT_IMAGE="ghcr.io/rust-lang/buildkit:buildx-stable-1"
     # On non-CI jobs, we try to download a pre-built image from the rust-lang-ci
     # ghcr.io registry. If it is not possible, we fall back to building the image
     # locally.
@@ -140,7 +141,9 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
     elif [[ "$PR_CI_JOB" == "1" ]];
     then
         # Enable a new Docker driver so that --cache-from works with a registry backend
-        docker buildx create --use --driver docker-container
+        # Use a custom image to avoid DockerHub rate limits
+        docker buildx create --use --driver docker-container \
+          --driver-opt image=${GHCR_BUILDKIT_IMAGE}
 
         # Build the image using registry caching backend
         retry docker \
@@ -156,7 +159,9 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
             --password-stdin
 
         # Enable a new Docker driver so that --cache-from/to works with a registry backend
-        docker buildx create --use --driver docker-container
+        # Use a custom image to avoid DockerHub rate limits
+        docker buildx create --use --driver docker-container \
+          --driver-opt image=${GHCR_BUILDKIT_IMAGE}
 
         # Build the image using registry caching backend
         retry docker \
diff --git a/src/ci/docker/scripts/build-zstd.sh b/src/ci/docker/scripts/build-zstd.sh
index a3d37ccc311..cffa7151e38 100755
--- a/src/ci/docker/scripts/build-zstd.sh
+++ b/src/ci/docker/scripts/build-zstd.sh
@@ -25,5 +25,11 @@ cd zstd-$ZSTD
 CFLAGS=-fPIC hide_output make -j$(nproc) VERBOSE=1
 hide_output make install
 
+# It doesn't seem to be possible to move destination directory
+# of the `make install` above. We thus copy the built artifacts
+# manually to our custom rustroot, so that it can be found through
+# LD_LIBRARY_PATH.
+cp /usr/local/lib/libzstd* /rustroot/lib64
+
 cd ..
 rm -rf zstd-$ZSTD
diff --git a/src/ci/docker/scripts/rfl-build.sh b/src/ci/docker/scripts/rfl-build.sh
index 8776e0f0be9..3eb85ab215e 100755
--- a/src/ci/docker/scripts/rfl-build.sh
+++ b/src/ci/docker/scripts/rfl-build.sh
@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-LINUX_VERSION=v6.13-rc1
+LINUX_VERSION=50e57739141b41f731ab31f8380821c7969f9dc4
 
 # Build rustc, rustdoc, cargo, clippy-driver and rustfmt
 ../x.py build --stage 2 library rustdoc clippy rustfmt
@@ -28,7 +28,7 @@ rm -rf linux || true
 # Download Linux at a specific commit
 mkdir -p linux
 git -C linux init
-git -C linux remote add origin https://github.com/Rust-for-Linux/linux.git
+git -C linux remote add origin https://github.com/Darksonn/linux.git
 git -C linux fetch --depth 1 origin ${LINUX_VERSION}
 git -C linux checkout FETCH_HEAD
 
diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py
index b7dac412dbe..c93766ef33a 100755
--- a/src/ci/github-actions/ci.py
+++ b/src/ci/github-actions/ci.py
@@ -249,7 +249,7 @@ def run_workflow_locally(job_data: Dict[str, Any], job_name: str, pr_jobs: bool)
     env = os.environ.copy()
     env.update(custom_env)
 
-    subprocess.run(args, env=env)
+    subprocess.run(args, env=env, check=True)
 
 
 def calculate_job_matrix(job_data: Dict[str, Any]):
diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
index 799ea3e9ad2..341d82599fd 100644
--- a/src/ci/github-actions/jobs.yml
+++ b/src/ci/github-actions/jobs.yml
@@ -5,7 +5,7 @@ runners:
     env: { }
 
   - &job-linux-4c
-    os: ubuntu-22.04
+    os: ubuntu-24.04
     # Free some disk space to avoid running out of space during the build.
     free_disk: true
     <<: *base-job
@@ -35,17 +35,25 @@ runners:
     os: windows-2022
     <<: *base-job
 
+  - &job-windows-25
+    os: windows-2025
+    <<: *base-job
+
   - &job-windows-8c
     os: windows-2022-8core-32gb
     <<: *base-job
 
-  - &job-windows-16c
-    os: windows-2022-16core-64gb
+  - &job-windows-25-8c
+    os: windows-2025-8core-32gb
     <<: *base-job
 
   - &job-aarch64-linux
+    # Free some disk space to avoid running out of space during the build.
+    free_disk: true
     os: ubuntu-22.04-arm
 
+  - &job-aarch64-linux-8c
+    os: ubuntu-22.04-arm64-8core-32gb
 envs:
   env-x86_64-apple-tests: &env-x86_64-apple-tests
     SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact
@@ -78,7 +86,7 @@ envs:
     #   builds)
     # - not running `opt-dist`'s post-optimization smoke tests on the resulting toolchain
     #
-    # If you *want* these to happen however, temporarily uncomment it before triggering a try build.
+    # If you *want* these to happen however, temporarily comment it before triggering a try build.
     DIST_TRY_BUILD: 1
 
   auto:
@@ -138,7 +146,7 @@ auto:
   - name: dist-aarch64-linux
     env:
       CODEGEN_BACKENDS: llvm,cranelift
-    <<: *job-aarch64-linux
+    <<: *job-aarch64-linux-8c
 
   - name: dist-android
     <<: *job-linux-4c
@@ -159,10 +167,10 @@ auto:
     <<: *job-linux-4c
 
   - name: dist-loongarch64-linux
-    <<: *job-linux-4c
+    <<: *job-linux-4c-largedisk
 
   - name: dist-loongarch64-musl
-    <<: *job-linux-4c
+    <<: *job-linux-4c-largedisk
 
   - name: dist-ohos
     <<: *job-linux-4c
@@ -442,23 +450,29 @@ auto:
   #  Windows Builders  #
   ######################
 
-  - name: x86_64-msvc
+  - name: x86_64-msvc-1
     env:
       RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-      SCRIPT: make ci-msvc
-    <<: *job-windows-8c
+      SCRIPT: make ci-msvc-py
+    <<: *job-windows-25
+
+  - name: x86_64-msvc-2
+    env:
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+      SCRIPT: make ci-msvc-ps1
+    <<: *job-windows-25
 
   # i686-msvc is split into two jobs to run tests in parallel.
   - name: i686-msvc-1
     env:
       RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-      SCRIPT: make ci-msvc-py-set1
+      SCRIPT: make ci-msvc-py
     <<: *job-windows
 
   - name: i686-msvc-2
     env:
       RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-      SCRIPT: make ci-msvc-ps1-set2
+      SCRIPT: make ci-msvc-ps1
     <<: *job-windows
 
   # x86_64-msvc-ext is split into multiple jobs to run tests in parallel.
@@ -511,18 +525,16 @@ auto:
     env:
       RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
       SCRIPT: make ci-mingw
-      # We are intentionally allowing an old toolchain on this builder (and that's
-      # incompatible with LLVM downloads today).
+      # There is no dist-i686-mingw-alt, so there is no prebuilt LLVM with assertions
       NO_DOWNLOAD_CI_LLVM: 1
-    <<: *job-windows-8c
+    <<: *job-windows-25-8c
 
   # x86_64-mingw is split into two jobs to run tests in parallel.
   - name: x86_64-mingw-1
     env:
       SCRIPT: make ci-mingw-x
       RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
-      # We are intentionally allowing an old toolchain on this builder (and that's
-      # incompatible with LLVM downloads today).
+      # There is no dist-x86_64-mingw-alt, so there is no prebuilt LLVM with assertions
       NO_DOWNLOAD_CI_LLVM: 1
     <<: *job-windows
 
@@ -530,8 +542,7 @@ auto:
     env:
       SCRIPT: make ci-mingw-bootstrap
       RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
-      # We are intentionally allowing an old toolchain on this builder (and that's
-      # incompatible with LLVM downloads today).
+      # There is no dist-x86_64-mingw-alt, so there is no prebuilt LLVM with assertions
       NO_DOWNLOAD_CI_LLVM: 1
     <<: *job-windows
 
@@ -547,7 +558,7 @@ auto:
       SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
       DIST_REQUIRE_ALL_TOOLS: 1
       CODEGEN_BACKENDS: llvm,cranelift
-    <<: *job-windows-8c
+    <<: *job-windows-25-8c
 
   - name: dist-i686-msvc
     env:
@@ -579,9 +590,6 @@ auto:
       RUST_CONFIGURE_ARGS: >-
         --build=i686-pc-windows-gnu
         --enable-full-tools
-      # We are intentionally allowing an old toolchain on this builder (and that's
-      # incompatible with LLVM downloads today).
-      NO_DOWNLOAD_CI_LLVM: 1
       SCRIPT: python x.py dist bootstrap --include-default-paths
       DIST_REQUIRE_ALL_TOOLS: 1
       CODEGEN_BACKENDS: llvm,cranelift
@@ -593,9 +601,6 @@ auto:
       RUST_CONFIGURE_ARGS: >-
         --build=x86_64-pc-windows-gnu
         --enable-full-tools
-      # We are intentionally allowing an old toolchain on this builder (and that's
-      # incompatible with LLVM downloads today).
-      NO_DOWNLOAD_CI_LLVM: 1
       DIST_REQUIRE_ALL_TOOLS: 1
       CODEGEN_BACKENDS: llvm,cranelift
     <<: *job-windows
diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh
new file mode 100755
index 00000000000..8850e168145
--- /dev/null
+++ b/src/ci/scripts/free-disk-space.sh
@@ -0,0 +1,176 @@
+#!/bin/bash
+set -euo pipefail
+
+# Free disk space on Linux GitHub action runners
+# Script inspired by https://github.com/jlumbroso/free-disk-space
+
+isX86() {
+    local arch
+    arch=$(uname -m)
+    if [ "$arch" = "x86_64" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# print a line of the specified character
+printSeparationLine() {
+    for ((i = 0; i < 80; i++)); do
+        printf "%s" "$1"
+    done
+    printf "\n"
+}
+
+# compute available space
+# REF: https://unix.stackexchange.com/a/42049/60849
+# REF: https://stackoverflow.com/a/450821/408734
+getAvailableSpace() {
+    df -a | awk 'NR > 1 {avail+=$4} END {print avail}'
+}
+
+# make Kb human readable (assume the input is Kb)
+# REF: https://unix.stackexchange.com/a/44087/60849
+formatByteCount() {
+    numfmt --to=iec-i --suffix=B --padding=7 "$1"'000'
+}
+
+# macro to output saved space
+printSavedSpace() {
+    # Disk space before the operation
+    local before=${1}
+    local title=${2:-}
+
+    local after
+    after=$(getAvailableSpace)
+    local saved=$((after - before))
+
+    echo ""
+    printSeparationLine "*"
+    if [ -n "${title}" ]; then
+        echo "=> ${title}: Saved $(formatByteCount "$saved")"
+    else
+        echo "=> Saved $(formatByteCount "$saved")"
+    fi
+    printSeparationLine "*"
+    echo ""
+}
+
+# macro to print output of df with caption
+printDF() {
+    local caption=${1}
+
+    printSeparationLine "="
+    echo "${caption}"
+    echo ""
+    echo "$ df -h"
+    echo ""
+    df -h
+    printSeparationLine "="
+}
+
+removeDir() {
+    dir=${1}
+
+    local before
+    if [ ! -d "$dir" ]; then
+        echo "::warning::Directory $dir does not exist, skipping."
+    else
+        before=$(getAvailableSpace)
+        sudo rm -rf "$dir"
+        printSavedSpace "$before" "Removed $dir"
+    fi
+}
+
+removeUnusedDirectories() {
+    local dirs_to_remove=(
+        "/usr/local/lib/android"
+        "/usr/share/dotnet"
+
+        # Haskell runtime
+        "/usr/local/.ghcup"
+    )
+
+    for dir in "${dirs_to_remove[@]}"; do
+        removeDir "$dir"
+    done
+}
+
+execAndMeasureSpaceChange() {
+    local operation=${1} # Function to execute
+    local title=${2}
+
+    local before
+    before=$(getAvailableSpace)
+    $operation
+
+    printSavedSpace "$before" "$title"
+}
+
+# Remove large packages
+# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
+cleanPackages() {
+    local packages=(
+        '^aspnetcore-.*'
+        '^dotnet-.*'
+        '^llvm-.*'
+        '^mongodb-.*'
+        '^mysql-.*'
+        'azure-cli'
+        'firefox'
+        'libgl1-mesa-dri'
+        'mono-devel'
+        'php.*'
+    )
+
+    if isX86; then
+        packages+=(
+            'google-chrome-stable'
+            'google-cloud-cli'
+            'google-cloud-sdk'
+            'powershell'
+        )
+    fi
+
+    sudo apt-get -qq remove -y --fix-missing "${packages[@]}"
+
+    sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed"
+    sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed"
+}
+
+# Remove Docker images
+cleanDocker() {
+    echo "=> Removing the following docker images:"
+    sudo docker image ls
+    echo "=> Removing docker images..."
+    sudo docker image prune --all --force || true
+}
+
+# Remove Swap storage
+cleanSwap() {
+    sudo swapoff -a || true
+    sudo rm -rf /mnt/swapfile || true
+    free -h
+}
+
+# Display initial disk space stats
+
+AVAILABLE_INITIAL=$(getAvailableSpace)
+
+printDF "BEFORE CLEAN-UP:"
+echo ""
+
+execAndMeasureSpaceChange cleanPackages "Unused packages"
+execAndMeasureSpaceChange cleanDocker "Docker images"
+execAndMeasureSpaceChange cleanSwap "Swap storage"
+
+removeUnusedDirectories
+
+# Output saved space statistic
+echo ""
+printDF "AFTER CLEAN-UP:"
+
+echo ""
+echo ""
+
+printSavedSpace "$AVAILABLE_INITIAL" "Total saved"