diff options
| author | Laurențiu Nicola <lnicola@dend.ro> | 2025-02-10 07:49:43 +0200 |
|---|---|---|
| committer | Laurențiu Nicola <lnicola@dend.ro> | 2025-02-10 07:49:43 +0200 |
| commit | 15cd1f011c4f854f7a7cfbe2e727cc4b02b1d254 (patch) | |
| tree | bdd9d79bd54ffe237db9c987d6c1e2ab12349e16 /src/ci | |
| parent | 879dc387cde7ac4b85aa5a185158c9c3c488c5e7 (diff) | |
| parent | d9a4a47b8b3dc0bdff83360cea2013200d60d49c (diff) | |
Merge from rust-lang/rust
Diffstat (limited to 'src/ci')
| -rw-r--r-- | src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile | 1 | ||||
| -rw-r--r-- | src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 3 | ||||
| -rw-r--r-- | src/ci/docker/host-x86_64/mingw-check/Dockerfile | 5 | ||||
| -rw-r--r-- | src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version | 2 | ||||
| -rwxr-xr-x | src/ci/docker/run.sh | 9 | ||||
| -rwxr-xr-x | src/ci/docker/scripts/build-zstd.sh | 6 | ||||
| -rwxr-xr-x | src/ci/docker/scripts/rfl-build.sh | 4 | ||||
| -rwxr-xr-x | src/ci/github-actions/ci.py | 2 | ||||
| -rw-r--r-- | src/ci/github-actions/jobs.yml | 57 | ||||
| -rwxr-xr-x | src/ci/scripts/free-disk-space.sh | 176 |
10 files changed, 231 insertions, 34 deletions
diff --git a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile index 6f33c632181..2b8a3f829c6 100644 --- a/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile +++ b/src/ci/docker/host-aarch64/dist-aarch64-linux/Dockerfile @@ -20,6 +20,7 @@ RUN yum upgrade -y && \ gcc-c++ \ git \ glibc-devel \ + glibc-static \ libedit-devel \ libstdc++-devel \ make \ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index dde6fe7f6d0..0b4682ac32b 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -21,6 +21,8 @@ RUN yum upgrade -y && \ git \ glibc-devel.i686 \ glibc-devel.x86_64 \ + glibc-static.i686 \ + glibc-static.x86_64 \ libedit-devel \ libstdc++-devel.i686 \ libstdc++-devel.x86_64 \ @@ -34,6 +36,7 @@ RUN yum upgrade -y && \ python3 \ unzip \ wget \ + flex \ xz \ zlib-devel.i686 \ zlib-devel.x86_64 \ diff --git a/src/ci/docker/host-x86_64/mingw-check/Dockerfile b/src/ci/docker/host-x86_64/mingw-check/Dockerfile index d408cd518a0..9234c6dc921 100644 --- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile @@ -29,7 +29,7 @@ ENV PATH="/node/bin:${PATH}" # Install es-check # Pin its version to prevent unrelated CI failures due to future es-check versions. -RUN npm install es-check@6.1.1 eslint@8.6.0 -g +RUN npm install es-check@6.1.1 eslint@8.6.0 typescript@5.7.3 -g COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh @@ -68,4 +68,5 @@ ENV SCRIPT \ es-check es2019 ../src/librustdoc/html/static/js/*.js && \ eslint -c ../src/librustdoc/html/static/.eslintrc.js ../src/librustdoc/html/static/js/*.js && \ eslint -c ../src/tools/rustdoc-js/.eslintrc.js ../src/tools/rustdoc-js/tester.js && \ - eslint -c ../src/tools/rustdoc-gui/.eslintrc.js ../src/tools/rustdoc-gui/tester.js + eslint -c ../src/tools/rustdoc-gui/.eslintrc.js ../src/tools/rustdoc-gui/tester.js && \ + tsc --project ../src/librustdoc/html/static/js/tsconfig.json diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version index 7211b157c69..a158e5b6253 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version @@ -1 +1 @@ -0.18.2 \ No newline at end of file +0.20.2 \ No newline at end of file diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index d1bc0519bc1..d2697ac27ab 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -123,6 +123,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then build_args+=("--build-arg" "SCRIPT_ARG=${DOCKER_SCRIPT}") fi + GHCR_BUILDKIT_IMAGE="ghcr.io/rust-lang/buildkit:buildx-stable-1" # On non-CI jobs, we try to download a pre-built image from the rust-lang-ci # ghcr.io registry. If it is not possible, we fall back to building the image # locally. @@ -140,7 +141,9 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then elif [[ "$PR_CI_JOB" == "1" ]]; then # Enable a new Docker driver so that --cache-from works with a registry backend - docker buildx create --use --driver docker-container + # Use a custom image to avoid DockerHub rate limits + docker buildx create --use --driver docker-container \ + --driver-opt image=${GHCR_BUILDKIT_IMAGE} # Build the image using registry caching backend retry docker \ @@ -156,7 +159,9 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then --password-stdin # Enable a new Docker driver so that --cache-from/to works with a registry backend - docker buildx create --use --driver docker-container + # Use a custom image to avoid DockerHub rate limits + docker buildx create --use --driver docker-container \ + --driver-opt image=${GHCR_BUILDKIT_IMAGE} # Build the image using registry caching backend retry docker \ diff --git a/src/ci/docker/scripts/build-zstd.sh b/src/ci/docker/scripts/build-zstd.sh index a3d37ccc311..cffa7151e38 100755 --- a/src/ci/docker/scripts/build-zstd.sh +++ b/src/ci/docker/scripts/build-zstd.sh @@ -25,5 +25,11 @@ cd zstd-$ZSTD CFLAGS=-fPIC hide_output make -j$(nproc) VERBOSE=1 hide_output make install +# It doesn't seem to be possible to move destination directory +# of the `make install` above. We thus copy the built artifacts +# manually to our custom rustroot, so that it can be found through +# LD_LIBRARY_PATH. +cp /usr/local/lib/libzstd* /rustroot/lib64 + cd .. rm -rf zstd-$ZSTD diff --git a/src/ci/docker/scripts/rfl-build.sh b/src/ci/docker/scripts/rfl-build.sh index 8776e0f0be9..3eb85ab215e 100755 --- a/src/ci/docker/scripts/rfl-build.sh +++ b/src/ci/docker/scripts/rfl-build.sh @@ -2,7 +2,7 @@ set -euo pipefail -LINUX_VERSION=v6.13-rc1 +LINUX_VERSION=50e57739141b41f731ab31f8380821c7969f9dc4 # Build rustc, rustdoc, cargo, clippy-driver and rustfmt ../x.py build --stage 2 library rustdoc clippy rustfmt @@ -28,7 +28,7 @@ rm -rf linux || true # Download Linux at a specific commit mkdir -p linux git -C linux init -git -C linux remote add origin https://github.com/Rust-for-Linux/linux.git +git -C linux remote add origin https://github.com/Darksonn/linux.git git -C linux fetch --depth 1 origin ${LINUX_VERSION} git -C linux checkout FETCH_HEAD diff --git a/src/ci/github-actions/ci.py b/src/ci/github-actions/ci.py index b7dac412dbe..c93766ef33a 100755 --- a/src/ci/github-actions/ci.py +++ b/src/ci/github-actions/ci.py @@ -249,7 +249,7 @@ def run_workflow_locally(job_data: Dict[str, Any], job_name: str, pr_jobs: bool) env = os.environ.copy() env.update(custom_env) - subprocess.run(args, env=env) + subprocess.run(args, env=env, check=True) def calculate_job_matrix(job_data: Dict[str, Any]): diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 799ea3e9ad2..341d82599fd 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -5,7 +5,7 @@ runners: env: { } - &job-linux-4c - os: ubuntu-22.04 + os: ubuntu-24.04 # Free some disk space to avoid running out of space during the build. free_disk: true <<: *base-job @@ -35,17 +35,25 @@ runners: os: windows-2022 <<: *base-job + - &job-windows-25 + os: windows-2025 + <<: *base-job + - &job-windows-8c os: windows-2022-8core-32gb <<: *base-job - - &job-windows-16c - os: windows-2022-16core-64gb + - &job-windows-25-8c + os: windows-2025-8core-32gb <<: *base-job - &job-aarch64-linux + # Free some disk space to avoid running out of space during the build. + free_disk: true os: ubuntu-22.04-arm + - &job-aarch64-linux-8c + os: ubuntu-22.04-arm64-8core-32gb envs: env-x86_64-apple-tests: &env-x86_64-apple-tests SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact @@ -78,7 +86,7 @@ envs: # builds) # - not running `opt-dist`'s post-optimization smoke tests on the resulting toolchain # - # If you *want* these to happen however, temporarily uncomment it before triggering a try build. + # If you *want* these to happen however, temporarily comment it before triggering a try build. DIST_TRY_BUILD: 1 auto: @@ -138,7 +146,7 @@ auto: - name: dist-aarch64-linux env: CODEGEN_BACKENDS: llvm,cranelift - <<: *job-aarch64-linux + <<: *job-aarch64-linux-8c - name: dist-android <<: *job-linux-4c @@ -159,10 +167,10 @@ auto: <<: *job-linux-4c - name: dist-loongarch64-linux - <<: *job-linux-4c + <<: *job-linux-4c-largedisk - name: dist-loongarch64-musl - <<: *job-linux-4c + <<: *job-linux-4c-largedisk - name: dist-ohos <<: *job-linux-4c @@ -442,23 +450,29 @@ auto: # Windows Builders # ###################### - - name: x86_64-msvc + - name: x86_64-msvc-1 env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-msvc - <<: *job-windows-8c + SCRIPT: make ci-msvc-py + <<: *job-windows-25 + + - name: x86_64-msvc-2 + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-msvc-ps1 + <<: *job-windows-25 # i686-msvc is split into two jobs to run tests in parallel. - name: i686-msvc-1 env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-msvc-py-set1 + SCRIPT: make ci-msvc-py <<: *job-windows - name: i686-msvc-2 env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-msvc-ps1-set2 + SCRIPT: make ci-msvc-ps1 <<: *job-windows # x86_64-msvc-ext is split into multiple jobs to run tests in parallel. @@ -511,18 +525,16 @@ auto: env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw - # We are intentionally allowing an old toolchain on this builder (and that's - # incompatible with LLVM downloads today). + # There is no dist-i686-mingw-alt, so there is no prebuilt LLVM with assertions NO_DOWNLOAD_CI_LLVM: 1 - <<: *job-windows-8c + <<: *job-windows-25-8c # x86_64-mingw is split into two jobs to run tests in parallel. - name: x86_64-mingw-1 env: SCRIPT: make ci-mingw-x RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - # We are intentionally allowing an old toolchain on this builder (and that's - # incompatible with LLVM downloads today). + # There is no dist-x86_64-mingw-alt, so there is no prebuilt LLVM with assertions NO_DOWNLOAD_CI_LLVM: 1 <<: *job-windows @@ -530,8 +542,7 @@ auto: env: SCRIPT: make ci-mingw-bootstrap RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - # We are intentionally allowing an old toolchain on this builder (and that's - # incompatible with LLVM downloads today). + # There is no dist-x86_64-mingw-alt, so there is no prebuilt LLVM with assertions NO_DOWNLOAD_CI_LLVM: 1 <<: *job-windows @@ -547,7 +558,7 @@ auto: SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 CODEGEN_BACKENDS: llvm,cranelift - <<: *job-windows-8c + <<: *job-windows-25-8c - name: dist-i686-msvc env: @@ -579,9 +590,6 @@ auto: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-gnu --enable-full-tools - # We are intentionally allowing an old toolchain on this builder (and that's - # incompatible with LLVM downloads today). - NO_DOWNLOAD_CI_LLVM: 1 SCRIPT: python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 CODEGEN_BACKENDS: llvm,cranelift @@ -593,9 +601,6 @@ auto: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-gnu --enable-full-tools - # We are intentionally allowing an old toolchain on this builder (and that's - # incompatible with LLVM downloads today). - NO_DOWNLOAD_CI_LLVM: 1 DIST_REQUIRE_ALL_TOOLS: 1 CODEGEN_BACKENDS: llvm,cranelift <<: *job-windows diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh new file mode 100755 index 00000000000..8850e168145 --- /dev/null +++ b/src/ci/scripts/free-disk-space.sh @@ -0,0 +1,176 @@ +#!/bin/bash +set -euo pipefail + +# Free disk space on Linux GitHub action runners +# Script inspired by https://github.com/jlumbroso/free-disk-space + +isX86() { + local arch + arch=$(uname -m) + if [ "$arch" = "x86_64" ]; then + return 0 + else + return 1 + fi +} + +# print a line of the specified character +printSeparationLine() { + for ((i = 0; i < 80; i++)); do + printf "%s" "$1" + done + printf "\n" +} + +# compute available space +# REF: https://unix.stackexchange.com/a/42049/60849 +# REF: https://stackoverflow.com/a/450821/408734 +getAvailableSpace() { + df -a | awk 'NR > 1 {avail+=$4} END {print avail}' +} + +# make Kb human readable (assume the input is Kb) +# REF: https://unix.stackexchange.com/a/44087/60849 +formatByteCount() { + numfmt --to=iec-i --suffix=B --padding=7 "$1"'000' +} + +# macro to output saved space +printSavedSpace() { + # Disk space before the operation + local before=${1} + local title=${2:-} + + local after + after=$(getAvailableSpace) + local saved=$((after - before)) + + echo "" + printSeparationLine "*" + if [ -n "${title}" ]; then + echo "=> ${title}: Saved $(formatByteCount "$saved")" + else + echo "=> Saved $(formatByteCount "$saved")" + fi + printSeparationLine "*" + echo "" +} + +# macro to print output of df with caption +printDF() { + local caption=${1} + + printSeparationLine "=" + echo "${caption}" + echo "" + echo "$ df -h" + echo "" + df -h + printSeparationLine "=" +} + +removeDir() { + dir=${1} + + local before + if [ ! -d "$dir" ]; then + echo "::warning::Directory $dir does not exist, skipping." + else + before=$(getAvailableSpace) + sudo rm -rf "$dir" + printSavedSpace "$before" "Removed $dir" + fi +} + +removeUnusedDirectories() { + local dirs_to_remove=( + "/usr/local/lib/android" + "/usr/share/dotnet" + + # Haskell runtime + "/usr/local/.ghcup" + ) + + for dir in "${dirs_to_remove[@]}"; do + removeDir "$dir" + done +} + +execAndMeasureSpaceChange() { + local operation=${1} # Function to execute + local title=${2} + + local before + before=$(getAvailableSpace) + $operation + + printSavedSpace "$before" "$title" +} + +# Remove large packages +# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh +cleanPackages() { + local packages=( + '^aspnetcore-.*' + '^dotnet-.*' + '^llvm-.*' + '^mongodb-.*' + '^mysql-.*' + 'azure-cli' + 'firefox' + 'libgl1-mesa-dri' + 'mono-devel' + 'php.*' + ) + + if isX86; then + packages+=( + 'google-chrome-stable' + 'google-cloud-cli' + 'google-cloud-sdk' + 'powershell' + ) + fi + + sudo apt-get -qq remove -y --fix-missing "${packages[@]}" + + sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed" + sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed" +} + +# Remove Docker images +cleanDocker() { + echo "=> Removing the following docker images:" + sudo docker image ls + echo "=> Removing docker images..." + sudo docker image prune --all --force || true +} + +# Remove Swap storage +cleanSwap() { + sudo swapoff -a || true + sudo rm -rf /mnt/swapfile || true + free -h +} + +# Display initial disk space stats + +AVAILABLE_INITIAL=$(getAvailableSpace) + +printDF "BEFORE CLEAN-UP:" +echo "" + +execAndMeasureSpaceChange cleanPackages "Unused packages" +execAndMeasureSpaceChange cleanDocker "Docker images" +execAndMeasureSpaceChange cleanSwap "Swap storage" + +removeUnusedDirectories + +# Output saved space statistic +echo "" +printDF "AFTER CLEAN-UP:" + +echo "" +echo "" + +printSavedSpace "$AVAILABLE_INITIAL" "Total saved" |
