diff options
| -rw-r--r-- | .github/workflows/ci.yml | 9 | ||||
| -rw-r--r-- | src/ci/github-actions/jobs.yml | 13 | ||||
| -rwxr-xr-x | src/ci/scripts/free-disk-space-linux.sh | 265 | ||||
| -rw-r--r-- | src/ci/scripts/free-disk-space-windows.ps1 | 35 | ||||
| -rwxr-xr-x | src/ci/scripts/free-disk-space.sh | 268 |
5 files changed, 316 insertions, 274 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc8ac539a3a..ac73bcabb97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: with: fetch-depth: 2 - # Free up disk space on Linux by removing preinstalled components that + # Free up disk space on Linux and Windows by removing preinstalled components that # we do not need. We do this to enable some of the less resource # intensive jobs to run on free runners, which however also have # less disk space. @@ -125,6 +125,13 @@ jobs: run: src/ci/scripts/free-disk-space.sh if: matrix.free_disk + # If we don't need to free up disk space then just report how much space we have + - name: print disk usage + run: | + echo "disk usage:" + df -h + if: matrix.free_disk == false + # Rust Log Analyzer can't currently detect the PR number of a GitHub # Actions build on its own, so a hint in the log message is needed to # point it in the right direction. diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 0a6ebe44b3d..541272dc937 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -31,20 +31,11 @@ runners: <<: *base-job - &job-windows - os: windows-2022 - <<: *base-job - - # NOTE: windows-2025 has less disk space available than windows-2022, - # because the D drive is missing. - - &job-windows-25 os: windows-2025 + free_disk: true <<: *base-job - &job-windows-8c - os: windows-2022-8core-32gb - <<: *base-job - - - &job-windows-25-8c os: windows-2025-8core-32gb <<: *base-job @@ -655,7 +646,7 @@ auto: SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths DIST_REQUIRE_ALL_TOOLS: 1 CODEGEN_BACKENDS: llvm,cranelift - <<: *job-windows-25-8c + <<: *job-windows-8c - name: dist-i686-msvc env: diff --git a/src/ci/scripts/free-disk-space-linux.sh b/src/ci/scripts/free-disk-space-linux.sh new file mode 100755 index 00000000000..32649fe0d9b --- /dev/null +++ b/src/ci/scripts/free-disk-space-linux.sh @@ -0,0 +1,265 @@ +#!/bin/bash +set -euo pipefail + +# Free disk space on Linux GitHub action runners +# Script inspired by https://github.com/jlumbroso/free-disk-space + +isX86() { + local arch + arch=$(uname -m) + if [ "$arch" = "x86_64" ]; then + return 0 + else + return 1 + fi +} + +# Check if we're on a GitHub hosted runner. +# In aws codebuild, the variable RUNNER_ENVIRONMENT is "self-hosted". +isGitHubRunner() { + # `:-` means "use the value of RUNNER_ENVIRONMENT if it exists, otherwise use an empty string". + if [[ "${RUNNER_ENVIRONMENT:-}" == "github-hosted" ]]; then + return 0 + else + return 1 + fi +} + +# print a line of the specified character +printSeparationLine() { + for ((i = 0; i < 80; i++)); do + printf "%s" "$1" + done + printf "\n" +} + +# compute available space +# REF: https://unix.stackexchange.com/a/42049/60849 +# REF: https://stackoverflow.com/a/450821/408734 +getAvailableSpace() { + df -a | awk 'NR > 1 {avail+=$4} END {print avail}' +} + +# make Kb human readable (assume the input is Kb) +# REF: https://unix.stackexchange.com/a/44087/60849 +formatByteCount() { + numfmt --to=iec-i --suffix=B --padding=7 "${1}000" +} + +# macro to output saved space +printSavedSpace() { + # Disk space before the operation + local before=${1} + local title=${2:-} + + local after + after=$(getAvailableSpace) + local saved=$((after - before)) + + if [ "$saved" -lt 0 ]; then + echo "::warning::Saved space is negative: $saved. Using '0' as saved space." + saved=0 + fi + + echo "" + printSeparationLine "*" + if [ -n "${title}" ]; then + echo "=> ${title}: Saved $(formatByteCount "$saved")" + else + echo "=> Saved $(formatByteCount "$saved")" + fi + printSeparationLine "*" + echo "" +} + +# macro to print output of df with caption +printDF() { + local caption=${1} + + printSeparationLine "=" + echo "${caption}" + echo "" + echo "$ df -h" + echo "" + df -h + printSeparationLine "=" +} + +removeUnusedFilesAndDirs() { + local to_remove=( + "/usr/share/java" + ) + + if isGitHubRunner; then + to_remove+=( + "/usr/local/aws-sam-cli" + "/usr/local/doc/cmake" + "/usr/local/julia"* + "/usr/local/lib/android" + "/usr/local/share/chromedriver-"* + "/usr/local/share/chromium" + "/usr/local/share/cmake-"* + "/usr/local/share/edge_driver" + "/usr/local/share/emacs" + "/usr/local/share/gecko_driver" + "/usr/local/share/icons" + "/usr/local/share/powershell" + "/usr/local/share/vcpkg" + "/usr/local/share/vim" + "/usr/share/apache-maven-"* + "/usr/share/gradle-"* + "/usr/share/kotlinc" + "/usr/share/miniconda" + "/usr/share/php" + "/usr/share/ri" + "/usr/share/swift" + + # binaries + "/usr/local/bin/azcopy" + "/usr/local/bin/bicep" + "/usr/local/bin/ccmake" + "/usr/local/bin/cmake-"* + "/usr/local/bin/cmake" + "/usr/local/bin/cpack" + "/usr/local/bin/ctest" + "/usr/local/bin/helm" + "/usr/local/bin/kind" + "/usr/local/bin/kustomize" + "/usr/local/bin/minikube" + "/usr/local/bin/packer" + "/usr/local/bin/phpunit" + "/usr/local/bin/pulumi-"* + "/usr/local/bin/pulumi" + "/usr/local/bin/stack" + + # Haskell runtime + "/usr/local/.ghcup" + + # Azure + "/opt/az" + "/usr/share/az_"* + ) + + if [ -n "${AGENT_TOOLSDIRECTORY:-}" ]; then + # Environment variable set by GitHub Actions + to_remove+=( + "${AGENT_TOOLSDIRECTORY}" + ) + else + echo "::warning::AGENT_TOOLSDIRECTORY is not set. Skipping removal." + fi + else + # Remove folders and files present in AWS CodeBuild + to_remove+=( + # binaries + "/usr/local/bin/ecs-cli" + "/usr/local/bin/eksctl" + "/usr/local/bin/kubectl" + + "${HOME}/.gradle" + "${HOME}/.dotnet" + "${HOME}/.goenv" + "${HOME}/.phpenv" + + ) + fi + + for element in "${to_remove[@]}"; do + if [ ! -e "$element" ]; then + # The file or directory doesn't exist. + # Maybe it was removed in a newer version of the runner or it's not present in a + # specific architecture (e.g. ARM). + echo "::warning::Directory or file $element does not exist, skipping." + fi + done + + # Remove all files and directories at once to save time. + sudo rm -rf "${to_remove[@]}" +} + +execAndMeasureSpaceChange() { + local operation=${1} # Function to execute + local title=${2} + + local before + before=$(getAvailableSpace) + $operation + + printSavedSpace "$before" "$title" +} + +# Remove large packages +# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh +cleanPackages() { + local packages=( + '^aspnetcore-.*' + '^dotnet-.*' + '^llvm-.*' + '^mongodb-.*' + 'firefox' + 'libgl1-mesa-dri' + 'mono-devel' + 'php.*' + ) + + if isGitHubRunner; then + packages+=( + azure-cli + ) + + if isX86; then + packages+=( + 'google-chrome-stable' + 'google-cloud-cli' + 'google-cloud-sdk' + 'powershell' + ) + fi + else + packages+=( + 'google-chrome-stable' + ) + fi + + sudo apt-get -qq remove -y --fix-missing "${packages[@]}" + + sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed" + sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed" +} + +# Remove Docker images. +# Ubuntu 22 runners have docker images already installed. +# They aren't present in ubuntu 24 runners. +cleanDocker() { + echo "=> Removing the following docker images:" + sudo docker image ls + echo "=> Removing docker images..." + sudo docker image prune --all --force || true +} + +# Remove Swap storage +cleanSwap() { + sudo swapoff -a || true + sudo rm -rf /mnt/swapfile || true + free -h +} + +# Display initial disk space stats + +AVAILABLE_INITIAL=$(getAvailableSpace) + +printDF "BEFORE CLEAN-UP:" +echo "" +execAndMeasureSpaceChange cleanPackages "Unused packages" +execAndMeasureSpaceChange cleanDocker "Docker images" +execAndMeasureSpaceChange cleanSwap "Swap storage" +execAndMeasureSpaceChange removeUnusedFilesAndDirs "Unused files and directories" + +# Output saved space statistic +echo "" +printDF "AFTER CLEAN-UP:" + +echo "" +echo "" + +printSavedSpace "$AVAILABLE_INITIAL" "Total saved" diff --git a/src/ci/scripts/free-disk-space-windows.ps1 b/src/ci/scripts/free-disk-space-windows.ps1 new file mode 100644 index 00000000000..8a4677bd2ab --- /dev/null +++ b/src/ci/scripts/free-disk-space-windows.ps1 @@ -0,0 +1,35 @@ +# Free disk space on Windows GitHub action runners. + +$ErrorActionPreference = 'Stop' + +Get-Volume | Out-String | Write-Output + +$available = $(Get-Volume C).SizeRemaining + +$dirs = 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm', +'C:\rtools45', 'C:\ghcup', 'C:\Program Files (x86)\Android', +'C:\Program Files\Google\Chrome', 'C:\Program Files (x86)\Microsoft\Edge', +'C:\Program Files\Mozilla Firefox', 'C:\Program Files\MySQL', 'C:\Julia', +'C:\Program Files\MongoDB', 'C:\Program Files\Azure Cosmos DB Emulator', +'C:\Program Files\PostgreSQL', 'C:\Program Files\Unity Hub', +'C:\Strawberry', 'C:\hostedtoolcache\windows\Java_Temurin-Hotspot_jdk' + +foreach ($dir in $dirs) { + Start-ThreadJob -InputObject $dir { + Remove-Item -Recurse -Force -LiteralPath $input + } | Out-Null +} + +foreach ($job in Get-Job) { + Wait-Job $job | Out-Null + if ($job.Error) { + Write-Output "::warning file=$PSCommandPath::$($job.Error)" + } + Remove-Job $job +} + +Get-Volume | Out-String | Write-Output + +$saved = ($(Get-Volume C).SizeRemaining - $available) / 1gb +$savedRounded = [math]::Round($saved, 3) +Write-Output "total space saved: $savedRounded GB" diff --git a/src/ci/scripts/free-disk-space.sh b/src/ci/scripts/free-disk-space.sh index 173f64858b3..062ad801cd8 100755 --- a/src/ci/scripts/free-disk-space.sh +++ b/src/ci/scripts/free-disk-space.sh @@ -1,266 +1,10 @@ #!/bin/bash set -euo pipefail -# Free disk space on Linux GitHub action runners -# Script inspired by https://github.com/jlumbroso/free-disk-space +script_dir=$(dirname "$0") -isX86() { - local arch - arch=$(uname -m) - if [ "$arch" = "x86_64" ]; then - return 0 - else - return 1 - fi -} - -# Check if we're on a GitHub hosted runner. -# In aws codebuild, the variable RUNNER_ENVIRONMENT is "self-hosted". -isGitHubRunner() { - # `:-` means "use the value of RUNNER_ENVIRONMENT if it exists, otherwise use an empty string". - if [[ "${RUNNER_ENVIRONMENT:-}" == "github-hosted" ]]; then - return 0 - else - return 1 - fi -} - -# print a line of the specified character -printSeparationLine() { - for ((i = 0; i < 80; i++)); do - printf "%s" "$1" - done - printf "\n" -} - -# compute available space -# REF: https://unix.stackexchange.com/a/42049/60849 -# REF: https://stackoverflow.com/a/450821/408734 -getAvailableSpace() { - df -a | awk 'NR > 1 {avail+=$4} END {print avail}' -} - -# make Kb human readable (assume the input is Kb) -# REF: https://unix.stackexchange.com/a/44087/60849 -formatByteCount() { - numfmt --to=iec-i --suffix=B --padding=7 "${1}000" -} - -# macro to output saved space -printSavedSpace() { - # Disk space before the operation - local before=${1} - local title=${2:-} - - local after - after=$(getAvailableSpace) - local saved=$((after - before)) - - if [ "$saved" -lt 0 ]; then - echo "::warning::Saved space is negative: $saved. Using '0' as saved space." - saved=0 - fi - - echo "" - printSeparationLine "*" - if [ -n "${title}" ]; then - echo "=> ${title}: Saved $(formatByteCount "$saved")" - else - echo "=> Saved $(formatByteCount "$saved")" - fi - printSeparationLine "*" - echo "" -} - -# macro to print output of df with caption -printDF() { - local caption=${1} - - printSeparationLine "=" - echo "${caption}" - echo "" - echo "$ df -h" - echo "" - df -h - printSeparationLine "=" -} - -removeUnusedFilesAndDirs() { - local to_remove=( - "/usr/share/java" - ) - - if isGitHubRunner; then - to_remove+=( - "/usr/local/aws-sam-cli" - "/usr/local/doc/cmake" - "/usr/local/julia"* - "/usr/local/lib/android" - "/usr/local/share/chromedriver-"* - "/usr/local/share/chromium" - "/usr/local/share/cmake-"* - "/usr/local/share/edge_driver" - "/usr/local/share/emacs" - "/usr/local/share/gecko_driver" - "/usr/local/share/icons" - "/usr/local/share/powershell" - "/usr/local/share/vcpkg" - "/usr/local/share/vim" - "/usr/share/apache-maven-"* - "/usr/share/gradle-"* - "/usr/share/kotlinc" - "/usr/share/miniconda" - "/usr/share/php" - "/usr/share/ri" - "/usr/share/swift" - - # binaries - "/usr/local/bin/azcopy" - "/usr/local/bin/bicep" - "/usr/local/bin/ccmake" - "/usr/local/bin/cmake-"* - "/usr/local/bin/cmake" - "/usr/local/bin/cpack" - "/usr/local/bin/ctest" - "/usr/local/bin/helm" - "/usr/local/bin/kind" - "/usr/local/bin/kustomize" - "/usr/local/bin/minikube" - "/usr/local/bin/packer" - "/usr/local/bin/phpunit" - "/usr/local/bin/pulumi-"* - "/usr/local/bin/pulumi" - "/usr/local/bin/stack" - - # Haskell runtime - "/usr/local/.ghcup" - - # Azure - "/opt/az" - "/usr/share/az_"* - ) - - if [ -n "${AGENT_TOOLSDIRECTORY:-}" ]; then - # Environment variable set by GitHub Actions - to_remove+=( - "${AGENT_TOOLSDIRECTORY}" - ) - else - echo "::warning::AGENT_TOOLSDIRECTORY is not set. Skipping removal." - fi - else - # Remove folders and files present in AWS CodeBuild - to_remove+=( - # binaries - "/usr/local/bin/ecs-cli" - "/usr/local/bin/eksctl" - "/usr/local/bin/kubectl" - - "${HOME}/.gradle" - "${HOME}/.dotnet" - "${HOME}/.goenv" - "${HOME}/.phpenv" - - ) - fi - - for element in "${to_remove[@]}"; do - if [ ! -e "$element" ]; then - # The file or directory doesn't exist. - # Maybe it was removed in a newer version of the runner or it's not present in a - # specific architecture (e.g. ARM). - echo "::warning::Directory or file $element does not exist, skipping." - fi - done - - # Remove all files and directories at once to save time. - sudo rm -rf "${to_remove[@]}" -} - -execAndMeasureSpaceChange() { - local operation=${1} # Function to execute - local title=${2} - - local before - before=$(getAvailableSpace) - $operation - - printSavedSpace "$before" "$title" -} - -# Remove large packages -# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh -cleanPackages() { - local packages=( - '^aspnetcore-.*' - '^dotnet-.*' - '^llvm-.*' - '^mongodb-.*' - 'firefox' - 'libgl1-mesa-dri' - 'mono-devel' - 'php.*' - ) - - if isGitHubRunner; then - packages+=( - azure-cli - ) - - if isX86; then - packages+=( - 'google-chrome-stable' - 'google-cloud-cli' - 'google-cloud-sdk' - 'powershell' - ) - fi - else - packages+=( - 'google-chrome-stable' - ) - fi - - sudo apt-get -qq remove -y --fix-missing "${packages[@]}" - - sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed" - sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed" -} - -# Remove Docker images. -# Ubuntu 22 runners have docker images already installed. -# They aren't present in ubuntu 24 runners. -cleanDocker() { - echo "=> Removing the following docker images:" - sudo docker image ls - echo "=> Removing docker images..." - sudo docker image prune --all --force || true -} - -# Remove Swap storage -cleanSwap() { - sudo swapoff -a || true - sudo rm -rf /mnt/swapfile || true - free -h -} - -# Display initial disk space stats - -AVAILABLE_INITIAL=$(getAvailableSpace) - -printDF "BEFORE CLEAN-UP:" -echo "" - -execAndMeasureSpaceChange cleanPackages "Unused packages" -execAndMeasureSpaceChange cleanDocker "Docker images" -execAndMeasureSpaceChange cleanSwap "Swap storage" -execAndMeasureSpaceChange removeUnusedFilesAndDirs "Unused files and directories" - -# Output saved space statistic -echo "" -printDF "AFTER CLEAN-UP:" - -echo "" -echo "" - -printSavedSpace "$AVAILABLE_INITIAL" "Total saved" +if [[ "${RUNNER_OS:-}" == "Windows" ]]; then + pwsh $script_dir/free-disk-space-windows.ps1 +else + $script_dir/free-disk-space-linux.sh +fi |