Auto merge of #70190 - pietroalbini:gha, r=Mark-Simulacrum

Add GitHub Actions configuration

This PR adds the GitHub Actions configuration to the rust-lang/rust repository. The configuration will be run in parallel with Azure Pipelines until the evaluation finishes: the infrastructure team will then decide whether to switch.

Since GitHub Actions doesn't currently have any way to include pieces of configuration, this also adds the `src/tools/expand-yaml-anchors` tool, which serves as a sort of templating system. Otherwise the configuration is a mostly straight port from the Azure Pipelines configuration (thanks to all the PRs opened in the past).

There are still a few small things I need to fix before we can land this, but it's mostly complete and ready for an initial review.

r? @Mark-Simulacrum

6 files changed, 59 insertions, 26 deletions

diff --git a/src/ci/scripts/install-awscli.sh b/src/ci/scripts/install-awscli.sh
index e2118793850..f9b759fe343 100755
--- a/src/ci/scripts/install-awscli.sh
+++ b/src/ci/scripts/install-awscli.sh
@@ -28,7 +28,7 @@ if isLinux; then
     pipflags="--user"
 
     sudo apt-get install -y python3-setuptools
-    echo "##vso[task.prependpath]$HOME/.local/bin"
+    ciCommandAddPath "${HOME}/.local/bin"
 fi
 
 mkdir -p "${DEPS_DIR}"
diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh
index 98373df7fce..78728dd7d00 100755
--- a/src/ci/scripts/install-mingw.sh
+++ b/src/ci/scripts/install-mingw.sh
@@ -50,8 +50,8 @@ if isWindows; then
     esac
 
     if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then
-        pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \
-            mingw-w64-$arch-gcc mingw-w64-$arch-python2
+        pacman -S --noconfirm --needed mingw-w64-$arch-toolchain \
+            mingw-w64-$arch-cmake mingw-w64-$arch-gcc mingw-w64-$arch-python2
         ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin"
     else
         mingw_dir="mingw${bits}"
diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh
index 9e899ba9d89..3c3b5007f86 100755
--- a/src/ci/scripts/install-msys2.sh
+++ b/src/ci/scripts/install-msys2.sh
@@ -22,4 +22,7 @@ if isWindows; then
     rm msys2.nupkg chocolatey-core.extension.nupkg
     mkdir -p "$(ciCheckoutPath)/msys2/home/${USERNAME}"
     ciCommandAddPath "$(ciCheckoutPath)/msys2/usr/bin"
+
+    echo "switching shell to use our own bash"
+    ciCommandSetEnv CI_OVERRIDE_SHELL "$(ciCheckoutPath)/msys2/usr/bin/bash.exe"
 fi
diff --git a/src/ci/scripts/setup-environment.sh b/src/ci/scripts/setup-environment.sh
index d134fcd47ba..411ef6f9b28 100755
--- a/src/ci/scripts/setup-environment.sh
+++ b/src/ci/scripts/setup-environment.sh
@@ -11,16 +11,34 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
 # Since matrix variables are readonly in Azure Pipelines, we take
 # INITIAL_RUST_CONFIGURE_ARGS and establish RUST_CONFIGURE_ARGS
 # which downstream steps can alter
-# macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]],
-# which was introduced in Bash 4.2
-if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then
-    INITIAL_RUST_CONFIG=""
-    echo "No initial Rust configure args set"
-else
-    INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}"
-    ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}"
+if isAzurePipelines; then
+    # macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]],
+    # which was introduced in Bash 4.2
+    if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then
+        INITIAL_RUST_CONFIG=""
+        echo "No initial Rust configure args set"
+    else
+        INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}"
+        ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}"
+    fi
 fi
 
+# Load extra environment variables
+vars="${EXTRA_VARIABLES-}"
+echo "${vars}" | jq '' >/dev/null  # Validate JSON and exit on errors
+for key in $(echo "${vars}" | jq "keys[]" -r); do
+    # On Windows, for whatever reason, $key contains the BOM character in it,
+    # and that messes up `jq ".${key}"`. This line strips the BOM from the key.
+    #
+    # https://unix.stackexchange.com/a/381263
+    key="$(echo "${key}" | sed '1s/^\xEF\xBB\xBF//')"
+
+    echo "adding extra environment variable ${key}"
+    value="$(echo "${vars}" | jq ".${key}" -r)"
+    export "${key}"="${value}"
+    ciCommandSetEnv "${key}" "${value}"
+done
+
 # Builders starting with `dist-` are dist builders, but if they also end with
 # `-alt` they are alternate dist builders.
 if [[ "${CI_JOB_NAME}" = dist-* ]]; then
diff --git a/src/ci/scripts/symlink-build-dir.sh b/src/ci/scripts/symlink-build-dir.sh
new file mode 100755
index 00000000000..c77059c00ac
--- /dev/null
+++ b/src/ci/scripts/symlink-build-dir.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# We've had multiple issues with the default disk running out of disk space
+# during builds, and it looks like other disks mounted in the VMs have more
+# space available. This script synlinks the build directory to those other
+# disks, in the CI providers and OSes affected by this.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
+
+if isWindows && isAzurePipelines; then
+    cmd //c "mkdir c:\\MORE_SPACE"
+    cmd //c "mklink /J build c:\\MORE_SPACE"
+elif isLinux && isGitHubActions; then
+    sudo mkdir -p /mnt/more-space
+    sudo chown -R "$(whoami):" /mnt/more-space
+
+    # Switch the whole workspace to the /mnt partition, which has more space.
+    # We don't just symlink the `obj` directory as doing that creates problems
+    # with the docker container.
+    current_dir="$(readlink -f "$(pwd)")"
+    cd /tmp
+    mv "${current_dir}" /mnt/more-space/workspace
+    ln -s /mnt/more-space/workspace "${current_dir}"
+    cd "${current_dir}"
+fi
diff --git a/src/ci/scripts/windows-symlink-build-dir.sh b/src/ci/scripts/windows-symlink-build-dir.sh
deleted file mode 100755
index e57128c70f5..00000000000
--- a/src/ci/scripts/windows-symlink-build-dir.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-# We've had issues with the default drive in use running out of space during a
-# build, and it looks like the `C:` drive has more space than the default `D:`
-# drive. We should probably confirm this with the azure pipelines team at some
-# point, but this seems to fix our "disk space full" problems.
-
-set -euo pipefail
-IFS=$'\n\t'
-
-source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
-
-if isWindows; then
-    cmd //c "mkdir c:\\MORE_SPACE"
-    cmd //c "mklink /J build c:\\MORE_SPACE"
-fi