Auto merge of #70190 - pietroalbini:gha, r=Mark-Simulacrum

Add GitHub Actions configuration

This PR adds the GitHub Actions configuration to the rust-lang/rust repository. The configuration will be run in parallel with Azure Pipelines until the evaluation finishes: the infrastructure team will then decide whether to switch.

Since GitHub Actions doesn't currently have any way to include pieces of configuration, this also adds the `src/tools/expand-yaml-anchors` tool, which serves as a sort of templating system. Otherwise the configuration is a mostly straight port from the Azure Pipelines configuration (thanks to all the PRs opened in the past).

There are still a few small things I need to fix before we can land this, but it's mostly complete and ready for an initial review.

r? @Mark-Simulacrum

22 files changed, 1157 insertions, 29 deletions

diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index dd519506d42..243cd3fa199 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -21,6 +21,7 @@ use crate::doc;
 use crate::flags::Subcommand;
 use crate::install;
 use crate::native;
+use crate::run;
 use crate::test;
 use crate::tool;
 use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir};
@@ -313,6 +314,7 @@ pub enum Kind {
     Dist,
     Doc,
     Install,
+    Run,
 }
 
 impl<'a> Builder<'a> {
@@ -353,6 +355,7 @@ impl<'a> Builder<'a> {
             }
             Kind::Test => describe!(
                 crate::toolstate::ToolStateCheck,
+                test::ExpandYamlAnchors,
                 test::Tidy,
                 test::Ui,
                 test::CompileFail,
@@ -454,6 +457,7 @@ impl<'a> Builder<'a> {
                 install::Src,
                 install::Rustc
             ),
+            Kind::Run => describe!(run::ExpandYamlAnchors,),
         }
     }
 
@@ -507,6 +511,7 @@ impl<'a> Builder<'a> {
             Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]),
             Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
             Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
+            Subcommand::Run { ref paths } => (Kind::Run, &paths[..]),
             Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(),
         };
 
diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs
index d8831c6d9e5..eda26f7df1f 100644
--- a/src/bootstrap/flags.rs
+++ b/src/bootstrap/flags.rs
@@ -86,6 +86,9 @@ pub enum Subcommand {
     Install {
         paths: Vec<PathBuf>,
     },
+    Run {
+        paths: Vec<PathBuf>,
+    },
 }
 
 impl Default for Subcommand {
@@ -113,6 +116,7 @@ Subcommands:
     clean       Clean out build directories
     dist        Build distribution artifacts
     install     Install distribution artifacts
+    run         Run tools contained in this repository
 
 To learn more about a subcommand, run `./x.py <subcommand> -h`",
         );
@@ -188,6 +192,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
                 || (s == "clean")
                 || (s == "dist")
                 || (s == "install")
+                || (s == "run")
         });
         let subcommand = match subcommand {
             Some(s) => s,
@@ -400,6 +405,18 @@ Arguments:
         ./x.py doc --stage 1",
                 );
             }
+            "run" => {
+                subcommand_help.push_str(
+                    "\n
+Arguments:
+    This subcommand accepts a number of paths to tools to build and run. For
+    example:
+
+        ./x.py run src/tool/expand-yaml-anchors
+
+    At least a tool needs to be called.",
+                );
+            }
             _ => {}
         };
         // Get any optional paths which occur after the subcommand
@@ -468,6 +485,13 @@ Arguments:
             "fmt" => Subcommand::Format { check: matches.opt_present("check") },
             "dist" => Subcommand::Dist { paths },
             "install" => Subcommand::Install { paths },
+            "run" => {
+                if paths.is_empty() {
+                    println!("\nrun requires at least a path!\n");
+                    usage(1, &opts, &subcommand_help, &extra_help);
+                }
+                Subcommand::Run { paths }
+            }
             _ => {
                 usage(1, &opts, &subcommand_help, &extra_help);
             }
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index a476d25f102..6436fa75655 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -140,6 +140,7 @@ mod format;
 mod install;
 mod metadata;
 mod native;
+mod run;
 mod sanity;
 mod test;
 mod tool;
diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs
new file mode 100644
index 00000000000..90053471427
--- /dev/null
+++ b/src/bootstrap/run.rs
@@ -0,0 +1,43 @@
+use crate::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::tool::Tool;
+use std::process::Command;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ExpandYamlAnchors;
+
+impl Step for ExpandYamlAnchors {
+    type Output = ();
+
+    /// Runs the `expand-yaml_anchors` tool.
+    ///
+    /// This tool in `src/tools` read the CI configuration files written in YAML and expands the
+    /// anchors in them, since GitHub Actions doesn't support them.
+    fn run(self, builder: &Builder<'_>) {
+        builder.info("Expanding YAML anchors in the GitHub Actions configuration");
+        try_run(
+            builder,
+            &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src),
+        );
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/expand-yaml-anchors")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(ExpandYamlAnchors);
+    }
+}
+
+fn try_run(builder: &Builder<'_>, cmd: &mut Command) -> bool {
+    if !builder.fail_fast {
+        if !builder.try_run(cmd) {
+            let mut failures = builder.delayed_failures.borrow_mut();
+            failures.push(format!("{:?}", cmd));
+            return false;
+        }
+    } else {
+        builder.run(cmd);
+    }
+    true
+}
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index b52fbe4666e..5b946b05735 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -750,6 +750,35 @@ impl Step for Tidy {
     }
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct ExpandYamlAnchors;
+
+impl Step for ExpandYamlAnchors {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+
+    /// Ensure the `generate-ci-config` tool was run locally.
+    ///
+    /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the
+    /// appropriate configuration for all our CI providers. This step ensures the tool was called
+    /// by the user before committing CI changes.
+    fn run(self, builder: &Builder<'_>) {
+        builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded");
+        try_run(
+            builder,
+            &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src),
+        );
+    }
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/expand-yaml-anchors")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(ExpandYamlAnchors);
+    }
+}
+
 fn testdir(builder: &Builder<'_>, host: Interned<String>) -> PathBuf {
     builder.out.join(host).join("test")
 }
diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs
index c8ccba467e5..52f750f448e 100644
--- a/src/bootstrap/tool.rs
+++ b/src/bootstrap/tool.rs
@@ -378,6 +378,7 @@ bootstrap_tool!(
     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
     RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
+    ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
 );
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml
index 13db4725ded..56fe3864204 100644
--- a/src/ci/azure-pipelines/auto.yml
+++ b/src/ci/azure-pipelines/auto.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 #
 # Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows.
 #
diff --git a/src/ci/azure-pipelines/master.yml b/src/ci/azure-pipelines/master.yml
index 9c5a15a3cf4..485b80398c8 100644
--- a/src/ci/azure-pipelines/master.yml
+++ b/src/ci/azure-pipelines/master.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 #
 # Azure Pipelines job to publish toolstate. Only triggers on pushes to master.
 #
diff --git a/src/ci/azure-pipelines/pr.yml b/src/ci/azure-pipelines/pr.yml
index 1f0be53677d..37c1779b799 100644
--- a/src/ci/azure-pipelines/pr.yml
+++ b/src/ci/azure-pipelines/pr.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 #
 # Azure Pipelines pull request build for Rust
 #
diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml
index ee9425aa1c5..85ff3e52a84 100644
--- a/src/ci/azure-pipelines/steps/run.yml
+++ b/src/ci/azure-pipelines/steps/run.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 # FIXME(linux): need to configure core dumps, enable them, and then dump
 # backtraces on failure from all core dumps:
 #
@@ -59,8 +70,8 @@ steps:
   displayName: Install InnoSetup
   condition: and(succeeded(), not(variables.SKIP_JOB))
 
-- bash: src/ci/scripts/windows-symlink-build-dir.sh
-  displayName: Ensure the build happens on C:\ instead of D:\
+- bash: src/ci/scripts/symlink-build-dir.sh
+  displayName: Ensure the build happens on a partition with enough space
   condition: and(succeeded(), not(variables.SKIP_JOB))
 
 - bash: src/ci/scripts/disable-git-crlf-conversion.sh
diff --git a/src/ci/azure-pipelines/try.yml b/src/ci/azure-pipelines/try.yml
index 698608795e2..38a0685e0f7 100644
--- a/src/ci/azure-pipelines/try.yml
+++ b/src/ci/azure-pipelines/try.yml
@@ -1,3 +1,14 @@
+#####################################
+##    READ BEFORE CHANGING THIS    ##
+#####################################
+
+# We're in the process of evaluating GitHub Actions as a possible replacement
+# for Azure Pipelines, and at the moment the configuration is duplicated
+# between the two CI providers. Be sure to also change the configuration in
+# src/ci/github-actions when changing this file.
+
+#####################################
+
 pr: none
 trigger:
 - try
diff --git a/src/ci/docker/mingw-check/Dockerfile b/src/ci/docker/mingw-check/Dockerfile
index e973ba2e33c..1293717f975 100644
--- a/src/ci/docker/mingw-check/Dockerfile
+++ b/src/ci/docker/mingw-check/Dockerfile
@@ -22,7 +22,8 @@ RUN sh /scripts/sccache.sh
 COPY mingw-check/validate-toolstate.sh /scripts/
 
 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
-ENV SCRIPT python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
+ENV SCRIPT python2.7 ../x.py test src/tools/expand-yaml-anchors && \
+           python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
            python2.7 ../x.py build --stage 0 src/tools/build-manifest && \
            python2.7 ../x.py test --stage 0 src/tools/compiletest && \
            python2.7 ../x.py test src/tools/tidy && \
diff --git a/src/ci/exec-with-shell.py b/src/ci/exec-with-shell.py
new file mode 100755
index 00000000000..26ce69e33d9
--- /dev/null
+++ b/src/ci/exec-with-shell.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+# A simple wrapper that forwards the arguments to bash, unless the
+# CI_OVERRIDE_SHELL environment variable is present: in that case the content
+# of that environment variable is used as the shell path.
+
+import os
+import sys
+import subprocess
+
+try:
+    shell = os.environ["CI_OVERRIDE_SHELL"]
+except KeyError:
+    shell = "bash"
+
+res = subprocess.call([shell] + sys.argv[1:])
+sys.exit(res)
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
new file mode 100644
index 00000000000..81a334cd487
--- /dev/null
+++ b/src/ci/github-actions/ci.yml
@@ -0,0 +1,709 @@
+######################################################
+#   WARNING! Action needed when changing this file   #
+######################################################
+
+# Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
+# CI configuration stored on the repository. To work around that this file is
+# expanded by a tool in the repository, and the expansion is committed as well.
+#
+# After you make any change to the file you'll need to run this command:
+#
+#   ./x.py run src/tools/expand-yaml-anchors
+#
+# ...and commit the file it updated in addition to this one. If you forget this
+# step CI will fail.
+
+---
+
+###############################
+#   YAML Anchors Definition   #
+###############################
+
+# This key contains most of the YAML anchors that will be used later in the
+# document. YAML anchors allows us to greatly reduce duplication inside the CI
+# configuration by reusing parts of the configuration.
+#
+# YAML anchors work by defining an anchor with `&anchor-name` and reusing its
+# content in another place with `*anchor-name`. The special `<<` map key merges
+# the content of the map with the content of the anchor (or list of anchors).
+#
+# The expand-yaml-anchors tool will automatically remove this block from the
+# output YAML file.
+x--expand-yaml-anchors--remove:
+
+  - &shared-ci-variables
+    CI_JOB_NAME: ${{ matrix.name }}
+
+  - &public-variables
+    SCCACHE_BUCKET: rust-lang-gha-caches
+    TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
+
+  - &prod-variables
+    SCCACHE_BUCKET: rust-lang-gha-caches
+    DEPLOY_BUCKET: rust-lang-gha
+    TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
+    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
+    TOOLSTATE_PUBLISH: 1
+    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
+    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
+    # rotate them in a single branch while keeping the old key in another
+    # branch, which wouldn't be possible if the key was named with the kind
+    # (caches, artifacts...).
+    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
+    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
+
+  - &base-job
+    env: {}
+
+  - &job-linux-xl
+    os: ubuntu-latest-xl
+    <<: *base-job
+
+  - &job-macos-xl
+    os: macos-latest  # We don't have an XL builder for this
+    <<: *base-job
+
+  - &job-windows-xl
+    os: windows-latest-xl
+    <<: *base-job
+
+  - &step
+    if: success() && !env.SKIP_JOB
+
+  - &step-run
+    <<: *step
+    # While on Linux and macOS builders it just forwards the arguments to the
+    # system bash, this wrapper allows switching from the host's bash.exe to
+    # the one we install along with MSYS2 mid-build on Windows.
+    #
+    # Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL
+    # environment variable is set pointing to our MSYS2's bash.exe. From that
+    # moment the host's bash.exe will not be called anymore.
+    #
+    # This is needed because we can't launch our own bash.exe from the host
+    # bash.exe, as that would load two different cygwin1.dll in memory, causing
+    # "cygwin heap mismatch" errors.
+    shell: python src/ci/exec-with-shell.py {0}
+
+  - &base-ci-job
+    timeout-minutes: 600
+    runs-on: "${{ matrix.os }}"
+    env: *shared-ci-variables
+    steps:
+      - name: disable git crlf conversion
+        run: git config --global core.autocrlf false
+        shell: bash
+
+      - name: checkout the source code
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 2
+
+      - name: configure GitHub Actions to kill the build when outdated
+        uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
+        with:
+          github_token: "${{ secrets.github_token }}"
+        <<: *step
+
+      - name: add extra environment variables
+        run: src/ci/scripts/setup-environment.sh
+        env:
+          # Since it's not possible to merge `${{ matrix.env }}` with the other
+          # variables in `job.<name>.env`, the variables defined in the matrix
+          # are passed to the `setup-environment.sh` script encoded in JSON,
+          # which then uses log commands to actually set them.
+          EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
+        <<: *step-run
+
+      - name: decide whether to skip this job
+        run: src/ci/scripts/should-skip-this.sh
+        <<: *step-run
+
+      - name: collect CPU statistics
+        run: src/ci/scripts/collect-cpu-stats.sh
+        <<: *step-run
+
+      - name: show the current environment
+        run: src/ci/scripts/dump-environment.sh
+        <<: *step-run
+
+      - name: install awscli
+        run: src/ci/scripts/install-awscli.sh
+        <<: *step-run
+
+      - name: install sccache
+        run: src/ci/scripts/install-sccache.sh
+        <<: *step-run
+
+      - name: install clang
+        run: src/ci/scripts/install-clang.sh
+        <<: *step-run
+
+      - name: install WIX
+        run: src/ci/scripts/install-wix.sh
+        <<: *step-run
+
+      - name: install InnoSetup
+        run: src/ci/scripts/install-innosetup.sh
+        <<: *step-run
+
+      - name: ensure the build happens on a partition with enough space
+        run: src/ci/scripts/symlink-build-dir.sh
+        <<: *step-run
+
+      - name: disable git crlf conversion
+        run: src/ci/scripts/disable-git-crlf-conversion.sh
+        <<: *step-run
+
+      - name: install MSYS2
+        run: src/ci/scripts/install-msys2.sh
+        <<: *step-run
+
+      - name: install MSYS2 packages
+        run: src/ci/scripts/install-msys2-packages.sh
+        <<: *step-run
+
+      - name: install MinGW
+        run: src/ci/scripts/install-mingw.sh
+        <<: *step-run
+
+      - name: install ninja
+        run: src/ci/scripts/install-ninja.sh
+        <<: *step-run
+
+      - name: enable ipv6 on Docker
+        run: src/ci/scripts/enable-docker-ipv6.sh
+        <<: *step-run
+
+      # Disable automatic line ending conversion (again). On Windows, when we're
+      # installing dependencies, something switches the git configuration directory or
+      # re-enables autocrlf. We've not tracked down the exact cause -- and there may
+      # be multiple -- but this should ensure submodules are checked out with the
+      # appropriate line endings.
+      - name: disable git crlf conversion
+        run: src/ci/scripts/disable-git-crlf-conversion.sh
+        <<: *step-run
+
+      - name: checkout submodules
+        run: src/ci/scripts/checkout-submodules.sh
+        <<: *step-run
+
+      - name: ensure line endings are correct
+        run: src/ci/scripts/verify-line-endings.sh
+        <<: *step-run
+
+      - name: run the build
+        run: src/ci/scripts/run-build-from-ci.sh
+        env:
+          AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
+          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
+        <<: *step-run
+
+      - name: upload artifacts to S3
+        run: src/ci/scripts/upload-artifacts.sh
+        env:
+          AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
+        # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
+        # builders *should* have the AWS credentials available. Still, explicitly
+        # adding the condition is helpful as this way CI will not silently skip
+        # deploying artifacts from a dist builder if the variables are misconfigured,
+        # erroring about invalid credentials instead.
+        if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
+        <<: *step-run
+
+  # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
+  # Check out their documentation for more information on why they're needed.
+
+  - &base-outcome-job
+    name: bors build finished
+    runs-on: ubuntu-latest
+
+  - &base-success-job
+    steps:
+      - name: mark the job as a success
+        run: exit 0
+    <<: *base-outcome-job
+
+  - &base-failure-job
+    steps:
+      - name: mark the job as a failure
+        run: exit 1
+    <<: *base-outcome-job
+
+###########################
+#   Builders definition   #
+###########################
+
+name: CI
+on:
+  push:
+    branches:
+      - auto
+      - try
+      - master
+  pull_request:
+    branches:
+      - "**"
+
+jobs:
+  pr:
+    <<: *base-ci-job
+    name: PR
+    env:
+      <<: [*shared-ci-variables, *public-variables]
+    if: github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        name:
+          - mingw-check
+          - x86_64-gnu-llvm-7
+          - x86_64-gnu-tools
+        include:
+          - name: mingw-check
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu-llvm-7
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu-tools
+            env:
+              CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
+            <<: *job-linux-xl
+
+  try:
+    <<: *base-ci-job
+    name: try
+    env:
+      <<: [*shared-ci-variables, *prod-variables]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'
+    strategy:
+      matrix:
+        name:
+          - dist-x86_64-linux
+          - dist-x86_64-linux-alt
+        include:
+          - name: dist-x86_64-linux
+            <<: *job-linux-xl
+
+          - name: dist-x86_64-linux-alt
+            env:
+              IMAGE: dist-x86_64-linux
+            <<: *job-linux-xl
+
+  auto:
+    <<: *base-ci-job
+    name: auto
+    env:
+      <<: [*shared-ci-variables, *prod-variables]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
+    strategy:
+      matrix:
+        name:
+          - arm-android
+          - armhf-gnu
+          - dist-aarch64-linux
+          - dist-android
+          - dist-arm-linux
+          - dist-armhf-linux
+          - dist-armv7-linux
+          - dist-i586-gnu-i586-i686-musl
+          - dist-i686-freebsd
+          - dist-i686-linux
+          - dist-i686-mingw
+          - dist-i686-msvc
+          - dist-mips-linux
+          - dist-mips64-linux
+          - dist-mips64el-linux
+          - dist-mipsel-linux
+          - dist-powerpc-linux
+          - dist-powerpc64-linux
+          - dist-powerpc64le-linux
+          - dist-s390x-linux
+          - dist-various-1
+          - dist-various-2
+          - dist-x86_64-apple
+          - dist-x86_64-apple-alt
+          - dist-x86_64-freebsd
+          - dist-x86_64-linux
+          - dist-x86_64-linux-alt
+          - dist-x86_64-mingw
+          - dist-x86_64-msvc
+          - dist-x86_64-msvc-alt
+          - dist-x86_64-musl
+          - dist-x86_64-netbsd
+          - i686-gnu
+          - i686-gnu-nopt
+          - i686-mingw-1
+          - i686-mingw-2
+          - i686-msvc-1
+          - i686-msvc-2
+          - mingw-check
+          - test-various
+          - wasm32
+          - x86_64-apple
+          - x86_64-gnu
+          - x86_64-gnu-aux
+          - x86_64-gnu-debug
+          - x86_64-gnu-distcheck
+          - x86_64-gnu-full-bootstrap
+          - x86_64-gnu-llvm-7
+          - x86_64-gnu-nopt
+          - x86_64-gnu-tools
+          - x86_64-mingw-1
+          - x86_64-mingw-2
+          - x86_64-msvc-1
+          - x86_64-msvc-2
+          - x86_64-msvc-aux
+          - x86_64-msvc-cargo
+          - x86_64-msvc-tools
+        include:
+          #############################
+          #   Linux/Docker builders   #
+          #############################
+
+          - name: arm-android
+            <<: *job-linux-xl
+
+          - name: armhf-gnu
+            <<: *job-linux-xl
+
+          - name: dist-aarch64-linux
+            <<: *job-linux-xl
+
+          - name: dist-android
+            <<: *job-linux-xl
+
+          - name: dist-arm-linux
+            <<: *job-linux-xl
+
+          - name: dist-armhf-linux
+            <<: *job-linux-xl
+
+          - name: dist-armv7-linux
+            <<: *job-linux-xl
+
+          - name: dist-i586-gnu-i586-i686-musl
+            <<: *job-linux-xl
+
+          - name: dist-i686-freebsd
+            <<: *job-linux-xl
+
+          - name: dist-i686-linux
+            <<: *job-linux-xl
+
+          - name: dist-mips-linux
+            <<: *job-linux-xl
+
+          - name: dist-mips64-linux
+            <<: *job-linux-xl
+
+          - name: dist-mips64el-linux
+            <<: *job-linux-xl
+
+          - name: dist-mipsel-linux
+            <<: *job-linux-xl
+
+          - name: dist-powerpc-linux
+            <<: *job-linux-xl
+
+          - name: dist-powerpc64-linux
+            <<: *job-linux-xl
+
+          - name: dist-powerpc64le-linux
+            <<: *job-linux-xl
+
+          - name: dist-s390x-linux
+            <<: *job-linux-xl
+
+          - name: dist-various-1
+            <<: *job-linux-xl
+
+          - name: dist-various-2
+            <<: *job-linux-xl
+
+          - name: dist-x86_64-freebsd
+            <<: *job-linux-xl
+
+          - name: dist-x86_64-linux
+            <<: *job-linux-xl
+
+          - name: dist-x86_64-linux-alt
+            env:
+              IMAGE: dist-x86_64-linux
+            <<: *job-linux-xl
+
+          - name: dist-x86_64-musl
+            <<: *job-linux-xl
+
+          - name: dist-x86_64-netbsd
+            <<: *job-linux-xl
+
+          - name: i686-gnu
+            <<: *job-linux-xl
+
+          - name: i686-gnu-nopt
+            <<: *job-linux-xl
+
+          - name: mingw-check
+            <<: *job-linux-xl
+
+          - name: test-various
+            <<: *job-linux-xl
+
+          - name: wasm32
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu-aux
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu-debug
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu-distcheck
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu-full-bootstrap
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu-llvm-7
+            env:
+              RUST_BACKTRACE: 1
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu-nopt
+            <<: *job-linux-xl
+
+          - name: x86_64-gnu-tools
+            env:
+              DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
+            <<: *job-linux-xl
+
+          ####################
+          #  macOS Builders  #
+          ####################
+
+          - name: dist-x86_64-apple
+            env:
+              SCRIPT: ./x.py dist
+              RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
+              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+              MACOSX_DEPLOYMENT_TARGET: 10.7
+              NO_LLVM_ASSERTIONS: 1
+              NO_DEBUG_ASSERTIONS: 1
+              DIST_REQUIRE_ALL_TOOLS: 1
+            <<: *job-macos-xl
+
+          - name: dist-x86_64-apple-alt
+            env:
+              SCRIPT: ./x.py dist
+              RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
+              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+              MACOSX_DEPLOYMENT_TARGET: 10.7
+              NO_LLVM_ASSERTIONS: 1
+              NO_DEBUG_ASSERTIONS: 1
+            <<: *job-macos-xl
+
+          - name: x86_64-apple
+            env:
+              SCRIPT: ./x.py test
+              RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
+              RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
+              MACOSX_DEPLOYMENT_TARGET: 10.8
+              MACOSX_STD_DEPLOYMENT_TARGET: 10.7
+              NO_LLVM_ASSERTIONS: 1
+              NO_DEBUG_ASSERTIONS: 1
+            <<: *job-macos-xl
+
+          ######################
+          #  Windows Builders  #
+          ######################
+
+          - name: x86_64-msvc-1
+            env:
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+              SCRIPT: make ci-subset-1
+              # FIXME(#59637)
+              NO_DEBUG_ASSERTIONS: 1
+              NO_LLVM_ASSERTIONS: 1
+            <<: *job-windows-xl
+
+          - name: x86_64-msvc-2
+            env:
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+              SCRIPT: make ci-subset-2
+            <<: *job-windows-xl
+
+          - name: i686-msvc-1
+            env:
+              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
+              SCRIPT: make ci-subset-1
+              # FIXME(#59637)
+              NO_DEBUG_ASSERTIONS: 1
+              NO_LLVM_ASSERTIONS: 1
+            <<: *job-windows-xl
+
+          - name: i686-msvc-2
+            env:
+              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
+              SCRIPT: make ci-subset-2
+              # FIXME(#59637)
+              NO_DEBUG_ASSERTIONS: 1
+              NO_LLVM_ASSERTIONS: 1
+            <<: *job-windows-xl
+
+          - name: x86_64-msvc-aux
+            env:
+              RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
+            <<: *job-windows-xl
+
+          - name: x86_64-msvc-cargo
+            env:
+              SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
+              VCVARS_BAT: vcvars64.bat
+              # FIXME(#59637)
+              NO_DEBUG_ASSERTIONS: 1
+              NO_LLVM_ASSERTIONS: 1
+            <<: *job-windows-xl
+
+          - name: x86_64-msvc-tools
+            env:
+              SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
+            <<: *job-windows-xl
+
+          # 32/64-bit MinGW builds.
+          #
+          # We are using MinGW with posix threads since LLVM does not compile with
+          # the win32 threads version due to missing support for C++'s std::thread.
+          #
+          # Instead of relying on the MinGW version installed on appveryor we download
+          # and install one ourselves so we won't be surprised by changes to appveyor's
+          # build image.
+          #
+          # Finally, note that the downloads below are all in the `rust-lang-ci` S3
+          # bucket, but they cleraly didn't originate there! The downloads originally
+          # came from the mingw-w64 SourceForge download site. Unfortunately
+          # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
+
+          - name: i686-mingw-1
+            env:
+              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+              SCRIPT: make ci-mingw-subset-1
+              CUSTOM_MINGW: 1
+              # FIXME(#59637)
+              NO_DEBUG_ASSERTIONS: 1
+              NO_LLVM_ASSERTIONS: 1
+            <<: *job-windows-xl
+
+          - name: i686-mingw-2
+            env:
+              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+              SCRIPT: make ci-mingw-subset-2
+              CUSTOM_MINGW: 1
+            <<: *job-windows-xl
+
+          - name: x86_64-mingw-1
+            env:
+              SCRIPT: make ci-mingw-subset-1
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
+              CUSTOM_MINGW: 1
+              # FIXME(#59637)
+              NO_DEBUG_ASSERTIONS: 1
+              NO_LLVM_ASSERTIONS: 1
+            <<: *job-windows-xl
+
+          - name: x86_64-mingw-2
+            env:
+              SCRIPT: make ci-mingw-subset-2
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
+              CUSTOM_MINGW: 1
+            <<: *job-windows-xl
+
+          - name: dist-x86_64-msvc
+            env:
+              RUST_CONFIGURE_ARGS: >-
+                --build=x86_64-pc-windows-msvc
+                --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
+                --enable-full-tools
+                --enable-profiler
+              SCRIPT: python x.py dist
+              DIST_REQUIRE_ALL_TOOLS: 1
+            <<: *job-windows-xl
+
+          - name: dist-i686-msvc
+            env:
+              RUST_CONFIGURE_ARGS: >-
+                --build=i686-pc-windows-msvc
+                --target=i586-pc-windows-msvc
+                --enable-full-tools
+                --enable-profiler
+              SCRIPT: python x.py dist
+              DIST_REQUIRE_ALL_TOOLS: 1
+            <<: *job-windows-xl
+
+          - name: dist-i686-mingw
+            env:
+              RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
+              SCRIPT: python x.py dist
+              CUSTOM_MINGW: 1
+              DIST_REQUIRE_ALL_TOOLS: 1
+            <<: *job-windows-xl
+
+          - name: dist-x86_64-mingw
+            env:
+              SCRIPT: python x.py dist
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
+              CUSTOM_MINGW: 1
+              DIST_REQUIRE_ALL_TOOLS: 1
+            <<: *job-windows-xl
+
+          - name: dist-x86_64-msvc-alt
+            env:
+              RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
+              SCRIPT: python x.py dist
+            <<: *job-windows-xl
+
+  master:
+    name: master
+    runs-on: ubuntu-latest
+    env:
+      <<: [*prod-variables]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
+    steps:
+      - name: checkout the source code
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 2
+
+      - name: publish toolstate
+        run: src/ci/publish_toolstate.sh
+        env:
+          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
+        <<: *step-run
+
+  # These jobs don't actually test anything, but they're used to tell bors the
+  # build completed, as there is no practical way to detect when a workflow is
+  # successful listening to webhooks only.
+  try-success:
+    needs: [try]
+    if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
+    <<: *base-success-job
+  try-failure:
+    needs: [try]
+    if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
+    <<: *base-failure-job
+  auto-success:
+    needs: [auto]
+    if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
+    <<: *base-success-job
+  auto-failure:
+    needs: [auto]
+    if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
+    <<: *base-failure-job
diff --git a/src/ci/scripts/install-awscli.sh b/src/ci/scripts/install-awscli.sh
index e2118793850..f9b759fe343 100755
--- a/src/ci/scripts/install-awscli.sh
+++ b/src/ci/scripts/install-awscli.sh
@@ -28,7 +28,7 @@ if isLinux; then
     pipflags="--user"
 
     sudo apt-get install -y python3-setuptools
-    echo "##vso[task.prependpath]$HOME/.local/bin"
+    ciCommandAddPath "${HOME}/.local/bin"
 fi
 
 mkdir -p "${DEPS_DIR}"
diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh
index 98373df7fce..78728dd7d00 100755
--- a/src/ci/scripts/install-mingw.sh
+++ b/src/ci/scripts/install-mingw.sh
@@ -50,8 +50,8 @@ if isWindows; then
     esac
 
     if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then
-        pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \
-            mingw-w64-$arch-gcc mingw-w64-$arch-python2
+        pacman -S --noconfirm --needed mingw-w64-$arch-toolchain \
+            mingw-w64-$arch-cmake mingw-w64-$arch-gcc mingw-w64-$arch-python2
         ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin"
     else
         mingw_dir="mingw${bits}"
diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh
index 9e899ba9d89..3c3b5007f86 100755
--- a/src/ci/scripts/install-msys2.sh
+++ b/src/ci/scripts/install-msys2.sh
@@ -22,4 +22,7 @@ if isWindows; then
     rm msys2.nupkg chocolatey-core.extension.nupkg
     mkdir -p "$(ciCheckoutPath)/msys2/home/${USERNAME}"
     ciCommandAddPath "$(ciCheckoutPath)/msys2/usr/bin"
+
+    echo "switching shell to use our own bash"
+    ciCommandSetEnv CI_OVERRIDE_SHELL "$(ciCheckoutPath)/msys2/usr/bin/bash.exe"
 fi
diff --git a/src/ci/scripts/setup-environment.sh b/src/ci/scripts/setup-environment.sh
index d134fcd47ba..411ef6f9b28 100755
--- a/src/ci/scripts/setup-environment.sh
+++ b/src/ci/scripts/setup-environment.sh
@@ -11,16 +11,34 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
 # Since matrix variables are readonly in Azure Pipelines, we take
 # INITIAL_RUST_CONFIGURE_ARGS and establish RUST_CONFIGURE_ARGS
 # which downstream steps can alter
-# macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]],
-# which was introduced in Bash 4.2
-if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then
-    INITIAL_RUST_CONFIG=""
-    echo "No initial Rust configure args set"
-else
-    INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}"
-    ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}"
+if isAzurePipelines; then
+    # macOS ships with Bash 3.16, so we cannot use [[ -v FOO ]],
+    # which was introduced in Bash 4.2
+    if [[ -z "${INITIAL_RUST_CONFIGURE_ARGS+x}" ]]; then
+        INITIAL_RUST_CONFIG=""
+        echo "No initial Rust configure args set"
+    else
+        INITIAL_RUST_CONFIG="${INITIAL_RUST_CONFIGURE_ARGS}"
+        ciCommandSetEnv RUST_CONFIGURE_ARGS "${INITIAL_RUST_CONFIG}"
+    fi
 fi
 
+# Load extra environment variables
+vars="${EXTRA_VARIABLES-}"
+echo "${vars}" | jq '' >/dev/null  # Validate JSON and exit on errors
+for key in $(echo "${vars}" | jq "keys[]" -r); do
+    # On Windows, for whatever reason, $key contains the BOM character in it,
+    # and that messes up `jq ".${key}"`. This line strips the BOM from the key.
+    #
+    # https://unix.stackexchange.com/a/381263
+    key="$(echo "${key}" | sed '1s/^\xEF\xBB\xBF//')"
+
+    echo "adding extra environment variable ${key}"
+    value="$(echo "${vars}" | jq ".${key}" -r)"
+    export "${key}"="${value}"
+    ciCommandSetEnv "${key}" "${value}"
+done
+
 # Builders starting with `dist-` are dist builders, but if they also end with
 # `-alt` they are alternate dist builders.
 if [[ "${CI_JOB_NAME}" = dist-* ]]; then
diff --git a/src/ci/scripts/symlink-build-dir.sh b/src/ci/scripts/symlink-build-dir.sh
new file mode 100755
index 00000000000..c77059c00ac
--- /dev/null
+++ b/src/ci/scripts/symlink-build-dir.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# We've had multiple issues with the default disk running out of disk space
+# during builds, and it looks like other disks mounted in the VMs have more
+# space available. This script synlinks the build directory to those other
+# disks, in the CI providers and OSes affected by this.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
+
+if isWindows && isAzurePipelines; then
+    cmd //c "mkdir c:\\MORE_SPACE"
+    cmd //c "mklink /J build c:\\MORE_SPACE"
+elif isLinux && isGitHubActions; then
+    sudo mkdir -p /mnt/more-space
+    sudo chown -R "$(whoami):" /mnt/more-space
+
+    # Switch the whole workspace to the /mnt partition, which has more space.
+    # We don't just symlink the `obj` directory as doing that creates problems
+    # with the docker container.
+    current_dir="$(readlink -f "$(pwd)")"
+    cd /tmp
+    mv "${current_dir}" /mnt/more-space/workspace
+    ln -s /mnt/more-space/workspace "${current_dir}"
+    cd "${current_dir}"
+fi
diff --git a/src/ci/scripts/windows-symlink-build-dir.sh b/src/ci/scripts/windows-symlink-build-dir.sh
deleted file mode 100755
index e57128c70f5..00000000000
--- a/src/ci/scripts/windows-symlink-build-dir.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-# We've had issues with the default drive in use running out of space during a
-# build, and it looks like the `C:` drive has more space than the default `D:`
-# drive. We should probably confirm this with the azure pipelines team at some
-# point, but this seems to fix our "disk space full" problems.
-
-set -euo pipefail
-IFS=$'\n\t'
-
-source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
-
-if isWindows; then
-    cmd //c "mkdir c:\\MORE_SPACE"
-    cmd //c "mklink /J build c:\\MORE_SPACE"
-fi
diff --git a/src/tools/expand-yaml-anchors/Cargo.toml b/src/tools/expand-yaml-anchors/Cargo.toml
new file mode 100644
index 00000000000..2c63e28b693
--- /dev/null
+++ b/src/tools/expand-yaml-anchors/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+name = "expand-yaml-anchors"
+version = "0.1.0"
+authors = ["Pietro Albini <pietro@pietroalbini.org>"]
+edition = "2018"
+
+[dependencies]
+yaml-rust = "0.4.3"
+yaml-merge-keys = "0.4.0"
diff --git a/src/tools/expand-yaml-anchors/src/main.rs b/src/tools/expand-yaml-anchors/src/main.rs
new file mode 100644
index 00000000000..f2ed8aa409a
--- /dev/null
+++ b/src/tools/expand-yaml-anchors/src/main.rs
@@ -0,0 +1,202 @@
+use std::error::Error;
+use std::path::{Path, PathBuf};
+use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
+
+/// List of directories containing files to expand. The first tuple element is the source
+/// directory, while the second tuple element is the destination directory.
+#[rustfmt::skip]
+static TO_EXPAND: &[(&str, &str)] = &[
+    ("src/ci/github-actions", ".github/workflows"),
+];
+
+/// Name of a special key that will be removed from all the maps in expanded configuration files.
+/// This key can then be used to contain shared anchors.
+static REMOVE_MAP_KEY: &str = "x--expand-yaml-anchors--remove";
+
+/// Message that will be included at the top of all the expanded files. {source} will be replaced
+/// with the source filename relative to the base path.
+static HEADER_MESSAGE: &str = "\
+#############################################################
+#   WARNING: automatically generated file, DO NOT CHANGE!   #
+#############################################################
+
+# This file was automatically generated by the expand-yaml-anchors tool. The
+# source file that generated this one is:
+#
+#   {source}
+#
+# Once you make changes to that file you need to run:
+#
+#   ./x.py run src/tools/expand-yaml-anchors/
+#
+# The CI build will fail if the tool is not run after changes to this file.
+
+";
+
+enum Mode {
+    Check,
+    Generate,
+}
+
+struct App {
+    mode: Mode,
+    base: PathBuf,
+}
+
+impl App {
+    fn from_args() -> Result<Self, Box<dyn Error>> {
+        // Parse CLI arguments
+        let args = std::env::args().skip(1).collect::<Vec<_>>();
+        let (mode, base) = match args.iter().map(|s| s.as_str()).collect::<Vec<_>>().as_slice() {
+            &["generate", ref base] => (Mode::Generate, PathBuf::from(base)),
+            &["check", ref base] => (Mode::Check, PathBuf::from(base)),
+            _ => {
+                eprintln!("usage: expand-yaml-anchors <source-dir> <dest-dir>");
+                std::process::exit(1);
+            }
+        };
+
+        Ok(App { mode, base })
+    }
+
+    fn run(&self) -> Result<(), Box<dyn Error>> {
+        for (source, dest) in TO_EXPAND {
+            let source = self.base.join(source);
+            let dest = self.base.join(dest);
+            for entry in std::fs::read_dir(&source)? {
+                let path = entry?.path();
+                if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") {
+                    continue;
+                }
+
+                let dest_path = dest.join(path.file_name().unwrap());
+                self.expand(&path, &dest_path).with_context(|| match self.mode {
+                    Mode::Generate => format!(
+                        "failed to expand {} into {}",
+                        self.path(&path),
+                        self.path(&dest_path)
+                    ),
+                    Mode::Check => format!("{} is not up to date", self.path(&dest_path)),
+                })?;
+            }
+        }
+        Ok(())
+    }
+
+    fn expand(&self, source: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
+        let content = std::fs::read_to_string(source)
+            .with_context(|| format!("failed to read {}", self.path(source)))?;
+
+        let mut buf = HEADER_MESSAGE.replace("{source}", &self.path(source).to_string());
+
+        let documents = YamlLoader::load_from_str(&content)
+            .with_context(|| format!("failed to parse {}", self.path(source)))?;
+        for mut document in documents.into_iter() {
+            document = yaml_merge_keys::merge_keys(document)
+                .with_context(|| format!("failed to expand {}", self.path(source)))?;
+            document = filter_document(document);
+
+            YamlEmitter::new(&mut buf).dump(&document).map_err(|err| WithContext {
+                context: "failed to serialize the expanded yaml".into(),
+                source: Box::new(err),
+            })?;
+            buf.push('\n');
+        }
+
+        match self.mode {
+            Mode::Check => {
+                let old = std::fs::read_to_string(dest)
+                    .with_context(|| format!("failed to read {}", self.path(dest)))?;
+                if old != buf {
+                    return Err(Box::new(StrError(format!(
+                        "{} and {} are different",
+                        self.path(source),
+                        self.path(dest),
+                    ))));
+                }
+            }
+            Mode::Generate => {
+                std::fs::write(dest, buf.as_bytes())
+                    .with_context(|| format!("failed to write to {}", self.path(dest)))?;
+            }
+        }
+        Ok(())
+    }
+
+    fn path<'a>(&self, path: &'a Path) -> impl std::fmt::Display + 'a {
+        path.strip_prefix(&self.base).unwrap_or(path).display()
+    }
+}
+
+fn filter_document(document: Yaml) -> Yaml {
+    match document {
+        Yaml::Hash(map) => Yaml::Hash(
+            map.into_iter()
+                .filter(|(key, _)| {
+                    if let Yaml::String(string) = &key { string != REMOVE_MAP_KEY } else { true }
+                })
+                .map(|(key, value)| (filter_document(key), filter_document(value)))
+                .collect(),
+        ),
+        Yaml::Array(vec) => {
+            Yaml::Array(vec.into_iter().map(|item| filter_document(item)).collect())
+        }
+        other => other,
+    }
+}
+
+fn main() {
+    if let Err(err) = App::from_args().and_then(|app| app.run()) {
+        eprintln!("error: {}", err);
+
+        let mut source = err.as_ref() as &dyn Error;
+        while let Some(err) = source.source() {
+            eprintln!("caused by: {}", err);
+            source = err;
+        }
+
+        std::process::exit(1);
+    }
+}
+
+#[derive(Debug)]
+struct StrError(String);
+
+impl Error for StrError {}
+
+impl std::fmt::Display for StrError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.0, f)
+    }
+}
+
+#[derive(Debug)]
+struct WithContext {
+    context: String,
+    source: Box<dyn Error>,
+}
+
+impl std::fmt::Display for WithContext {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "{}", self.context)
+    }
+}
+
+impl Error for WithContext {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        Some(self.source.as_ref())
+    }
+}
+
+pub(crate) trait ResultExt<T> {
+    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>>;
+}
+
+impl<T, E: Into<Box<dyn Error>>> ResultExt<T> for Result<T, E> {
+    fn with_context<F: FnOnce() -> String>(self, f: F) -> Result<T, Box<dyn Error>> {
+        match self {
+            Ok(ok) => Ok(ok),
+            Err(err) => Err(WithContext { source: err.into(), context: f() }.into()),
+        }
+    }
+}