Rollup merge of #66090 - pietroalbini:ci-improvements, r=alexcrichton

Misc CI improvements

This PR contains some misc improvements to our CI configuration:

* The environment variables for MinGW builders were greatly simplified, with just `CUSTOM_MINGW=1` to tell the install scripts to install the vendored copy. All the others (`MINGW_URL`, `MINGW_DIR`, `MINGW_ARCHIVE` and `MSYS_BITS`) are detected either from the builder name or the environment.
* Collecting CPU stats and running the build were moved into scripts.
* Toolstate scripts validation was previously a separate step, ran just when `IMAGE=mingw-check`. This moves the validation code inside the actual image.
* Vendored copies are now fetched from https://ci-mirrors.rust-lang.org instead of directly from the bucket.

r? @alexcrichton

12 files changed, 38 insertions, 9 deletions

diff --git a/src/ci/docker/armhf-gnu/Dockerfile b/src/ci/docker/armhf-gnu/Dockerfile
index 9493b336987..5373612279b 100644
--- a/src/ci/docker/armhf-gnu/Dockerfile
+++ b/src/ci/docker/armhf-gnu/Dockerfile
@@ -72,7 +72,7 @@ RUN arm-linux-gnueabihf-gcc addentropy.c -o rootfs/addentropy -static
 
 # TODO: What is this?!
 # Source of the file: https://github.com/vfdev-5/qemu-rpi2-vexpress/raw/master/vexpress-v2p-ca15-tc1.dtb
-RUN curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/vexpress-v2p-ca15-tc1.dtb
+RUN curl -O https://ci-mirrors.rust-lang.org/rustc/vexpress-v2p-ca15-tc1.dtb
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
diff --git a/src/ci/docker/dist-armv7-linux/crosstool-ng.sh b/src/ci/docker/dist-armv7-linux/crosstool-ng.sh
index ae737d9677d..fb067a79a5c 100644
--- a/src/ci/docker/dist-armv7-linux/crosstool-ng.sh
+++ b/src/ci/docker/dist-armv7-linux/crosstool-ng.sh
@@ -1,7 +1,7 @@
 set -ex
 
 # Mirrored from https://github.com/crosstool-ng/crosstool-ng/archive/crosstool-ng-1.24.0.tar.gz
-url="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/crosstool-ng-1.24.0.tar.gz"
+url="https://ci-mirrors.rust-lang.org/rustc/crosstool-ng-1.24.0.tar.gz"
 curl -Lf $url | tar xzf -
 cd crosstool-ng-crosstool-ng-1.24.0
 ./bootstrap
diff --git a/src/ci/docker/dist-various-1/install-mips-musl.sh b/src/ci/docker/dist-various-1/install-mips-musl.sh
index 29cfb5d9608..9584258d234 100755
--- a/src/ci/docker/dist-various-1/install-mips-musl.sh
+++ b/src/ci/docker/dist-various-1/install-mips-musl.sh
@@ -5,7 +5,7 @@ mkdir /usr/local/mips-linux-musl
 # originally from
 # https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/
 # OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2
-URL="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc"
+URL="https://ci-mirrors.rust-lang.org/rustc"
 FILE="OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2"
 curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mips-linux-musl --strip-components=2
 
diff --git a/src/ci/docker/dist-various-1/install-mipsel-musl.sh b/src/ci/docker/dist-various-1/install-mipsel-musl.sh
index de8c359d167..50a8e554b16 100755
--- a/src/ci/docker/dist-various-1/install-mipsel-musl.sh
+++ b/src/ci/docker/dist-various-1/install-mipsel-musl.sh
@@ -5,7 +5,7 @@ mkdir /usr/local/mipsel-linux-musl
 # Note that this originally came from:
 # https://downloads.openwrt.org/snapshots/trunk/malta/generic/
 # OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2
-URL="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc"
+URL="https://ci-mirrors.rust-lang.org/rustc"
 FILE="OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2"
 curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mipsel-linux-musl --strip-components=2
 
diff --git a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
index c63ea6facca..17aa78945cf 100755
--- a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
+++ b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
@@ -5,7 +5,7 @@
 set -ex
 
 # Originally from https://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/clang%2Bllvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \
+curl https://ci-mirrors.rust-lang.org/rustc/clang%2Bllvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \
   tar xJf -
 export PATH=`pwd`/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-14.04/bin:$PATH
 
diff --git a/src/ci/docker/dist-x86_64-linux/build-curl.sh b/src/ci/docker/dist-x86_64-linux/build-curl.sh
index 8200bbe2fdc..0c7eb5fdac9 100755
--- a/src/ci/docker/dist-x86_64-linux/build-curl.sh
+++ b/src/ci/docker/dist-x86_64-linux/build-curl.sh
@@ -5,6 +5,9 @@ source shared.sh
 
 VERSION=7.66.0
 
+# This needs to be downloaded directly from S3, it can't go through the CDN.
+# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1
+# (without paying an absurd amount of money).
 curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/curl-$VERSION.tar.xz \
   | xz --decompress \
   | tar xf -
diff --git a/src/ci/docker/dist-x86_64-linux/build-openssl.sh b/src/ci/docker/dist-x86_64-linux/build-openssl.sh
index be8a6c93945..d8f6bdb51b8 100755
--- a/src/ci/docker/dist-x86_64-linux/build-openssl.sh
+++ b/src/ci/docker/dist-x86_64-linux/build-openssl.sh
@@ -4,6 +4,10 @@ set -ex
 source shared.sh
 
 VERSION=1.0.2k
+
+# This needs to be downloaded directly from S3, it can't go through the CDN.
+# That's because the CDN is backed by CloudFront, which requires SNI and TLSv1
+# (without paying an absurd amount of money).
 URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz
 
 curl $URL | tar xzf -
diff --git a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh
index 797f674b954..f8697c698b9 100755
--- a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh
+++ b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh
@@ -25,7 +25,7 @@ cd netbsd
 
 mkdir -p /x-tools/x86_64-unknown-netbsd/sysroot
 
-URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
+URL=https://ci-mirrors.rust-lang.org/rustc
 
 # Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/source/sets/*.tgz
 curl $URL/2018-03-01-netbsd-src.tgz | tar xzf -
diff --git a/src/ci/docker/mingw-check/Dockerfile b/src/ci/docker/mingw-check/Dockerfile
index 24e2dea4ca7..b2d96aed2a9 100644
--- a/src/ci/docker/mingw-check/Dockerfile
+++ b/src/ci/docker/mingw-check/Dockerfile
@@ -19,7 +19,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
+COPY mingw-check/validate-toolstate.sh /scripts/
+
 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
 ENV SCRIPT python2.7 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
            python2.7 ../x.py build --stage 0 src/tools/build-manifest && \
-           python2.7 ../x.py test --stage 0 src/tools/compiletest
+           python2.7 ../x.py test --stage 0 src/tools/compiletest && \
+           /scripts/validate-toolstate.sh
diff --git a/src/ci/docker/mingw-check/validate-toolstate.sh b/src/ci/docker/mingw-check/validate-toolstate.sh
new file mode 100755
index 00000000000..2ebf1d6d5ae
--- /dev/null
+++ b/src/ci/docker/mingw-check/validate-toolstate.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# A quick smoke test to make sure publish_tooolstate.py works.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+rm -rf rust-toolstate
+git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git
+cd rust-toolstate
+python2.7 "../../src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" \
+    "$(git log --format=%s -n1 HEAD)" "" ""
+# Only check maintainers if this build is supposed to publish toolstate.
+# Builds that are not supposed to publish don't have the access token.
+if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then
+  TOOLSTATE_VALIDATE_MAINTAINERS_REPO=rust-lang/rust python2.7 \
+      "../../src/tools/publish_toolstate.py"
+fi
+cd ..
+rm -rf rust-toolstate
diff --git a/src/ci/docker/scripts/freebsd-toolchain.sh b/src/ci/docker/scripts/freebsd-toolchain.sh
index 70155e770a9..5670e10be23 100755
--- a/src/ci/docker/scripts/freebsd-toolchain.sh
+++ b/src/ci/docker/scripts/freebsd-toolchain.sh
@@ -59,7 +59,7 @@ done
 
 # Originally downloaded from:
 # https://download.freebsd.org/ftp/releases/${freebsd_arch}/${freebsd_version}-RELEASE/base.txz
-URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz
+URL=https://ci-mirrors.rust-lang.org/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz
 curl "$URL" | tar xJf - -C "$sysroot" --wildcards "${files_to_extract[@]}"
 
 # Fix up absolute symlinks from the system image.  This can be removed
diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh
index efeb0ed0d72..552afbfee7c 100644
--- a/src/ci/docker/scripts/sccache.sh
+++ b/src/ci/docker/scripts/sccache.sh
@@ -1,6 +1,6 @@
 set -ex
 
 curl -fo /usr/local/bin/sccache \
-  https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl
+  https://ci-mirrors.rust-lang.org/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl
 
 chmod +x /usr/local/bin/sccache