Auto merge of #60777 - pietroalbini:azure-pipelines, r=alexcrichton

Add Azure Pipelines configuration

Huge thanks to ＠johnterickson and ＠willsmythe for writing the initial config! :heart:
I applied some changes to the initial config and disabled most of the builders since we're not going to run all of them during the initial step for the evaluation.

[More details about our plans for the Azure Pipelines evaluation.](https://internals.rust-lang.org/t/update-on-the-ci-investigation/10056)

r? ＠alexcrichton ＠kennytm
cc ＠rust-lang/infra ＠ethomson ＠rylev

7 files changed, 50 insertions, 18 deletions

diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in
index 07be27c2f5a..ea05b30ecef 100644
--- a/src/bootstrap/mk/Makefile.in
+++ b/src/bootstrap/mk/Makefile.in
@@ -80,9 +80,9 @@ TESTS_IN_2 := \
 	src/test/run-pass-fulldeps \
 	src/tools/linkchecker
 
-appveyor-subset-1:
+ci-subset-1:
 	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %)
-appveyor-subset-2:
+ci-subset-2:
 	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2)
 
 
diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs
index a162c65672f..f22f0559265 100644
--- a/src/bootstrap/util.rs
+++ b/src/bootstrap/util.rs
@@ -326,6 +326,8 @@ pub enum CiEnv {
     Travis,
     /// The AppVeyor environment, for Windows builds.
     AppVeyor,
+    /// The Azure Pipelines environment, for Linux (including Docker), Windows, and macOS builds.
+    AzurePipelines,
 }
 
 impl CiEnv {
@@ -335,6 +337,8 @@ impl CiEnv {
             CiEnv::Travis
         } else if env::var("APPVEYOR").ok().map_or(false, |e| &*e == "True") {
             CiEnv::AppVeyor
+        } else if env::var("TF_BUILD").ok().map_or(false, |e| &*e == "True") {
+            CiEnv::AzurePipelines
         } else {
             CiEnv::None
         }
diff --git a/src/ci/docker/dist-various-2/Dockerfile b/src/ci/docker/dist-various-2/Dockerfile
index e7d6f201b44..bf449c83f12 100644
--- a/src/ci/docker/dist-various-2/Dockerfile
+++ b/src/ci/docker/dist-various-2/Dockerfile
@@ -32,9 +32,10 @@ COPY dist-various-2/build-cloudabi-toolchain.sh /tmp/
 RUN /tmp/build-cloudabi-toolchain.sh x86_64-unknown-cloudabi
 COPY dist-various-2/build-fuchsia-toolchain.sh /tmp/
 RUN /tmp/build-fuchsia-toolchain.sh
-COPY dist-various-2/build-solaris-toolchain.sh /tmp/
-RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386
-RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc
+# FIXME(#61022) - reenable solaris
+# COPY dist-various-2/build-solaris-toolchain.sh /tmp/
+# RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386
+# RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc
 COPY dist-various-2/build-x86_64-fortanix-unknown-sgx-toolchain.sh /tmp/
 # We pass the commit id of the port of LLVM's libunwind to the build script.
 # Any update to the commit id here, should cause the container image to be re-built from this point on.
@@ -73,10 +74,11 @@ ENV CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS \
 
 ENV TARGETS=x86_64-fuchsia
 ENV TARGETS=$TARGETS,aarch64-fuchsia
-ENV TARGETS=$TARGETS,sparcv9-sun-solaris
 ENV TARGETS=$TARGETS,wasm32-unknown-unknown
 ENV TARGETS=$TARGETS,wasm32-wasi
-ENV TARGETS=$TARGETS,x86_64-sun-solaris
+# FIXME(#61022) - reenable solaris
+# ENV TARGETS=$TARGETS,sparcv9-sun-solaris
+# ENV TARGETS=$TARGETS,x86_64-sun-solaris
 ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnux32
 ENV TARGETS=$TARGETS,x86_64-unknown-cloudabi
 ENV TARGETS=$TARGETS,x86_64-fortanix-unknown-sgx
diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index 98a765e1cec..c9642dbf60c 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -40,9 +40,12 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       docker --version >> $hash_key
       cksum=$(sha512sum $hash_key | \
         awk '{print $1}')
+
       s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
-      url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
-      echo "Attempting to download $s3url"
+      url="https://$SCCACHE_BUCKET.s3.amazonaws.com/docker/$cksum"
+      upload="aws s3 cp - $s3url"
+
+      echo "Attempting to download $url"
       rm -f /tmp/rustci_docker_cache
       set +e
       retry curl -y 30 -Y 10 --connect-timeout 30 -f -L -C - -o /tmp/rustci_docker_cache "$url"
@@ -65,17 +68,17 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       -f "$dockerfile" \
       "$context"
 
-    if [ "$s3url" != "" ]; then
+    if [ "$upload" != "" ]; then
       digest=$(docker inspect rust-ci --format '{{.Id}}')
       echo "Built container $digest"
       if ! grep -q "$digest" <(echo "$loaded_images"); then
-        echo "Uploading finished image to $s3url"
+        echo "Uploading finished image to $url"
         set +e
         docker history -q rust-ci | \
           grep -v missing | \
           xargs docker save | \
           gzip | \
-          aws s3 cp - $s3url
+          $upload
         set -e
       else
         echo "Looks like docker image is the same as before, not uploading"
@@ -87,8 +90,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       echo "$digest" >>"$info"
     fi
 elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
-    if [ -n "$TRAVIS_OS_NAME" ]; then
-        echo Cannot run disabled images on travis!
+    if isCI; then
+        echo Cannot run disabled images on CI!
         exit 1
     fi
     # retry messes with the pipe from tar to docker. Not needed on non-travis
@@ -140,8 +143,11 @@ exec docker \
   --env DEPLOY \
   --env DEPLOY_ALT \
   --env LOCAL_USER_ID=`id -u` \
+  --env CI \
   --env TRAVIS \
   --env TRAVIS_BRANCH \
+  --env TF_BUILD \
+  --env BUILD_SOURCEBRANCHNAME \
   --env TOOLSTATE_REPO_ACCESS_TOKEN \
   --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \
   --volume "$HOME/.cargo:/cargo" \
diff --git a/src/ci/run.sh b/src/ci/run.sh
index c8d9ffd92fa..8ead6e0e4bd 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -23,7 +23,9 @@ fi
 ci_dir=`cd $(dirname $0) && pwd`
 source "$ci_dir/shared.sh"
 
-if [ "$TRAVIS" != "true" ] || [ "$TRAVIS_BRANCH" == "auto" ]; then
+branch_name=$(getCIBranch)
+
+if [ ! isCI ] || [ "$branch_name" = "auto" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
 fi
 
@@ -44,7 +46,7 @@ fi
 # FIXME: need a scheme for changing this `nightly` value to `beta` and `stable`
 #        either automatically or manually.
 export RUST_RELEASE_CHANNEL=nightly
-if [ "$DEPLOY$DEPLOY_ALT" != "" ]; then
+if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --release-channel=$RUST_RELEASE_CHANNEL"
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-static-stdcpp"
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.remap-debuginfo"
@@ -110,7 +112,7 @@ travis_time_finish
 # Display the CPU and memory information. This helps us know why the CI timing
 # is fluctuating.
 travis_fold start log-system-info
-if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+if isOSX; then
     system_profiler SPHardwareDataType || true
     sysctl hw || true
     ncpus=$(sysctl -n hw.ncpu)
@@ -139,3 +141,5 @@ else
   do_make all
   do_make "$RUST_CHECK_TARGET"
 fi
+
+sccache --show-stats || true
diff --git a/src/ci/shared.sh b/src/ci/shared.sh
index 3ba64ad4120..1e667fca587 100644
--- a/src/ci/shared.sh
+++ b/src/ci/shared.sh
@@ -24,6 +24,22 @@ function retry {
   done
 }
 
+function isCI {
+  [ "$CI" = "true" ] || [ "$TRAVIS" = "true" ] || [ "$TF_BUILD" = "True" ]
+}
+
+function isOSX {
+  [ "$TRAVIS_OS_NAME" = "osx" ] || [ "$AGENT_OS" = "Darwin" ]
+}
+
+function getCIBranch {
+  if [ "$TRAVIS" = "true" ]; then
+    echo "$TRAVIS_BRANCH"
+  else
+    echo "$BUILD_SOURCEBRANCHNAME"
+  fi;
+}
+
 if ! declare -F travis_fold; then
   if [ "${TRAVIS-false}" = 'true' ]; then
     # This is a trimmed down copy of
diff --git a/src/tools/cargotest/main.rs b/src/tools/cargotest/main.rs
index 3126b44f0d6..14035eedbb4 100644
--- a/src/tools/cargotest/main.rs
+++ b/src/tools/cargotest/main.rs
@@ -61,7 +61,7 @@ const TEST_REPOS: &'static [Test] = &[
     Test {
         name: "webrender",
         repo: "https://github.com/servo/webrender",
-        sha: "57250b2b8fa63934f80e5376a29f7dcb3f759ad6",
+        sha: "cdadd068f4c7218bd983d856981d561e605270ab",
         lock: None,
         packages: &[],
     },