From 699376ada01dc6223e5365e79ae11c79502fd2ef Mon Sep 17 00:00:00 2001
From: John Erickson <jerick@microsoft.com>
Date: Mon, 6 May 2019 11:15:52 -0700
Subject: Add Azure Pipelines

---
 src/bootstrap/README.md                      |  3 +-
 src/bootstrap/mk/Makefile.in                 | 74 ++++++++++++++++++++++++++--
 src/bootstrap/native.rs                      |  9 +++-
 src/bootstrap/util.rs                        |  4 ++
 src/ci/docker/i686-gnu-1/Dockerfile          | 22 +++++++++
 src/ci/docker/i686-gnu-2/Dockerfile          | 21 ++++++++
 src/ci/docker/i686-gnu/Dockerfile            | 27 ----------
 src/ci/docker/run.sh                         | 39 ++++++++++++---
 src/ci/docker/scripts/sccache.sh             |  3 +-
 src/ci/docker/x86_64-gnu-tools/checktools.sh |  2 +-
 src/ci/run.sh                                | 46 +++++++++--------
 src/ci/shared.sh                             | 16 ++++++
 src/test/run-pass/fds-are-cloexec.rs         |  1 +
 13 files changed, 202 insertions(+), 65 deletions(-)
 create mode 100644 src/ci/docker/i686-gnu-1/Dockerfile
 create mode 100644 src/ci/docker/i686-gnu-2/Dockerfile
 delete mode 100644 src/ci/docker/i686-gnu/Dockerfile

(limited to 'src')

diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md
index 1e01d68fb3a..c47c0f765f0 100644
--- a/src/bootstrap/README.md
+++ b/src/bootstrap/README.md
@@ -39,8 +39,7 @@ The script accepts commands, flags, and arguments to determine what to do:
   ```
 
   If files are dirty that would normally be rebuilt from stage 0, that can be
-  overridden using `--keep-stage 0`. Using `--keep-stage n` will skip all steps
-  that belong to stage n or earlier:
+  overridden using `--keep-stage 0`.
 
   ```
   # keep old build products for stage 0 and build stage 1
diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in
index 07be27c2f5a..29b0cdaef63 100644
--- a/src/bootstrap/mk/Makefile.in
+++ b/src/bootstrap/mk/Makefile.in
@@ -14,9 +14,10 @@ endif
 
 BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py
 
-all:
+build:
 	$(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS)
-	$(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS)
+
+all: build doc
 
 help:
 	$(Q)echo 'Welcome to the rustbuild build system!'
@@ -80,10 +81,75 @@ TESTS_IN_2 := \
 	src/test/run-pass-fulldeps \
 	src/tools/linkchecker
 
-appveyor-subset-1:
+MIN_TEST := \
+	src/test/debuginfo
+
+ci-subset-1:
 	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %)
-appveyor-subset-2:
+ci-subset-2:
 	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2)
 
+ci-test-prepare: 
+	$(Q)$(BOOTSTRAP) test $(MIN_TEST)
+
+ci-resume-subset-1:
+	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %)
+ci-resume-subset-2:
+	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2)
+
+TESTS_IN_B := \
+	src/tools/linkchecker
+
+TESTS_IN_C := \
+	src/test/run-pass \
+	src/test/run-pass-fulldeps
+
+TESTS_IN_D := \
+	src/test/compile-fail \
+	src/test/rustdoc \
+	src/test/pretty
+
+TESTS_IN_E := \
+	src/test/ui 
+
+TESTS_IN_F := \
+	src/test/run-fail \
+	src/liballoc \
+	src/libcore
+
+TESTS_IN_G := \
+	src/tools/rustdoc \
+	src/test/rustdoc-js-std \
+	src/test/run-make-fulldeps \
+	src/libstd
+
+TESTS_IN_H := \
+	src/librustc_driver
+
+ci-resume-subset-A:
+	$(Q)$(BOOTSTRAP) test \
+		$(KEEP_STAGE) \
+		$(TESTS_IN_B:%=--exclude %) \
+		$(TESTS_IN_C:%=--exclude %) \
+		$(TESTS_IN_D:%=--exclude %) \
+		$(TESTS_IN_E:%=--exclude %) \
+		$(TESTS_IN_F:%=--exclude %) \
+		$(TESTS_IN_G:%=--exclude %) \
+		$(TESTS_IN_H:%=--exclude %) \
+		$(MIN_TEST:%=--exclude %) 
+ci-resume-subset-B:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_B)
+ci-resume-subset-C:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_C)
+ci-resume-subset-D:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_D)
+ci-resume-subset-E:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_E)
+ci-resume-subset-F:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_F)
+ci-resume-subset-G:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_G)
+ci-resume-subset-H:
+	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_H)
 
 .PHONY: dist
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index 5777331b9bf..62a8b7aad03 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -81,6 +81,13 @@ impl Step for Llvm {
             (info, "src/llvm-project/llvm", builder.llvm_out(target), dir.join("bin"))
         };
 
+        let build_llvm_config = llvm_config_ret_dir
+            .join(exe("llvm-config", &*builder.config.build));
+
+        if env::var_os("SKIP_LLVM_BUILD").is_some() {
+            return build_llvm_config
+        }
+
         if !llvm_info.is_git() {
             println!(
                 "git could not determine the LLVM submodule commit hash. \
@@ -88,8 +95,6 @@ impl Step for Llvm {
             );
         }
 
-        let build_llvm_config = llvm_config_ret_dir
-            .join(exe("llvm-config", &*builder.config.build));
         let done_stamp = out_dir.join("llvm-finished-building");
 
         if let Some(llvm_commit) = llvm_info.sha() {
diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs
index a162c65672f..e79700d92d8 100644
--- a/src/bootstrap/util.rs
+++ b/src/bootstrap/util.rs
@@ -326,6 +326,8 @@ pub enum CiEnv {
     Travis,
     /// The AppVeyor environment, for Windows builds.
     AppVeyor,
+    /// The Azure Pipelines environment, for Linux (including Docker), Window, and macOS builds.
+    AzurePipelines,
 }
 
 impl CiEnv {
@@ -335,6 +337,8 @@ impl CiEnv {
             CiEnv::Travis
         } else if env::var("APPVEYOR").ok().map_or(false, |e| &*e == "True") {
             CiEnv::AppVeyor
+        } else if env::var("TF_BUILD").ok().map_or(false, |e| &*e == "True") {
+            CiEnv::AzurePipelines
         } else {
             CiEnv::None
         }
diff --git a/src/ci/docker/i686-gnu-1/Dockerfile b/src/ci/docker/i686-gnu-1/Dockerfile
new file mode 100644
index 00000000000..1a04b8af4c1
--- /dev/null
+++ b/src/ci/docker/i686-gnu-1/Dockerfile
@@ -0,0 +1,22 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++-multilib \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  sudo \
+  gdb \
+  xz-utils
+
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
+ENV SCRIPT make ci-subset-1
+
diff --git a/src/ci/docker/i686-gnu-2/Dockerfile b/src/ci/docker/i686-gnu-2/Dockerfile
new file mode 100644
index 00000000000..b1a0ea6f0e9
--- /dev/null
+++ b/src/ci/docker/i686-gnu-2/Dockerfile
@@ -0,0 +1,21 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++-multilib \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  sudo \
+  gdb \
+  xz-utils
+
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
+ENV SCRIPT make ci-subset-2
diff --git a/src/ci/docker/i686-gnu/Dockerfile b/src/ci/docker/i686-gnu/Dockerfile
deleted file mode 100644
index 17441ddb454..00000000000
--- a/src/ci/docker/i686-gnu/Dockerfile
+++ /dev/null
@@ -1,27 +0,0 @@
-FROM ubuntu:16.04
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  g++-multilib \
-  make \
-  file \
-  curl \
-  ca-certificates \
-  python2.7 \
-  git \
-  cmake \
-  sudo \
-  gdb \
-  xz-utils
-
-
-COPY scripts/sccache.sh /scripts/
-RUN sh /scripts/sccache.sh
-
-ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
-# Exclude some tests that are unlikely to be platform specific, to speed up
-# this slow job.
-ENV SCRIPT python2.7 ../x.py test \
-  --exclude src/bootstrap \
-  --exclude src/test/rustdoc-js \
-  --exclude src/tools/error_index_generator \
-  --exclude src/tools/linkchecker
diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index 98a765e1cec..cd913e30d11 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# ignore-tidy-linelength
+
 set -e
 
 export MSYS_NO_PATHCONV=1
@@ -40,9 +42,24 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       docker --version >> $hash_key
       cksum=$(sha512sum $hash_key | \
         awk '{print $1}')
-      s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
-      url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
-      echo "Attempting to download $s3url"
+
+      if [ "$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT" != "" ]; then
+        # install azcopy
+        echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod/ xenial main" > azure.list
+        sudo cp ./azure.list /etc/apt/sources.list.d/
+        sudo apt-key adv --keyserver packages.microsoft.com --recv-keys EB3E94ADBE1229CF
+        sudo apt-get update
+        sudo apt-get install azcopy
+
+        url="https://$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$DOCKER_LAYER_CACHE_AZURE_STORAGE_CONTAINER/$cksum"
+        upload="azcopy --quiet --destination $url --dest-key $DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY"
+      else
+        s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
+        url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
+        upload="aws s3 cp - $s3url"
+      fi
+
+      echo "Attempting to download $url"
       rm -f /tmp/rustci_docker_cache
       set +e
       retry curl -y 30 -Y 10 --connect-timeout 30 -f -L -C - -o /tmp/rustci_docker_cache "$url"
@@ -65,17 +82,17 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       -f "$dockerfile" \
       "$context"
 
-    if [ "$s3url" != "" ]; then
+    if [ "$upload" != "" ]; then
       digest=$(docker inspect rust-ci --format '{{.Id}}')
       echo "Built container $digest"
       if ! grep -q "$digest" <(echo "$loaded_images"); then
-        echo "Uploading finished image to $s3url"
+        echo "Uploading finished image to $url"
         set +e
         docker history -q rust-ci | \
           grep -v missing | \
           xargs docker save | \
           gzip | \
-          aws s3 cp - $s3url
+          $upload
         set -e
       else
         echo "Looks like docker image is the same as before, not uploading"
@@ -87,8 +104,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       echo "$digest" >>"$info"
     fi
 elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
-    if [ -n "$TRAVIS_OS_NAME" ]; then
-        echo Cannot run disabled images on travis!
+    if isCI; then
+        echo Cannot run disabled images on CI!
         exit 1
     fi
     # retry messes with the pipe from tar to docker. Not needed on non-travis
@@ -117,6 +134,9 @@ if [ "$SCCACHE_BUCKET" != "" ]; then
     args="$args --env SCCACHE_REGION"
     args="$args --env AWS_ACCESS_KEY_ID"
     args="$args --env AWS_SECRET_ACCESS_KEY"
+elif [ "$SCCACHE_AZURE_CONNECTION_STRING" != "" ]; then
+    args="$args --env SCCACHE_AZURE_CONNECTION_STRING"
+    args="$args --env SCCACHE_AZURE_BLOB_CONTAINER"
 else
     mkdir -p $HOME/.cache/sccache
     args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache"
@@ -140,8 +160,11 @@ exec docker \
   --env DEPLOY \
   --env DEPLOY_ALT \
   --env LOCAL_USER_ID=`id -u` \
+  --env CI \
   --env TRAVIS \
   --env TRAVIS_BRANCH \
+  --env TF_BUILD \
+  --env BUILD_SOURCEBRANCHNAME \
   --env TOOLSTATE_REPO_ACCESS_TOKEN \
   --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \
   --volume "$HOME/.cargo:/cargo" \
diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh
index 4c03419894e..d6f2f4fadb2 100644
--- a/src/ci/docker/scripts/sccache.sh
+++ b/src/ci/docker/scripts/sccache.sh
@@ -1,6 +1,7 @@
 set -ex
 
+echo Use sccache built with Azure Storage support
 curl -fo /usr/local/bin/sccache \
-  https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl
+  https://rustlangtools.blob.core.windows.net/public/stable-x86_64-unknown-linux-musl.sccache
 
 chmod +x /usr/local/bin/sccache
diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh
index af0198705a2..c7e64cd8c3e 100755
--- a/src/ci/docker/x86_64-gnu-tools/checktools.sh
+++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh
@@ -6,7 +6,7 @@ X_PY="$1"
 TOOLSTATE_FILE="$(realpath $2)"
 OS="$3"
 COMMIT="$(git rev-parse HEAD)"
-CHANGED_FILES="$(git diff --name-status HEAD HEAD^)"
+CHANGED_FILES="$(git diff --name-status HEAD HEAD^ || echo)"
 SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))"
 # ^ Number of days after the last promotion of beta.
 #   Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens.
diff --git a/src/ci/run.sh b/src/ci/run.sh
index 42d0d7db596..08ede122d65 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -23,7 +23,9 @@ fi
 ci_dir=`cd $(dirname $0) && pwd`
 source "$ci_dir/shared.sh"
 
-if [ "$TRAVIS" != "true" ] || [ "$TRAVIS_BRANCH" == "auto" ]; then
+branch_name=$(getCIBranch)
+
+if [ ! isCI ] || [ "$branch_name" = "auto" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
 fi
 
@@ -44,7 +46,7 @@ fi
 # FIXME: need a scheme for changing this `nightly` value to `beta` and `stable`
 #        either automatically or manually.
 export RUST_RELEASE_CHANNEL=nightly
-if [ "$DEPLOY$DEPLOY_ALT" != "" ]; then
+if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --release-channel=$RUST_RELEASE_CHANNEL"
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-static-stdcpp"
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.remap-debuginfo"
@@ -88,28 +90,30 @@ if [ "$RUN_CHECK_WITH_PARALLEL_QUERIES" != "" ]; then
   rm -rf build
 fi
 
-travis_fold start configure
-travis_time_start
-$SRC/configure $RUST_CONFIGURE_ARGS
-travis_fold end configure
-travis_time_finish
-
-travis_fold start make-prepare
-travis_time_start
-retry make prepare
-travis_fold end make-prepare
-travis_time_finish
-
-travis_fold start check-bootstrap
-travis_time_start
-make check-bootstrap
-travis_fold end check-bootstrap
-travis_time_finish
+if [ "$SKIP_PREP" != "1" ]; then
+  travis_fold start configure
+  travis_time_start
+  $SRC/configure $RUST_CONFIGURE_ARGS
+  travis_fold end configure
+  travis_time_finish
+
+  travis_fold start make-prepare
+  travis_time_start
+  retry make prepare
+  travis_fold end make-prepare
+  travis_time_finish
+
+  travis_fold start check-bootstrap
+  travis_time_start
+  make check-bootstrap
+  travis_fold end check-bootstrap
+  travis_time_finish
+fi
 
 # Display the CPU and memory information. This helps us know why the CI timing
 # is fluctuating.
 travis_fold start log-system-info
-if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+if isOSX; then
     system_profiler SPHardwareDataType || true
     sysctl hw || true
     ncpus=$(sysctl -n hw.ncpu)
@@ -138,3 +142,5 @@ else
   do_make all
   do_make "$RUST_CHECK_TARGET"
 fi
+
+sccache --show-stats
diff --git a/src/ci/shared.sh b/src/ci/shared.sh
index 3ba64ad4120..1e667fca587 100644
--- a/src/ci/shared.sh
+++ b/src/ci/shared.sh
@@ -24,6 +24,22 @@ function retry {
   done
 }
 
+function isCI {
+  [ "$CI" = "true" ] || [ "$TRAVIS" = "true" ] || [ "$TF_BUILD" = "True" ]
+}
+
+function isOSX {
+  [ "$TRAVIS_OS_NAME" = "osx" ] || [ "$AGENT_OS" = "Darwin" ]
+}
+
+function getCIBranch {
+  if [ "$TRAVIS" = "true" ]; then
+    echo "$TRAVIS_BRANCH"
+  else
+    echo "$BUILD_SOURCEBRANCHNAME"
+  fi;
+}
+
 if ! declare -F travis_fold; then
   if [ "${TRAVIS-false}" = 'true' ]; then
     # This is a trimmed down copy of
diff --git a/src/test/run-pass/fds-are-cloexec.rs b/src/test/run-pass/fds-are-cloexec.rs
index 3bc0ceb5cf2..bcfc3245db3 100644
--- a/src/test/run-pass/fds-are-cloexec.rs
+++ b/src/test/run-pass/fds-are-cloexec.rs
@@ -3,6 +3,7 @@
 // ignore-cloudabi no processes
 // ignore-emscripten no processes
 // ignore-haiku
+// ignore-macos
 
 #![feature(rustc_private)]
 
-- 
cgit 1.4.1-3-g733a5


From 869ddd8d0019683c007956bae17fc07cd30f39f7 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Fri, 10 May 2019 10:17:52 +0200
Subject: ci: still use aws for caches

---
 .azure-pipelines/auto.fanout.yml      |  4 ++--
 .azure-pipelines/auto.yml             |  5 -----
 .azure-pipelines/steps/macos.yml      | 10 +++++-----
 .azure-pipelines/steps/run-script.yml |  6 +++---
 .azure-pipelines/steps/windows.yml    |  6 +++---
 src/ci/docker/run.sh                  | 23 +++--------------------
 src/ci/docker/scripts/sccache.sh      |  3 +--
 7 files changed, 17 insertions(+), 40 deletions(-)

(limited to 'src')

diff --git a/.azure-pipelines/auto.fanout.yml b/.azure-pipelines/auto.fanout.yml
index e12836257d9..def19a87076 100644
--- a/.azure-pipelines/auto.fanout.yml
+++ b/.azure-pipelines/auto.fanout.yml
@@ -160,7 +160,7 @@ jobs:
 
   - script: |
       REM md sccache
-      REM powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe"
+      REM powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc"
       echo ##vso[task.prependpath]%CD%\sccache
     displayName: Download and install sccache
 
@@ -392,7 +392,7 @@ jobs:
       mkdir -p $HOME/rustsrc    
       echo "##vso[task.setvariable variable=PATH;]$PATH"
 
-      curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache
+      curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin
       chmod +x /usr/local/bin/sccache
 
       curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin
diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml
index 5f4c155ce98..189536cff5f 100644
--- a/.azure-pipelines/auto.yml
+++ b/.azure-pipelines/auto.yml
@@ -1,11 +1,6 @@
 #
 # Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows.
 # 
-# Notes:
-# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled
-# and with openssl statically-linked for Linux.
-# Here's the build (with a backpointer to source) of where the bits came from:
-# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275
 
 pr: none
 # trigger:
diff --git a/.azure-pipelines/steps/macos.yml b/.azure-pipelines/steps/macos.yml
index a217df6f308..e88b5613e46 100644
--- a/.azure-pipelines/steps/macos.yml
+++ b/.azure-pipelines/steps/macos.yml
@@ -3,21 +3,21 @@ steps:
 
 - bash: |
     export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/
-    mkdir -p $HOME/rustsrc    
+    mkdir -p $HOME/rustsrc
     echo "##vso[task.setvariable variable=PATH;]$PATH"
 
-    curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache
+    curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin
     chmod +x /usr/local/bin/sccache
 
     curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin
     chmod +x /usr/local/bin/stamp
-    
+
     export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang
     echo "##vso[task.setvariable variable=CC]$CC"
-    
+
     export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++
     echo "##vso[task.setvariable variable=CXX]$CXX"
-    
+
     echo "##vso[task.setvariable variable=AR]ar"
   displayName: Prep
 
diff --git a/.azure-pipelines/steps/run-script.yml b/.azure-pipelines/steps/run-script.yml
index 4ab3e2e06d3..cb8e8a629a7 100644
--- a/.azure-pipelines/steps/run-script.yml
+++ b/.azure-pipelines/steps/run-script.yml
@@ -5,7 +5,7 @@ steps:
     date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true)
   displayName: Log time information (before)
 
-- bash: |    
+- bash: |
     which sccache
     stamp sh -x -c "$RUN_SCRIPT"
   env:
@@ -15,8 +15,8 @@ steps:
 
     # Explicitly decrypt secret variables
     # See https://docs.microsoft.com/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables
-    SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET)
-    DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET)
+    AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID)
+    AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY)
   displayName: Run script
 
 - bash: |
diff --git a/.azure-pipelines/steps/windows.yml b/.azure-pipelines/steps/windows.yml
index 5dcba664bee..e3c06d543a9 100644
--- a/.azure-pipelines/steps/windows.yml
+++ b/.azure-pipelines/steps/windows.yml
@@ -106,7 +106,7 @@ steps:
 
 - script: |
     md sccache
-    powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe"
+    powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc"
     echo ##vso[task.prependpath]%CD%\sccache
   displayName: Download and install sccache
 
@@ -155,6 +155,6 @@ steps:
 
     # explicitly decrypt secret variables
     # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch
-    SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET)
-    DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET)
+    AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID)
+    AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY)
   displayName: Run script
diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index cd913e30d11..0e1485601e9 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 
-# ignore-tidy-linelength
-
 set -e
 
 export MSYS_NO_PATHCONV=1
@@ -43,21 +41,9 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       cksum=$(sha512sum $hash_key | \
         awk '{print $1}')
 
-      if [ "$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT" != "" ]; then
-        # install azcopy
-        echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod/ xenial main" > azure.list
-        sudo cp ./azure.list /etc/apt/sources.list.d/
-        sudo apt-key adv --keyserver packages.microsoft.com --recv-keys EB3E94ADBE1229CF
-        sudo apt-get update
-        sudo apt-get install azcopy
-
-        url="https://$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$DOCKER_LAYER_CACHE_AZURE_STORAGE_CONTAINER/$cksum"
-        upload="azcopy --quiet --destination $url --dest-key $DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY"
-      else
-        s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
-        url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
-        upload="aws s3 cp - $s3url"
-      fi
+      s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
+      url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
+      upload="aws s3 cp - $s3url"
 
       echo "Attempting to download $url"
       rm -f /tmp/rustci_docker_cache
@@ -134,9 +120,6 @@ if [ "$SCCACHE_BUCKET" != "" ]; then
     args="$args --env SCCACHE_REGION"
     args="$args --env AWS_ACCESS_KEY_ID"
     args="$args --env AWS_SECRET_ACCESS_KEY"
-elif [ "$SCCACHE_AZURE_CONNECTION_STRING" != "" ]; then
-    args="$args --env SCCACHE_AZURE_CONNECTION_STRING"
-    args="$args --env SCCACHE_AZURE_BLOB_CONTAINER"
 else
     mkdir -p $HOME/.cache/sccache
     args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache"
diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh
index d6f2f4fadb2..4c03419894e 100644
--- a/src/ci/docker/scripts/sccache.sh
+++ b/src/ci/docker/scripts/sccache.sh
@@ -1,7 +1,6 @@
 set -ex
 
-echo Use sccache built with Azure Storage support
 curl -fo /usr/local/bin/sccache \
-  https://rustlangtools.blob.core.windows.net/public/stable-x86_64-unknown-linux-musl.sccache
+  https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl
 
 chmod +x /usr/local/bin/sccache
-- 
cgit 1.4.1-3-g733a5


From 7dfd8ceb4fe5fceb426ab6868312cc0890e28837 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Sun, 12 May 2019 17:55:48 +0200
Subject: ci: remove fanout from the azure pipelines config

---
 .azure-pipelines/auto.fanout.yml | 521 ---------------------------------------
 .azure-pipelines/auto.yml        |   4 +-
 src/bootstrap/README.md          |   3 +-
 src/bootstrap/mk/Makefile.in     |  70 +-----
 src/bootstrap/native.rs          |   9 +-
 src/ci/run.sh                    |  36 ++-
 6 files changed, 25 insertions(+), 618 deletions(-)
 delete mode 100644 .azure-pipelines/auto.fanout.yml

(limited to 'src')

diff --git a/.azure-pipelines/auto.fanout.yml b/.azure-pipelines/auto.fanout.yml
deleted file mode 100644
index def19a87076..00000000000
--- a/.azure-pipelines/auto.fanout.yml
+++ /dev/null
@@ -1,521 +0,0 @@
-#
-# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows.
-# 
-# Notes:
-# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled
-# and with openssl statically-linked for Linux.
-# Here's the build (with a backpointer to source) of where the bits came from:
-# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275
-
-name: fanout 
-pr: none
-trigger:
-- auto
-
-variables:
-- group: caching
-
-jobs:
-# WINDOWS JOBS
-- job: Windows
-  timeoutInMinutes: 180
-  variables:
-    TAR_COMMAND: tar
-  pool:
-    vmImage: 'vs2017-win2016'
-  steps:
-  - template: steps/windows.yml
-  strategy:
-    matrix:
-      # 32/64 bit MSVC tests
-      x86_64-msvc-1:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        SCRIPT: make ci-subset-1
-      x86_64-msvc-2:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        SCRIPT: make ci-subset-2
-
-- job: Windows_build  
-  timeoutInMinutes: 180
-  variables:
-    TAR_COMMAND: tar
-  pool:
-    vmImage: 'vs2017-win2016'
-  steps:
-  - template: steps/windows.yml
-  - script: |
-      echo on
-      echo "##vso[task.setvariable variable=TAR_PATH]%TEMP%\drop.tar"
-  - bash: |
-      set -x
-      $TAR_COMMAND --dereference --exclude=.git --exclude=citools $SUBMODULES_EXCLUDES -cf $TEMP/drop.tar .
-      $TAR_COMMAND --append --file=$TEMP/drop.tar citools
-  - task: PublishPipelineArtifact@0
-    inputs:
-      artifactName: $(System.JobDisplayName)
-      targetPath: $(TAR_PATH)
-  strategy:
-    matrix:
-      # 32/64 bit MSVC tests
-      x86_64-msvc-build:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        SCRIPT: make ci-test-prepare
-
-- job: Windows_test
-  dependsOn: Windows_build
-  timeoutInMinutes: 180
-  variables:
-    TAR_COMMAND: tar
-    SKIP_PREP: 1
-    KEEP_STAGE: '--keep-stage 0 --keep-stage 1'
-    SKIP_LLVM_BUILD: 1
-  pool:
-    vmImage: 'vs2017-win2016'
-  steps:
-  - checkout: self
-    fetchDepth: 2
-  - script: |
-      set MSYS_PATH=%CD%\citools\msys64
-      set PATH=%MSYS_PATH%\usr\bin;%PATH%
-      where rev
-      rev --help
-      where make
-      echo ##vso[task.prependpath]%MSYS_PATH%\usr\bin
-    displayName: Install msys2
-
-  # # If we need to download a custom MinGW, do so here and set the path
-  # # appropriately.
-  # #
-  # # Note that this *also* means that we're not using what is typically
-  # # /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where
-  # # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we
-  # # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe`
-  # # file exists in there (which it doesn't by default).
-  # - script: |
-  #     powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%"
-  #     7z x -y %MINGW_ARCHIVE% > nul
-  #     echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin
-  #   condition: and(succeeded(), ne(variables['MINGW_URL'],''))
-  #   displayName: Download custom MinGW
-
-  # If we're compiling for MSVC then we, like most other distribution builders,
-  # switch to clang as the compiler. This'll allow us eventually to enable LTO
-  # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think
-  # clang has an output mode compatible with MinGW that we need. If it does we
-  # should switch to clang for MinGW as well!
-  #
-  # Note that the LLVM installer is an NSIS installer
-  #
-  # Original downloaded here came from
-  # http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe
-  - script: |
-      # powershell -Command "iwr -outf LLVM-7.0.0-win64.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/LLVM-7.0.0-win64.exe"
-      # .\LLVM-7.0.0-win64.exe /S /NCRC /D=C:\clang-rust
-      set CLANG_DIR=%CD%\citools\clang-rust
-      set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe
-      echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS%
-    condition: and(succeeded(), eq(variables['MINGW_URL'],''))
-    displayName: Download clang
-
-  # # Here we do a pretty heinous thing which is to mangle the MinGW installation
-  # # we just had above. Currently, as of this writing, we're using MinGW-w64
-  # # builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it appears to
-  # # be the first version which contains a fix for #40546, builds randomly
-  # # failing during LLVM due to ar.exe/ranlib.exe failures.
-  # #
-  # # Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds
-  # # to contain a regression in gdb (#40184). As a result if we were to use the
-  # # gdb provided (7.11.1) then we would fail all debuginfo tests.
-  # #
-  # # In order to fix spurious failures (pretty high priority) we use 6.3.0. To
-  # # avoid disabling gdb tests we download an *old* version of gdb, specifically
-  # # that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb
-  # # with the 6.2.0 gdb to get tests passing.
-  # #
-  # # Note that we don't literally overwrite the gdb.exe binary because it appears
-  # # to just use gdborig.exe, so that's the binary we deal with instead.
-  # - script: |
-  #     echo ON
-  #     powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe"
-  #     mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe
-  #   condition: and(succeeded(), ne(variables['MINGW_URL'],''))
-  #   displayName: Override with 6.3.0 gdb with 6.2.0 gdb
-
-  # Otherwise pull in the MinGW installed on appveyor
-  - script: |
-      echo Find mingw
-      set PATH | findstr /i msys
-      set PATH | findstr /i mingw
-      echo ##vso[task.prependpath]C:\msys64\mingw%MSYS_BITS%\bin
-    condition: and(succeeded(), eq(variables['MINGW_URL'],''))
-    displayName: Add MinGW to path
-
-  - script: |
-      copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe
-      echo ##vso[task.prependpath]C:\Python27amd64
-    displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes
-
-  - script: |
-      REM md sccache
-      REM powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc"
-      echo ##vso[task.prependpath]%CD%\sccache
-    displayName: Download and install sccache
-
-  # Note that this is originally from the github releases patch of Ninja
-  - script: |
-      REM md ninja
-      REM powershell -Command "iwr -outf 2017-03-15-ninja-win.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-15-ninja-win.zip"
-      REM 7z x -oninja 2017-03-15-ninja-win.zip
-      REM del 2017-03-15-ninja-win.zip
-      set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja
-      echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS%
-      echo ##vso[task.prependpath]%CD%\ninja
-    displayName: Download and install ninja
-  
-  - script: |
-      IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" (
-        IF NOT EXIST build (
-          mkdir c:\MORE_SPACE
-          mklink /J build c:\MORE_SPACE
-        )
-      )
-  - script: |
-      echo on
-      echo ##vso[task.setvariable variable=TAR_FOLDER_PATH]%TEMP%
-  - template: steps/show-environment-variables.yml
-  - task: DownloadPipelineArtifact@1
-    inputs:
-      artifactName: $(ARTIFACT_NAME)
-      downloadPath: $(TAR_FOLDER_PATH)
-  # - task: DownloadPipelineArtifact@1
-  #   inputs:
-  #     artifactName: $(ARTIFACT_NAME)
-  #     downloadPath: $(TAR_FOLDER_PATH)
-  #     buildType: specific
-  #     project: rust
-  #     pipeline: 8
-  #     buildVersionToDownload: specific
-  #     buildId: 200
-  #     # https://dev.azure.com/rust-lang-azure/rust/_build/results?buildId=145
-
-  # - template: steps/macos.yml 
-  - bash: $TAR_COMMAND -tvf $TEMP/drop.tar
-  - bash: $TAR_COMMAND --touch -xf $TEMP/drop.tar
-  - bash: rm $TEMP/drop.tar
-  - script: |
-      REM mkdir handle
-      REM powershell -Command "iwr -outf 2017-05-15-Handle.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-05-15-Handle.zip"
-      REM 7z x -ohandle 2017-05-15-Handle.zip
-      REM del 2017-05-15-Handle.zip
-      set PATH=%PATH%;%CD%\handle
-      handle.exe -accepteula -help
-      echo ##vso[task.setvariable variable=PATH]%PATH%
-    displayName: Help debug handle issues
-  - script: |
-      REM echo force the specific VS version https://github.com/johnterickson/rust/issues/8
-      IF "%VCVARS_BAT%" NEQ "" ( 
-        CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\%VCVARS_BAT%"
-      )
-
-      where sccache
-      where rev
-      set | findstr /v SCCACHE_AZURE_CONNECTION_STRING
-
-      if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc
-      
-      sh src/ci/run.sh
-    env:      
-      CI: true
-      CI_JOB_NAME: $(System.JobDisplayName)
-      SRC: .
-      NO_CCACHE: 1
-
-      # explicitly decrypt secret variables
-      # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch
-      SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET)
-      DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET)
-    displayName: Run script
-  
-  strategy:
-    matrix:
-      # 32/64 bit MSVC tests
-      x86_64-msvc-A:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        ARTIFACT_NAME: x86_64-msvc-build
-        SCRIPT: make ci-resume-subset-A
-      x86_64-msvc-B:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        ARTIFACT_NAME: x86_64-msvc-build
-        SCRIPT: make ci-resume-subset-B
-      x86_64-msvc-C:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        ARTIFACT_NAME: x86_64-msvc-build
-        SCRIPT: make ci-resume-subset-C
-      x86_64-msvc-D:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        ARTIFACT_NAME: x86_64-msvc-build
-        SCRIPT: make ci-resume-subset-D
-      x86_64-msvc-E:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        ARTIFACT_NAME: x86_64-msvc-build
-        SCRIPT: make ci-resume-subset-E
-      x86_64-msvc-F:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        ARTIFACT_NAME: x86_64-msvc-build
-        SCRIPT: make ci-resume-subset-F
-      x86_64-msvc-G:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        ARTIFACT_NAME: x86_64-msvc-build
-        SCRIPT: make ci-resume-subset-G
-      x86_64-msvc-H:
-        MSYS_BITS: 64
-        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-        ARTIFACT_NAME: x86_64-msvc-build
-        SCRIPT: make ci-resume-subset-H
-
-# macOS JOBS
-- job: macOS
-  timeoutInMinutes: 180
-  variables:
-    TAR_COMMAND: gtar
-  pool:
-    vmImage: macos-10.13
-  steps:
-  - checkout: self
-    fetchDepth: 2
-  - template: steps/macos.yml  
-  strategy:
-    matrix:
-      x86_64-apple:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-
-- job: macOS_build
-  timeoutInMinutes: 180
-  variables:
-    TAR_COMMAND: gtar
-  pool:
-    vmImage: macos-10.13
-  steps:
-  - checkout: self
-    fetchDepth: 2
-  - template: steps/macos.yml
-  - script: |
-      set -x
-
-      git submodule
-      export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/')
-      echo $SUBMODULES_EXCLUDES
-
-      $TAR_COMMAND --exclude=.git $SUBMODULES_EXCLUDES -cf /tmp/drop.tar .
-  - task: PublishPipelineArtifact@0
-    inputs:
-      artifactName: $(System.JobDisplayName)
-      targetPath: '/tmp/drop.tar'
-
-  strategy:
-    matrix:
-      # macOS builders. These are placed near the beginning because they are very
-      # slow to run.
-
-      # OSX builders running tests, these run the full test suite.
-      # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some
-      # runners that run `//ignore-debug` tests.
-      #
-      # Note that the compiler is compiled to target 10.8 here because the Xcode
-      # version that we're using, 8.2, cannot compile LLVM for OSX 10.7.
-      x86_64-apple-build:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-        SCRIPT: make ci-test-prepare
-
-- job: macOS_test
-  dependsOn: macOS_build
-  timeoutInMinutes: 180
-  variables:
-    TAR_COMMAND: gtar
-    SKIP_PREP: 1
-    KEEP_STAGE: '--keep-stage 0 --keep-stage 1'
-    SKIP_LLVM_BUILD: 1
-  pool:
-    vmImage: macos-10.13
-  steps:
-  - checkout: self
-    fetchDepth: 2
-  - bash: brew install gnu-tar
-    displayName: install a tar that works well
-
-  - template: steps/show-disk-usage.yml
-
-  - task: DownloadPipelineArtifact@1
-    inputs:
-      artifactName: $(ARTIFACT_NAME)
-      downloadPath: $(System.DefaultWorkingDirectory)
-  # - task: DownloadPipelineArtifact@1
-  #   inputs:
-  #     artifactName: $(ARTIFACT_NAME)
-  #     downloadPath: $(System.DefaultWorkingDirectory)
-  #     buildType: specific
-  #     project: rust
-  #     pipeline: 8
-  #     buildVersionToDownload: specific
-  #     buildId: 200
-
-  # - template: steps/macos.yml 
-  - template: steps/show-disk-usage.yml
-  - bash: $TAR_COMMAND -tvf ./drop.tar
-  - bash: $TAR_COMMAND -xf ./drop.tar
-  - bash: rm ./drop.tar
-  - template: steps/show-disk-usage.yml
-  - bash: |
-      export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/
-      mkdir -p $HOME/rustsrc    
-      echo "##vso[task.setvariable variable=PATH;]$PATH"
-
-      curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin
-      chmod +x /usr/local/bin/sccache
-
-      curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin
-      chmod +x /usr/local/bin/stamp
-      
-      export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang
-      echo "##vso[task.setvariable variable=CC]$CC"
-      
-      export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++
-      echo "##vso[task.setvariable variable=CXX]$CXX"
-      
-      echo "##vso[task.setvariable variable=AR]ar"
-    displayName: Prep
-
-  - bash: brew install gnu-tar
-    displayName: install a tar that works well
-
-  - bash: |
-      brew update
-      brew install xz
-      brew install swig
-    condition: and(succeeded(), eq(variables['RUST_CHECK_TARGET'],'dist'))
-    displayName: Install xz and swigw
-
-  - bash: |
-      export RUN_SCRIPT="src/ci/run.sh"
-      echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT"
-    displayName: Prepare run script (init and run)
-
-  - template: steps/show-environment-variables.yml
-
-  - template: steps/verify-publish-toolstate.yml
-
-  - template: steps/run-script.yml
-
-  strategy:
-    matrix:
-      # macOS builders. These are placed near the beginning because they are very
-      # slow to run.
-
-      # OSX builders running tests, these run the full test suite.
-      # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some
-      # runners that run `//ignore-debug` tests.
-      #
-      # Note that the compiler is compiled to target 10.8 here because the Xcode
-      # version that we're using, 8.2, cannot compile LLVM for OSX 10.7.
-      x86_64-apple-A:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-        ARTIFACT_NAME: x86_64-apple-build
-        SCRIPT: make ci-resume-subset-A
-      x86_64-apple-B:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-        ARTIFACT_NAME: x86_64-apple-build
-        SCRIPT: make ci-resume-subset-B
-      x86_64-apple-C:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-        ARTIFACT_NAME: x86_64-apple-build
-        SCRIPT: make ci-resume-subset-C
-      x86_64-apple-D:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-        ARTIFACT_NAME: x86_64-apple-build
-        SCRIPT: make ci-resume-subset-D
-      x86_64-apple-E:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-        ARTIFACT_NAME: x86_64-apple-build
-        SCRIPT: make ci-resume-subset-E
-      x86_64-apple-F:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-        ARTIFACT_NAME: x86_64-apple-build
-        SCRIPT: make ci-resume-subset-F
-      x86_64-apple-G:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-        ARTIFACT_NAME: x86_64-apple-build
-        SCRIPT: make ci-resume-subset-G
-      x86_64-apple-H:
-        RUST_CHECK_TARGET: check
-        RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
-        RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
-        MACOSX_DEPLOYMENT_TARGET: 10.8
-        MACOSX_STD_DEPLOYMENT_TARGET: 10.7
-        NO_LLVM_ASSERTIONS: 1
-        NO_DEBUG_ASSERTIONS: 1
-        ARTIFACT_NAME: x86_64-apple-build
-        SCRIPT: make ci-resume-subset-H
diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml
index 189536cff5f..facc592422b 100644
--- a/.azure-pipelines/auto.yml
+++ b/.azure-pipelines/auto.yml
@@ -3,8 +3,8 @@
 # 
 
 pr: none
-# trigger:
-# - auto
+ trigger:
+ - auto
 
 variables:
 - group: caching
diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md
index c47c0f765f0..1e01d68fb3a 100644
--- a/src/bootstrap/README.md
+++ b/src/bootstrap/README.md
@@ -39,7 +39,8 @@ The script accepts commands, flags, and arguments to determine what to do:
   ```
 
   If files are dirty that would normally be rebuilt from stage 0, that can be
-  overridden using `--keep-stage 0`.
+  overridden using `--keep-stage 0`. Using `--keep-stage n` will skip all steps
+  that belong to stage n or earlier:
 
   ```
   # keep old build products for stage 0 and build stage 1
diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in
index 29b0cdaef63..ea05b30ecef 100644
--- a/src/bootstrap/mk/Makefile.in
+++ b/src/bootstrap/mk/Makefile.in
@@ -14,10 +14,9 @@ endif
 
 BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py
 
-build:
+all:
 	$(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS)
-
-all: build doc
+	$(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS)
 
 help:
 	$(Q)echo 'Welcome to the rustbuild build system!'
@@ -81,75 +80,10 @@ TESTS_IN_2 := \
 	src/test/run-pass-fulldeps \
 	src/tools/linkchecker
 
-MIN_TEST := \
-	src/test/debuginfo
-
 ci-subset-1:
 	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %)
 ci-subset-2:
 	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2)
 
-ci-test-prepare: 
-	$(Q)$(BOOTSTRAP) test $(MIN_TEST)
-
-ci-resume-subset-1:
-	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %)
-ci-resume-subset-2:
-	$(Q)$(BOOTSTRAP) test $(TESTS_IN_2)
-
-TESTS_IN_B := \
-	src/tools/linkchecker
-
-TESTS_IN_C := \
-	src/test/run-pass \
-	src/test/run-pass-fulldeps
-
-TESTS_IN_D := \
-	src/test/compile-fail \
-	src/test/rustdoc \
-	src/test/pretty
-
-TESTS_IN_E := \
-	src/test/ui 
-
-TESTS_IN_F := \
-	src/test/run-fail \
-	src/liballoc \
-	src/libcore
-
-TESTS_IN_G := \
-	src/tools/rustdoc \
-	src/test/rustdoc-js-std \
-	src/test/run-make-fulldeps \
-	src/libstd
-
-TESTS_IN_H := \
-	src/librustc_driver
-
-ci-resume-subset-A:
-	$(Q)$(BOOTSTRAP) test \
-		$(KEEP_STAGE) \
-		$(TESTS_IN_B:%=--exclude %) \
-		$(TESTS_IN_C:%=--exclude %) \
-		$(TESTS_IN_D:%=--exclude %) \
-		$(TESTS_IN_E:%=--exclude %) \
-		$(TESTS_IN_F:%=--exclude %) \
-		$(TESTS_IN_G:%=--exclude %) \
-		$(TESTS_IN_H:%=--exclude %) \
-		$(MIN_TEST:%=--exclude %) 
-ci-resume-subset-B:
-	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_B)
-ci-resume-subset-C:
-	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_C)
-ci-resume-subset-D:
-	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_D)
-ci-resume-subset-E:
-	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_E)
-ci-resume-subset-F:
-	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_F)
-ci-resume-subset-G:
-	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_G)
-ci-resume-subset-H:
-	$(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_H)
 
 .PHONY: dist
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index 62a8b7aad03..5777331b9bf 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -81,13 +81,6 @@ impl Step for Llvm {
             (info, "src/llvm-project/llvm", builder.llvm_out(target), dir.join("bin"))
         };
 
-        let build_llvm_config = llvm_config_ret_dir
-            .join(exe("llvm-config", &*builder.config.build));
-
-        if env::var_os("SKIP_LLVM_BUILD").is_some() {
-            return build_llvm_config
-        }
-
         if !llvm_info.is_git() {
             println!(
                 "git could not determine the LLVM submodule commit hash. \
@@ -95,6 +88,8 @@ impl Step for Llvm {
             );
         }
 
+        let build_llvm_config = llvm_config_ret_dir
+            .join(exe("llvm-config", &*builder.config.build));
         let done_stamp = out_dir.join("llvm-finished-building");
 
         if let Some(llvm_commit) = llvm_info.sha() {
diff --git a/src/ci/run.sh b/src/ci/run.sh
index 08ede122d65..a51c2da3cbd 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -90,25 +90,23 @@ if [ "$RUN_CHECK_WITH_PARALLEL_QUERIES" != "" ]; then
   rm -rf build
 fi
 
-if [ "$SKIP_PREP" != "1" ]; then
-  travis_fold start configure
-  travis_time_start
-  $SRC/configure $RUST_CONFIGURE_ARGS
-  travis_fold end configure
-  travis_time_finish
-
-  travis_fold start make-prepare
-  travis_time_start
-  retry make prepare
-  travis_fold end make-prepare
-  travis_time_finish
-
-  travis_fold start check-bootstrap
-  travis_time_start
-  make check-bootstrap
-  travis_fold end check-bootstrap
-  travis_time_finish
-fi
+travis_fold start configure
+travis_time_start
+$SRC/configure $RUST_CONFIGURE_ARGS
+travis_fold end configure
+travis_time_finish
+
+travis_fold start make-prepare
+travis_time_start
+retry make prepare
+travis_fold end make-prepare
+travis_time_finish
+
+travis_fold start check-bootstrap
+travis_time_start
+make check-bootstrap
+travis_fold end check-bootstrap
+travis_time_finish
 
 # Display the CPU and memory information. This helps us know why the CI timing
 # is fluctuating.
-- 
cgit 1.4.1-3-g733a5


From aab0bb47fa102b57a7e7c35af78c3f0cb79d05eb Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Sun, 12 May 2019 18:00:38 +0200
Subject: ci: merge the i1686-gnu images

---
 .azure-pipelines/auto.yml           |  6 ++----
 src/ci/docker/i686-gnu-1/Dockerfile | 22 ----------------------
 src/ci/docker/i686-gnu-2/Dockerfile | 21 ---------------------
 src/ci/docker/i686-gnu/Dockerfile   | 27 +++++++++++++++++++++++++++
 4 files changed, 29 insertions(+), 47 deletions(-)
 delete mode 100644 src/ci/docker/i686-gnu-1/Dockerfile
 delete mode 100644 src/ci/docker/i686-gnu-2/Dockerfile
 create mode 100644 src/ci/docker/i686-gnu/Dockerfile

(limited to 'src')

diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml
index facc592422b..e8959e3f17c 100644
--- a/.azure-pipelines/auto.yml
+++ b/.azure-pipelines/auto.yml
@@ -126,10 +126,8 @@ jobs:
 
       asmjs:
         IMAGE: asmjs
-      i686-gnu-1:
-        IMAGE: i686-gnu-1
-      i686-gnu-2:
-        IMAGE: i686-gnu-2
+      i686-gnu:
+        IMAGE: i686-gnu
       i686-gnu-nopt:
         IMAGE: i686-gnu-nopt
       test-various:
diff --git a/src/ci/docker/i686-gnu-1/Dockerfile b/src/ci/docker/i686-gnu-1/Dockerfile
deleted file mode 100644
index 1a04b8af4c1..00000000000
--- a/src/ci/docker/i686-gnu-1/Dockerfile
+++ /dev/null
@@ -1,22 +0,0 @@
-FROM ubuntu:16.04
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  g++-multilib \
-  make \
-  file \
-  curl \
-  ca-certificates \
-  python2.7 \
-  git \
-  cmake \
-  sudo \
-  gdb \
-  xz-utils
-
-
-COPY scripts/sccache.sh /scripts/
-RUN sh /scripts/sccache.sh
-
-ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
-ENV SCRIPT make ci-subset-1
-
diff --git a/src/ci/docker/i686-gnu-2/Dockerfile b/src/ci/docker/i686-gnu-2/Dockerfile
deleted file mode 100644
index b1a0ea6f0e9..00000000000
--- a/src/ci/docker/i686-gnu-2/Dockerfile
+++ /dev/null
@@ -1,21 +0,0 @@
-FROM ubuntu:16.04
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  g++-multilib \
-  make \
-  file \
-  curl \
-  ca-certificates \
-  python2.7 \
-  git \
-  cmake \
-  sudo \
-  gdb \
-  xz-utils
-
-
-COPY scripts/sccache.sh /scripts/
-RUN sh /scripts/sccache.sh
-
-ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
-ENV SCRIPT make ci-subset-2
diff --git a/src/ci/docker/i686-gnu/Dockerfile b/src/ci/docker/i686-gnu/Dockerfile
new file mode 100644
index 00000000000..17441ddb454
--- /dev/null
+++ b/src/ci/docker/i686-gnu/Dockerfile
@@ -0,0 +1,27 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++-multilib \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  sudo \
+  gdb \
+  xz-utils
+
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
+# Exclude some tests that are unlikely to be platform specific, to speed up
+# this slow job.
+ENV SCRIPT python2.7 ../x.py test \
+  --exclude src/bootstrap \
+  --exclude src/test/rustdoc-js \
+  --exclude src/tools/error_index_generator \
+  --exclude src/tools/linkchecker
-- 
cgit 1.4.1-3-g733a5


From 94f7660ab376c91c19eee67af7a9a41b268679b6 Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Sun, 12 May 2019 18:16:42 +0200
Subject: ci: re-enable fds-are-cloexec test on macOS

---
 src/test/run-pass/fds-are-cloexec.rs | 1 -
 1 file changed, 1 deletion(-)

(limited to 'src')

diff --git a/src/test/run-pass/fds-are-cloexec.rs b/src/test/run-pass/fds-are-cloexec.rs
index bcfc3245db3..3bc0ceb5cf2 100644
--- a/src/test/run-pass/fds-are-cloexec.rs
+++ b/src/test/run-pass/fds-are-cloexec.rs
@@ -3,7 +3,6 @@
 // ignore-cloudabi no processes
 // ignore-emscripten no processes
 // ignore-haiku
-// ignore-macos
 
 #![feature(rustc_private)]
 
-- 
cgit 1.4.1-3-g733a5


From 4965ffdb7a0dd825e251bcd1cc83c20ee6c8e45d Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Mon, 13 May 2019 15:19:29 +0200
Subject: Update src/bootstrap/util.rs

Co-Authored-By: Jake Goulding <shepmaster@mac.com>
---
 src/bootstrap/util.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs
index e79700d92d8..f22f0559265 100644
--- a/src/bootstrap/util.rs
+++ b/src/bootstrap/util.rs
@@ -326,7 +326,7 @@ pub enum CiEnv {
     Travis,
     /// The AppVeyor environment, for Windows builds.
     AppVeyor,
-    /// The Azure Pipelines environment, for Linux (including Docker), Window, and macOS builds.
+    /// The Azure Pipelines environment, for Linux (including Docker), Windows, and macOS builds.
     AzurePipelines,
 }
 
-- 
cgit 1.4.1-3-g733a5


From 30b008f5337332e0a1f941113d14f90fa876315e Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Wed, 15 May 2019 19:37:02 +0200
Subject: ci: remove stray echo

---
 src/ci/docker/x86_64-gnu-tools/checktools.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh
index c7e64cd8c3e..af0198705a2 100755
--- a/src/ci/docker/x86_64-gnu-tools/checktools.sh
+++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh
@@ -6,7 +6,7 @@ X_PY="$1"
 TOOLSTATE_FILE="$(realpath $2)"
 OS="$3"
 COMMIT="$(git rev-parse HEAD)"
-CHANGED_FILES="$(git diff --name-status HEAD HEAD^ || echo)"
+CHANGED_FILES="$(git diff --name-status HEAD HEAD^)"
 SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))"
 # ^ Number of days after the last promotion of beta.
 #   Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens.
-- 
cgit 1.4.1-3-g733a5


From 0843207d1b8e079aebbb06527abb26bbd083e54d Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Mon, 20 May 2019 12:55:01 -0700
Subject: Make sure sccache not present doesn't kill the build

---
 src/ci/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/ci/run.sh b/src/ci/run.sh
index a51c2da3cbd..c996dcb14af 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -141,4 +141,4 @@ else
   do_make "$RUST_CHECK_TARGET"
 fi
 
-sccache --show-stats
+sccache --show-stats || true
-- 
cgit 1.4.1-3-g733a5


From 751597f0c79a7602db1481b44ebfd5c9a554b591 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Mon, 20 May 2019 12:58:53 -0700
Subject: Check out rustfmt submodule through tarballs

This takes 30+ seconds to check out on Windows, so let's speed it up
through a tarball like we do other big git repositories.
---
 src/ci/init_repo.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/ci/init_repo.sh b/src/ci/init_repo.sh
index 3dfd3381576..b1e4f931c44 100755
--- a/src/ci/init_repo.sh
+++ b/src/ci/init_repo.sh
@@ -46,7 +46,7 @@ function fetch_github_commit_archive {
     rm $cached
 }
 
-included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example"
+included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example src/tools/rustfmt"
 modules="$(git config --file .gitmodules --get-regexp '\.path$' | cut -d' ' -f2)"
 modules=($modules)
 use_git=""
-- 
cgit 1.4.1-3-g733a5


From 67db2304771d3c7065a68f6a486048b770b8e63e Mon Sep 17 00:00:00 2001
From: Pietro Albini <pietro@pietroalbini.org>
Date: Tue, 21 May 2019 10:51:37 +0200
Subject: ci: fix tidy

---
 src/ci/init_repo.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/ci/init_repo.sh b/src/ci/init_repo.sh
index b1e4f931c44..10215d863be 100755
--- a/src/ci/init_repo.sh
+++ b/src/ci/init_repo.sh
@@ -46,7 +46,8 @@ function fetch_github_commit_archive {
     rm $cached
 }
 
-included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example src/tools/rustfmt"
+included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example"
+included="${included} src/tools/rustfmt"
 modules="$(git config --file .gitmodules --get-regexp '\.path$' | cut -d' ' -f2)"
 modules=($modules)
 use_git=""
-- 
cgit 1.4.1-3-g733a5


From 8d42b87ac041dc05c9ae8c060aaa8864826639d4 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Tue, 21 May 2019 12:20:50 -0700
Subject: Use new-style s3 urls in docker download script

---
 src/ci/docker/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index 0e1485601e9..c9642dbf60c 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -42,7 +42,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
         awk '{print $1}')
 
       s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
-      url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
+      url="https://$SCCACHE_BUCKET.s3.amazonaws.com/docker/$cksum"
       upload="aws s3 cp - $s3url"
 
       echo "Attempting to download $url"
-- 
cgit 1.4.1-3-g733a5


From e764f475ca7fffd6167ea991afc7d1b2b3f642dc Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Tue, 21 May 2019 15:34:56 -0700
Subject: Disable solaris target since toolchain no longer builds

---
 src/ci/docker/dist-various-2/Dockerfile | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/ci/docker/dist-various-2/Dockerfile b/src/ci/docker/dist-various-2/Dockerfile
index 1d6a0269c5b..04db5e9c057 100644
--- a/src/ci/docker/dist-various-2/Dockerfile
+++ b/src/ci/docker/dist-various-2/Dockerfile
@@ -32,9 +32,10 @@ COPY dist-various-2/build-cloudabi-toolchain.sh /tmp/
 RUN /tmp/build-cloudabi-toolchain.sh x86_64-unknown-cloudabi
 COPY dist-various-2/build-fuchsia-toolchain.sh /tmp/
 RUN /tmp/build-fuchsia-toolchain.sh
-COPY dist-various-2/build-solaris-toolchain.sh /tmp/
-RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386
-RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc
+# FIXME(#61022) - reenable solaris
+# COPY dist-various-2/build-solaris-toolchain.sh /tmp/
+# RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386
+# RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc
 COPY dist-various-2/build-x86_64-fortanix-unknown-sgx-toolchain.sh /tmp/
 # We pass the commit id of the port of LLVM's libunwind to the build script.
 # Any update to the commit id here, should cause the container image to be re-built from this point on.
@@ -73,10 +74,11 @@ ENV CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS \
 
 ENV TARGETS=x86_64-fuchsia
 ENV TARGETS=$TARGETS,aarch64-fuchsia
-ENV TARGETS=$TARGETS,sparcv9-sun-solaris
 ENV TARGETS=$TARGETS,wasm32-unknown-unknown
 ENV TARGETS=$TARGETS,wasm32-unknown-wasi
-ENV TARGETS=$TARGETS,x86_64-sun-solaris
+# FIXME(#61022) - reenable solaris
+# ENV TARGETS=$TARGETS,sparcv9-sun-solaris
+# ENV TARGETS=$TARGETS,x86_64-sun-solaris
 ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnux32
 ENV TARGETS=$TARGETS,x86_64-unknown-cloudabi
 ENV TARGETS=$TARGETS,x86_64-fortanix-unknown-sgx
-- 
cgit 1.4.1-3-g733a5


From e5418f61deb65a2d5a2bd48320e05d9cf689cff6 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Tue, 21 May 2019 15:57:47 -0700
Subject: Update webrender we're testing

The one locked here has a dependency on an old enough `cmake` crate that
it's not detecting visual studio correctly. Let's update webrender which
updates `cmake` which should be able to detect Visual Studio correctly.
---
 src/tools/cargotest/main.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/tools/cargotest/main.rs b/src/tools/cargotest/main.rs
index 3126b44f0d6..14035eedbb4 100644
--- a/src/tools/cargotest/main.rs
+++ b/src/tools/cargotest/main.rs
@@ -61,7 +61,7 @@ const TEST_REPOS: &'static [Test] = &[
     Test {
         name: "webrender",
         repo: "https://github.com/servo/webrender",
-        sha: "57250b2b8fa63934f80e5376a29f7dcb3f759ad6",
+        sha: "cdadd068f4c7218bd983d856981d561e605270ab",
         lock: None,
         packages: &[],
     },
-- 
cgit 1.4.1-3-g733a5


From a8ac80b7bcb46d199c196ec0313ca7fd70aee857 Mon Sep 17 00:00:00 2001
From: Alex Crichton <alex@alexcrichton.com>
Date: Wed, 22 May 2019 07:09:25 -0700
Subject: Final cleanups

* Clean up the `install-windows-build-deps.yml` file and add some more
  comments where appropriate.

* Add some comments to `run.yml`

* Don't fast path the `rustfmt` submodule, but we'll take care of that
  later if necessary.
---
 .../steps/install-windows-build-deps.yml           | 57 ++++++++++------------
 .azure-pipelines/steps/run.yml                     |  2 +
 src/ci/init_repo.sh                                |  1 -
 3 files changed, 27 insertions(+), 33 deletions(-)

(limited to 'src')

diff --git a/.azure-pipelines/steps/install-windows-build-deps.yml b/.azure-pipelines/steps/install-windows-build-deps.yml
index e54f7b6eb67..037c8daa2a8 100644
--- a/.azure-pipelines/steps/install-windows-build-deps.yml
+++ b/.azure-pipelines/steps/install-windows-build-deps.yml
@@ -1,21 +1,20 @@
 steps:
-# FIXME: are these still needed?
-# - bash: |
-#     set -x
-#     git submodule
-#     export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/')
-#     echo "##vso[task.setvariable variable=SUBMODULES_EXCLUDES;]$SUBMODULES_EXCLUDES"
-
-# FIXME: needs a comment to justify its existence
+# We've had issues with the default drive in use running out of space during a
+# build, and it looks like the `C:` drive has more space than the default `D:`
+# drive. We should probably confirm this with the azure pipelines team at some
+# point, but this seems to fix our "disk space full" problems.
 - script: |
-    REM echo hack as drive D is too small
-    IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" (
-      mkdir c:\MORE_SPACE
-      mklink /J build c:\MORE_SPACE
-    )
+    mkdir c:\MORE_SPACE
+    mklink /J build c:\MORE_SPACE
   displayName: "Ensure build happens on C:/ instead of D:/"
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 
+# Download and install MSYS2, needed primarily for the test suite (run-make) but
+# also used by the MinGW toolchain for assembling things.
+#
+# FIXME: we should probe the default azure image and see if we can use the MSYS2
+# toolchain there. (if there's even one there). For now though this gets the job
+# done.
 - script: |
     set MSYS_PATH=%CD%\citools\msys64
     choco install msys2 --params="/InstallDir:%MSYS_PATH% /NoPath" -y
@@ -37,23 +36,11 @@ steps:
 # If we need to download a custom MinGW, do so here and set the path
 # appropriately.
 #
-# Note that this *also* means that we're not using what is typically
-# /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where
-# /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we
-# move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe`
-# file exists in there (which it doesn't by default).
-- script: |
-    powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%"
-    7z x -y %MINGW_ARCHIVE% > nul
-    echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin
-  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), ne(variables['MINGW_URL'],''))
-  displayName: Download custom MinGW
-
-# Here we do a pretty heinous thing which is to mangle the MinGW installation
-# we just had above. Currently, as of this writing, we're using MinGW-w64
-# builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it appears to
-# be the first version which contains a fix for #40546, builds randomly
-# failing during LLVM due to ar.exe/ranlib.exe failures.
+# Here we also do a pretty heinous thing which is to mangle the MinGW
+# installation we just downloaded. Currently, as of this writing, we're using
+# MinGW-w64 builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it
+# appears to be the first version which contains a fix for #40546, builds
+# randomly failing during LLVM due to ar.exe/ranlib.exe failures.
 #
 # Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds
 # to contain a regression in gdb (#40184). As a result if we were to use the
@@ -67,11 +54,13 @@ steps:
 # Note that we don't literally overwrite the gdb.exe binary because it appears
 # to just use gdborig.exe, so that's the binary we deal with instead.
 - script: |
-    echo ON
+    powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%"
+    7z x -y %MINGW_ARCHIVE% > nul
     powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe"
     mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe
+    echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), ne(variables['MINGW_URL'],''))
-  displayName: Override with 6.3.0 gdb with 6.2.0 gdb
+  displayName: Download custom MinGW
 
 # Otherwise pull in the MinGW installed on appveyor
 - script: |
@@ -79,6 +68,10 @@ steps:
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],''))
   displayName: Add MinGW to path
 
+# Make sure we use the native python interpreter instead of some msys equivalent
+# one way or another. The msys interpreters seem to have weird path conversions
+# baked in which break LLVM's build system one way or another, so let's use the
+# native version which keeps everything as native as possible.
 - script: |
     copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe
     echo ##vso[task.prependpath]C:\Python27amd64
diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml
index f7ee490b1fd..47a4c183085 100644
--- a/.azure-pipelines/steps/run.yml
+++ b/.azure-pipelines/steps/run.yml
@@ -114,6 +114,8 @@ steps:
     AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
   displayName: Run build
 
+# If we're a deploy builder, use the `aws` command to publish everything to our
+# bucket.
 - bash: |
     set -e
     if [ "$AGENT_OS" = "Linux" ]; then
diff --git a/src/ci/init_repo.sh b/src/ci/init_repo.sh
index 10215d863be..3dfd3381576 100755
--- a/src/ci/init_repo.sh
+++ b/src/ci/init_repo.sh
@@ -47,7 +47,6 @@ function fetch_github_commit_archive {
 }
 
 included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example"
-included="${included} src/tools/rustfmt"
 modules="$(git config --file .gitmodules --get-regexp '\.path$' | cut -d' ' -f2)"
 modules=($modules)
 use_git=""
-- 
cgit 1.4.1-3-g733a5