travis: Make more network requests retryable

This commit attempts to move more network operations to being retryable through
various operations. For example git submodule updates, downloading snapshots,
etc, are now all in retryable steps.

Hopefully this commit can cut down on the number of network failures we've been
seeing!

6 files changed, 48 insertions, 11 deletions

diff --git a/.travis.yml b/.travis.yml
index cc032edb908..9e90a1d2ddb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -45,7 +45,7 @@ matrix:
       os: osx
       osx_image: xcode8.2
       install: &osx_install_sccache >
-        curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
+        travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
           chmod +x /usr/local/bin/sccache
     - env: >
         RUST_CHECK_TARGET=check
@@ -63,7 +63,7 @@ matrix:
       os: osx
       osx_image: xcode8.2
       install: >
-        curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
+        travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
           chmod +x /usr/local/bin/sccache &&
           brew uninstall --ignore-dependencies openssl &&
           brew install openssl --universal --without-test
diff --git a/appveyor.yml b/appveyor.yml
index ad6b5316226..f76fc0f6466 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -103,7 +103,7 @@ install:
   # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we
   # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe`
   # file exists in there (which it doesn't by default).
-  - if defined MINGW_URL appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE%
+  - if defined MINGW_URL appveyor-retry appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE%
   - if defined MINGW_URL 7z x -y %MINGW_ARCHIVE% > nul
   - if defined MINGW_URL set PATH=%CD%\%MINGW_DIR%\bin;C:\msys64\usr\bin;%PATH%
 
@@ -115,12 +115,12 @@ install:
   - set PATH=C:\Python27;%PATH%
 
   # Download and install sccache
-  - appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-pc-windows-msvc
+  - appveyor-retry appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-pc-windows-msvc
   - mv 2017-02-24-sccache-x86_64-pc-windows-msvc sccache
   - set PATH=%PATH%;%CD%
 
   # Install InnoSetup to get `iscc` used to produce installers
-  - choco install -y InnoSetup
+  - appveyor-retry choco install -y InnoSetup
   - set PATH="C:\Program Files (x86)\Inno Setup 5";%PATH%
 
   # Help debug some handle issues on AppVeyor
@@ -131,7 +131,7 @@ install:
   - handle.exe -accepteula -help
 
 test_script:
-  - git submodule update --init
+  - appveyor-retry sh -c 'git submodule deinit -f . && git submodule update --init'
   - set SRC=.
   - set NO_CCACHE=1
   - sh src/ci/run.sh
diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in
index 536095503e0..457ac825832 100644
--- a/src/bootstrap/mk/Makefile.in
+++ b/src/bootstrap/mk/Makefile.in
@@ -71,6 +71,8 @@ install:
 	$(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS)
 tidy:
 	$(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS)
+prepare:
+	$(Q)$(BOOTSTRAP) build nonexistent/path/to/trigger/cargo/metadata
 
 check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu:
 	$(Q)$(BOOTSTRAP) test --target arm-linux-androideabi
diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index 892c5baa5c6..1e61f216910 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2016 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@@ -19,7 +19,9 @@ ci_dir="`dirname $docker_dir`"
 src_dir="`dirname $ci_dir`"
 root_dir="`dirname $src_dir`"
 
-docker \
+source "$ci_dir/shared.sh"
+
+retry docker \
   build \
   --rm \
   -t rust-ci \
diff --git a/src/ci/run.sh b/src/ci/run.sh
index c4b25565993..53d16f6239e 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2016 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@@ -20,6 +20,9 @@ if [ "$NO_CHANGE_USER" = "" ]; then
   fi
 fi
 
+ci_dir=`cd $(dirname $0) && pwd`
+source "$ci_dir/shared.sh"
+
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules"
@@ -54,9 +57,8 @@ else
   fi
 fi
 
-set -ex
-
 $SRC/configure $RUST_CONFIGURE_ARGS
+retry make prepare
 
 if [ "$TRAVIS_OS_NAME" = "osx" ]; then
     ncpus=$(sysctl -n hw.ncpu)
@@ -64,6 +66,8 @@ else
     ncpus=$(grep processor /proc/cpuinfo | wc -l)
 fi
 
+set -x
+
 if [ ! -z "$SCRIPT" ]; then
   sh -x -c "$SCRIPT"
 else
diff --git a/src/ci/shared.sh b/src/ci/shared.sh
new file mode 100644
index 00000000000..ecd9b7e98a4
--- /dev/null
+++ b/src/ci/shared.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# See http://unix.stackexchange.com/questions/82598
+function retry {
+  local n=1
+  local max=5
+  local delay=15
+  while true; do
+    echo "Attempting:" "$@"
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+      else
+        echo "The command has failed after $n attempts."
+        exit 1
+      fi
+    }
+  done
+}