Retry when downloading the Docker cache.

Prevent spuriously needing to rebuild the docker image when the network
was down.

Also, adjusted the retry function to insert a sleep between retries,
because retrying immediately will often just hit the same issue.

2 files changed, 5 insertions, 2 deletions

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index c470ae7eb30..3465e386cd9 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -36,8 +36,10 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
       url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
       echo "Attempting to download $s3url"
+      rm -f /tmp/rustci_docker_cache
       set +e
-      loaded_images=$(curl $url | docker load | sed 's/.* sha/sha/')
+      retry curl -f -L -C - -o /tmp/rustci_docker_cache "$url"
+      loaded_images=$(docker load -i /tmp/rustci_docker_cache | sed 's/.* sha/sha/')
       set -e
       echo "Downloaded containers:\n$loaded_images"
     fi
diff --git a/src/ci/shared.sh b/src/ci/shared.sh
index 4a08683e3ee..bb6945f0fd6 100644
--- a/src/ci/shared.sh
+++ b/src/ci/shared.sh
@@ -21,11 +21,12 @@ function retry {
   while true; do
     "$@" && break || {
       if [[ $n -lt $max ]]; then
+        sleep $n  # don't retry immediately
         ((n++))
         echo "Command failed. Attempt $n/$max:"
       else
         echo "The command has failed after $n attempts."
-        exit 1
+        return 1
       fi
     }
   done