about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorbors <bors@rust-lang.org>2016-12-16 07:26:41 +0000
committerbors <bors@rust-lang.org>2016-12-16 07:26:41 +0000
commitd250169cb5a96481a3e7c8f9fe05de49f60e5ae5 (patch)
tree59a8e128fbe416b32d364585b9d316878f2d4abd /src
parentc6d8ab09e39bacae2c008995bbb669442115517f (diff)
parent96a5fc76dcce1bd6669a9e288721ee6aad521096 (diff)
downloadrust-d250169cb5a96481a3e7c8f9fe05de49f60e5ae5.tar.gz
rust-d250169cb5a96481a3e7c8f9fe05de49f60e5ae5.zip
Auto merge of #38359 - alexcrichton:sccache, r=brson
rustbuild: Add sccache support

This commit adds support for sccache, a ccache-like compiler which works on MSVC
and stores results into an S3 bucket. This also switches over all Travis and
AppVeyor automation to using sccache to ensure a shared and unified cache over
time which can be shared across builders.

The support for sccache manifests as a new `--enable-sccache` option which
instructs us to configure LLVM differently to use a 'sccache' binary instead of
a 'ccache' binary. All docker images for Travis builds are updated to download
Mozilla's tooltool builds of sccache onto various containers and systems.
Additionally a new `rust-lang-ci-sccache` bucket is configured to hold all of
our ccache goodies.

---

Note that this does not currently change Windows [due to previously written up issues](https://github.com/rust-lang/rust/issues/38119#issuecomment-266631585). Despite that, however, I was curious to get timings for the builds on Travis to see what ranges we're working with. As a result, this is a WIP PR I'm using to gauge build times and such.
Diffstat (limited to 'src')
-rw-r--r--src/bootstrap/config.rs33
-rw-r--r--src/bootstrap/config.toml.example2
-rw-r--r--src/bootstrap/native.rs6
-rw-r--r--src/bootstrap/sanity.rs4
-rw-r--r--src/ci/docker/arm-android/Dockerfile7
-rw-r--r--src/ci/docker/cross/Dockerfile7
-rw-r--r--src/ci/docker/i686-gnu-nopt/Dockerfile7
-rw-r--r--src/ci/docker/i686-gnu/Dockerfile7
-rwxr-xr-xsrc/ci/docker/run.sh6
-rw-r--r--src/ci/docker/x86_64-freebsd/Dockerfile4
-rw-r--r--src/ci/docker/x86_64-gnu-cargotest/Dockerfile7
-rw-r--r--src/ci/docker/x86_64-gnu-debug/Dockerfile7
-rw-r--r--src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile7
-rw-r--r--src/ci/docker/x86_64-gnu-make/Dockerfile7
-rw-r--r--src/ci/docker/x86_64-gnu-nopt/Dockerfile7
-rw-r--r--src/ci/docker/x86_64-gnu/Dockerfile7
-rw-r--r--src/ci/docker/x86_64-musl/Dockerfile4
-rwxr-xr-xsrc/ci/run.sh6
18 files changed, 110 insertions, 25 deletions
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 60f65f62300..b8abcb28c4e 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -38,7 +38,7 @@ use util::push_exe_path;
 /// `src/bootstrap/config.toml.example`.
 #[derive(Default)]
 pub struct Config {
-    pub ccache: bool,
+    pub ccache: Option<String>,
     pub ninja: bool,
     pub verbose: bool,
     pub submodules: bool,
@@ -138,7 +138,7 @@ struct Build {
 /// TOML representation of how the LLVM build is configured.
 #[derive(RustcDecodable, Default)]
 struct Llvm {
-    ccache: Option<bool>,
+    ccache: Option<StringOrBool>,
     ninja: Option<bool>,
     assertions: Option<bool>,
     optimize: Option<bool>,
@@ -147,6 +147,18 @@ struct Llvm {
     static_libstdcpp: Option<bool>,
 }
 
+#[derive(RustcDecodable)]
+enum StringOrBool {
+    String(String),
+    Bool(bool),
+}
+
+impl Default for StringOrBool {
+    fn default() -> StringOrBool {
+        StringOrBool::Bool(false)
+    }
+}
+
 /// TOML representation of how the Rust build is configured.
 #[derive(RustcDecodable, Default)]
 struct Rust {
@@ -247,7 +259,15 @@ impl Config {
         set(&mut config.vendor, build.vendor);
 
         if let Some(ref llvm) = toml.llvm {
-            set(&mut config.ccache, llvm.ccache);
+            match llvm.ccache {
+                Some(StringOrBool::String(ref s)) => {
+                    config.ccache = Some(s.to_string())
+                }
+                Some(StringOrBool::Bool(true)) => {
+                    config.ccache = Some("ccache".to_string());
+                }
+                Some(StringOrBool::Bool(false)) | None => {}
+            }
             set(&mut config.ninja, llvm.ninja);
             set(&mut config.llvm_assertions, llvm.assertions);
             set(&mut config.llvm_optimize, llvm.optimize);
@@ -338,7 +358,6 @@ impl Config {
             }
 
             check! {
-                ("CCACHE", self.ccache),
                 ("MANAGE_SUBMODULES", self.submodules),
                 ("COMPILER_DOCS", self.compiler_docs),
                 ("DOCS", self.docs),
@@ -475,6 +494,12 @@ impl Config {
                     let path = parse_configure_path(value);
                     self.python = Some(path);
                 }
+                "CFG_ENABLE_CCACHE" if value == "1" => {
+                    self.ccache = Some("ccache".to_string());
+                }
+                "CFG_ENABLE_SCCACHE" if value == "1" => {
+                    self.ccache = Some("sccache".to_string());
+                }
                 _ => {}
             }
         }
diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example
index b6774b3af20..9e50e71bf7d 100644
--- a/src/bootstrap/config.toml.example
+++ b/src/bootstrap/config.toml.example
@@ -25,6 +25,8 @@
 
 # Indicates whether ccache is used when building LLVM
 #ccache = false
+# or alternatively ...
+#ccache = "/path/to/ccache"
 
 # If an external LLVM root is specified, we automatically check the version by
 # default to make sure it's within the range that we're expecting, but setting
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index ffa3fe1cbf2..6ba6b8e6c86 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -109,10 +109,10 @@ pub fn llvm(build: &Build, target: &str) {
 
     // MSVC handles compiler business itself
     if !target.contains("msvc") {
-        if build.config.ccache {
-           cfg.define("CMAKE_C_COMPILER", "ccache")
+        if let Some(ref ccache) = build.config.ccache {
+           cfg.define("CMAKE_C_COMPILER", ccache)
               .define("CMAKE_C_COMPILER_ARG1", build.cc(target))
-              .define("CMAKE_CXX_COMPILER", "ccache")
+              .define("CMAKE_CXX_COMPILER", ccache)
               .define("CMAKE_CXX_COMPILER_ARG1", build.cxx(target));
         } else {
            cfg.define("CMAKE_C_COMPILER", build.cc(target))
diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs
index f3fe22698bb..2992099687d 100644
--- a/src/bootstrap/sanity.rs
+++ b/src/bootstrap/sanity.rs
@@ -223,4 +223,8 @@ $ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake
     if build.lldb_version.is_some() {
         build.lldb_python_dir = run(Command::new("lldb").arg("-P")).ok();
     }
+
+    if let Some(ref s) = build.config.ccache {
+        need_cmd(s.as_ref());
+    }
 }
diff --git a/src/ci/docker/arm-android/Dockerfile b/src/ci/docker/arm-android/Dockerfile
index 5489fb7304d..8911b4ff0cb 100644
--- a/src/ci/docker/arm-android/Dockerfile
+++ b/src/ci/docker/arm-android/Dockerfile
@@ -16,7 +16,8 @@ RUN dpkg --add-architecture i386 && \
   expect \
   openjdk-9-jre \
   sudo \
-  libstdc++6:i386
+  libstdc++6:i386 \
+  xz-utils
 
 WORKDIR /android/
 ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools
@@ -33,6 +34,10 @@ COPY start-emulator.sh /android/
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/android/start-emulator.sh"]
 
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
+
 ENV TARGETS=arm-linux-androideabi
 ENV TARGETS=$TARGETS,i686-linux-android
 ENV TARGETS=$TARGETS,aarch64-linux-android
diff --git a/src/ci/docker/cross/Dockerfile b/src/ci/docker/cross/Dockerfile
index eeb6c79d69b..08b436313f6 100644
--- a/src/ci/docker/cross/Dockerfile
+++ b/src/ci/docker/cross/Dockerfile
@@ -21,7 +21,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gcc-powerpc-linux-gnu libc6-dev-powerpc-cross \
   gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross \
   gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross \
-  gcc-s390x-linux-gnu libc6-dev-s390x-cross
+  gcc-s390x-linux-gnu libc6-dev-s390x-cross \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
diff --git a/src/ci/docker/i686-gnu-nopt/Dockerfile b/src/ci/docker/i686-gnu-nopt/Dockerfile
index 51161997e22..1da33c94c7b 100644
--- a/src/ci/docker/i686-gnu-nopt/Dockerfile
+++ b/src/ci/docker/i686-gnu-nopt/Dockerfile
@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
diff --git a/src/ci/docker/i686-gnu/Dockerfile b/src/ci/docker/i686-gnu/Dockerfile
index 1603d353275..9e5b0e0435e 100644
--- a/src/ci/docker/i686-gnu/Dockerfile
+++ b/src/ci/docker/i686-gnu/Dockerfile
@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index ff5345d3aac..ce8b49a92dc 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -25,7 +25,6 @@ docker \
   -t rust-ci \
   "`dirname "$script"`/$image"
 
-mkdir -p $HOME/.ccache
 mkdir -p $HOME/.cargo
 mkdir -p $root_dir/obj
 
@@ -35,8 +34,9 @@ exec docker \
   --volume "$root_dir/obj:/checkout/obj" \
   --workdir /checkout/obj \
   --env SRC=/checkout \
-  --env CCACHE_DIR=/ccache \
-  --volume "$HOME/.ccache:/ccache" \
+  --env SCCACHE_BUCKET=$SCCACHE_BUCKET \
+  --env AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+  --env AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
   --env CARGO_HOME=/cargo \
   --env LOCAL_USER_ID=`id -u` \
   --volume "$HOME/.cargo:/cargo" \
diff --git a/src/ci/docker/x86_64-freebsd/Dockerfile b/src/ci/docker/x86_64-freebsd/Dockerfile
index 629cfbc3a9c..75f3174e2c0 100644
--- a/src/ci/docker/x86_64-freebsd/Dockerfile
+++ b/src/ci/docker/x86_64-freebsd/Dockerfile
@@ -23,6 +23,10 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
+
 ENV \
     AR_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-ar \
     CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-gcc
diff --git a/src/ci/docker/x86_64-gnu-cargotest/Dockerfile b/src/ci/docker/x86_64-gnu-cargotest/Dockerfile
index 78aa510ec04..0d9835e86d2 100644
--- a/src/ci/docker/x86_64-gnu-cargotest/Dockerfile
+++ b/src/ci/docker/x86_64-gnu-cargotest/Dockerfile
@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   libssl-dev \
-  sudo
+  sudo \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
diff --git a/src/ci/docker/x86_64-gnu-debug/Dockerfile b/src/ci/docker/x86_64-gnu-debug/Dockerfile
index 7d00f11cbb8..eec88442293 100644
--- a/src/ci/docker/x86_64-gnu-debug/Dockerfile
+++ b/src/ci/docker/x86_64-gnu-debug/Dockerfile
@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
diff --git a/src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile b/src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile
index 95484c660e6..4c9198d88eb 100644
--- a/src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile
+++ b/src/ci/docker/x86_64-gnu-llvm-3.7/Dockerfile
@@ -14,7 +14,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gdb \
   llvm-3.7-tools \
   libedit-dev \
-  zlib1g-dev
+  zlib1g-dev \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
diff --git a/src/ci/docker/x86_64-gnu-make/Dockerfile b/src/ci/docker/x86_64-gnu-make/Dockerfile
index 4a5c5b57ea7..1c503aea13d 100644
--- a/src/ci/docker/x86_64-gnu-make/Dockerfile
+++ b/src/ci/docker/x86_64-gnu-make/Dockerfile
@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
diff --git a/src/ci/docker/x86_64-gnu-nopt/Dockerfile b/src/ci/docker/x86_64-gnu-nopt/Dockerfile
index c2a5449d2e3..66de6ea13ac 100644
--- a/src/ci/docker/x86_64-gnu-nopt/Dockerfile
+++ b/src/ci/docker/x86_64-gnu-nopt/Dockerfile
@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
diff --git a/src/ci/docker/x86_64-gnu/Dockerfile b/src/ci/docker/x86_64-gnu/Dockerfile
index 54bbc9b3421..3d71b7ffb9a 100644
--- a/src/ci/docker/x86_64-gnu/Dockerfile
+++ b/src/ci/docker/x86_64-gnu/Dockerfile
@@ -11,7 +11,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   cmake \
   ccache \
   sudo \
-  gdb
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
 
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
diff --git a/src/ci/docker/x86_64-musl/Dockerfile b/src/ci/docker/x86_64-musl/Dockerfile
index b3068cacd04..96b38067cbb 100644
--- a/src/ci/docker/x86_64-musl/Dockerfile
+++ b/src/ci/docker/x86_64-musl/Dockerfile
@@ -23,6 +23,10 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
     rm dumb-init_*.deb
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
+
 ENV RUST_CONFIGURE_ARGS \
       --target=x86_64-unknown-linux-musl \
       --musl-root-x86_64=/musl-x86_64
diff --git a/src/ci/run.sh b/src/ci/run.sh
index 10f2d15da34..152694346aa 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -26,17 +26,13 @@ if [ "$NO_VENDOR" = "" ]; then
   ENABLE_VENDOR=--enable-vendor
 fi
 
-if [ "$NO_CCACHE" = "" ]; then
-  ENABLE_CCACHE=--enable-ccache
-fi
-
 set -ex
 
 $SRC/configure \
   --disable-manage-submodules \
   --enable-debug-assertions \
   --enable-quiet-tests \
-  $ENABLE_CCACHE \
+  --enable-sccache \
   $ENABLE_VENDOR \
   $ENABLE_LLVM_ASSERTIONS \
   $RUST_CONFIGURE_ARGS
generated by cgit-pink 1.4.1-3-g733a5 (git 2.37.1) at 2026-05-09 23:37:48 +0000