Auto merge of #38708 - alexcrichton:add-distcheck, r=brson

Gate on distcheck on Travis

This commit adds a new entry to the Travis matrix to gate on distcheck, the illustrious test process that has historically taken *8 hours* to complete and also breaks all the time on nightly. By adding it to Travis we should hope to never see nightly breakage (like https://github.com/rust-lang/rust/issues/38690) because of this ever again!

"But wait, surely we can't wait 8 hours for all PRs!" you might be thinking, and you are indeed correct. The distcheck added here is much more optimized for speed than the old buildbot instances for a number of reasons:

* We're not building *two host compilers* beforehand. The current distcheck bot does a cross for i686 Linux and x86_64 Linux before it actually runs distcheck, building 6 compilers and LLVM twice. None of this is done in parallel as well (e.g. `-j1`). Not doing any of this work will be a huge win!
* We're using sccache to compile LLVM, so it should be much faster. Distcheck on the bots didn't cache LLVM well and rebuilt it every time.

All in all, this version of "distcheck" should be exactly like other matrix entries that run tests except that it's a *little* slower to start as it has to create the source tarball then rebuild the build system in the distcheck dir. Overall this should be well under the 2 hours that Android is currently taking anyway.

Closes https://github.com/rust-lang/rust/issues/38691

7 files changed, 52 insertions, 7 deletions

diff --git a/.travis.yml b/.travis.yml
index b19877e3843..26cabf92bda 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,6 +26,7 @@ matrix:
     - env: IMAGE=x86_64-gnu-make
     - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1
     - env: IMAGE=x86_64-musl
+    - env: IMAGE=x86_64-gnu-distcheck
 
     # OSX builders
     - env: >
diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index a032f57a416..6db1afa54a6 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -562,6 +562,7 @@ pub fn distcheck(build: &Build) {
        .current_dir(&dir);
     build.run(&mut cmd);
     build.run(Command::new("./configure")
+                     .args(&build.config.configure_args)
                      .current_dir(&dir));
     build.run(Command::new(build_helper::make(&build.config.build))
                      .arg("check")
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 9767afd73ca..11c5b374ed9 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -94,6 +94,7 @@ pub struct Config {
     pub nodejs: Option<PathBuf>,
     pub gdb: Option<PathBuf>,
     pub python: Option<PathBuf>,
+    pub configure_args: Vec<String>,
 }
 
 /// Per-target configuration stored in the global configuration structure.
@@ -519,6 +520,11 @@ impl Config {
                 "CFG_ENABLE_SCCACHE" if value == "1" => {
                     self.ccache = Some("sccache".to_string());
                 }
+                "CFG_CONFIGURE_ARGS" if value.len() > 0 => {
+                    self.configure_args = value.split_whitespace()
+                                               .map(|s| s.to_string())
+                                               .collect();
+                }
                 _ => {}
             }
         }
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 6e3174ed2f6..428c3da7764 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -397,6 +397,13 @@ pub fn rust_src(build: &Build, host: &str) {
             }
         }
 
+        // If we're inside the vendor directory then we need to preserve
+        // everything as Cargo's vendoring support tracks all checksums and we
+        // want to be sure we don't accidentally leave out a file.
+        if spath.contains("vendor") {
+            return true
+        }
+
         let excludes = [
             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs
index c9e756b6f99..2ab3776ada0 100644
--- a/src/bootstrap/util.rs
+++ b/src/bootstrap/util.rs
@@ -76,9 +76,9 @@ pub fn cp_r(src: &Path, dst: &Path) {
 /// Copies the `src` directory recursively to `dst`. Both are assumed to exist
 /// when this function is called. Unwanted files or directories can be skipped
 /// by returning `false` from the filter function.
-pub fn cp_filtered<F: Fn(&Path) -> bool>(src: &Path, dst: &Path, filter: &F) {
+pub fn cp_filtered(src: &Path, dst: &Path, filter: &Fn(&Path) -> bool) {
     // Inner function does the actual work
-    fn recurse<F: Fn(&Path) -> bool>(src: &Path, dst: &Path, relative: &Path, filter: &F) {
+    fn recurse(src: &Path, dst: &Path, relative: &Path, filter: &Fn(&Path) -> bool) {
         for f in t!(fs::read_dir(src)) {
             let f = t!(f);
             let path = f.path();
diff --git a/src/ci/docker/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/x86_64-gnu-distcheck/Dockerfile
new file mode 100644
index 00000000000..880c70866b0
--- /dev/null
+++ b/src/ci/docker/x86_64-gnu-distcheck/Dockerfile
@@ -0,0 +1,26 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++ \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  sudo \
+  gdb \
+  xz-utils
+
+ENV SCCACHE_DIGEST=7237e38e029342fa27b7ac25412cb9d52554008b12389727320bd533fd7f05b6a96d55485f305caf95e5c8f5f97c3313e10012ccad3e752aba2518f3522ba783
+RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST | \
+      tar xJf - -C /usr/local/bin --strip-components=1
+
+RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
+    dpkg -i dumb-init_*.deb && \
+    rm dumb-init_*.deb
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu
+ENV XPY_RUN test distcheck
diff --git a/src/ci/run.sh b/src/ci/run.sh
index 152694346aa..3dc3cf91501 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -43,10 +43,14 @@ else
     ncpus=$(nproc)
 fi
 
-make -j $ncpus tidy
-make -j $ncpus
-if [ ! -z "$XPY_CHECK" ]; then
-  exec python2.7 $SRC/x.py $XPY_CHECK
+if [ ! -z "$XPY_RUN" ]; then
+  exec python2.7 $SRC/x.py $XPY_RUN
 else
-  exec make $RUST_CHECK_TARGET -j $ncpus
+  make -j $ncpus tidy
+  make -j $ncpus
+  if [ ! -z "$XPY_CHECK" ]; then
+    exec python2.7 $SRC/x.py $XPY_CHECK
+  else
+    exec make $RUST_CHECK_TARGET -j $ncpus
+  fi
 fi