Auto merge of #39728 - eddyb:vendeur-tres-bien, r=alexcrichton

Automate vendoring by invoking cargo-vendor when building src dist tarballs.

This avoids #39633 bringing the `src/vendor` checked into git by #37524, past 200,000 lines of code.

I believe the strategy of having rustbuild run `cargo vendor` during the `dist src` step is sound.

However, the only way to be sure `cargo-vendor` exists is to run `cargo install --force cargo-vendor`, which will recompile it every time (not passing `--force` means you can't tell between "already exists" and "build error"). ~~This is quite suboptimal and I'd like to somehow do it in each `Dockerfile` that would need it.~~

* [ ] Cache `CARGO_HOME` (i.e. `~/.cargo`) between CI runs
  * `bin/cargo-vendor` and the actual caches are the relevant bits
* [x] Do not build `cargo-vendor` all the time
  * ~~Maybe detect `~/.cargo/bin/cargo-vendor` already exists?~~
  * ~~Could also try to build it in a `Dockerfile` but do we have `cargo`/`rustc` there?~~
  * Final solution: check `cargo install --list` for a line starting with `cargo-vendor `

cc @rust-lang/tools

6 files changed, 48 insertions, 9 deletions

diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index caf2402f40c..ee3f663dbd5 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -294,6 +294,8 @@ class RustBuild(object):
             raise Exception("no cargo executable found at `%s`" % self.cargo())
         args = [self.cargo(), "build", "--manifest-path",
                 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
+        if self.use_locked_deps:
+            args.append("--locked")
         if self.use_vendored_sources:
             args.append("--frozen")
         self.run(args, env)
@@ -455,6 +457,9 @@ def main():
     rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
                               'CFG_ENABLE_VENDOR' in rb.config_mk
 
+    rb.use_locked_deps = '\nlocked-deps = true' in rb.config_toml or \
+                         'CFG_ENABLE_LOCKED_DEPS' in rb.config_mk
+
     if 'SUDO_USER' in os.environ and not rb.use_vendored_sources:
         if os.environ.get('USER') != os.environ['SUDO_USER']:
             rb.use_vendored_sources = True
diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index cbfbcbe4f0c..00758460bec 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -108,8 +108,12 @@ pub fn cargotest(build: &Build, stage: u32, host: &str) {
 pub fn tidy(build: &Build, host: &str) {
     println!("tidy check ({})", host);
     let compiler = Compiler::new(0, host);
-    build.run(build.tool_cmd(&compiler, "tidy")
-                   .arg(build.src.join("src")));
+    let mut cmd = build.tool_cmd(&compiler, "tidy");
+    cmd.arg(build.src.join("src"));
+    if !build.config.vendor {
+        cmd.arg("--no-vendor");
+    }
+    build.run(&mut cmd);
 }
 
 fn testdir(build: &Build, host: &str) -> PathBuf {
@@ -643,6 +647,7 @@ pub fn distcheck(build: &Build) {
     build.run(&mut cmd);
     build.run(Command::new("./configure")
                      .args(&build.config.configure_args)
+                     .arg("--enable-vendor")
                      .current_dir(&dir));
     build.run(Command::new(build_helper::make(&build.config.build))
                      .arg("check")
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 47f505ad37e..e95416be4b6 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -44,6 +44,7 @@ pub struct Config {
     pub submodules: bool,
     pub compiler_docs: bool,
     pub docs: bool,
+    pub locked_deps: bool,
     pub vendor: bool,
     pub target_config: HashMap<String, Target>,
     pub full_bootstrap: bool,
@@ -145,6 +146,7 @@ struct Build {
     docs: Option<bool>,
     submodules: Option<bool>,
     gdb: Option<String>,
+    locked_deps: Option<bool>,
     vendor: Option<bool>,
     nodejs: Option<String>,
     python: Option<String>,
@@ -294,6 +296,7 @@ impl Config {
         set(&mut config.compiler_docs, build.compiler_docs);
         set(&mut config.docs, build.docs);
         set(&mut config.submodules, build.submodules);
+        set(&mut config.locked_deps, build.locked_deps);
         set(&mut config.vendor, build.vendor);
         set(&mut config.full_bootstrap, build.full_bootstrap);
         set(&mut config.extended, build.extended);
@@ -440,6 +443,7 @@ impl Config {
                 ("LOCAL_REBUILD", self.local_rebuild),
                 ("NINJA", self.ninja),
                 ("CODEGEN_TESTS", self.codegen_tests),
+                ("LOCKED_DEPS", self.locked_deps),
                 ("VENDOR", self.vendor),
                 ("FULL_BOOTSTRAP", self.full_bootstrap),
                 ("EXTENDED", self.extended),
diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example
index 5f4303a728c..f95e890f346 100644
--- a/src/bootstrap/config.toml.example
+++ b/src/bootstrap/config.toml.example
@@ -108,6 +108,10 @@
 # Note that Python 2 is currently required.
 #python = "python2.7"
 
+# Force Cargo to check that Cargo.lock describes the precise dependency
+# set that all the Cargo.toml files create, instead of updating it.
+#locked-deps = false
+
 # Indicate whether the vendored sources are used for Rust dependencies or not
 #vendor = false
 
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 2da2892150b..52a7c63c904 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -360,6 +360,8 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
     t!(fs::remove_dir_all(&image));
 }
 
+const CARGO_VENDOR_VERSION: &'static str = "0.1.4";
+
 /// Creates the `rust-src` installer component and the plain source tarball
 pub fn rust_src(build: &Build) {
     println!("Dist src");
@@ -404,13 +406,6 @@ pub fn rust_src(build: &Build) {
             }
         }
 
-        // If we're inside the vendor directory then we need to preserve
-        // everything as Cargo's vendoring support tracks all checksums and we
-        // want to be sure we don't accidentally leave out a file.
-        if spath.contains("vendor") {
-            return true
-        }
-
         let excludes = [
             "CVS", "RCS", "SCCS", ".git", ".gitignore", ".gitmodules",
             ".gitattributes", ".cvsignore", ".svn", ".arch-ids", "{arch}",
@@ -433,6 +428,29 @@ pub fn rust_src(build: &Build) {
         copy(&build.src.join(item), &dst_src.join(item));
     }
 
+    // Get cargo-vendor installed, if it isn't already.
+    let mut has_cargo_vendor = false;
+    let mut cmd = Command::new(&build.cargo);
+    for line in output(cmd.arg("install").arg("--list")).lines() {
+        has_cargo_vendor |= line.starts_with("cargo-vendor ");
+    }
+    if !has_cargo_vendor {
+        let mut cmd = Command::new(&build.cargo);
+        cmd.arg("install")
+           .arg("--force")
+           .arg("--debug")
+           .arg("--vers").arg(CARGO_VENDOR_VERSION)
+           .arg("cargo-vendor")
+           .env("RUSTC", &build.rustc);
+        build.run(&mut cmd);
+    }
+
+    // Vendor all Cargo dependencies
+    let mut cmd = Command::new(&build.cargo);
+    cmd.arg("vendor")
+       .current_dir(&dst_src.join("src"));
+    build.run(&mut cmd);
+
     // Create source tarball in rust-installer format
     let mut cmd = Command::new("sh");
     cmd.arg(sanitize_sh(&build.src.join("src/rust-installer/gen-installer.sh")))
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index ba6b34343f0..7bd611eb53e 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -528,6 +528,9 @@ impl Build {
         if self.config.rust_optimize && cmd != "bench" {
             cargo.arg("--release");
         }
+        if self.config.locked_deps {
+            cargo.arg("--locked");
+        }
         if self.config.vendor || self.is_sudo {
             cargo.arg("--frozen");
         }