Auto merge of #37039 - Manishearth:rollup, r=Manishearth

Rollup of 6 pull requests

- Successful merges: #36937, #37016, #37028, #37029, #37031, #37034
- Failed merges: #37027

3 files changed, 75 insertions, 212 deletions

diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock
deleted file mode 100644
index babbb6a16ba..00000000000
--- a/src/bootstrap/Cargo.lock
+++ /dev/null
@@ -1,180 +0,0 @@
-[root]
-name = "bootstrap"
-version = "0.0.0"
-dependencies = [
- "build_helper 0.1.0",
- "cmake 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)",
- "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "gcc 0.3.35 (git+https://github.com/alexcrichton/gcc-rs)",
- "getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",
- "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
- "md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 0.1.73 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
- "toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "aho-corasick"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "build_helper"
-version = "0.1.0"
-
-[[package]]
-name = "cmake"
-version = "0.1.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "filetime"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "gcc"
-version = "0.3.35"
-source = "git+https://github.com/alexcrichton/gcc-rs#8ff5360b6e0dc4f3c9d3f71036f1ff403c68469d"
-
-[[package]]
-name = "gcc"
-version = "0.3.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "getopts"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "kernel32-sys"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "md5"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "memchr"
-version = "0.1.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "num_cpus"
-version = "0.2.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "regex"
-version = "0.1.73"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "aho-corasick 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex-syntax 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "thread_local 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "rustc-serialize"
-version = "0.3.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "thread-id"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "thread_local"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "toml"
-version = "0.1.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "utf8-ranges"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "winapi"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "winapi-build"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[metadata]
-"checksum aho-corasick 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2b3fb52b09c1710b961acb35390d514be82e4ac96a9969a8e38565a29b878dc9"
-"checksum cmake 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "dfcf5bcece56ef953b8ea042509e9dcbdfe97820b7e20d86beb53df30ed94978"
-"checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922"
-"checksum gcc 0.3.35 (git+https://github.com/alexcrichton/gcc-rs)" = "<none>"
-"checksum gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)" = "91ecd03771effb0c968fd6950b37e89476a578aaf1c70297d8e92b6516ec3312"
-"checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685"
-"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
-"checksum libc 0.2.15 (registry+https://github.com/rust-lang/crates.io-index)" = "23e3757828fa702a20072c37ff47938e9dd331b92fac6e223d26d4b7a55f7ee2"
-"checksum md5 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a5539a8dee9b4ae308c9c406a379838b435a8f2c84cf9fedc6d5a576be9888db"
-"checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20"
-"checksum num_cpus 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "cee7e88156f3f9e19bdd598f8d6c9db7bf4078f99f8381f43a55b09648d1a6e3"
-"checksum regex 0.1.73 (registry+https://github.com/rust-lang/crates.io-index)" = "56b7ee9f764ecf412c6e2fff779bca4b22980517ae335a21aeaf4e32625a5df2"
-"checksum regex-syntax 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "31040aad7470ad9d8c46302dcffba337bb4289ca5da2e3cd6e37b64109a85199"
-"checksum rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)" = "6159e4e6e559c81bd706afe9c8fd68f547d3e851ce12e76b1de7914bab61691b"
-"checksum thread-id 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a9539db560102d1cef46b8b78ce737ff0bb64e7e18d35b2a5688f7d097d0ff03"
-"checksum thread_local 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "55dd963dbaeadc08aa7266bf7f91c3154a7805e32bb94b820b769d2ef3b4744d"
-"checksum toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "0590d72182e50e879c4da3b11c6488dae18fccb1ae0c7a3eda18e16795844796"
-"checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f"
-"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
-"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index b8417218a22..b8d0eb3ff99 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -13,19 +13,44 @@
 //! This file implements the various regression test suites that we execute on
 //! our CI.
 
+use std::collections::{HashMap, HashSet};
 use std::env;
-use std::fs::{self, File};
-use std::io::prelude::*;
+use std::fs;
 use std::path::{PathBuf, Path};
 use std::process::Command;
 
 use build_helper::output;
+use rustc_serialize::json;
 
 use {Build, Compiler, Mode};
 use util::{self, dylib_path, dylib_path_var};
 
 const ADB_TEST_DIR: &'static str = "/data/tmp";
 
+#[derive(RustcDecodable)]
+struct Output {
+    packages: Vec<Package>,
+    resolve: Resolve,
+}
+
+#[derive(RustcDecodable)]
+struct Package {
+    id: String,
+    name: String,
+    source: Option<String>,
+}
+
+#[derive(RustcDecodable)]
+struct Resolve {
+    nodes: Vec<ResolveNode>,
+}
+
+#[derive(RustcDecodable)]
+struct ResolveNode {
+    id: String,
+    dependencies: Vec<String>,
+}
+
 /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
 ///
 /// This tool in `src/tools` will verify the validity of all our links in the
@@ -263,56 +288,74 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) {
 /// It essentially is the driver for running `cargo test`.
 ///
 /// Currently this runs all tests for a DAG by passing a bunch of `-p foo`
-/// arguments, and those arguments are discovered from `Cargo.lock`.
+/// arguments, and those arguments are discovered from `cargo metadata`.
 pub fn krate(build: &Build,
              compiler: &Compiler,
              target: &str,
              mode: Mode) {
-    let (name, path, features) = match mode {
-        Mode::Libstd => ("libstd", "src/rustc/std_shim", build.std_features()),
-        Mode::Libtest => ("libtest", "src/rustc/test_shim", String::new()),
-        Mode::Librustc => ("librustc", "src/rustc", build.rustc_features()),
+    let (name, path, features, root) = match mode {
+        Mode::Libstd => {
+            ("libstd", "src/rustc/std_shim", build.std_features(), "std_shim")
+        }
+        Mode::Libtest => {
+            ("libtest", "src/rustc/test_shim", String::new(), "test_shim")
+        }
+        Mode::Librustc => {
+            ("librustc", "src/rustc", build.rustc_features(), "rustc-main")
+        }
         _ => panic!("can only test libraries"),
     };
     println!("Testing {} stage{} ({} -> {})", name, compiler.stage,
              compiler.host, target);
 
+    // Run `cargo metadata` to figure out what crates we're testing.
+    //
+    // Down below we're going to call `cargo test`, but to test the right set
+    // of packages we're going to have to know what `-p` arguments to pass it
+    // to know what crates to test. Here we run `cargo metadata` to learn about
+    // the dependency graph and what `-p` arguments there are.
+    let mut cargo = Command::new(&build.cargo);
+    cargo.arg("metadata")
+         .arg("--manifest-path").arg(build.src.join(path).join("Cargo.toml"));
+    let output = output(&mut cargo);
+    let output: Output = json::decode(&output).unwrap();
+    let id2pkg = output.packages.iter()
+                        .map(|pkg| (&pkg.id, pkg))
+                        .collect::<HashMap<_, _>>();
+    let id2deps = output.resolve.nodes.iter()
+                        .map(|node| (&node.id, &node.dependencies))
+                        .collect::<HashMap<_, _>>();
+
     // Build up the base `cargo test` command.
+    //
+    // Pass in some standard flags then iterate over the graph we've discovered
+    // in `cargo metadata` with the maps above and figure out what `-p`
+    // arguments need to get passed.
     let mut cargo = build.cargo(compiler, mode, target, "test");
     cargo.arg("--manifest-path")
          .arg(build.src.join(path).join("Cargo.toml"))
          .arg("--features").arg(features);
 
-    // Generate a list of `-p` arguments to pass to the `cargo test` invocation
-    // by crawling the corresponding Cargo.lock file.
-    let lockfile = build.src.join(path).join("Cargo.lock");
-    let mut contents = String::new();
-    t!(t!(File::open(&lockfile)).read_to_string(&mut contents));
-    let mut lines = contents.lines();
-    while let Some(line) = lines.next() {
-        let prefix = "name = \"";
-        if !line.starts_with(prefix) {
+    let mut visited = HashSet::new();
+    let root_pkg = output.packages.iter().find(|p| p.name == root).unwrap();
+    let mut next = vec![&root_pkg.id];
+    while let Some(id) = next.pop() {
+        // Skip any packages with sources listed, as these come from crates.io
+        // and we shouldn't be testing them.
+        if id2pkg[id].source.is_some() {
             continue
         }
-        lines.next(); // skip `version = ...`
-
-        // skip crates.io or otherwise non-path crates
-        if let Some(line) = lines.next() {
-            if line.starts_with("source") {
-                continue
-            }
-        }
-
-        let crate_name = &line[prefix.len()..line.len() - 1];
-
         // Right now jemalloc is our only target-specific crate in the sense
         // that it's not present on all platforms. Custom skip it here for now,
         // but if we add more this probably wants to get more generalized.
-        if crate_name.contains("jemalloc") {
-            continue
+        if !id.contains("jemalloc") {
+            cargo.arg("-p").arg(&id2pkg[id].name);
+        }
+        for dep in id2deps[id] {
+            if visited.insert(dep) {
+                next.push(dep);
+            }
         }
-
-        cargo.arg("-p").arg(crate_name);
     }
 
     // The tests are going to run with the *target* libraries, so we need to
diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs
index 22539b31ef2..3bf0f211921 100644
--- a/src/bootstrap/step.rs
+++ b/src/bootstrap/step.rs
@@ -524,7 +524,7 @@ impl<'a> Step<'a> {
                      self.target(compiler.host).tool_compiletest(compiler.stage)]
             }
             Source::CheckDocs { compiler } => {
-                vec![self.libstd(compiler)]
+                vec![self.libtest(compiler)]
             }
             Source::CheckErrorIndex { compiler } => {
                 vec![self.libstd(compiler),