Update miri to rustc changes

26 files changed, 1004 insertions, 608 deletions

diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml
index 3f1d03b1872..bbbbf0e1915 100644
--- a/src/bootstrap/Cargo.toml
+++ b/src/bootstrap/Cargo.toml
@@ -34,7 +34,7 @@ cmake = "0.1.23"
 filetime = "0.1"
 num_cpus = "1.0"
 getopts = "0.2"
-cc = "1.0"
+cc = "1.0.1"
 libc = "0.2"
 serde = "1.0.8"
 serde_derive = "1.0.8"
diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md
index e543b8c070b..9ff681ac680 100644
--- a/src/bootstrap/README.md
+++ b/src/bootstrap/README.md
@@ -39,7 +39,7 @@ The script accepts commands, flags, and arguments to determine what to do:
   ```
 
   If files are dirty that would normally be rebuilt from stage 0, that can be
-  overidden using `--keep-stage 0`. Using `--keep-stage n` will skip all steps
+  overridden using `--keep-stage 0`. Using `--keep-stage n` will skip all steps
   that belong to stage n or earlier:
 
   ```
@@ -126,17 +126,17 @@ install a nightly, presumably using `rustup`. You will then want to
 configure your directory to use this build, like so:
 
 ```
-# configure to use local rust instead of downloding a beta.
+# configure to use local rust instead of downloading a beta.
 # `--local-rust-root` is optional here. If elided, we will
 # use whatever rustc we find on your PATH.
-> configure --enable-rustbuild --local-rust-root=~/.cargo/ --enable-local-rebuild
+> ./configure --local-rust-root=~/.cargo/ --enable-local-rebuild
 ```
 
 After that, you can use the `--incremental` flag to actually do
 incremental builds:
 
 ```
-> ../x.py build --incremental
+> ./x.py build --incremental
 ```
 
 The `--incremental` flag will store incremental compilation artifacts
diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs
index 848b10d312c..631c9f72f35 100644
--- a/src/bootstrap/bin/rustc.rs
+++ b/src/bootstrap/bin/rustc.rs
@@ -31,8 +31,6 @@ extern crate bootstrap;
 
 use std::env;
 use std::ffi::OsString;
-use std::io;
-use std::io::prelude::*;
 use std::str::FromStr;
 use std::path::PathBuf;
 use std::process::{Command, ExitStatus};
@@ -122,19 +120,14 @@ fn main() {
             cmd.arg("-L").arg(&root);
         }
 
-        // Pass down extra flags, commonly used to configure `-Clinker` when
-        // cross compiling.
-        if let Ok(s) = env::var("RUSTC_FLAGS") {
-            cmd.args(&s.split(" ").filter(|s| !s.is_empty()).collect::<Vec<_>>());
+        // Override linker if necessary.
+        if let Ok(target_linker) = env::var("RUSTC_TARGET_LINKER") {
+            cmd.arg(format!("-Clinker={}", target_linker));
         }
 
         // Pass down incremental directory, if any.
         if let Ok(dir) = env::var("RUSTC_INCREMENTAL") {
             cmd.arg(format!("-Zincremental={}", dir));
-
-            if verbose > 0 {
-                cmd.arg("-Zincremental-info");
-            }
         }
 
         let crate_name = args.windows(2)
@@ -182,12 +175,16 @@ fn main() {
         if let Ok(s) = env::var("RUSTC_CODEGEN_UNITS") {
             cmd.arg("-C").arg(format!("codegen-units={}", s));
         }
+        if env::var("RUSTC_THINLTO").is_ok() {
+            cmd.arg("-Ccodegen-units=16").arg("-Zthinlto");
+        }
 
         // Emit save-analysis info.
         if env::var("RUSTC_SAVE_ANALYSIS") == Ok("api".to_string()) {
             cmd.arg("-Zsave-analysis");
             cmd.env("RUST_SAVE_ANALYSIS_CONFIG",
-                    "{\"output_file\": null,\"full_docs\": false,\"pub_only\": true,\
+                    "{\"output_file\": null,\"full_docs\": false,\
+                     \"pub_only\": true,\"reachable_only\": false,\
                      \"distro_crate\": true,\"signatures\": false,\"borrow_data\": false}");
         }
 
@@ -258,6 +255,11 @@ fn main() {
         if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() {
             cmd.arg("-Z").arg("force-unstable-if-unmarked");
         }
+    } else {
+        // Override linker if necessary.
+        if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") {
+            cmd.arg(format!("-Clinker={}", host_linker));
+        }
     }
 
     let color = match env::var("RUSTC_COLOR") {
@@ -270,7 +272,7 @@ fn main() {
     }
 
     if verbose > 1 {
-        writeln!(&mut io::stderr(), "rustc command: {:?}", cmd).unwrap();
+        eprintln!("rustc command: {:?}", cmd);
     }
 
     // Actually run the compiler!
diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs
index d7d72d5dd56..4e975adc972 100644
--- a/src/bootstrap/bin/rustdoc.rs
+++ b/src/bootstrap/bin/rustdoc.rs
@@ -47,6 +47,17 @@ fn main() {
     if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() {
         cmd.arg("-Z").arg("force-unstable-if-unmarked");
     }
+    if let Some(linker) = env::var_os("RUSTC_TARGET_LINKER") {
+        cmd.arg("--linker").arg(linker).arg("-Z").arg("unstable-options");
+    }
+
+    // Bootstrap's Cargo-command builder sets this variable to the current Rust version; let's pick
+    // it up so we can make rustdoc print this into the docs
+    if let Some(version) = env::var_os("RUSTDOC_CRATE_VERSION") {
+        // This "unstable-options" can be removed when `--crate-version` is stabilized
+        cmd.arg("-Z").arg("unstable-options")
+           .arg("--crate-version").arg(version);
+    }
 
     std::process::exit(match cmd.status() {
         Ok(s) => s.code().unwrap_or(1),
diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index 4a8c3dcebcb..707aceebb1e 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -8,7 +8,7 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 import argparse
 import contextlib
 import datetime
@@ -294,7 +294,7 @@ def default_build_triple():
             raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
         # only the n64 ABI is supported, indicate it
         ostype += 'abi64'
-    elif cputype == 'sparcv9':
+    elif cputype == 'sparcv9' or cputype == 'sparc64':
         pass
     else:
         err = "unknown cpu type: {}".format(cputype)
@@ -302,6 +302,7 @@ def default_build_triple():
 
     return "{}-{}".format(cputype, ostype)
 
+
 class RustBuild(object):
     """Provide all the methods required to build Rust"""
     def __init__(self):
@@ -498,7 +499,7 @@ class RustBuild(object):
 
         If the key does not exists, the result is None:
 
-        >>> rb.get_toml("key3") == None
+        >>> rb.get_toml("key3") is None
         True
         """
         for line in self.config_toml.splitlines():
@@ -531,7 +532,7 @@ class RustBuild(object):
         """
         config = self.get_toml(program)
         if config:
-            return config
+            return os.path.expanduser(config)
         return os.path.join(self.bin_root(), "bin", "{}{}".format(
             program, self.exe_suffix()))
 
@@ -647,7 +648,8 @@ class RustBuild(object):
                       if not ((module.endswith("llvm") and
                                self.get_toml('llvm-config')) or
                               (module.endswith("jemalloc") and
-                               self.get_toml('jemalloc')))]
+                               (self.get_toml('use-jemalloc') == "false" or
+                                self.get_toml('jemalloc'))))]
         run(["git", "submodule", "update",
              "--init", "--recursive"] + submodules,
             cwd=self.rust_root, verbose=self.verbose)
diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py
index 32ea4b4abe6..4db7e2ec016 100644
--- a/src/bootstrap/bootstrap_test.py
+++ b/src/bootstrap/bootstrap_test.py
@@ -10,6 +10,7 @@
 
 """Bootstrap tests"""
 
+from __future__ import absolute_import, division, print_function
 import os
 import doctest
 import unittest
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index ffd959d86f5..dcffc83c4b6 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -261,9 +261,10 @@ impl<'a> Builder<'a> {
                 doc::Reference, doc::Rustdoc, doc::CargoBook),
             Kind::Dist => describe!(dist::Docs, dist::Mingw, dist::Rustc, dist::DebuggerScripts,
                 dist::Std, dist::Analysis, dist::Src, dist::PlainSourceTarball, dist::Cargo,
-                dist::Rls, dist::Extended, dist::HashSign, dist::DontDistWithMiriEnabled),
+                dist::Rls, dist::Rustfmt, dist::Extended, dist::HashSign,
+                dist::DontDistWithMiriEnabled),
             Kind::Install => describe!(install::Docs, install::Std, install::Cargo, install::Rls,
-                install::Analysis, install::Src, install::Rustc),
+                install::Rustfmt, install::Analysis, install::Src, install::Rustc),
         }
     }
 
@@ -306,7 +307,7 @@ impl<'a> Builder<'a> {
             Subcommand::Bench { ref paths, .. } => (Kind::Bench, &paths[..]),
             Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]),
             Subcommand::Install { ref paths } => (Kind::Install, &paths[..]),
-            Subcommand::Clean => panic!(),
+            Subcommand::Clean { .. } => panic!(),
         };
 
         let builder = Builder {
@@ -413,12 +414,15 @@ impl<'a> Builder<'a> {
     pub fn rustdoc_cmd(&self, host: Interned<String>) -> Command {
         let mut cmd = Command::new(&self.out.join("bootstrap/debug/rustdoc"));
         let compiler = self.compiler(self.top_stage, host);
-        cmd
-            .env("RUSTC_STAGE", compiler.stage.to_string())
-            .env("RUSTC_SYSROOT", self.sysroot(compiler))
-            .env("RUSTC_LIBDIR", self.sysroot_libdir(compiler, self.build.build))
-            .env("CFG_RELEASE_CHANNEL", &self.build.config.channel)
-            .env("RUSTDOC_REAL", self.rustdoc(host));
+        cmd.env("RUSTC_STAGE", compiler.stage.to_string())
+           .env("RUSTC_SYSROOT", self.sysroot(compiler))
+           .env("RUSTC_LIBDIR", self.sysroot_libdir(compiler, self.build.build))
+           .env("CFG_RELEASE_CHANNEL", &self.build.config.channel)
+           .env("RUSTDOC_REAL", self.rustdoc(host))
+           .env("RUSTDOC_CRATE_VERSION", self.build.rust_version());
+        if let Some(linker) = self.build.linker(host) {
+            cmd.env("RUSTC_TARGET_LINKER", linker);
+        }
         cmd
     }
 
@@ -468,8 +472,6 @@ impl<'a> Builder<'a> {
              .env("RUSTC", self.out.join("bootstrap/debug/rustc"))
              .env("RUSTC_REAL", self.rustc(compiler))
              .env("RUSTC_STAGE", stage.to_string())
-             .env("RUSTC_CODEGEN_UNITS",
-                  self.config.rust_codegen_units.to_string())
              .env("RUSTC_DEBUG_ASSERTIONS",
                   self.config.rust_debug_assertions.to_string())
              .env("RUSTC_SYSROOT", self.sysroot(compiler))
@@ -481,15 +483,23 @@ impl<'a> Builder<'a> {
              } else {
                  PathBuf::from("/path/to/nowhere/rustdoc/not/required")
              })
-             .env("TEST_MIRI", self.config.test_miri.to_string())
-             .env("RUSTC_FLAGS", self.rustc_flags(target).join(" "));
+             .env("TEST_MIRI", self.config.test_miri.to_string());
+
+        if let Some(n) = self.config.rust_codegen_units {
+            cargo.env("RUSTC_CODEGEN_UNITS", n.to_string());
+        }
+
+        if let Some(host_linker) = self.build.linker(compiler.host) {
+            cargo.env("RUSTC_HOST_LINKER", host_linker);
+        }
+        if let Some(target_linker) = self.build.linker(target) {
+            cargo.env("RUSTC_TARGET_LINKER", target_linker);
+        }
+        cargo.env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string())
+            .env("RUSTC_DEBUGINFO_LINES", self.config.rust_debuginfo_lines.to_string());
 
         if mode != Mode::Tool {
-            // Tools don't get debuginfo right now, e.g. cargo and rls don't
-            // get compiled with debuginfo.
-            cargo.env("RUSTC_DEBUGINFO", self.config.rust_debuginfo.to_string())
-                 .env("RUSTC_DEBUGINFO_LINES", self.config.rust_debuginfo_lines.to_string())
-                 .env("RUSTC_FORCE_UNSTABLE", "1");
+            cargo.env("RUSTC_FORCE_UNSTABLE", "1");
 
             // Currently the compiler depends on crates from crates.io, and
             // then other crates can depend on the compiler (e.g. proc-macro
@@ -556,17 +566,35 @@ impl<'a> Builder<'a> {
 
         cargo.env("RUSTC_VERBOSE", format!("{}", self.verbosity));
 
-        // Specify some various options for build scripts used throughout
-        // the build.
+        // Throughout the build Cargo can execute a number of build scripts
+        // compiling C/C++ code and we need to pass compilers, archivers, flags, etc
+        // obtained previously to those build scripts.
+        // Build scripts use either the `cc` crate or `configure/make` so we pass
+        // the options through environment variables that are fetched and understood by both.
         //
         // FIXME: the guard against msvc shouldn't need to be here
         if !target.contains("msvc") {
-            cargo.env(format!("CC_{}", target), self.cc(target))
-                 .env(format!("AR_{}", target), self.ar(target).unwrap()) // only msvc is None
-                 .env(format!("CFLAGS_{}", target), self.cflags(target).join(" "));
+            let cc = self.cc(target);
+            cargo.env(format!("CC_{}", target), cc)
+                 .env("CC", cc);
+
+            let cflags = self.cflags(target).join(" ");
+            cargo.env(format!("CFLAGS_{}", target), cflags.clone())
+                 .env("CFLAGS", cflags.clone());
+
+            if let Some(ar) = self.ar(target) {
+                let ranlib = format!("{} s", ar.display());
+                cargo.env(format!("AR_{}", target), ar)
+                     .env("AR", ar)
+                     .env(format!("RANLIB_{}", target), ranlib.clone())
+                     .env("RANLIB", ranlib);
+            }
 
             if let Ok(cxx) = self.cxx(target) {
-                 cargo.env(format!("CXX_{}", target), cxx);
+                cargo.env(format!("CXX_{}", target), cxx)
+                     .env("CXX", cxx)
+                     .env(format!("CXXFLAGS_{}", target), cflags.clone())
+                     .env("CXXFLAGS", cflags);
             }
         }
 
@@ -574,6 +602,9 @@ impl<'a> Builder<'a> {
             cargo.env("RUSTC_SAVE_ANALYSIS", "api".to_string());
         }
 
+        // For `cargo doc` invocations, make rustdoc print the Rust version into the docs
+        cargo.env("RUSTDOC_CRATE_VERSION", self.build.rust_version());
+
         // Environment variables *required* throughout the build
         //
         // FIXME: should update code to not require this env var
@@ -582,12 +613,20 @@ impl<'a> Builder<'a> {
         // Set this for all builds to make sure doc builds also get it.
         cargo.env("CFG_RELEASE_CHANNEL", &self.build.config.channel);
 
-        if self.is_verbose() {
+        if self.is_very_verbose() {
             cargo.arg("-v");
         }
-        // FIXME: cargo bench does not accept `--release`
-        if self.config.rust_optimize && cmd != "bench" {
-            cargo.arg("--release");
+        if self.config.rust_optimize {
+            // FIXME: cargo bench does not accept `--release`
+            if cmd != "bench" {
+                cargo.arg("--release");
+            }
+
+            if self.config.rust_codegen_units.is_none() &&
+               self.build.is_rust_llvm(compiler.host)
+            {
+                cargo.env("RUSTC_THINLTO", "1");
+            }
         }
         if self.config.locked_deps {
             cargo.arg("--locked");
diff --git a/src/bootstrap/cc_detect.rs b/src/bootstrap/cc_detect.rs
index 08df65c7611..e531fdaf292 100644
--- a/src/bootstrap/cc_detect.rs
+++ b/src/bootstrap/cc_detect.rs
@@ -31,20 +31,51 @@
 //! ever be probed for. Instead the compilers found here will be used for
 //! everything.
 
+use std::collections::HashSet;
+use std::{env, iter};
+use std::path::{Path, PathBuf};
 use std::process::Command;
-use std::iter;
 
-use build_helper::{cc2ar, output};
+use build_helper::output;
 use cc;
 
 use Build;
 use config::Target;
 use cache::Interned;
 
+// The `cc` crate doesn't provide a way to obtain a path to the detected archiver,
+// so use some simplified logic here. First we respect the environment variable `AR`, then
+// try to infer the archiver path from the C compiler path.
+// In the future this logic should be replaced by calling into the `cc` crate.
+fn cc2ar(cc: &Path, target: &str) -> Option<PathBuf> {
+    if let Some(ar) = env::var_os("AR") {
+        Some(PathBuf::from(ar))
+    } else if target.contains("msvc") {
+        None
+    } else if target.contains("musl") {
+        Some(PathBuf::from("ar"))
+    } else if target.contains("openbsd") {
+        Some(PathBuf::from("ar"))
+    } else {
+        let parent = cc.parent().unwrap();
+        let file = cc.file_name().unwrap().to_str().unwrap();
+        for suffix in &["gcc", "cc", "clang"] {
+            if let Some(idx) = file.rfind(suffix) {
+                let mut file = file[..idx].to_owned();
+                file.push_str("ar");
+                return Some(parent.join(&file));
+            }
+        }
+        Some(parent.join(file))
+    }
+}
+
 pub fn find(build: &mut Build) {
     // For all targets we're going to need a C compiler for building some shims
     // and such as well as for being a linker for Rust code.
-    for target in build.targets.iter().chain(&build.hosts).cloned().chain(iter::once(build.build)) {
+    let targets = build.targets.iter().chain(&build.hosts).cloned().chain(iter::once(build.build))
+                               .collect::<HashSet<_>>();
+    for target in targets.into_iter() {
         let mut cfg = cc::Build::new();
         cfg.cargo_metadata(false).opt_level(0).warnings(false).debug(false)
            .target(&target).host(&build.build);
@@ -53,20 +84,27 @@ pub fn find(build: &mut Build) {
         if let Some(cc) = config.and_then(|c| c.cc.as_ref()) {
             cfg.compiler(cc);
         } else {
-            set_compiler(&mut cfg, "gcc", target, config, build);
+            set_compiler(&mut cfg, Language::C, target, config, build);
         }
 
         let compiler = cfg.get_compiler();
-        let ar = cc2ar(compiler.path(), &target);
+        let ar = if let ar @ Some(..) = config.and_then(|c| c.ar.clone()) {
+            ar
+        } else {
+            cc2ar(compiler.path(), &target)
+        };
+
         build.verbose(&format!("CC_{} = {:?}", &target, compiler.path()));
-        if let Some(ref ar) = ar {
+        build.cc.insert(target, compiler);
+        if let Some(ar) = ar {
             build.verbose(&format!("AR_{} = {:?}", &target, ar));
+            build.ar.insert(target, ar);
         }
-        build.cc.insert(target, (compiler, ar));
     }
 
     // For all host triples we need to find a C++ compiler as well
-    for host in build.hosts.iter().cloned().chain(iter::once(build.build)) {
+    let hosts = build.hosts.iter().cloned().chain(iter::once(build.build)).collect::<HashSet<_>>();
+    for host in hosts.into_iter() {
         let mut cfg = cc::Build::new();
         cfg.cargo_metadata(false).opt_level(0).warnings(false).debug(false).cpp(true)
            .target(&host).host(&build.build);
@@ -74,7 +112,7 @@ pub fn find(build: &mut Build) {
         if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) {
             cfg.compiler(cxx);
         } else {
-            set_compiler(&mut cfg, "g++", host, config, build);
+            set_compiler(&mut cfg, Language::CPlusPlus, host, config, build);
         }
         let compiler = cfg.get_compiler();
         build.verbose(&format!("CXX_{} = {:?}", host, compiler.path()));
@@ -83,7 +121,7 @@ pub fn find(build: &mut Build) {
 }
 
 fn set_compiler(cfg: &mut cc::Build,
-                gnu_compiler: &str,
+                compiler: Language,
                 target: Interned<String>,
                 config: Option<&Target>,
                 build: &Build) {
@@ -94,7 +132,7 @@ fn set_compiler(cfg: &mut cc::Build,
         t if t.contains("android") => {
             if let Some(ndk) = config.and_then(|c| c.ndk.as_ref()) {
                 let target = target.replace("armv7", "arm");
-                let compiler = format!("{}-{}", target, gnu_compiler);
+                let compiler = format!("{}-{}", target, compiler.clang());
                 cfg.compiler(ndk.join("bin").join(compiler));
             }
         }
@@ -103,6 +141,7 @@ fn set_compiler(cfg: &mut cc::Build,
         // which is a gcc version from ports, if this is the case.
         t if t.contains("openbsd") => {
             let c = cfg.get_compiler();
+            let gnu_compiler = compiler.gcc();
             if !c.path().ends_with(gnu_compiler) {
                 return
             }
@@ -145,3 +184,29 @@ fn set_compiler(cfg: &mut cc::Build,
         _ => {}
     }
 }
+
+/// The target programming language for a native compiler.
+enum Language {
+    /// The compiler is targeting C.
+    C,
+    /// The compiler is targeting C++.
+    CPlusPlus,
+}
+
+impl Language {
+    /// Obtains the name of a compiler in the GCC collection.
+    fn gcc(self) -> &'static str {
+        match self {
+            Language::C => "gcc",
+            Language::CPlusPlus => "g++",
+        }
+    }
+
+    /// Obtains the name of a compiler in the clang suite.
+    fn clang(self) -> &'static str {
+        match self {
+            Language::C => "clang",
+            Language::CPlusPlus => "clang++",
+        }
+    }
+}
diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs
index 6ed504dfe74..0485ebe17fb 100644
--- a/src/bootstrap/channel.rs
+++ b/src/bootstrap/channel.rs
@@ -24,7 +24,7 @@ use Build;
 use config::Config;
 
 // The version number
-pub const CFG_RELEASE_NUM: &str = "1.22.0";
+pub const CFG_RELEASE_NUM: &str = "1.24.0";
 
 // An optional number to put after the label, e.g. '.2' -> '-beta.2'
 // Be sure to make this starts with a dot to conform to semver pre-release
diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index 6e276f44668..ee780d1245e 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -65,19 +65,21 @@ impl fmt::Display for TestKind {
     }
 }
 
-fn try_run_expecting(build: &Build, cmd: &mut Command, expect: BuildExpectation) {
+fn try_run_expecting(build: &Build, cmd: &mut Command, expect: BuildExpectation) -> bool {
     if !build.fail_fast {
         if !build.try_run(cmd, expect) {
             let mut failures = build.delayed_failures.borrow_mut();
             failures.push(format!("{:?}", cmd));
+            return false;
         }
     } else {
         build.run_expecting(cmd, expect);
     }
+    true
 }
 
 fn try_run(build: &Build, cmd: &mut Command) {
-    try_run_expecting(build, cmd, BuildExpectation::None)
+    try_run_expecting(build, cmd, BuildExpectation::None);
 }
 
 fn try_run_quiet(build: &Build, cmd: &mut Command) {
@@ -246,19 +248,24 @@ impl Step for Rls {
         let compiler = builder.compiler(stage, host);
 
         builder.ensure(tool::Rls { compiler, target: self.host });
-        let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test");
-        cargo.arg("--manifest-path").arg(build.src.join("src/tools/rls/Cargo.toml"));
+        let mut cargo = tool::prepare_tool_cargo(builder,
+                                                 compiler,
+                                                 host,
+                                                 "test",
+                                                 "src/tools/rls");
 
         // Don't build tests dynamically, just a pain to work with
         cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
 
         builder.add_rustc_lib_path(compiler, &mut cargo);
 
-        try_run_expecting(
+        if try_run_expecting(
             build,
             &mut cargo,
             builder.build.config.toolstate.rls.passes(ToolState::Testing),
-        );
+        ) {
+            build.save_toolstate("rls", ToolState::Testing);
+        }
     }
 }
 
@@ -291,24 +298,30 @@ impl Step for Rustfmt {
         let compiler = builder.compiler(stage, host);
 
         builder.ensure(tool::Rustfmt { compiler, target: self.host });
-        let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test");
-        cargo.arg("--manifest-path").arg(build.src.join("src/tools/rustfmt/Cargo.toml"));
+        let mut cargo = tool::prepare_tool_cargo(builder,
+                                                 compiler,
+                                                 host,
+                                                 "test",
+                                                 "src/tools/rustfmt");
 
         // Don't build tests dynamically, just a pain to work with
         cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
 
         builder.add_rustc_lib_path(compiler, &mut cargo);
 
-        try_run_expecting(
+        if try_run_expecting(
             build,
             &mut cargo,
             builder.build.config.toolstate.rustfmt.passes(ToolState::Testing),
-        );
+        ) {
+            build.save_toolstate("rustfmt", ToolState::Testing);
+        }
     }
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Miri {
+    stage: u32,
     host: Interned<String>,
 }
 
@@ -324,6 +337,7 @@ impl Step for Miri {
 
     fn make_run(run: RunConfig) {
         run.builder.ensure(Miri {
+            stage: run.builder.top_stage,
             host: run.target,
         });
     }
@@ -331,33 +345,40 @@ impl Step for Miri {
     /// Runs `cargo test` for miri.
     fn run(self, builder: &Builder) {
         let build = builder.build;
+        let stage = self.stage;
         let host = self.host;
-        let compiler = builder.compiler(1, host);
-
-        let miri = builder.ensure(tool::Miri { compiler, target: self.host });
-        let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test");
-        cargo.arg("--manifest-path").arg(build.src.join("src/tools/miri/Cargo.toml"));
-
-        // Don't build tests dynamically, just a pain to work with
-        cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
-        // miri tests need to know about the stage sysroot
-        cargo.env("MIRI_SYSROOT", builder.sysroot(compiler));
-        cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler));
-        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
-        cargo.env("MIRI_PATH", miri);
-
-        builder.add_rustc_lib_path(compiler, &mut cargo);
+        let compiler = builder.compiler(stage, host);
 
-        try_run_expecting(
-            build,
-            &mut cargo,
-            builder.build.config.toolstate.miri.passes(ToolState::Testing),
-        );
+        if let Some(miri) = builder.ensure(tool::Miri { compiler, target: self.host }) {
+            let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test");
+            cargo.arg("--manifest-path").arg(build.src.join("src/tools/miri/Cargo.toml"));
+
+            // Don't build tests dynamically, just a pain to work with
+            cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
+            // miri tests need to know about the stage sysroot
+            cargo.env("MIRI_SYSROOT", builder.sysroot(compiler));
+            cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler));
+            cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
+            cargo.env("MIRI_PATH", miri);
+
+            builder.add_rustc_lib_path(compiler, &mut cargo);
+
+            if try_run_expecting(
+                build,
+                &mut cargo,
+                builder.build.config.toolstate.miri.passes(ToolState::Testing),
+            ) {
+                build.save_toolstate("miri", ToolState::Testing);
+            }
+        } else {
+            eprintln!("failed to test miri: could not build");
+        }
     }
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Clippy {
+    stage: u32,
     host: Interned<String>,
 }
 
@@ -372,6 +393,7 @@ impl Step for Clippy {
 
     fn make_run(run: RunConfig) {
         run.builder.ensure(Clippy {
+            stage: run.builder.top_stage,
             host: run.target,
         });
     }
@@ -379,25 +401,37 @@ impl Step for Clippy {
     /// Runs `cargo test` for clippy.
     fn run(self, builder: &Builder) {
         let build = builder.build;
+        let stage = self.stage;
         let host = self.host;
-        let compiler = builder.compiler(1, host);
-
-        let _clippy = builder.ensure(tool::Clippy { compiler, target: self.host });
-        let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test");
-        cargo.arg("--manifest-path").arg(build.src.join("src/tools/clippy/Cargo.toml"));
-
-        // Don't build tests dynamically, just a pain to work with
-        cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
-        // clippy tests need to know about the stage sysroot
-        cargo.env("SYSROOT", builder.sysroot(compiler));
-
-        builder.add_rustc_lib_path(compiler, &mut cargo);
+        let compiler = builder.compiler(stage, host);
 
-        try_run_expecting(
-            build,
-            &mut cargo,
-            builder.build.config.toolstate.clippy.passes(ToolState::Testing),
-        );
+        if let Some(clippy) = builder.ensure(tool::Clippy { compiler, target: self.host }) {
+            let mut cargo = builder.cargo(compiler, Mode::Tool, host, "test");
+            cargo.arg("--manifest-path").arg(build.src.join("src/tools/clippy/Cargo.toml"));
+
+            // Don't build tests dynamically, just a pain to work with
+            cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
+            // clippy tests need to know about the stage sysroot
+            cargo.env("SYSROOT", builder.sysroot(compiler));
+            cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler));
+            cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
+            let host_libs = builder.stage_out(compiler, Mode::Tool).join(builder.cargo_dir());
+            cargo.env("HOST_LIBS", host_libs);
+            // clippy tests need to find the driver
+            cargo.env("CLIPPY_DRIVER_PATH", clippy);
+
+            builder.add_rustc_lib_path(compiler, &mut cargo);
+
+            if try_run_expecting(
+                build,
+                &mut cargo,
+                builder.build.config.toolstate.clippy.passes(ToolState::Testing),
+            ) {
+                build.save_toolstate("clippy-driver", ToolState::Testing);
+            }
+        } else {
+            eprintln!("failed to test clippy: could not build");
+        }
     }
 }
 
@@ -736,12 +770,14 @@ impl Step for Compiletest {
             flags.push("-g".to_string());
         }
 
-        let mut hostflags = build.rustc_flags(compiler.host);
-        hostflags.extend(flags.clone());
+        if let Some(linker) = build.linker(target) {
+            cmd.arg("--linker").arg(linker);
+        }
+
+        let hostflags = flags.clone();
         cmd.arg("--host-rustcflags").arg(hostflags.join(" "));
 
-        let mut targetflags = build.rustc_flags(target);
-        targetflags.extend(flags);
+        let mut targetflags = flags.clone();
         targetflags.push(format!("-Lnative={}",
                                  build.test_helpers_out(target).display()));
         cmd.arg("--target-rustcflags").arg(targetflags.join(" "));
@@ -795,6 +831,9 @@ impl Step for Compiletest {
                 .arg("--cflags").arg(build.cflags(target).join(" "))
                 .arg("--llvm-components").arg(llvm_components.trim())
                 .arg("--llvm-cxxflags").arg(llvm_cxxflags.trim());
+                if let Some(ar) = build.ar(target) {
+                    cmd.arg("--ar").arg(ar);
+                }
             }
         }
         if suite == "run-make" && !build.config.llvm_enabled {
@@ -820,7 +859,7 @@ impl Step for Compiletest {
         // Note that if we encounter `PATH` we make sure to append to our own `PATH`
         // rather than stomp over it.
         if target.contains("msvc") {
-            for &(ref k, ref v) in build.cc[&target].0.env() {
+            for &(ref k, ref v) in build.cc[&target].env() {
                 if k != "PATH" {
                     cmd.env(k, v);
                 }
@@ -1185,7 +1224,8 @@ impl Step for Crate {
                     // ends up messing with various mtime calculations and such.
                     if !name.contains("jemalloc") &&
                        *name != *"build_helper" &&
-                       !(name.starts_with("rustc_") && name.ends_with("san")) {
+                       !(name.starts_with("rustc_") && name.ends_with("san")) &&
+                       name != "dlmalloc" {
                         cargo.arg("-p").arg(&format!("{}:0.0.0", name));
                     }
                     for dep in build.crates[&name].deps.iter() {
@@ -1218,6 +1258,17 @@ impl Step for Crate {
         if target.contains("emscripten") {
             cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target)),
                       build.config.nodejs.as_ref().expect("nodejs not configured"));
+        } else if target.starts_with("wasm32") {
+            // On the wasm32-unknown-unknown target we're using LTO which is
+            // incompatible with `-C prefer-dynamic`, so disable that here
+            cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
+
+            let node = build.config.nodejs.as_ref()
+                .expect("nodejs not configured");
+            let runner = format!("{} {}/src/etc/wasm32-shim.js",
+                                 node.display(),
+                                 build.src.display());
+            cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target)), &runner);
         } else if build.remote_tested(target) {
             cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target)),
                       format!("{} run",
diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs
index 119340a0190..87f194fb7d2 100644
--- a/src/bootstrap/clean.rs
+++ b/src/bootstrap/clean.rs
@@ -13,7 +13,7 @@
 //! Responsible for cleaning out a build directory of all old and stale
 //! artifacts to prepare for a fresh build. Currently doesn't remove the
 //! `build/cache` directory (download cache) or the `build/$target/llvm`
-//! directory as we want that cached between builds.
+//! directory unless the --all flag is present.
 
 use std::fs;
 use std::io::{self, ErrorKind};
@@ -21,24 +21,29 @@ use std::path::Path;
 
 use Build;
 
-pub fn clean(build: &Build) {
+pub fn clean(build: &Build, all: bool) {
     rm_rf("tmp".as_ref());
-    rm_rf(&build.out.join("tmp"));
-    rm_rf(&build.out.join("dist"));
 
-    for host in &build.hosts {
-        let entries = match build.out.join(host).read_dir() {
-            Ok(iter) => iter,
-            Err(_) => continue,
-        };
+    if all {
+        rm_rf(&build.out);
+    } else {
+        rm_rf(&build.out.join("tmp"));
+        rm_rf(&build.out.join("dist"));
 
-        for entry in entries {
-            let entry = t!(entry);
-            if entry.file_name().to_str() == Some("llvm") {
-                continue
+        for host in &build.hosts {
+            let entries = match build.out.join(host).read_dir() {
+                Ok(iter) => iter,
+                Err(_) => continue,
+            };
+
+            for entry in entries {
+                let entry = t!(entry);
+                if entry.file_name().to_str() == Some("llvm") {
+                    continue
+                }
+                let path = t!(entry.path().canonicalize());
+                rm_rf(&path);
             }
-            let path = t!(entry.path().canonicalize());
-            rm_rf(&path);
         }
     }
 }
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index 335e1690a2e..db013691bb1 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -29,7 +29,7 @@ use build_helper::{output, mtime, up_to_date};
 use filetime::FileTime;
 use serde_json;
 
-use util::{exe, libdir, is_dylib, copy};
+use util::{exe, libdir, is_dylib, copy, read_stamp_file, CiEnv};
 use {Build, Compiler, Mode};
 use native;
 use tool;
@@ -102,13 +102,13 @@ impl Step for Std {
             copy_musl_third_party_objects(build, target, &libdir);
         }
 
-        let out_dir = build.cargo_out(compiler, Mode::Libstd, target);
+        let out_dir = build.stage_out(compiler, Mode::Libstd);
         build.clear_if_dirty(&out_dir, &builder.rustc(compiler));
         let mut cargo = builder.cargo(compiler, Mode::Libstd, target, "build");
         std_cargo(build, &compiler, target, &mut cargo);
         run_cargo(build,
-                &mut cargo,
-                &libstd_stamp(build, compiler, target));
+                  &mut cargo,
+                  &libstd_stamp(build, compiler, target));
 
         builder.ensure(StdLink {
             compiler: builder.compiler(compiler.stage, build.build),
@@ -354,13 +354,13 @@ impl Step for Test {
         let _folder = build.fold_output(|| format!("stage{}-test", compiler.stage));
         println!("Building stage{} test artifacts ({} -> {})", compiler.stage,
                 &compiler.host, target);
-        let out_dir = build.cargo_out(compiler, Mode::Libtest, target);
+        let out_dir = build.stage_out(compiler, Mode::Libtest);
         build.clear_if_dirty(&out_dir, &libstd_stamp(build, compiler, target));
         let mut cargo = builder.cargo(compiler, Mode::Libtest, target, "build");
         test_cargo(build, &compiler, target, &mut cargo);
         run_cargo(build,
-                &mut cargo,
-                &libtest_stamp(build, compiler, target));
+                  &mut cargo,
+                  &libtest_stamp(build, compiler, target));
 
         builder.ensure(TestLink {
             compiler: builder.compiler(compiler.stage, build.build),
@@ -480,8 +480,9 @@ impl Step for Rustc {
         println!("Building stage{} compiler artifacts ({} -> {})",
                  compiler.stage, &compiler.host, target);
 
-        let out_dir = build.cargo_out(compiler, Mode::Librustc, target);
-        build.clear_if_dirty(&out_dir, &libtest_stamp(build, compiler, target));
+        let stage_out = builder.stage_out(compiler, Mode::Librustc);
+        build.clear_if_dirty(&stage_out, &libstd_stamp(build, compiler, target));
+        build.clear_if_dirty(&stage_out, &libtest_stamp(build, compiler, target));
 
         let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "build");
         rustc_cargo(build, &compiler, target, &mut cargo);
@@ -560,9 +561,6 @@ pub fn rustc_cargo(build: &Build,
     if let Some(ref s) = build.config.rustc_default_linker {
         cargo.env("CFG_DEFAULT_LINKER", s);
     }
-    if let Some(ref s) = build.config.rustc_default_ar {
-        cargo.env("CFG_DEFAULT_AR", s);
-    }
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
@@ -760,15 +758,7 @@ impl Step for Assemble {
 /// `sysroot_dst` provided.
 fn add_to_sysroot(sysroot_dst: &Path, stamp: &Path) {
     t!(fs::create_dir_all(&sysroot_dst));
-    let mut contents = Vec::new();
-    t!(t!(File::open(stamp)).read_to_end(&mut contents));
-    // This is the method we use for extracting paths from the stamp file passed to us. See
-    // run_cargo for more information (in this file).
-    for part in contents.split(|b| *b == 0) {
-        if part.is_empty() {
-            continue
-        }
-        let path = Path::new(t!(str::from_utf8(part)));
+    for path in read_stamp_file(stamp) {
         copy(&path, &sysroot_dst.join(path.file_name().unwrap()));
     }
 }
@@ -802,7 +792,7 @@ fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path) {
     cargo.arg("--message-format").arg("json")
          .stdout(Stdio::piped());
 
-    if stderr_isatty() {
+    if stderr_isatty() && build.ci_env == CiEnv::None {
         // since we pass message-format=json to cargo, we need to tell the rustc
         // wrapper to give us colored output if necessary. This is because we
         // only want Cargo's JSON output, not rustcs.
@@ -870,10 +860,19 @@ fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path) {
             // have a hash in the name, but there's a version of this file in
             // the `deps` folder which *does* have a hash in the name. That's
             // the one we'll want to we'll probe for it later.
-            toplevel.push((filename.file_stem().unwrap()
-                                    .to_str().unwrap().to_string(),
-                            filename.extension().unwrap().to_owned()
-                                    .to_str().unwrap().to_string()));
+            //
+            // We do not use `Path::file_stem` or `Path::extension` here,
+            // because some generated files may have multiple extensions e.g.
+            // `std-<hash>.dll.lib` on Windows. The aforementioned methods only
+            // split the file name by the last extension (`.lib`) while we need
+            // to split by all extensions (`.dll.lib`).
+            let expected_len = t!(filename.metadata()).len();
+            let filename = filename.file_name().unwrap().to_str().unwrap();
+            let mut parts = filename.splitn(2, '.');
+            let file_stem = parts.next().unwrap().to_owned();
+            let extension = parts.next().unwrap().to_owned();
+
+            toplevel.push((file_stem, extension, expected_len));
         }
     }
 
@@ -893,11 +892,12 @@ fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path) {
         .map(|e| t!(e))
         .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata())))
         .collect::<Vec<_>>();
-    for (prefix, extension) in toplevel {
-        let candidates = contents.iter().filter(|&&(_, ref filename, _)| {
+    for (prefix, extension, expected_len) in toplevel {
+        let candidates = contents.iter().filter(|&&(_, ref filename, ref meta)| {
             filename.starts_with(&prefix[..]) &&
                 filename[prefix.len()..].starts_with("-") &&
-                filename.ends_with(&extension[..])
+                filename.ends_with(&extension[..]) &&
+                meta.len() == expected_len
         });
         let max = candidates.max_by_key(|&&(_, _, ref metadata)| {
             FileTime::from_last_modification_time(metadata)
@@ -941,6 +941,8 @@ fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path) {
     let max = max.unwrap();
     let max_path = max_path.unwrap();
     if stamp_contents == new_contents && max <= stamp_mtime {
+        build.verbose(&format!("not updating {:?}; contents equal and {} <= {}",
+                stamp, max, stamp_mtime));
         return
     }
     if max > stamp_mtime {
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index c8b2ed042c1..9dd37d8e456 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -76,19 +76,18 @@ pub struct Config {
     pub llvm_static_stdcpp: bool,
     pub llvm_link_shared: bool,
     pub llvm_targets: Option<String>,
-    pub llvm_experimental_targets: Option<String>,
+    pub llvm_experimental_targets: String,
     pub llvm_link_jobs: Option<u32>,
 
     // rust codegen options
     pub rust_optimize: bool,
-    pub rust_codegen_units: u32,
+    pub rust_codegen_units: Option<u32>,
     pub rust_debug_assertions: bool,
     pub rust_debuginfo: bool,
     pub rust_debuginfo_lines: bool,
     pub rust_debuginfo_only_std: bool,
     pub rust_rpath: bool,
     pub rustc_default_linker: Option<String>,
-    pub rustc_default_ar: Option<String>,
     pub rust_optimize_tests: bool,
     pub rust_debuginfo_tests: bool,
     pub rust_dist_src: bool,
@@ -113,6 +112,8 @@ pub struct Config {
     pub channel: String,
     pub quiet_tests: bool,
     pub test_miri: bool,
+    pub save_toolstates: Option<PathBuf>,
+
     // Fallback musl-root for all targets
     pub musl_root: Option<PathBuf>,
     pub prefix: Option<PathBuf>,
@@ -144,6 +145,8 @@ pub struct Target {
     pub jemalloc: Option<PathBuf>,
     pub cc: Option<PathBuf>,
     pub cxx: Option<PathBuf>,
+    pub ar: Option<PathBuf>,
+    pub linker: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
     pub crt_static: Option<bool>,
     pub musl_root: Option<PathBuf>,
@@ -206,6 +209,11 @@ struct Install {
     bindir: Option<String>,
     libdir: Option<String>,
     mandir: Option<String>,
+
+    // standard paths, currently unused
+    datadir: Option<String>,
+    infodir: Option<String>,
+    localstatedir: Option<String>,
 }
 
 /// TOML representation of how the LLVM build is configured.
@@ -262,7 +270,6 @@ struct Rust {
     use_jemalloc: Option<bool>,
     backtrace: Option<bool>,
     default_linker: Option<String>,
-    default_ar: Option<String>,
     channel: Option<String>,
     musl_root: Option<String>,
     rpath: Option<bool>,
@@ -274,6 +281,7 @@ struct Rust {
     dist_src: Option<bool>,
     quiet_tests: Option<bool>,
     test_miri: Option<bool>,
+    save_toolstates: Option<String>,
 }
 
 /// TOML representation of how each build target is configured.
@@ -284,6 +292,8 @@ struct TomlTarget {
     jemalloc: Option<String>,
     cc: Option<String>,
     cxx: Option<String>,
+    ar: Option<String>,
+    linker: Option<String>,
     android_ndk: Option<String>,
     crt_static: Option<bool>,
     musl_root: Option<String>,
@@ -297,6 +307,7 @@ impl Config {
         let mut config = Config::default();
         config.llvm_enabled = true;
         config.llvm_optimize = true;
+        config.llvm_version_check = true;
         config.use_jemalloc = true;
         config.backtrace = true;
         config.rust_optimize = true;
@@ -304,7 +315,6 @@ impl Config {
         config.submodules = true;
         config.docs = true;
         config.rust_rpath = true;
-        config.rust_codegen_units = 1;
         config.channel = "dev".to_string();
         config.codegen_tests = true;
         config.ignore_git = false;
@@ -440,7 +450,8 @@ impl Config {
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
             set(&mut config.llvm_link_shared, llvm.link_shared);
             config.llvm_targets = llvm.targets.clone();
-            config.llvm_experimental_targets = llvm.experimental_targets.clone();
+            config.llvm_experimental_targets = llvm.experimental_targets.clone()
+                .unwrap_or("WebAssembly".to_string());
             config.llvm_link_jobs = llvm.link_jobs;
         }
 
@@ -464,12 +475,12 @@ impl Config {
             set(&mut config.quiet_tests, rust.quiet_tests);
             set(&mut config.test_miri, rust.test_miri);
             config.rustc_default_linker = rust.default_linker.clone();
-            config.rustc_default_ar = rust.default_ar.clone();
             config.musl_root = rust.musl_root.clone().map(PathBuf::from);
+            config.save_toolstates = rust.save_toolstates.clone().map(PathBuf::from);
 
             match rust.codegen_units {
-                Some(0) => config.rust_codegen_units = num_cpus::get() as u32,
-                Some(n) => config.rust_codegen_units = n,
+                Some(0) => config.rust_codegen_units = Some(num_cpus::get() as u32),
+                Some(n) => config.rust_codegen_units = Some(n),
                 None => {}
             }
         }
@@ -487,8 +498,10 @@ impl Config {
                 if let Some(ref s) = cfg.android_ndk {
                     target.ndk = Some(env::current_dir().unwrap().join(s));
                 }
-                target.cxx = cfg.cxx.clone().map(PathBuf::from);
                 target.cc = cfg.cc.clone().map(PathBuf::from);
+                target.cxx = cfg.cxx.clone().map(PathBuf::from);
+                target.ar = cfg.ar.clone().map(PathBuf::from);
+                target.linker = cfg.linker.clone().map(PathBuf::from);
                 target.crt_static = cfg.crt_static.clone();
                 target.musl_root = cfg.musl_root.clone().map(PathBuf::from);
                 target.qemu_rootfs = cfg.qemu_rootfs.clone().map(PathBuf::from);
@@ -520,7 +533,7 @@ impl Config {
         // Now that we've reached the end of our configuration, infer the
         // default values for all options that we haven't otherwise stored yet.
 
-        let default = config.channel == "nightly";
+        let default = false;
         config.llvm_assertions = llvm_assertions.unwrap_or(default);
 
         let default = match &config.channel[..] {
diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py
index 67337bf4421..48ca2838e4f 100755
--- a/src/bootstrap/configure.py
+++ b/src/bootstrap/configure.py
@@ -11,6 +11,7 @@
 
 # ignore-tidy-linelength
 
+from __future__ import absolute_import, division, print_function
 import sys
 import os
 rust_dir = os.path.dirname(os.path.abspath(__file__))
@@ -19,21 +20,26 @@ rust_dir = os.path.dirname(rust_dir)
 sys.path.append(os.path.join(rust_dir, "src", "bootstrap"))
 import bootstrap
 
-class Option:
+
+class Option(object):
     def __init__(self, name, rustbuild, desc, value):
         self.name = name
         self.rustbuild = rustbuild
         self.desc = desc
         self.value = value
 
+
 options = []
 
+
 def o(*args):
     options.append(Option(*args, value=False))
 
+
 def v(*args):
     options.append(Option(*args, value=True))
 
+
 o("debug", "rust.debug", "debug mode; disables optimization unless `--enable-optimize` given")
 o("docs", "build.docs", "build standard library documentation")
 o("compiler-docs", "build.compiler-docs", "build compiler documentation")
@@ -71,6 +77,7 @@ o("debuginfo", "rust.debuginfo", "build with debugger metadata")
 o("debuginfo-lines", "rust.debuginfo-lines", "build with line number debugger metadata")
 o("debuginfo-only-std", "rust.debuginfo-only-std", "build only libstd with debugging information")
 o("debug-jemalloc", "rust.debug-jemalloc", "build jemalloc with --enable-debug --enable-fill")
+v("save-toolstates", "rust.save-toolstates", "save build and test status of external tools into this file")
 
 v("prefix", "install.prefix", "set installation prefix")
 v("localstatedir", "install.localstatedir", "local state directory")
@@ -119,9 +126,8 @@ v("experimental-targets", "llvm.experimental-targets",
   "experimental LLVM targets to build")
 v("release-channel", "rust.channel", "the name of the release channel to build")
 
-# Used on systems where "cc" and "ar" are unavailable
+# Used on systems where "cc" is unavailable
 v("default-linker", "rust.default-linker", "the default linker")
-v("default-ar", "rust.default-ar", "the default ar")
 
 # Many of these are saved below during the "writing configuration" step
 # (others are conditionally saved).
@@ -136,13 +142,16 @@ v("target", None, "GNUs ./configure syntax LLVM target triples")
 
 v("set", None, "set arbitrary key/value pairs in TOML configuration")
 
+
 def p(msg):
     print("configure: " + msg)
 
+
 def err(msg):
     print("configure: error: " + msg)
     sys.exit(1)
 
+
 if '--help' in sys.argv or '-h' in sys.argv:
     print('Usage: ./configure [options]')
     print('')
@@ -208,7 +217,7 @@ while i < len(sys.argv):
                 continue
 
         found = True
-        if not option.name in known_args:
+        if option.name not in known_args:
             known_args[option.name] = []
         known_args[option.name].append((option, value))
         break
@@ -217,7 +226,12 @@ while i < len(sys.argv):
         unknown_args.append(arg)
 p("")
 
-if 'option-checking' not in known_args or known_args['option-checking'][1]:
+# Note: here and a few other places, we use [-1] to apply the *last* value
+# passed.  But if option-checking is enabled, then the known_args loop will
+# also assert that options are only passed once.
+option_checking = ('option-checking' not in known_args
+                   or known_args['option-checking'][-1][1])
+if option_checking:
     if len(unknown_args) > 0:
         err("Option '" + unknown_args[0] + "' is not recognized")
     if len(need_value_args) > 0:
@@ -227,27 +241,30 @@ if 'option-checking' not in known_args or known_args['option-checking'][1]:
 # TOML we're going to write out
 config = {}
 
+
 def build():
     if 'build' in known_args:
-        return known_args['build'][0][1]
+        return known_args['build'][-1][1]
     return bootstrap.default_build_triple()
 
+
 def set(key, value):
-      s = "{:20} := {}".format(key, value)
-      if len(s) < 70:
-          p(s)
-      else:
-          p(s[:70] + " ...")
-
-      arr = config
-      parts = key.split('.')
-      for i, part in enumerate(parts):
-          if i == len(parts) - 1:
-              arr[part] = value
-          else:
-              if not part in arr:
-                  arr[part] = {}
-              arr = arr[part]
+    s = "{:20} := {}".format(key, value)
+    if len(s) < 70:
+        p(s)
+    else:
+        p(s[:70] + " ...")
+
+    arr = config
+    parts = key.split('.')
+    for i, part in enumerate(parts):
+        if i == len(parts) - 1:
+            arr[part] = value
+        else:
+            if part not in arr:
+                arr[part] = {}
+            arr = arr[part]
+
 
 for key in known_args:
     # The `set` option is special and can be passed a bunch of times
@@ -265,9 +282,9 @@ for key in known_args:
 
     # Ensure each option is only passed once
     arr = known_args[key]
-    if len(arr) > 1:
+    if option_checking and len(arr) > 1:
         err("Option '{}' provided more than once".format(key))
-    option, value = arr[0]
+    option, value = arr[-1]
 
     # If we have a clear avenue to set our value in rustbuild, do so
     if option.rustbuild is not None:
@@ -345,8 +362,9 @@ for target in configured_targets:
     targets[target] = sections['target'][:]
     targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", target)
 
+
 # Here we walk through the constructed configuration we have from the parsed
-# command line arguemnts. We then apply each piece of configuration by
+# command line arguments. We then apply each piece of configuration by
 # basically just doing a `sed` to change the various configuration line to what
 # we've got configure.
 def to_toml(value):
@@ -360,7 +378,8 @@ def to_toml(value):
     elif isinstance(value, str):
         return "'" + value + "'"
     else:
-        raise 'no toml'
+        raise RuntimeError('no toml')
+
 
 def configure_section(lines, config):
     for key in config:
@@ -375,10 +394,11 @@ def configure_section(lines, config):
         if not found:
             raise RuntimeError("failed to find config line for {}".format(key))
 
+
 for section_key in config:
     section_config = config[section_key]
-    if not section_key in sections:
-        raise RuntimeError("config key {} not in sections".format(key))
+    if section_key not in sections:
+        raise RuntimeError("config key {} not in sections".format(section_key))
 
     if section_key == 'target':
         for target in section_config:
@@ -407,11 +427,6 @@ with open('Makefile', 'w') as f:
     contents = contents.replace("$(CFG_PYTHON)", sys.executable)
     f.write(contents)
 
-# Finally, clean up with a bit of a help message
-relpath = os.path.dirname(__file__)
-if relpath == '':
-    relpath = '.'
-
 p("")
-p("run `python {}/x.py --help`".format(relpath))
+p("run `python {}/x.py --help`".format(rust_dir))
 p("")
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 3d4aa0413db..bfe2a64f5ba 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -39,6 +39,8 @@ pub fn pkgname(build: &Build, component: &str) -> String {
         format!("{}-{}", component, build.cargo_package_vers())
     } else if component == "rls" {
         format!("{}-{}", component, build.rls_package_vers())
+    } else if component == "rustfmt" {
+        format!("{}-{}", component, build.rustfmt_package_vers())
     } else {
         assert!(component.starts_with("rust"));
         format!("{}-{}", component, build.rust_package_vers())
@@ -176,7 +178,7 @@ fn make_win_dist(
         }
     }
 
-    let target_tools = ["gcc.exe", "ld.exe", "ar.exe", "dlltool.exe", "libwinpthread-1.dll"];
+    let target_tools = ["gcc.exe", "ld.exe", "dlltool.exe", "libwinpthread-1.dll"];
     let mut rustc_dlls = vec!["libstdc++-6.dll", "libwinpthread-1.dll"];
     if target_triple.starts_with("i686-") {
         rustc_dlls.push("libgcc_s_dw2-1.dll");
@@ -630,7 +632,7 @@ impl Step for Analysis {
         let image = tmpdir(build).join(format!("{}-{}-image", name, target));
 
         let src = build.stage_out(compiler, Mode::Libstd)
-            .join(target).join("release").join("deps");
+            .join(target).join(build.cargo_dir()).join("deps");
 
         let image_src = src.join("save-analysis");
         let dst = image.join("lib/rustlib").join(target).join("analysis");
@@ -670,6 +672,9 @@ fn copy_src_dirs(build: &Build, src_dirs: &[&str], exclude_dirs: &[&str], dst_di
              spath.ends_with(".s")) {
             return false
         }
+        if spath.contains("test/emscripten") || spath.contains("test\\emscripten") {
+            return false
+        }
 
         let full_path = Path::new(dir).join(path);
         if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) {
@@ -734,17 +739,16 @@ impl Step for Src {
         // (essentially libstd and all of its path dependencies)
         let std_src_dirs = [
             "src/build_helper",
+            "src/dlmalloc",
             "src/liballoc",
             "src/liballoc_jemalloc",
             "src/liballoc_system",
             "src/libbacktrace",
-            "src/libcollections",
             "src/libcompiler_builtins",
             "src/libcore",
             "src/liblibc",
             "src/libpanic_abort",
             "src/libpanic_unwind",
-            "src/librand",
             "src/librustc_asan",
             "src/librustc_lsan",
             "src/librustc_msan",
@@ -754,6 +758,7 @@ impl Step for Src {
             "src/libunwind",
             "src/rustc/compiler_builtins_shim",
             "src/rustc/libc_shim",
+            "src/rustc/dlmalloc_shim",
             "src/libtest",
             "src/libterm",
             "src/jemalloc",
@@ -1035,7 +1040,7 @@ pub struct Rls {
 }
 
 impl Step for Rls {
-    type Output = PathBuf;
+    type Output = Option<PathBuf>;
     const ONLY_BUILD_TARGETS: bool = true;
     const ONLY_HOSTS: bool = true;
 
@@ -1050,12 +1055,17 @@ impl Step for Rls {
         });
     }
 
-    fn run(self, builder: &Builder) -> PathBuf {
+    fn run(self, builder: &Builder) -> Option<PathBuf> {
         let build = builder.build;
         let stage = self.stage;
         let target = self.target;
         assert!(build.config.extended);
 
+        if !builder.config.toolstate.rls.testing() {
+            println!("skipping Dist RLS stage{} ({})", stage, target);
+            return None
+        }
+
         println!("Dist RLS stage{} ({})", stage, target);
         let src = build.src.join("src/tools/rls");
         let release_num = build.release_num("rls");
@@ -1068,10 +1078,13 @@ impl Step for Rls {
         t!(fs::create_dir_all(&image));
 
         // Prepare the image directory
+        // We expect RLS to build, because we've exited this step above if tool
+        // state for RLS isn't testing.
         let rls = builder.ensure(tool::Rls {
             compiler: builder.compiler(stage, build.build),
             target
-        });
+        }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
+
         install(&rls, &image.join("bin"), 0o755);
         let doc = image.join("share/doc/rls");
         install(&src.join("README.md"), &doc, 0o644);
@@ -1102,7 +1115,97 @@ impl Step for Rls {
            .arg("--component-name=rls-preview");
 
         build.run(&mut cmd);
-        distdir(build).join(format!("{}-{}.tar.gz", name, target))
+        Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
+    }
+}
+
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct Rustfmt {
+    pub stage: u32,
+    pub target: Interned<String>,
+}
+
+impl Step for Rustfmt {
+    type Output = Option<PathBuf>;
+    const ONLY_BUILD_TARGETS: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("rustfmt")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(Rustfmt {
+            stage: run.builder.top_stage,
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder) -> Option<PathBuf> {
+        let build = builder.build;
+        let stage = self.stage;
+        let target = self.target;
+        assert!(build.config.extended);
+
+        if !builder.config.toolstate.rustfmt.testing() {
+            println!("skipping Dist Rustfmt stage{} ({})", stage, target);
+            return None
+        }
+
+        println!("Dist Rustfmt stage{} ({})", stage, target);
+        let src = build.src.join("src/tools/rustfmt");
+        let release_num = build.release_num("rustfmt");
+        let name = pkgname(build, "rustfmt");
+        let version = build.rustfmt_info.version(build, &release_num);
+
+        let tmp = tmpdir(build);
+        let image = tmp.join("rustfmt-image");
+        drop(fs::remove_dir_all(&image));
+        t!(fs::create_dir_all(&image));
+
+        // Prepare the image directory
+        let rustfmt = builder.ensure(tool::Rustfmt {
+            compiler: builder.compiler(stage, build.build),
+            target
+        }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
+        let cargofmt = builder.ensure(tool::Cargofmt {
+            compiler: builder.compiler(stage, build.build),
+            target
+        }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
+
+        install(&rustfmt, &image.join("bin"), 0o755);
+        install(&cargofmt, &image.join("bin"), 0o755);
+        let doc = image.join("share/doc/rustfmt");
+        install(&src.join("README.md"), &doc, 0o644);
+        install(&src.join("LICENSE-MIT"), &doc, 0o644);
+        install(&src.join("LICENSE-APACHE"), &doc, 0o644);
+
+        // Prepare the overlay
+        let overlay = tmp.join("rustfmt-overlay");
+        drop(fs::remove_dir_all(&overlay));
+        t!(fs::create_dir_all(&overlay));
+        install(&src.join("README.md"), &overlay, 0o644);
+        install(&src.join("LICENSE-MIT"), &overlay, 0o644);
+        install(&src.join("LICENSE-APACHE"), &overlay, 0o644);
+        t!(t!(File::create(overlay.join("version"))).write_all(version.as_bytes()));
+
+        // Generate the installer tarball
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+           .arg("--product-name=Rust")
+           .arg("--rel-manifest-dir=rustlib")
+           .arg("--success-message=rustfmt-ready-to-fmt.")
+           .arg("--image-dir").arg(&image)
+           .arg("--work-dir").arg(&tmpdir(build))
+           .arg("--output-dir").arg(&distdir(build))
+           .arg("--non-installed-overlay").arg(&overlay)
+           .arg(format!("--package-name={}-{}", name, target))
+           .arg("--legacy-manifest-dirs=rustlib,cargo")
+           .arg("--component-name=rustfmt-preview");
+
+        build.run(&mut cmd);
+        Some(distdir(build).join(format!("{}-{}.tar.gz", name, target)))
     }
 }
 
@@ -1169,6 +1272,7 @@ impl Step for Extended {
             compiler: builder.compiler(stage, target),
         });
         let cargo_installer = builder.ensure(Cargo { stage, target });
+        let rustfmt_installer = builder.ensure(Rustfmt { stage, target });
         let rls_installer = builder.ensure(Rls { stage, target });
         let mingw_installer = builder.ensure(Mingw { host: target });
         let analysis_installer = builder.ensure(Analysis {
@@ -1202,8 +1306,13 @@ impl Step for Extended {
         // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering
         // the std files during uninstall. To do this ensure that rustc comes
         // before rust-std in the list below.
-        let mut tarballs = vec![rustc_installer, cargo_installer, rls_installer,
-                                analysis_installer, std_installer];
+        let mut tarballs = Vec::new();
+        tarballs.push(rustc_installer);
+        tarballs.push(cargo_installer);
+        tarballs.extend(rls_installer.clone());
+        tarballs.extend(rustfmt_installer.clone());
+        tarballs.push(analysis_installer);
+        tarballs.push(std_installer);
         if build.config.docs {
             tarballs.push(docs_installer);
         }
@@ -1245,35 +1354,41 @@ impl Step for Extended {
         }
         rtf.push_str("}");
 
+        fn filter(contents: &str, marker: &str) -> String {
+            let start = format!("tool-{}-start", marker);
+            let end = format!("tool-{}-end", marker);
+            let mut lines = Vec::new();
+            let mut omitted = false;
+            for line in contents.lines() {
+                if line.contains(&start) {
+                    omitted = true;
+                } else if line.contains(&end) {
+                    omitted = false;
+                } else if !omitted {
+                    lines.push(line);
+                }
+            }
+
+            lines.join("\n")
+        }
+
+        let xform = |p: &Path| {
+            let mut contents = String::new();
+            t!(t!(File::open(p)).read_to_string(&mut contents));
+            if rls_installer.is_none() {
+                contents = filter(&contents, "rls");
+            }
+            if rustfmt_installer.is_none() {
+                contents = filter(&contents, "rustfmt");
+            }
+            let ret = tmp.join(p.file_name().unwrap());
+            t!(t!(File::create(&ret)).write_all(contents.as_bytes()));
+            return ret
+        };
+
         if target.contains("apple-darwin") {
             let pkg = tmp.join("pkg");
             let _ = fs::remove_dir_all(&pkg);
-            t!(fs::create_dir_all(pkg.join("rustc")));
-            t!(fs::create_dir_all(pkg.join("cargo")));
-            t!(fs::create_dir_all(pkg.join("rust-docs")));
-            t!(fs::create_dir_all(pkg.join("rust-std")));
-            t!(fs::create_dir_all(pkg.join("rls")));
-            t!(fs::create_dir_all(pkg.join("rust-analysis")));
-
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rustc"), target)),
-                    &pkg.join("rustc"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "cargo"), target)),
-                    &pkg.join("cargo"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rust-docs"), target)),
-                    &pkg.join("rust-docs"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rust-std"), target)),
-                    &pkg.join("rust-std"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rls"), target)),
-                    &pkg.join("rls"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rust-analysis"), target)),
-                    &pkg.join("rust-analysis"));
-
-            install(&etc.join("pkg/postinstall"), &pkg.join("rustc"), 0o755);
-            install(&etc.join("pkg/postinstall"), &pkg.join("cargo"), 0o755);
-            install(&etc.join("pkg/postinstall"), &pkg.join("rust-docs"), 0o755);
-            install(&etc.join("pkg/postinstall"), &pkg.join("rust-std"), 0o755);
-            install(&etc.join("pkg/postinstall"), &pkg.join("rls"), 0o755);
-            install(&etc.join("pkg/postinstall"), &pkg.join("rust-analysis"), 0o755);
 
             let pkgbuild = |component: &str| {
                 let mut cmd = Command::new("pkgbuild");
@@ -1283,12 +1398,23 @@ impl Step for Extended {
                     .arg(pkg.join(component).with_extension("pkg"));
                 build.run(&mut cmd);
             };
-            pkgbuild("rustc");
-            pkgbuild("cargo");
-            pkgbuild("rust-docs");
-            pkgbuild("rust-std");
-            pkgbuild("rls");
-            pkgbuild("rust-analysis");
+
+            let prepare = |name: &str| {
+                t!(fs::create_dir_all(pkg.join(name)));
+                cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target)),
+                        &pkg.join(name));
+                install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755);
+                pkgbuild(name);
+            };
+            prepare("rustc");
+            prepare("cargo");
+            prepare("rust-docs");
+            prepare("rust-std");
+            prepare("rust-analysis");
+
+            if rls_installer.is_some() {
+                prepare("rls");
+            }
 
             // create an 'uninstall' package
             install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
@@ -1298,7 +1424,7 @@ impl Step for Extended {
             t!(t!(File::create(pkg.join("res/LICENSE.txt"))).write_all(license.as_bytes()));
             install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644);
             let mut cmd = Command::new("productbuild");
-            cmd.arg("--distribution").arg(etc.join("pkg/Distribution.xml"))
+            cmd.arg("--distribution").arg(xform(&etc.join("pkg/Distribution.xml")))
                 .arg("--resources").arg(pkg.join("res"))
                 .arg(distdir(build).join(format!("{}-{}.pkg",
                                                     pkgname(build, "rust"),
@@ -1310,46 +1436,34 @@ impl Step for Extended {
         if target.contains("windows") {
             let exe = tmp.join("exe");
             let _ = fs::remove_dir_all(&exe);
-            t!(fs::create_dir_all(exe.join("rustc")));
-            t!(fs::create_dir_all(exe.join("cargo")));
-            t!(fs::create_dir_all(exe.join("rls")));
-            t!(fs::create_dir_all(exe.join("rust-analysis")));
-            t!(fs::create_dir_all(exe.join("rust-docs")));
-            t!(fs::create_dir_all(exe.join("rust-std")));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rustc"), target))
-                        .join("rustc"),
-                    &exe.join("rustc"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "cargo"), target))
-                        .join("cargo"),
-                    &exe.join("cargo"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rust-docs"), target))
-                        .join("rust-docs"),
-                    &exe.join("rust-docs"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rust-std"), target))
-                        .join(format!("rust-std-{}", target)),
-                    &exe.join("rust-std"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rls"), target)).join("rls-preview"),
-                 &exe.join("rls"));
-            cp_r(&work.join(&format!("{}-{}", pkgname(build, "rust-analysis"), target))
-                        .join(format!("rust-analysis-{}", target)),
-                    &exe.join("rust-analysis"));
-
-            t!(fs::remove_file(exe.join("rustc/manifest.in")));
-            t!(fs::remove_file(exe.join("cargo/manifest.in")));
-            t!(fs::remove_file(exe.join("rust-docs/manifest.in")));
-            t!(fs::remove_file(exe.join("rust-std/manifest.in")));
-            t!(fs::remove_file(exe.join("rls/manifest.in")));
-            t!(fs::remove_file(exe.join("rust-analysis/manifest.in")));
 
+            let prepare = |name: &str| {
+                t!(fs::create_dir_all(exe.join(name)));
+                let dir = if name == "rust-std" || name == "rust-analysis" {
+                    format!("{}-{}", name, target)
+                } else if name == "rls" {
+                    "rls-preview".to_string()
+                } else {
+                    name.to_string()
+                };
+                cp_r(&work.join(&format!("{}-{}", pkgname(build, name), target))
+                            .join(dir),
+                        &exe.join(name));
+                t!(fs::remove_file(exe.join(name).join("manifest.in")));
+            };
+            prepare("rustc");
+            prepare("cargo");
+            prepare("rust-analysis");
+            prepare("rust-docs");
+            prepare("rust-std");
+            if rls_installer.is_some() {
+                prepare("rls");
+            }
             if target.contains("windows-gnu") {
-                t!(fs::create_dir_all(exe.join("rust-mingw")));
-                cp_r(&work.join(&format!("{}-{}", pkgname(build, "rust-mingw"), target))
-                            .join("rust-mingw"),
-                        &exe.join("rust-mingw"));
-                t!(fs::remove_file(exe.join("rust-mingw/manifest.in")));
+                prepare("rust-mingw");
             }
 
-            install(&etc.join("exe/rust.iss"), &exe, 0o644);
+            install(&xform(&etc.join("exe/rust.iss")), &exe, 0o644);
             install(&etc.join("exe/modpath.iss"), &exe, 0o644);
             install(&etc.join("exe/upgrade.iss"), &exe, 0o644);
             install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644);
@@ -1413,16 +1527,18 @@ impl Step for Extended {
                             .arg("-dr").arg("Std")
                             .arg("-var").arg("var.StdDir")
                             .arg("-out").arg(exe.join("StdGroup.wxs")));
-            build.run(Command::new(&heat)
-                            .current_dir(&exe)
-                            .arg("dir")
-                            .arg("rls")
-                            .args(&heat_flags)
-                            .arg("-cg").arg("RlsGroup")
-                            .arg("-dr").arg("Rls")
-                            .arg("-var").arg("var.RlsDir")
-                            .arg("-out").arg(exe.join("RlsGroup.wxs"))
-                            .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
+            if rls_installer.is_some() {
+                build.run(Command::new(&heat)
+                                .current_dir(&exe)
+                                .arg("dir")
+                                .arg("rls")
+                                .args(&heat_flags)
+                                .arg("-cg").arg("RlsGroup")
+                                .arg("-dr").arg("Rls")
+                                .arg("-var").arg("var.RlsDir")
+                                .arg("-out").arg(exe.join("RlsGroup.wxs"))
+                                .arg("-t").arg(etc.join("msi/remove-duplicates.xsl")));
+            }
             build.run(Command::new(&heat)
                             .current_dir(&exe)
                             .arg("dir")
@@ -1456,26 +1572,30 @@ impl Step for Extended {
                     .arg("-dDocsDir=rust-docs")
                     .arg("-dCargoDir=cargo")
                     .arg("-dStdDir=rust-std")
-                    .arg("-dRlsDir=rls")
                     .arg("-dAnalysisDir=rust-analysis")
                     .arg("-arch").arg(&arch)
                     .arg("-out").arg(&output)
                     .arg(&input);
                 add_env(build, &mut cmd, target);
 
+                if rls_installer.is_some() {
+                    cmd.arg("-dRlsDir=rls");
+                }
                 if target.contains("windows-gnu") {
                     cmd.arg("-dGccDir=rust-mingw");
                 }
                 build.run(&mut cmd);
             };
-            candle(&etc.join("msi/rust.wxs"));
+            candle(&xform(&etc.join("msi/rust.wxs")));
             candle(&etc.join("msi/ui.wxs"));
             candle(&etc.join("msi/rustwelcomedlg.wxs"));
             candle("RustcGroup.wxs".as_ref());
             candle("DocsGroup.wxs".as_ref());
             candle("CargoGroup.wxs".as_ref());
             candle("StdGroup.wxs".as_ref());
-            candle("RlsGroup.wxs".as_ref());
+            if rls_installer.is_some() {
+                candle("RlsGroup.wxs".as_ref());
+            }
             candle("AnalysisGroup.wxs".as_ref());
 
             if target.contains("windows-gnu") {
@@ -1499,10 +1619,13 @@ impl Step for Extended {
                 .arg("DocsGroup.wixobj")
                 .arg("CargoGroup.wixobj")
                 .arg("StdGroup.wixobj")
-                .arg("RlsGroup.wixobj")
                 .arg("AnalysisGroup.wixobj")
                 .current_dir(&exe);
 
+            if rls_installer.is_some() {
+                cmd.arg("RlsGroup.wixobj");
+            }
+
             if target.contains("windows-gnu") {
                 cmd.arg("GccGroup.wixobj");
             }
@@ -1586,6 +1709,7 @@ impl Step for HashSign {
         cmd.arg(build.rust_package_vers());
         cmd.arg(build.package_vers(&build.release_num("cargo")));
         cmd.arg(build.package_vers(&build.release_num("rls")));
+        cmd.arg(build.package_vers(&build.release_num("rustfmt")));
         cmd.arg(addr);
 
         t!(fs::create_dir_all(distdir(build)));
diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs
index b9a52a66793..3c12cfc4c7f 100644
--- a/src/bootstrap/doc.rs
+++ b/src/bootstrap/doc.rs
@@ -66,7 +66,7 @@ macro_rules! book {
 }
 
 book!(
-    Nomicon, "src/doc/book", "nomicon";
+    Nomicon, "src/doc/nomicon", "nomicon";
     Reference, "src/doc/reference", "reference";
     Rustdoc, "src/doc/rustdoc", "rustdoc";
 );
@@ -133,6 +133,52 @@ impl Step for UnstableBook {
 }
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct CargoBook {
+    target: Interned<String>,
+    name: Interned<String>,
+}
+
+impl Step for CargoBook {
+    type Output = ();
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        let builder = run.builder;
+        run.path("src/tools/cargo/src/doc/book").default_condition(builder.build.config.docs)
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(CargoBook {
+            target: run.target,
+            name: INTERNER.intern_str("cargo"),
+        });
+    }
+
+    fn run(self, builder: &Builder) {
+        let build = builder.build;
+
+        let target = self.target;
+        let name = self.name;
+        let src = build.src.join("src/tools/cargo/src/doc/book");
+
+        let out = build.doc_out(target);
+        t!(fs::create_dir_all(&out));
+
+        let out = out.join(name);
+
+        println!("Cargo Book ({}) - {}", target, name);
+
+        let _ = fs::remove_dir_all(&out);
+
+        build.run(builder.tool_cmd(Tool::Rustbook)
+                       .arg("build")
+                       .arg(&src)
+                       .arg("-d")
+                       .arg(out));
+    }
+}
+
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 struct RustbookSrc {
     target: Interned<String>,
     name: Interned<String>,
@@ -205,10 +251,12 @@ impl Step for TheBook {
     ///
     /// * Book (first edition)
     /// * Book (second edition)
+    /// * Version info and CSS
     /// * Index page
     /// * Redirect pages
     fn run(self, builder: &Builder) {
         let build = builder.build;
+        let compiler = self.compiler;
         let target = self.target;
         let name = self.name;
         // build book first edition
@@ -223,10 +271,16 @@ impl Step for TheBook {
             name: INTERNER.intern_string(format!("{}/second-edition", name)),
         });
 
+        // build the version info page and CSS
+        builder.ensure(Standalone {
+            compiler,
+            target,
+        });
+
         // build the index page
         let index = format!("{}/index.md", name);
         println!("Documenting book index ({})", target);
-        invoke_rustdoc(builder, self.compiler, target, &index);
+        invoke_rustdoc(builder, compiler, target, &index);
 
         // build the redirect pages
         println!("Documenting book redirect pages ({})", target);
@@ -235,56 +289,11 @@ impl Step for TheBook {
             let path = file.path();
             let path = path.to_str().unwrap();
 
-            invoke_rustdoc(builder, self.compiler, target, path);
+            invoke_rustdoc(builder, compiler, target, path);
         }
     }
 }
 
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
-pub struct CargoBook {
-    target: Interned<String>,
-}
-
-impl Step for CargoBook {
-    type Output = ();
-    const DEFAULT: bool = true;
-
-    fn should_run(run: ShouldRun) -> ShouldRun {
-        let builder = run.builder;
-        run.path("src/doc/cargo").default_condition(builder.build.config.docs)
-    }
-
-    fn make_run(run: RunConfig) {
-        run.builder.ensure(CargoBook {
-            target: run.target,
-        });
-    }
-
-    /// Create a placeholder for the cargo documentation so that doc.rust-lang.org/cargo will
-    /// redirect to doc.crates.io. We want to publish doc.rust-lang.org/cargo in the paper
-    /// version of the book, but we don't want to rush the process of switching cargo's docs
-    /// over to mdbook and deploying them. When the cargo book is ready, this implementation
-    /// should build the mdbook instead of this redirect page.
-    fn run(self, builder: &Builder) {
-        let build = builder.build;
-        let out = build.doc_out(self.target);
-
-        let cargo_dir = out.join("cargo");
-        t!(fs::create_dir_all(&cargo_dir));
-
-        let index = cargo_dir.join("index.html");
-        let redirect_html = r#"
-            <html>
-                <head>
-                    <meta http-equiv="refresh" content="0; URL='http://doc.crates.io'" />
-                </head>
-            </html>"#;
-
-        println!("Creating cargo book redirect page");
-        t!(t!(File::create(&index)).write_all(redirect_html.as_bytes()));
-    }
-}
-
 fn invoke_rustdoc(builder: &Builder, compiler: Compiler, target: Interned<String>, markdown: &str) {
     let build = builder.build;
     let out = build.doc_out(target);
@@ -293,25 +302,12 @@ fn invoke_rustdoc(builder: &Builder, compiler: Compiler, target: Interned<String
 
     let favicon = build.src.join("src/doc/favicon.inc");
     let footer = build.src.join("src/doc/footer.inc");
-
-    let version_input = build.src.join("src/doc/version_info.html.template");
     let version_info = out.join("version_info.html");
 
-    if !up_to_date(&version_input, &version_info) {
-        let mut info = String::new();
-        t!(t!(File::open(&version_input)).read_to_string(&mut info));
-        let info = info.replace("VERSION", &build.rust_release())
-                       .replace("SHORT_HASH", build.rust_info.sha_short().unwrap_or(""))
-                       .replace("STAMP", build.rust_info.sha().unwrap_or(""));
-        t!(t!(File::create(&version_info)).write_all(info.as_bytes()));
-    }
-
     let mut cmd = builder.rustdoc_cmd(compiler.host);
 
     let out = out.join("book");
 
-    t!(fs::copy(build.src.join("src/doc/rust.css"), out.join("rust.css")));
-
     cmd.arg("--html-after-content").arg(&footer)
         .arg("--html-before-content").arg(&version_info)
         .arg("--html-in-header").arg(&favicon)
@@ -320,7 +316,7 @@ fn invoke_rustdoc(builder: &Builder, compiler: Compiler, target: Interned<String
         .arg("-o").arg(&out)
         .arg(&path)
         .arg("--markdown-css")
-        .arg("rust.css");
+        .arg("../rust.css");
 
     build.run(&mut cmd);
 }
@@ -490,7 +486,7 @@ impl Step for Std {
         // for which docs must be built.
         if !build.config.compiler_docs {
             cargo.arg("--no-deps");
-            for krate in &["alloc", "collections", "core", "std", "std_unicode"] {
+            for krate in &["alloc", "core", "std", "std_unicode"] {
                 cargo.arg("-p").arg(krate);
                 // Create all crate output directories first to make sure rustdoc uses
                 // relative links.
@@ -623,11 +619,9 @@ impl Step for Rustc {
         compile::rustc_cargo(build, &compiler, target, &mut cargo);
 
         if build.config.compiler_docs {
-            // src/rustc/Cargo.toml contains bin crates called rustc and rustdoc
-            // which would otherwise overwrite the docs for the real rustc and
-            // rustdoc lib crates.
-            cargo.arg("-p").arg("rustc_driver")
-                 .arg("-p").arg("rustdoc");
+            // src/rustc/Cargo.toml contains a bin crate called rustc which
+            // would otherwise overwrite the docs for the real rustc lib crate.
+            cargo.arg("-p").arg("rustc_driver");
         } else {
             // Like with libstd above if compiler docs aren't enabled then we're not
             // documenting internal dependencies, so we have a whitelist.
diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs
index 7546d7fd4f0..b5d51598fab 100644
--- a/src/bootstrap/flags.rs
+++ b/src/bootstrap/flags.rs
@@ -60,7 +60,9 @@ pub enum Subcommand {
         paths: Vec<PathBuf>,
         test_args: Vec<String>,
     },
-    Clean,
+    Clean {
+        all: bool,
+    },
     Dist {
         paths: Vec<PathBuf>,
     },
@@ -134,9 +136,12 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`");
         let subcommand = match subcommand {
             Some(s) => s,
             None => {
-                // No subcommand -- show the general usage and subcommand help
+                // No or an invalid subcommand -- show the general usage and subcommand help
+                // An exit code will be 0 when no subcommand is given, and 1 in case of an invalid
+                // subcommand.
                 println!("{}\n", subcommand_help);
-                process::exit(1);
+                let exit_code = if args.is_empty() { 0 } else { 1 };
+                process::exit(exit_code);
             }
         };
 
@@ -147,6 +152,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`");
                 opts.optmulti("", "test-args", "extra arguments", "ARGS");
             },
             "bench" => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); },
+            "clean" => { opts.optflag("", "all", "clean all build artifacts"); },
             _ => { },
         };
 
@@ -250,7 +256,7 @@ Arguments:
             }
         });
 
-        // All subcommands can have an optional "Available paths" section
+        // All subcommands except `clean` can have an optional "Available paths" section
         if matches.opt_present("verbose") {
             let config = Config::parse(&["build".to_string()]);
             let mut build = Build::new(config);
@@ -258,9 +264,10 @@ Arguments:
 
             let maybe_rules_help = Builder::get_help(&build, subcommand.as_str());
             extra_help.push_str(maybe_rules_help.unwrap_or_default().as_str());
-        } else {
-            extra_help.push_str(format!("Run `./x.py {} -h -v` to see a list of available paths.",
-                     subcommand).as_str());
+        } else if subcommand.as_str() != "clean" {
+            extra_help.push_str(format!(
+                "Run `./x.py {} -h -v` to see a list of available paths.",
+                subcommand).as_str());
         }
 
         // User passed in -h/--help?
@@ -290,10 +297,13 @@ Arguments:
             }
             "clean" => {
                 if paths.len() > 0 {
-                    println!("\nclean takes no arguments\n");
+                    println!("\nclean does not take a path argument\n");
                     usage(1, &opts, &subcommand_help, &extra_help);
                 }
-                Subcommand::Clean
+
+                Subcommand::Clean {
+                    all: matches.opt_present("all"),
+                }
             }
             "dist" => {
                 Subcommand::Dist {
diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs
index 608924c9c28..743f32ece99 100644
--- a/src/bootstrap/install.rs
+++ b/src/bootstrap/install.rs
@@ -27,10 +27,8 @@ pub fn install_docs(builder: &Builder, stage: u32, host: Interned<String>) {
     install_sh(builder, "docs", "rust-docs", stage, Some(host));
 }
 
-pub fn install_std(builder: &Builder, stage: u32) {
-    for target in &builder.build.targets {
-        install_sh(builder, "std", "rust-std", stage, Some(*target));
-    }
+pub fn install_std(builder: &Builder, stage: u32, target: Interned<String>) {
+    install_sh(builder, "std", "rust-std", stage, Some(target));
 }
 
 pub fn install_cargo(builder: &Builder, stage: u32, host: Interned<String>) {
@@ -41,6 +39,10 @@ pub fn install_rls(builder: &Builder, stage: u32, host: Interned<String>) {
     install_sh(builder, "rls", "rls", stage, Some(host));
 }
 
+pub fn install_rustfmt(builder: &Builder, stage: u32, host: Interned<String>) {
+    install_sh(builder, "rustfmt", "rustfmt", stage, Some(host));
+}
+
 pub fn install_analysis(builder: &Builder, stage: u32, host: Interned<String>) {
     install_sh(builder, "analysis", "rust-analysis", stage, Some(host));
 }
@@ -175,19 +177,31 @@ install!((self, builder, _config),
         install_docs(builder, self.stage, self.target);
     };
     Std, "src/libstd", true, only_hosts: true, {
-        builder.ensure(dist::Std {
-            compiler: builder.compiler(self.stage, self.host),
-            target: self.target
-        });
-        install_std(builder, self.stage);
+        for target in &builder.build.targets {
+            builder.ensure(dist::Std {
+                compiler: builder.compiler(self.stage, self.host),
+                target: *target
+            });
+            install_std(builder, self.stage, *target);
+        }
     };
     Cargo, "cargo", _config.extended, only_hosts: true, {
         builder.ensure(dist::Cargo { stage: self.stage, target: self.target });
         install_cargo(builder, self.stage, self.target);
     };
     Rls, "rls", _config.extended, only_hosts: true, {
-        builder.ensure(dist::Rls { stage: self.stage, target: self.target });
-        install_rls(builder, self.stage, self.target);
+        if builder.ensure(dist::Rls { stage: self.stage, target: self.target }).is_some() {
+            install_rls(builder, self.stage, self.target);
+        } else {
+            println!("skipping Install RLS stage{} ({})", self.stage, self.target);
+        }
+    };
+    Rustfmt, "rustfmt", _config.extended, only_hosts: true, {
+        if builder.ensure(dist::Rustfmt { stage: self.stage, target: self.target }).is_some() {
+            install_rustfmt(builder, self.stage, self.target);
+        } else {
+            println!("skipping Install Rustfmt stage{} ({})", self.stage, self.target);
+        }
     };
     Analysis, "analysis", _config.extended, only_hosts: false, {
         builder.ensure(dist::Analysis {
diff --git a/src/bootstrap/job.rs b/src/bootstrap/job.rs
index 72a5d1338b8..fa3ba02482f 100644
--- a/src/bootstrap/job.rs
+++ b/src/bootstrap/job.rs
@@ -185,7 +185,7 @@ pub unsafe fn setup(build: &mut Build) {
                             0, FALSE, DUPLICATE_SAME_ACCESS);
 
     // If this failed, well at least we tried! An example of DuplicateHandle
-    // failing in the past has been when the wrong python2 package spawed this
+    // failing in the past has been when the wrong python2 package spawned this
     // build system (e.g. the `python2` package in MSYS instead of
     // `mingw-w64-x86_64-python2`. Not sure why it failed, but the "failure
     // mode" here is that we only clean everything up when the build system
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index 83aa08366df..2f00c313a0c 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -190,6 +190,7 @@ mod job {
 pub use config::Config;
 use flags::Subcommand;
 use cache::{Interned, INTERNER};
+use toolstate::ToolState;
 
 /// A structure representing a Rust compiler.
 ///
@@ -222,6 +223,7 @@ pub struct Build {
     rust_info: channel::GitInfo,
     cargo_info: channel::GitInfo,
     rls_info: channel::GitInfo,
+    rustfmt_info: channel::GitInfo,
     local_rebuild: bool,
     fail_fast: bool,
     verbosity: usize,
@@ -240,10 +242,11 @@ pub struct Build {
     lldb_python_dir: Option<String>,
 
     // Runtime state filled in later on
-    // target -> (cc, ar)
-    cc: HashMap<Interned<String>, (cc::Tool, Option<PathBuf>)>,
-    // host -> (cc, ar)
+    // C/C++ compilers and archiver for all targets
+    cc: HashMap<Interned<String>, cc::Tool>,
     cxx: HashMap<Interned<String>, cc::Tool>,
+    ar: HashMap<Interned<String>, PathBuf>,
+    // Misc
     crates: HashMap<Interned<String>, Crate>,
     is_sudo: bool,
     ci_env: CiEnv,
@@ -303,6 +306,7 @@ impl Build {
         let rust_info = channel::GitInfo::new(&config, &src);
         let cargo_info = channel::GitInfo::new(&config, &src.join("src/tools/cargo"));
         let rls_info = channel::GitInfo::new(&config, &src.join("src/tools/rls"));
+        let rustfmt_info = channel::GitInfo::new(&config, &src.join("src/tools/rustfmt"));
 
         Build {
             initial_rustc: config.initial_rustc.clone(),
@@ -322,8 +326,10 @@ impl Build {
             rust_info,
             cargo_info,
             rls_info,
+            rustfmt_info,
             cc: HashMap::new(),
             cxx: HashMap::new(),
+            ar: HashMap::new(),
             crates: HashMap::new(),
             lldb_version: None,
             lldb_python_dir: None,
@@ -345,8 +351,8 @@ impl Build {
             job::setup(self);
         }
 
-        if let Subcommand::Clean = self.config.cmd {
-            return clean::clean(self);
+        if let Subcommand::Clean { all } = self.config.cmd {
+            return clean::clean(self, all);
         }
 
         self.verbose("finding compilers");
@@ -383,16 +389,19 @@ impl Build {
     /// Clear out `dir` if `input` is newer.
     ///
     /// After this executes, it will also ensure that `dir` exists.
-    fn clear_if_dirty(&self, dir: &Path, input: &Path) {
+    fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool {
         let stamp = dir.join(".stamp");
+        let mut cleared = false;
         if mtime(&stamp) < mtime(input) {
             self.verbose(&format!("Dirty - {}", dir.display()));
             let _ = fs::remove_dir_all(dir);
+            cleared = true;
         } else if stamp.exists() {
-            return
+            return cleared;
         }
         t!(fs::create_dir_all(dir));
         t!(File::create(stamp));
+        cleared
     }
 
     /// Get the space-separated set of activated features for the standard
@@ -433,6 +442,12 @@ impl Build {
         if self.config.rust_optimize {"release"} else {"debug"}
     }
 
+    fn tools_dir(&self, compiler: Compiler) -> PathBuf {
+        let out = self.out.join(&*compiler.host).join(format!("stage{}-tools-bin", compiler.stage));
+        t!(fs::create_dir_all(&out));
+        out
+    }
+
     /// Get the directory for incremental by-products when using the
     /// given compiler.
     fn incremental_dir(&self, compiler: Compiler) -> PathBuf {
@@ -612,7 +627,7 @@ impl Build {
 
     /// Returns the path to the C compiler for the target specified.
     fn cc(&self, target: Interned<String>) -> &Path {
-        self.cc[&target].0.path()
+        self.cc[&target].path()
     }
 
     /// Returns a list of flags to pass to the C compiler for the target
@@ -620,7 +635,7 @@ impl Build {
     fn cflags(&self, target: Interned<String>) -> Vec<String> {
         // Filter out -O and /O (the optimization flags) that we picked up from
         // cc-rs because the build scripts will determine that for themselves.
-        let mut base = self.cc[&target].0.args().iter()
+        let mut base = self.cc[&target].args().iter()
                            .map(|s| s.to_string_lossy().into_owned())
                            .filter(|s| !s.starts_with("-O") && !s.starts_with("/O"))
                            .collect::<Vec<_>>();
@@ -644,7 +659,7 @@ impl Build {
 
     /// Returns the path to the `ar` archive utility for the target specified.
     fn ar(&self, target: Interned<String>) -> Option<&Path> {
-        self.cc[&target].1.as_ref().map(|p| &**p)
+        self.ar.get(&target).map(|p| &**p)
     }
 
     /// Returns the path to the C++ compiler for the target specified.
@@ -657,21 +672,17 @@ impl Build {
         }
     }
 
-    /// Returns flags to pass to the compiler to generate code for `target`.
-    fn rustc_flags(&self, target: Interned<String>) -> Vec<String> {
-        // New flags should be added here with great caution!
-        //
-        // It's quite unfortunate to **require** flags to generate code for a
-        // target, so it should only be passed here if absolutely necessary!
-        // Most default configuration should be done through target specs rather
-        // than an entry here.
-
-        let mut base = Vec::new();
-        if target != self.config.build && !target.contains("msvc") &&
-            !target.contains("emscripten") {
-            base.push(format!("-Clinker={}", self.cc(target).display()));
+    /// Returns the path to the linker for the given target if it needs to be overriden.
+    fn linker(&self, target: Interned<String>) -> Option<&Path> {
+        if let Some(linker) = self.config.target_config.get(&target)
+                                                       .and_then(|c| c.linker.as_ref()) {
+            Some(linker)
+        } else if target != self.config.build &&
+                  !target.contains("msvc") && !target.contains("emscripten") {
+            Some(self.cc(target))
+        } else {
+            None
         }
-        base
     }
 
     /// Returns if this target should statically link the C runtime, if specified
@@ -807,6 +818,11 @@ impl Build {
         self.package_vers(&self.release_num("rls"))
     }
 
+    /// Returns the value of `package_vers` above for rustfmt
+    fn rustfmt_package_vers(&self) -> String {
+        self.package_vers(&self.release_num("rustfmt"))
+    }
+
     /// Returns the `version` string associated with this compiler for Rust
     /// itself.
     ///
@@ -859,6 +875,30 @@ impl Build {
         }
     }
 
+    /// Updates the actual toolstate of a tool.
+    ///
+    /// The toolstates are saved to the file specified by the key
+    /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be
+    /// done. The file is updated immediately after this function completes.
+    pub fn save_toolstate(&self, tool: &str, state: ToolState) {
+        use std::io::{Seek, SeekFrom};
+
+        if let Some(ref path) = self.config.save_toolstates {
+            let mut file = t!(fs::OpenOptions::new()
+                .create(true)
+                .read(true)
+                .write(true)
+                .open(path));
+
+            let mut current_toolstates: HashMap<Box<str>, ToolState> =
+                serde_json::from_reader(&mut file).unwrap_or_default();
+            current_toolstates.insert(tool.into(), state);
+            t!(file.seek(SeekFrom::Start(0)));
+            t!(file.set_len(0));
+            t!(serde_json::to_writer(file, &current_toolstates));
+        }
+    }
+
     /// Get a list of crates from a root crate.
     ///
     /// Returns Vec<(crate, path to crate, is_root_crate)>
diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in
index 004f0c31024..925a361f0b2 100644
--- a/src/bootstrap/mk/Makefile.in
+++ b/src/bootstrap/mk/Makefile.in
@@ -52,11 +52,8 @@ check:
 	$(Q)$(BOOTSTRAP) test $(BOOTSTRAP_ARGS)
 check-aux:
 	$(Q)$(BOOTSTRAP) test \
-		src/tools/cargotest \
 		src/tools/cargo \
-		src/tools/rls \
-		src/tools/rustfmt \
-		src/tools/miri \
+		src/tools/cargotest \
 		src/test/pretty \
 		src/test/run-pass/pretty \
 		src/test/run-fail/pretty \
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index 00852d69305..a5408ee381b 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -110,10 +110,7 @@ impl Step for Llvm {
             None => "X86;ARM;AArch64;Mips;PowerPC;SystemZ;JSBackend;MSP430;Sparc;NVPTX;Hexagon",
         };
 
-        let llvm_exp_targets = match build.config.llvm_experimental_targets {
-            Some(ref s) => s,
-            None => "",
-        };
+        let llvm_exp_targets = &build.config.llvm_experimental_targets;
 
         let assertions = if build.config.llvm_assertions {"ON"} else {"OFF"};
 
@@ -227,6 +224,13 @@ impl Step for Llvm {
             cfg.build_arg("-j").build_arg(build.jobs().to_string());
             cfg.define("CMAKE_C_FLAGS", build.cflags(target).join(" "));
             cfg.define("CMAKE_CXX_FLAGS", build.cflags(target).join(" "));
+            if let Some(ar) = build.ar(target) {
+                if ar.is_absolute() {
+                    // LLVM build breaks if `CMAKE_AR` is a relative path, for some reason it
+                    // tries to resolve this path in the LLVM build directory.
+                    cfg.define("CMAKE_AR", sanitize_cc(ar));
+                }
+            }
         };
 
         configure_compilers(&mut cfg);
@@ -252,11 +256,14 @@ fn check_llvm_version(build: &Build, llvm_config: &Path) {
 
     let mut cmd = Command::new(llvm_config);
     let version = output(cmd.arg("--version"));
-    if version.starts_with("3.5") || version.starts_with("3.6") ||
-       version.starts_with("3.7") {
-        return
+    let mut parts = version.split('.').take(2)
+        .filter_map(|s| s.parse::<u32>().ok());
+    if let (Some(major), Some(minor)) = (parts.next(), parts.next()) {
+        if major > 3 || (major == 3 && minor >= 9) {
+            return
+        }
     }
-    panic!("\n\nbad LLVM version: {}, need >=3.5\n\n", version)
+    panic!("\n\nbad LLVM version: {}, need >=3.9\n\n", version)
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
@@ -309,13 +316,13 @@ impl Step for TestHelpers {
            .warnings(false)
            .debug(false)
            .file(build.src.join("src/rt/rust_test_helpers.c"))
-           .compile("librust_test_helpers.a");
+           .compile("rust_test_helpers");
     }
 }
 
-const OPENSSL_VERS: &'static str = "1.0.2k";
+const OPENSSL_VERS: &'static str = "1.0.2m";
 const OPENSSL_SHA256: &'static str =
-    "6b3977c61f2aedf0f96367dcfb5c6e578cf37e7b8d913b4ecb6643c3cb88d8c0";
+    "8c6ff15ec6b319b50788f42c7abc2890c08ba5a1cdcd3810eb9092deada37b0f";
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Openssl {
@@ -352,34 +359,51 @@ impl Step for Openssl {
             // originally from https://www.openssl.org/source/...
             let url = format!("https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/{}",
                               name);
-            let mut ok = false;
+            let mut last_error = None;
             for _ in 0..3 {
                 let status = Command::new("curl")
                                 .arg("-o").arg(&tmp)
+                                .arg("-f")  // make curl fail if the URL does not return HTTP 200
                                 .arg(&url)
                                 .status()
                                 .expect("failed to spawn curl");
-                if status.success() {
-                    ok = true;
-                    break
+
+                // Retry if download failed.
+                if !status.success() {
+                    last_error = Some(status.to_string());
+                    continue;
                 }
+
+                // Ensure the hash is correct.
+                let mut shasum = if target.contains("apple") || build.build.contains("netbsd") {
+                    let mut cmd = Command::new("shasum");
+                    cmd.arg("-a").arg("256");
+                    cmd
+                } else {
+                    Command::new("sha256sum")
+                };
+                let output = output(&mut shasum.arg(&tmp));
+                let found = output.split_whitespace().next().unwrap();
+
+                // If the hash is wrong, probably the download is incomplete or S3 served an error
+                // page. In any case, retry.
+                if found != OPENSSL_SHA256 {
+                    last_error = Some(format!(
+                        "downloaded openssl sha256 different\n\
+                         expected: {}\n\
+                         found:    {}\n",
+                        OPENSSL_SHA256,
+                        found
+                    ));
+                    continue;
+                }
+
+                // Everything is fine, so exit the retry loop.
+                last_error = None;
+                break;
             }
-            if !ok {
-                panic!("failed to download openssl source")
-            }
-            let mut shasum = if target.contains("apple") {
-                let mut cmd = Command::new("shasum");
-                cmd.arg("-a").arg("256");
-                cmd
-            } else {
-                Command::new("sha256sum")
-            };
-            let output = output(&mut shasum.arg(&tmp));
-            let found = output.split_whitespace().next().unwrap();
-            if found != OPENSSL_SHA256 {
-                panic!("downloaded openssl sha256 different\n\
-                        expected: {}\n\
-                        found:    {}\n", OPENSSL_SHA256, found);
+            if let Some(error) = last_error {
+                panic!("failed to download openssl source: {}", error);
             }
             t!(fs::rename(&tmp, &tarball));
         }
@@ -387,7 +411,7 @@ impl Step for Openssl {
         let dst = build.openssl_install_dir(target).unwrap();
         drop(fs::remove_dir_all(&obj));
         drop(fs::remove_dir_all(&dst));
-        build.run(Command::new("tar").arg("xf").arg(&tarball).current_dir(&out));
+        build.run(Command::new("tar").arg("zxf").arg(&tarball).current_dir(&out));
 
         let mut configure = Command::new("perl");
         configure.arg(obj.join("Configure"));
@@ -419,10 +443,12 @@ impl Step for Openssl {
             "powerpc64-unknown-linux-gnu" => "linux-ppc64",
             "powerpc64le-unknown-linux-gnu" => "linux-ppc64le",
             "s390x-unknown-linux-gnu" => "linux64-s390x",
+            "sparc64-unknown-linux-gnu" => "linux64-sparcv9",
             "sparc64-unknown-netbsd" => "BSD-sparc64",
             "x86_64-apple-darwin" => "darwin64-x86_64-cc",
             "x86_64-linux-android" => "linux-x86_64",
             "x86_64-unknown-freebsd" => "BSD-x86_64",
+            "x86_64-unknown-dragonfly" => "BSD-x86_64",
             "x86_64-unknown-linux-gnu" => "linux-x86_64",
             "x86_64-unknown-linux-musl" => "linux-x86_64",
             "x86_64-unknown-netbsd" => "BSD-x86_64",
diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs
index 8b23be69a85..bc275b7fc74 100644
--- a/src/bootstrap/sanity.rs
+++ b/src/bootstrap/sanity.rs
@@ -78,7 +78,7 @@ pub fn check(build: &mut Build) {
     }
 
     let mut cmd_finder = Finder::new();
-    // If we've got a git directory we're gona need git to update
+    // If we've got a git directory we're gonna need git to update
     // submodules and learn about various other aspects.
     if build.rust_info.is_git() {
         cmd_finder.must_have("git");
diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs
index a05e58e6a22..fa9bdc43c37 100644
--- a/src/bootstrap/tool.rs
+++ b/src/bootstrap/tool.rs
@@ -11,7 +11,7 @@
 use std::fs;
 use std::env;
 use std::path::PathBuf;
-use std::process::Command;
+use std::process::{Command, exit};
 
 use Mode;
 use Compiler;
@@ -38,24 +38,40 @@ impl Step for CleanTools {
         run.never()
     }
 
-    /// Build a tool in `src/tools`
-    ///
-    /// This will build the specified tool with the specified `host` compiler in
-    /// `stage` into the normal cargo output directory.
     fn run(self, builder: &Builder) {
         let build = builder.build;
         let compiler = self.compiler;
         let target = self.target;
         let mode = self.mode;
 
-        let stamp = match mode {
-            Mode::Libstd => libstd_stamp(build, compiler, target),
-            Mode::Libtest => libtest_stamp(build, compiler, target),
-            Mode::Librustc => librustc_stamp(build, compiler, target),
-            _ => panic!(),
+        // This is for the original compiler, but if we're forced to use stage 1, then
+        // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
+        // we copy the libs forward.
+        let tools_dir = build.stage_out(compiler, Mode::Tool);
+        let compiler = if builder.force_use_stage1(compiler, target) {
+            builder.compiler(1, compiler.host)
+        } else {
+            compiler
         };
-        let out_dir = build.cargo_out(compiler, Mode::Tool, target);
-        build.clear_if_dirty(&out_dir, &stamp);
+
+        for &cur_mode in &[Mode::Libstd, Mode::Libtest, Mode::Librustc] {
+            let stamp = match cur_mode {
+                Mode::Libstd => libstd_stamp(build, compiler, target),
+                Mode::Libtest => libtest_stamp(build, compiler, target),
+                Mode::Librustc => librustc_stamp(build, compiler, target),
+                _ => panic!(),
+            };
+
+            if build.clear_if_dirty(&tools_dir, &stamp) {
+                break;
+            }
+
+            // If we are a rustc tool, and std changed, we also need to clear ourselves out -- our
+            // dependencies depend on std. Therefore, we iterate up until our own mode.
+            if mode == cur_mode {
+                break;
+            }
+        }
     }
 }
 
@@ -70,7 +86,7 @@ struct ToolBuild {
 }
 
 impl Step for ToolBuild {
-    type Output = PathBuf;
+    type Output = Option<PathBuf>;
 
     fn should_run(run: ShouldRun) -> ShouldRun {
         run.never()
@@ -80,7 +96,7 @@ impl Step for ToolBuild {
     ///
     /// This will build the specified tool with the specified `host` compiler in
     /// `stage` into the normal cargo output directory.
-    fn run(self, builder: &Builder) -> PathBuf {
+    fn run(self, builder: &Builder) -> Option<PathBuf> {
         let build = builder.build;
         let compiler = self.compiler;
         let target = self.target;
@@ -99,8 +115,35 @@ impl Step for ToolBuild {
         println!("Building stage{} tool {} ({})", compiler.stage, tool, target);
 
         let mut cargo = prepare_tool_cargo(builder, compiler, target, "build", path);
-        build.run_expecting(&mut cargo, expectation);
-        build.cargo_out(compiler, Mode::Tool, target).join(exe(tool, &compiler.host))
+        let is_expected = build.try_run(&mut cargo, expectation);
+        // If the expectation is "Failing", `try_run` returning true actually
+        // means a build-failure is successfully observed, i.e. the tool is
+        // broken. Thus the XOR here.
+        // Sorry for the complicated logic, but we can remove this expectation
+        // logic after #45861 is fully fixed.
+        build.save_toolstate(tool, if is_expected ^ (expectation == BuildExpectation::Failing) {
+            ToolState::Compiling
+        } else {
+            ToolState::Broken
+        });
+
+        if !is_expected {
+            if expectation == BuildExpectation::None {
+                exit(1);
+            } else {
+                return None;
+            }
+        }
+
+        if expectation == BuildExpectation::Succeeding || expectation == BuildExpectation::None {
+            let cargo_out = build.cargo_out(compiler, Mode::Tool, target)
+                .join(exe(tool, &compiler.host));
+            let bin = build.tools_dir(compiler).join(exe(tool, &compiler.host));
+            copy(&cargo_out, &bin);
+            Some(bin)
+        } else {
+            None
+        }
     }
 }
 
@@ -169,12 +212,12 @@ macro_rules! tool {
             }
 
             pub fn tool_default_stage(&self, tool: Tool) -> u32 {
-                // Compile the error-index in the top stage as it depends on
-                // rustdoc, so we want to avoid recompiling rustdoc twice if we
-                // can. Otherwise compile everything else in stage0 as there's
-                // no need to rebootstrap everything
+                // Compile the error-index in the same stage as rustdoc to avoid
+                // recompiling rustdoc twice if we can. Otherwise compile
+                // everything else in stage0 as there's no need to rebootstrap
+                // everything.
                 match tool {
-                    Tool::ErrorIndex => self.top_stage,
+                    Tool::ErrorIndex if self.top_stage >= 2 => self.top_stage,
                     _ => 0,
                 }
             }
@@ -209,7 +252,7 @@ macro_rules! tool {
                     mode: $mode,
                     path: $path,
                     expectation: BuildExpectation::None,
-                })
+                }).expect("expected to build -- BuildExpectation::None")
             }
         }
         )+
@@ -257,7 +300,7 @@ impl Step for RemoteTestServer {
             mode: Mode::Libstd,
             path: "src/tools/remote-test-server",
             expectation: BuildExpectation::None,
-        })
+        }).expect("expected to build -- BuildExpectation::None")
     }
 }
 
@@ -375,74 +418,70 @@ impl Step for Cargo {
             mode: Mode::Librustc,
             path: "src/tools/cargo",
             expectation: BuildExpectation::None,
-        })
+        }).expect("BuildExpectation::None - expected to build")
     }
 }
 
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
-pub struct Clippy {
-    pub compiler: Compiler,
-    pub target: Interned<String>,
-}
+macro_rules! tool_extended {
+    (($sel:ident, $builder:ident),
+       $($name:ident,
+       $toolstate:ident,
+       $path:expr,
+       $tool_name:expr,
+       $extra_deps:block;)+) => {
+        $(
+            #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+        pub struct $name {
+            pub compiler: Compiler,
+            pub target: Interned<String>,
+        }
 
-impl Step for Clippy {
-    type Output = PathBuf;
-    const DEFAULT: bool = false;
-    const ONLY_HOSTS: bool = true;
+        impl Step for $name {
+            type Output = Option<PathBuf>;
+            const DEFAULT: bool = true;
+            const ONLY_HOSTS: bool = true;
 
-    fn should_run(run: ShouldRun) -> ShouldRun {
-        run.path("src/tools/clippy")
-    }
+            fn should_run(run: ShouldRun) -> ShouldRun {
+                let builder = run.builder;
+                run.path($path).default_condition(builder.build.config.extended)
+            }
 
-    fn make_run(run: RunConfig) {
-        run.builder.ensure(Clippy {
-            compiler: run.builder.compiler(run.builder.top_stage, run.builder.build.build),
-            target: run.target,
-        });
+            fn make_run(run: RunConfig) {
+                run.builder.ensure($name {
+                    compiler: run.builder.compiler(run.builder.top_stage, run.builder.build.build),
+                    target: run.target,
+                });
+            }
+
+            fn run($sel, $builder: &Builder) -> Option<PathBuf> {
+                $extra_deps
+                let toolstate = $builder.build.config.toolstate.$toolstate;
+                $builder.ensure(ToolBuild {
+                    compiler: $sel.compiler,
+                    target: $sel.target,
+                    tool: $tool_name,
+                    mode: Mode::Librustc,
+                    path: $path,
+                    expectation: toolstate.passes(ToolState::Compiling),
+                })
+            }
+        }
+        )+
     }
+}
 
-    fn run(self, builder: &Builder) -> PathBuf {
+tool_extended!((self, builder),
+    Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", {};
+    Clippy, clippy, "src/tools/clippy", "clippy-driver", {
         // Clippy depends on procedural macros (serde), which requires a full host
         // compiler to be available, so we need to depend on that.
         builder.ensure(compile::Rustc {
             compiler: self.compiler,
             target: builder.build.build,
         });
-        builder.ensure(ToolBuild {
-            compiler: self.compiler,
-            target: self.target,
-            tool: "clippy",
-            mode: Mode::Librustc,
-            path: "src/tools/clippy",
-            expectation: builder.build.config.toolstate.clippy.passes(ToolState::Compiling),
-        })
-    }
-}
-
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
-pub struct Rls {
-    pub compiler: Compiler,
-    pub target: Interned<String>,
-}
-
-impl Step for Rls {
-    type Output = PathBuf;
-    const DEFAULT: bool = true;
-    const ONLY_HOSTS: bool = true;
-
-    fn should_run(run: ShouldRun) -> ShouldRun {
-        let builder = run.builder;
-        run.path("src/tools/rls").default_condition(builder.build.config.extended)
-    }
-
-    fn make_run(run: RunConfig) {
-        run.builder.ensure(Rls {
-            compiler: run.builder.compiler(run.builder.top_stage, run.builder.build.build),
-            target: run.target,
-        });
-    }
-
-    fn run(self, builder: &Builder) -> PathBuf {
+    };
+    Miri, miri, "src/tools/miri", "miri", {};
+    Rls, rls, "src/tools/rls", "rls", {
         builder.ensure(native::Openssl {
             target: self.target,
         });
@@ -452,87 +491,9 @@ impl Step for Rls {
             compiler: self.compiler,
             target: builder.build.build,
         });
-        builder.ensure(ToolBuild {
-            compiler: self.compiler,
-            target: self.target,
-            tool: "rls",
-            mode: Mode::Librustc,
-            path: "src/tools/rls",
-            expectation: builder.build.config.toolstate.rls.passes(ToolState::Compiling),
-        })
-    }
-}
-
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
-pub struct Rustfmt {
-    pub compiler: Compiler,
-    pub target: Interned<String>,
-}
-
-impl Step for Rustfmt {
-    type Output = PathBuf;
-    const DEFAULT: bool = true;
-    const ONLY_HOSTS: bool = true;
-
-    fn should_run(run: ShouldRun) -> ShouldRun {
-        let builder = run.builder;
-        run.path("src/tools/rustfmt").default_condition(builder.build.config.extended)
-    }
-
-    fn make_run(run: RunConfig) {
-        run.builder.ensure(Rustfmt {
-            compiler: run.builder.compiler(run.builder.top_stage, run.builder.build.build),
-            target: run.target,
-        });
-    }
-
-    fn run(self, builder: &Builder) -> PathBuf {
-        builder.ensure(ToolBuild {
-            compiler: self.compiler,
-            target: self.target,
-            tool: "rustfmt",
-            mode: Mode::Librustc,
-            path: "src/tools/rustfmt",
-            expectation: builder.build.config.toolstate.rustfmt.passes(ToolState::Compiling),
-        })
-    }
-}
-
-
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
-pub struct Miri {
-    pub compiler: Compiler,
-    pub target: Interned<String>,
-}
-
-impl Step for Miri {
-    type Output = PathBuf;
-    const DEFAULT: bool = true;
-    const ONLY_HOSTS: bool = true;
-
-    fn should_run(run: ShouldRun) -> ShouldRun {
-        let build_miri = run.builder.build.config.test_miri;
-        run.path("src/tools/miri").default_condition(build_miri)
-    }
-
-    fn make_run(run: RunConfig) {
-        run.builder.ensure(Miri {
-            compiler: run.builder.compiler(run.builder.top_stage, run.builder.build.build),
-            target: run.target,
-        });
-    }
-
-    fn run(self, builder: &Builder) -> PathBuf {
-        builder.ensure(ToolBuild {
-            compiler: self.compiler,
-            target: self.target,
-            tool: "miri",
-            mode: Mode::Librustc,
-            path: "src/tools/miri",
-            expectation: builder.build.config.toolstate.miri.passes(ToolState::Compiling),
-        })
-    }
-}
+    };
+    Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", {};
+);
 
 impl<'a> Builder<'a> {
     /// Get a `Command` which is ready to run `tool` in `stage` built for
@@ -561,7 +522,7 @@ impl<'a> Builder<'a> {
         if compiler.host.contains("msvc") {
             let curpaths = env::var_os("PATH").unwrap_or_default();
             let curpaths = env::split_paths(&curpaths).collect::<Vec<_>>();
-            for &(ref k, ref v) in self.cc[&compiler.host].0.env() {
+            for &(ref k, ref v) in self.cc[&compiler.host].env() {
                 if k != "PATH" {
                     continue
                 }
diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs
index 8a113f6b4d2..00dbcc86af4 100644
--- a/src/bootstrap/toolstate.rs
+++ b/src/bootstrap/toolstate.rs
@@ -10,7 +10,7 @@
 
 use build_helper::BuildExpectation;
 
-#[derive(Copy, Clone, Debug, Deserialize, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
 /// Whether a tool can be compiled, tested or neither
 pub enum ToolState {
     /// The tool compiles successfully, but the test suite fails
@@ -31,6 +31,13 @@ impl ToolState {
             BuildExpectation::Failing
         }
     }
+
+    pub fn testing(&self) -> bool {
+        match *self {
+            ToolState::Testing => true,
+            _ => false,
+        }
+    }
 }
 
 impl Default for ToolState {
diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs
index a521dd09453..2506048858f 100644
--- a/src/bootstrap/util.rs
+++ b/src/bootstrap/util.rs
@@ -14,8 +14,9 @@
 //! not a lot of interesting happenings here unfortunately.
 
 use std::env;
-use std::fs;
-use std::io::{self, Write};
+use std::str;
+use std::fs::{self, File};
+use std::io::{self, Read, Write};
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::time::{SystemTime, Instant};
@@ -50,6 +51,22 @@ pub fn copy(src: &Path, dst: &Path) {
     t!(filetime::set_file_times(dst, atime, mtime));
 }
 
+pub fn read_stamp_file(stamp: &Path) -> Vec<PathBuf> {
+    let mut paths = Vec::new();
+    let mut contents = Vec::new();
+    t!(t!(File::open(stamp)).read_to_end(&mut contents));
+    // This is the method we use for extracting paths from the stamp file passed to us. See
+    // run_cargo for more information (in compile.rs).
+    for part in contents.split(|b| *b == 0) {
+        if part.is_empty() {
+            continue
+        }
+        let path = PathBuf::from(t!(str::from_utf8(part)));
+        paths.push(path);
+    }
+    paths
+}
+
 /// Copies the `src` directory recursively to `dst`. Both are assumed to exist
 /// when this function is called.
 pub fn cp_r(src: &Path, dst: &Path) {