Auto merge of #64316 - alexcrichton:cleanup-shim, r=Mark-Simulacrum

Delete most of `src/bootstrap/bin/rustc.rs`

This commit is an attempt at deleting as much of the `rustc.rs` shim that we have in rustbuild as possible.  This shim predates `RUSTFLAGS` and is as old as rustbuild itself. While useful for quick hacks, it subverts Cargo's knowledge of `rustc`, makes it more difficult to build crates out of rustbuild, and is generally a hazard/code smell due to its architecture.

Additionally since the inception of this script we've added a number of features to Cargo such as profile overrides and `RUSTFLAGS`. This commit attempts to use these features of Cargo as much as possible to delete almost all of `src/bootstrap/bin/rustc.rs`. It's hoped that all new configuration for the Rust compiler can be codified in rustbuild rather than in this shim, allowing Cargo to have more knowledge about what's going on and making it a bit easier to reproduce builds outside of Cargo itself.

This was primarily motivated by some recent work on std-aware Cargo, and is also generally a cleanup of the script itself. This internally resulted in a number of refactorings of rustbuild itself, and the commits should be readable one-at-a-time instead of having to digest them all at once.

8 files changed, 278 insertions, 312 deletions

diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs
index 84415baa3a1..475f2e90463 100644
--- a/src/bootstrap/bin/rustc.rs
+++ b/src/bootstrap/bin/rustc.rs
@@ -16,7 +16,6 @@
 //! never get replaced.
 
 use std::env;
-use std::ffi::OsString;
 use std::io;
 use std::path::PathBuf;
 use std::process::Command;
@@ -24,23 +23,7 @@ use std::str::FromStr;
 use std::time::Instant;
 
 fn main() {
-    let mut args = env::args_os().skip(1).collect::<Vec<_>>();
-
-    // Append metadata suffix for internal crates. See the corresponding entry
-    // in bootstrap/lib.rs for details.
-    if let Ok(s) = env::var("RUSTC_METADATA_SUFFIX") {
-        for i in 1..args.len() {
-            // Dirty code for borrowing issues
-            let mut new = None;
-            if let Some(current_as_str) = args[i].to_str() {
-                if (&*args[i - 1] == "-C" && current_as_str.starts_with("metadata")) ||
-                    current_as_str.starts_with("-Cmetadata") {
-                    new = Some(format!("{}-{}", current_as_str, s));
-                }
-            }
-            if let Some(new) = new { args[i] = new.into(); }
-        }
-    }
+    let args = env::args_os().skip(1).collect::<Vec<_>>();
 
     // Detect whether or not we're a build script depending on whether --target
     // is passed (a bit janky...)
@@ -93,48 +76,12 @@ fn main() {
         }
     }
 
-    // Non-zero stages must all be treated uniformly to avoid problems when attempting to uplift
-    // compiler libraries and such from stage 1 to 2.
-    //
-    // FIXME: the fact that core here is excluded is due to core_arch from our stdarch submodule
-    // being broken on the beta compiler with bootstrap passed, so this is a temporary workaround
-    // (we've just snapped, so there are no cfg(bootstrap) related annotations in core).
-    if stage == "0" {
-        if crate_name != Some("core") {
-            cmd.arg("--cfg").arg("bootstrap");
-        } else {
-            // NOTE(eddyb) see FIXME above, except now we need annotations again in core.
-            cmd.arg("--cfg").arg("boostrap_stdarch_ignore_this");
-        }
-    }
-
     // Print backtrace in case of ICE
     if env::var("RUSTC_BACKTRACE_ON_ICE").is_ok() && env::var("RUST_BACKTRACE").is_err() {
         cmd.env("RUST_BACKTRACE", "1");
     }
 
-    cmd.env("RUSTC_BREAK_ON_ICE", "1");
-
-    if let Ok(debuginfo_level) = env::var("RUSTC_DEBUGINFO_LEVEL") {
-        cmd.arg(format!("-Cdebuginfo={}", debuginfo_level));
-    }
-
-    if env::var_os("RUSTC_EXTERNAL_TOOL").is_none() {
-        // When extending this list, add the new lints to the RUSTFLAGS of the
-        // build_bootstrap function of src/bootstrap/bootstrap.py as well as
-        // some code doesn't go through this `rustc` wrapper.
-        cmd.arg("-Wrust_2018_idioms");
-        cmd.arg("-Wunused_lifetimes");
-        if use_internal_lints(crate_name) {
-            cmd.arg("-Zunstable-options");
-            cmd.arg("-Wrustc::internal");
-        }
-        if env::var_os("RUSTC_DENY_WARNINGS").is_some() {
-            cmd.arg("-Dwarnings");
-        }
-    }
-
-    if let Some(target) = target {
+    if target.is_some() {
         // The stage0 compiler has a special sysroot distinct from what we
         // actually downloaded, so we just always pass the `--sysroot` option,
         // unless one is already set.
@@ -142,43 +89,6 @@ fn main() {
             cmd.arg("--sysroot").arg(&sysroot);
         }
 
-        cmd.arg("-Zexternal-macro-backtrace");
-
-        // Link crates to the proc macro crate for the target, but use a host proc macro crate
-        // to actually run the macros
-        if env::var_os("RUST_DUAL_PROC_MACROS").is_some() {
-            cmd.arg("-Zdual-proc-macros");
-        }
-
-        // When we build Rust dylibs they're all intended for intermediate
-        // usage, so make sure we pass the -Cprefer-dynamic flag instead of
-        // linking all deps statically into the dylib.
-        if env::var_os("RUSTC_NO_PREFER_DYNAMIC").is_none() {
-            cmd.arg("-Cprefer-dynamic");
-        }
-
-        // Help the libc crate compile by assisting it in finding various
-        // sysroot native libraries.
-        if let Some(s) = env::var_os("MUSL_ROOT") {
-            if target.contains("musl") {
-                let mut root = OsString::from("native=");
-                root.push(&s);
-                root.push("/lib");
-                cmd.arg("-L").arg(&root);
-            }
-        }
-        if let Some(s) = env::var_os("WASI_ROOT") {
-            let mut root = OsString::from("native=");
-            root.push(&s);
-            root.push("/lib/wasm32-wasi");
-            cmd.arg("-L").arg(&root);
-        }
-
-        // Override linker if necessary.
-        if let Ok(target_linker) = env::var("RUSTC_TARGET_LINKER") {
-            cmd.arg(format!("-Clinker={}", target_linker));
-        }
-
         // If we're compiling specifically the `panic_abort` crate then we pass
         // the `-C panic=abort` option. Note that we do not do this for any
         // other crate intentionally as this is the only crate for now that we
@@ -205,82 +115,18 @@ fn main() {
 
         // The compiler builtins are pretty sensitive to symbols referenced in
         // libcore and such, so we never compile them with debug assertions.
+        //
+        // FIXME(rust-lang/cargo#7253) we should be doing this in `builder.rs`
+        // with env vars instead of doing it here in this script.
         if crate_name == Some("compiler_builtins") {
             cmd.arg("-C").arg("debug-assertions=no");
         } else {
             cmd.arg("-C").arg(format!("debug-assertions={}", debug_assertions));
         }
-
-        if let Ok(s) = env::var("RUSTC_CODEGEN_UNITS") {
-            cmd.arg("-C").arg(format!("codegen-units={}", s));
-        }
-
-        // Emit save-analysis info.
-        if env::var("RUSTC_SAVE_ANALYSIS") == Ok("api".to_string()) {
-            cmd.arg("-Zsave-analysis");
-            cmd.env("RUST_SAVE_ANALYSIS_CONFIG",
-                    "{\"output_file\": null,\"full_docs\": false,\
-                     \"pub_only\": true,\"reachable_only\": false,\
-                     \"distro_crate\": true,\"signatures\": false,\"borrow_data\": false}");
-        }
-
-        // Dealing with rpath here is a little special, so let's go into some
-        // detail. First off, `-rpath` is a linker option on Unix platforms
-        // which adds to the runtime dynamic loader path when looking for
-        // dynamic libraries. We use this by default on Unix platforms to ensure
-        // that our nightlies behave the same on Windows, that is they work out
-        // of the box. This can be disabled, of course, but basically that's why
-        // we're gated on RUSTC_RPATH here.
-        //
-        // Ok, so the astute might be wondering "why isn't `-C rpath` used
-        // here?" and that is indeed a good question to task. This codegen
-        // option is the compiler's current interface to generating an rpath.
-        // Unfortunately it doesn't quite suffice for us. The flag currently
-        // takes no value as an argument, so the compiler calculates what it
-        // should pass to the linker as `-rpath`. This unfortunately is based on
-        // the **compile time** directory structure which when building with
-        // Cargo will be very different than the runtime directory structure.
-        //
-        // All that's a really long winded way of saying that if we use
-        // `-Crpath` then the executables generated have the wrong rpath of
-        // something like `$ORIGIN/deps` when in fact the way we distribute
-        // rustc requires the rpath to be `$ORIGIN/../lib`.
-        //
-        // So, all in all, to set up the correct rpath we pass the linker
-        // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it
-        // fun to pass a flag to a tool to pass a flag to pass a flag to a tool
-        // to change a flag in a binary?
-        if env::var("RUSTC_RPATH") == Ok("true".to_string()) {
-            let rpath = if target.contains("apple") {
-
-                // Note that we need to take one extra step on macOS to also pass
-                // `-Wl,-instal_name,@rpath/...` to get things to work right. To
-                // do that we pass a weird flag to the compiler to get it to do
-                // so. Note that this is definitely a hack, and we should likely
-                // flesh out rpath support more fully in the future.
-                cmd.arg("-Z").arg("osx-rpath-install-name");
-                Some("-Wl,-rpath,@loader_path/../lib")
-            } else if !target.contains("windows") &&
-                      !target.contains("wasm32") &&
-                      !target.contains("fuchsia") {
-                Some("-Wl,-rpath,$ORIGIN/../lib")
-            } else {
-                None
-            };
-            if let Some(rpath) = rpath {
-                cmd.arg("-C").arg(format!("link-args={}", rpath));
-            }
-        }
-
-        if let Ok(s) = env::var("RUSTC_CRT_STATIC") {
-            if s == "true" {
-                cmd.arg("-C").arg("target-feature=+crt-static");
-            }
-            if s == "false" {
-                cmd.arg("-C").arg("target-feature=-crt-static");
-            }
-        }
     } else {
+        // FIXME(rust-lang/cargo#5754) we shouldn't be using special env vars
+        // here, but rather Cargo should know what flags to pass rustc itself.
+
         // Override linker if necessary.
         if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") {
             cmd.arg(format!("-Clinker={}", host_linker));
@@ -308,10 +154,6 @@ fn main() {
         cmd.arg("-Z").arg("force-unstable-if-unmarked");
     }
 
-    if env::var_os("RUSTC_PARALLEL_COMPILER").is_some() {
-        cmd.arg("--cfg").arg("parallel_compiler");
-    }
-
     if verbose > 1 {
         eprintln!(
             "rustc command: {:?}={:?} {:?}",
@@ -362,14 +204,6 @@ fn main() {
     std::process::exit(code);
 }
 
-// Rustc crates for which internal lints are in effect.
-fn use_internal_lints(crate_name: Option<&str>) -> bool {
-    crate_name.map_or(false, |crate_name| {
-        crate_name.starts_with("rustc") || crate_name.starts_with("syntax") ||
-        ["arena", "fmt_macros"].contains(&crate_name)
-    })
-}
-
 #[cfg(unix)]
 fn exec_cmd(cmd: &mut Command) -> io::Result<i32> {
     use std::os::unix::process::CommandExt;
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index b7873fd1d35..5d586f0c461 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -3,6 +3,7 @@ use std::cell::{Cell, RefCell};
 use std::collections::BTreeSet;
 use std::collections::HashMap;
 use std::env;
+use std::ffi::OsStr;
 use std::fmt::Debug;
 use std::fs;
 use std::hash::Hash;
@@ -682,7 +683,7 @@ impl<'a> Builder<'a> {
 
     /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic
     /// library lookup path.
-    pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Command) {
+    pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Cargo) {
         // Windows doesn't need dylib path munging because the dlls for the
         // compiler live next to the compiler and the system will find them
         // automatically.
@@ -690,7 +691,7 @@ impl<'a> Builder<'a> {
             return;
         }
 
-        add_lib_path(vec![self.rustc_libdir(compiler)], cmd);
+        add_lib_path(vec![self.rustc_libdir(compiler)], &mut cmd.command);
     }
 
     /// Gets a path to the compiler specified.
@@ -752,7 +753,7 @@ impl<'a> Builder<'a> {
         mode: Mode,
         target: Interned<String>,
         cmd: &str,
-    ) -> Command {
+    ) -> Cargo {
         let mut cargo = Command::new(&self.initial_cargo);
         let out_dir = self.stage_out(compiler, mode);
 
@@ -774,7 +775,17 @@ impl<'a> Builder<'a> {
 
         cargo
             .env("CARGO_TARGET_DIR", out_dir)
-            .arg(cmd);
+            .arg(cmd)
+            .arg("-Zconfig-profile");
+
+        let profile_var = |name: &str| {
+            let profile = if self.config.rust_optimize {
+                "RELEASE"
+            } else {
+                "DEV"
+            };
+            format!("CARGO_PROFILE_{}_{}", profile, name)
+        };
 
         // See comment in librustc_llvm/build.rs for why this is necessary, largely llvm-config
         // needs to not accidentally link to libLLVM in stage0/lib.
@@ -796,13 +807,29 @@ impl<'a> Builder<'a> {
             cargo.env("RUST_CHECK", "1");
         }
 
+        let stage;
+        if compiler.stage == 0 && self.local_rebuild {
+            // Assume the local-rebuild rustc already has stage1 features.
+            stage = 1;
+        } else {
+            stage = compiler.stage;
+        }
+
+        let mut rustflags = Rustflags::new(&target);
+        if stage != 0 {
+            rustflags.env("RUSTFLAGS_NOT_BOOTSTRAP");
+        } else {
+            rustflags.env("RUSTFLAGS_BOOTSTRAP");
+            rustflags.arg("--cfg=bootstrap");
+        }
+
         match mode {
             Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {},
             Mode::Rustc | Mode::Codegen | Mode::ToolRustc => {
                 // Build proc macros both for the host and the target
                 if target != compiler.host && cmd != "check" {
                     cargo.arg("-Zdual-proc-macros");
-                    cargo.env("RUST_DUAL_PROC_MACROS", "1");
+                    rustflags.arg("-Zdual-proc-macros");
                 }
             },
         }
@@ -852,37 +879,11 @@ impl<'a> Builder<'a> {
         }
         cargo.env("__CARGO_DEFAULT_LIB_METADATA", &metadata);
 
-        let stage;
-        if compiler.stage == 0 && self.local_rebuild {
-            // Assume the local-rebuild rustc already has stage1 features.
-            stage = 1;
-        } else {
-            stage = compiler.stage;
-        }
-
-        let mut extra_args = String::new();
-        if stage != 0 {
-            let s = env::var("RUSTFLAGS_NOT_BOOTSTRAP").unwrap_or_default();
-            extra_args.push_str(&s);
-        } else {
-            let s = env::var("RUSTFLAGS_BOOTSTRAP").unwrap_or_default();
-            extra_args.push_str(&s);
-        }
-
         if cmd == "clippy" {
-            extra_args.push_str("-Zforce-unstable-if-unmarked");
+            rustflags.arg("-Zforce-unstable-if-unmarked");
         }
 
-        if !extra_args.is_empty() {
-            cargo.env(
-                "RUSTFLAGS",
-                format!(
-                    "{} {}",
-                    env::var("RUSTFLAGS").unwrap_or_default(),
-                    extra_args
-                ),
-            );
-        }
+        rustflags.arg("-Zexternal-macro-backtrace");
 
         let want_rustdoc = self.doc_tests != DocTests::No;
 
@@ -919,7 +920,6 @@ impl<'a> Builder<'a> {
             )
             .env("RUSTC_SYSROOT", &sysroot)
             .env("RUSTC_LIBDIR", &libdir)
-            .env("RUSTC_RPATH", self.config.rust_rpath.to_string())
             .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc"))
             .env(
                 "RUSTDOC_REAL",
@@ -929,13 +929,63 @@ impl<'a> Builder<'a> {
                     PathBuf::from("/path/to/nowhere/rustdoc/not/required")
                 },
             )
-            .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir());
+            .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir())
+            .env("RUSTC_BREAK_ON_ICE", "1");
+
+        // Dealing with rpath here is a little special, so let's go into some
+        // detail. First off, `-rpath` is a linker option on Unix platforms
+        // which adds to the runtime dynamic loader path when looking for
+        // dynamic libraries. We use this by default on Unix platforms to ensure
+        // that our nightlies behave the same on Windows, that is they work out
+        // of the box. This can be disabled, of course, but basically that's why
+        // we're gated on RUSTC_RPATH here.
+        //
+        // Ok, so the astute might be wondering "why isn't `-C rpath` used
+        // here?" and that is indeed a good question to task. This codegen
+        // option is the compiler's current interface to generating an rpath.
+        // Unfortunately it doesn't quite suffice for us. The flag currently
+        // takes no value as an argument, so the compiler calculates what it
+        // should pass to the linker as `-rpath`. This unfortunately is based on
+        // the **compile time** directory structure which when building with
+        // Cargo will be very different than the runtime directory structure.
+        //
+        // All that's a really long winded way of saying that if we use
+        // `-Crpath` then the executables generated have the wrong rpath of
+        // something like `$ORIGIN/deps` when in fact the way we distribute
+        // rustc requires the rpath to be `$ORIGIN/../lib`.
+        //
+        // So, all in all, to set up the correct rpath we pass the linker
+        // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it
+        // fun to pass a flag to a tool to pass a flag to pass a flag to a tool
+        // to change a flag in a binary?
+        if self.config.rust_rpath {
+            let rpath = if target.contains("apple") {
+
+                // Note that we need to take one extra step on macOS to also pass
+                // `-Wl,-instal_name,@rpath/...` to get things to work right. To
+                // do that we pass a weird flag to the compiler to get it to do
+                // so. Note that this is definitely a hack, and we should likely
+                // flesh out rpath support more fully in the future.
+                rustflags.arg("-Zosx-rpath-install-name");
+                Some("-Wl,-rpath,@loader_path/../lib")
+            } else if !target.contains("windows") &&
+                      !target.contains("wasm32") &&
+                      !target.contains("fuchsia") {
+                Some("-Wl,-rpath,$ORIGIN/../lib")
+            } else {
+                None
+            };
+            if let Some(rpath) = rpath {
+                rustflags.arg(&format!("-Clink-args={}", rpath));
+            }
+        }
 
         if let Some(host_linker) = self.linker(compiler.host) {
             cargo.env("RUSTC_HOST_LINKER", host_linker);
         }
         if let Some(target_linker) = self.linker(target) {
-            cargo.env("RUSTC_TARGET_LINKER", target_linker);
+            let target = crate::envify(&target);
+            cargo.env(&format!("CARGO_TARGET_{}_LINKER", target), target_linker);
         }
         if !(["build", "check", "clippy", "fix", "rustc"].contains(&cmd)) && want_rustdoc {
             cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler));
@@ -947,32 +997,18 @@ impl<'a> Builder<'a> {
             Mode::ToolBootstrap | Mode::ToolStd |
             Mode::ToolRustc => self.config.rust_debuginfo_level_tools,
         };
-        cargo.env("RUSTC_DEBUGINFO_LEVEL", debuginfo_level.to_string());
+        cargo.env(profile_var("DEBUG"), debuginfo_level.to_string());
 
         if !mode.is_tool() {
             cargo.env("RUSTC_FORCE_UNSTABLE", "1");
-
-            // Currently the compiler depends on crates from crates.io, and
-            // then other crates can depend on the compiler (e.g., proc-macro
-            // crates). Let's say, for example that rustc itself depends on the
-            // bitflags crate. If an external crate then depends on the
-            // bitflags crate as well, we need to make sure they don't
-            // conflict, even if they pick the same version of bitflags. We'll
-            // want to make sure that e.g., a plugin and rustc each get their
-            // own copy of bitflags.
-
-            // Cargo ensures that this works in general through the -C metadata
-            // flag. This flag will frob the symbols in the binary to make sure
-            // they're different, even though the source code is the exact
-            // same. To solve this problem for the compiler we extend Cargo's
-            // already-passed -C metadata flag with our own. Our rustc.rs
-            // wrapper around the actual rustc will detect -C metadata being
-            // passed and frob it with this extra string we're passing in.
-            cargo.env("RUSTC_METADATA_SUFFIX", "rustc");
         }
 
         if let Some(x) = self.crt_static(target) {
-            cargo.env("RUSTC_CRT_STATIC", x.to_string());
+            if x {
+                rustflags.arg("-Ctarget-feature=+crt-static");
+            } else {
+                rustflags.arg("-Ctarget-feature=-crt-static");
+            }
         }
 
         if let Some(x) = self.crt_static(compiler.host) {
@@ -1031,8 +1067,21 @@ impl<'a> Builder<'a> {
 
         cargo.env("RUSTC_VERBOSE", self.verbosity.to_string());
 
-        if self.config.deny_warnings {
-            cargo.env("RUSTC_DENY_WARNINGS", "1");
+        if !mode.is_tool() {
+            // When extending this list, add the new lints to the RUSTFLAGS of the
+            // build_bootstrap function of src/bootstrap/bootstrap.py as well as
+            // some code doesn't go through this `rustc` wrapper.
+            rustflags.arg("-Wrust_2018_idioms");
+            rustflags.arg("-Wunused_lifetimes");
+
+            if self.config.deny_warnings {
+                rustflags.arg("-Dwarnings");
+            }
+        }
+
+        if let Mode::Rustc | Mode::Codegen = mode {
+            rustflags.arg("-Zunstable-options");
+            rustflags.arg("-Wrustc::internal");
         }
 
         // Throughout the build Cargo can execute a number of build scripts
@@ -1085,12 +1134,15 @@ impl<'a> Builder<'a> {
             }
         }
 
-        if (cmd == "build" || cmd == "rustc")
-            && mode == Mode::Std
+        if mode == Mode::Std
             && self.config.extended
             && compiler.is_final_stage(self)
         {
-            cargo.env("RUSTC_SAVE_ANALYSIS", "api".to_string());
+            rustflags.arg("-Zsave-analysis");
+            cargo.env("RUST_SAVE_ANALYSIS_CONFIG",
+                      "{\"output_file\": null,\"full_docs\": false,\
+                       \"pub_only\": true,\"reachable_only\": false,\
+                       \"distro_crate\": true,\"signatures\": false,\"borrow_data\": false}");
         }
 
         // For `cargo doc` invocations, make rustdoc print the Rust version into the docs
@@ -1146,7 +1198,7 @@ impl<'a> Builder<'a> {
         match (mode, self.config.rust_codegen_units_std, self.config.rust_codegen_units) {
             (Mode::Std, Some(n), _) |
             (_, _, Some(n)) => {
-                cargo.env("RUSTC_CODEGEN_UNITS", n.to_string());
+                cargo.env(profile_var("CODEGEN_UNITS"), n.to_string());
             }
             _ => {
                 // Don't set anything
@@ -1171,7 +1223,17 @@ impl<'a> Builder<'a> {
 
         self.ci_env.force_coloring_in_ci(&mut cargo);
 
-        cargo
+        // When we build Rust dylibs they're all intended for intermediate
+        // usage, so make sure we pass the -Cprefer-dynamic flag instead of
+        // linking all deps statically into the dylib.
+        if let Mode::Std | Mode::Rustc | Mode::Codegen = mode {
+            rustflags.arg("-Cprefer-dynamic");
+        }
+
+        Cargo {
+            command: cargo,
+            rustflags,
+        }
     }
 
     /// Ensure that a given step is built, returning its output. This will
@@ -1271,3 +1333,78 @@ impl<'a> Builder<'a> {
 
 #[cfg(test)]
 mod tests;
+
+#[derive(Debug)]
+struct Rustflags(String);
+
+impl Rustflags {
+    fn new(target: &str) -> Rustflags {
+        let mut ret = Rustflags(String::new());
+
+        // Inherit `RUSTFLAGS` by default ...
+        ret.env("RUSTFLAGS");
+
+        // ... and also handle target-specific env RUSTFLAGS if they're
+        // configured.
+        let target_specific = format!("CARGO_TARGET_{}_RUSTFLAGS", crate::envify(target));
+        ret.env(&target_specific);
+
+        ret
+    }
+
+    fn env(&mut self, env: &str) {
+        if let Ok(s) = env::var(env) {
+            for part in s.split_whitespace() {
+                self.arg(part);
+            }
+        }
+    }
+
+    fn arg(&mut self, arg: &str) -> &mut Self {
+        assert_eq!(arg.split_whitespace().count(), 1);
+        if self.0.len() > 0 {
+            self.0.push_str(" ");
+        }
+        self.0.push_str(arg);
+        self
+    }
+}
+
+#[derive(Debug)]
+pub struct Cargo {
+    command: Command,
+    rustflags: Rustflags,
+}
+
+impl Cargo {
+    pub fn rustflag(&mut self, arg: &str) -> &mut Cargo {
+        self.rustflags.arg(arg);
+        self
+    }
+
+    pub fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Cargo {
+        self.command.arg(arg.as_ref());
+        self
+    }
+
+    pub fn args<I, S>(&mut self, args: I) -> &mut Cargo
+        where I: IntoIterator<Item=S>, S: AsRef<OsStr>
+    {
+        for arg in args {
+            self.arg(arg.as_ref());
+        }
+        self
+    }
+
+    pub fn env(&mut self, key: impl AsRef<OsStr>, value: impl AsRef<OsStr>) -> &mut Cargo {
+        self.command.env(key.as_ref(), value.as_ref());
+        self
+    }
+}
+
+impl From<Cargo> for Command {
+    fn from(mut cargo: Cargo) -> Command {
+        cargo.command.env("RUSTFLAGS", &cargo.rustflags.0);
+        cargo.command
+    }
+}
diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index 205a80c3a3a..cadb9a7e441 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -52,7 +52,7 @@ impl Step for Std {
 
         builder.info(&format!("Checking std artifacts ({} -> {})", &compiler.host, target));
         run_cargo(builder,
-                  &mut cargo,
+                  cargo,
                   args(builder.kind),
                   &libstd_stamp(builder, compiler, target),
                   true);
@@ -100,7 +100,7 @@ impl Step for Rustc {
 
         builder.info(&format!("Checking compiler artifacts ({} -> {})", &compiler.host, target));
         run_cargo(builder,
-                  &mut cargo,
+                  cargo,
                   args(builder.kind),
                   &librustc_stamp(builder, compiler, target),
                   true);
@@ -152,7 +152,7 @@ impl Step for CodegenBackend {
         // We won't build LLVM if it's not available, as it shouldn't affect `check`.
 
         run_cargo(builder,
-                  &mut cargo,
+                  cargo,
                   args(builder.kind),
                   &codegen_backend_stamp(builder, compiler, target, backend),
                   true);
@@ -185,18 +185,18 @@ impl Step for Rustdoc {
 
         builder.ensure(Rustc { target });
 
-        let mut cargo = prepare_tool_cargo(builder,
-                                           compiler,
-                                           Mode::ToolRustc,
-                                           target,
-                                           cargo_subcommand(builder.kind),
-                                           "src/tools/rustdoc",
-                                           SourceType::InTree,
-                                           &[]);
+        let cargo = prepare_tool_cargo(builder,
+                                       compiler,
+                                       Mode::ToolRustc,
+                                       target,
+                                       cargo_subcommand(builder.kind),
+                                       "src/tools/rustdoc",
+                                       SourceType::InTree,
+                                       &[]);
 
         println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
-                  &mut cargo,
+                  cargo,
                   args(builder.kind),
                   &rustdoc_stamp(builder, compiler, target),
                   true);
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index 9a964457ef2..6ea32edfb20 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -21,6 +21,7 @@ use serde::Deserialize;
 use serde_json;
 
 use crate::dist;
+use crate::builder::Cargo;
 use crate::util::{exe, is_dylib};
 use crate::{Compiler, Mode, GitRepo};
 use crate::native;
@@ -98,7 +99,7 @@ impl Step for Std {
         builder.info(&format!("Building stage{} std artifacts ({} -> {})", compiler.stage,
                 &compiler.host, target));
         run_cargo(builder,
-                  &mut cargo,
+                  cargo,
                   vec![],
                   &libstd_stamp(builder, compiler, target),
                   false);
@@ -156,7 +157,7 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target:
 pub fn std_cargo(builder: &Builder<'_>,
                  compiler: &Compiler,
                  target: Interned<String>,
-                 cargo: &mut Command) {
+                 cargo: &mut Cargo) {
     if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") {
         cargo.env("MACOSX_DEPLOYMENT_TARGET", target);
     }
@@ -219,15 +220,19 @@ pub fn std_cargo(builder: &Builder<'_>,
             .arg("--manifest-path")
             .arg(builder.src.join("src/libtest/Cargo.toml"));
 
+        // Help the libc crate compile by assisting it in finding various
+        // sysroot native libraries.
         if target.contains("musl") {
             if let Some(p) = builder.musl_root(target) {
-                cargo.env("MUSL_ROOT", p);
+                let root = format!("native={}/lib", p.to_str().unwrap());
+                cargo.rustflag("-L").rustflag(&root);
             }
         }
 
         if target.ends_with("-wasi") {
             if let Some(p) = builder.wasi_root(target) {
-                cargo.env("WASI_ROOT", p);
+                let root = format!("native={}/lib/wasm32-wasi", p.to_str().unwrap());
+                cargo.rustflag("-L").rustflag(&root);
             }
         }
     }
@@ -430,7 +435,7 @@ impl Step for Rustc {
         builder.info(&format!("Building stage{} compiler artifacts ({} -> {})",
                  compiler.stage, &compiler.host, target));
         run_cargo(builder,
-                  &mut cargo,
+                  cargo,
                   vec![],
                   &librustc_stamp(builder, compiler, target),
                   false);
@@ -443,14 +448,14 @@ impl Step for Rustc {
     }
 }
 
-pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Command) {
+pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo) {
     cargo.arg("--features").arg(builder.rustc_features())
          .arg("--manifest-path")
          .arg(builder.src.join("src/rustc/Cargo.toml"));
     rustc_cargo_env(builder, cargo);
 }
 
-pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Command) {
+pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo) {
     // Set some configuration variables picked up by build scripts and
     // the compiler alike
     cargo.env("CFG_RELEASE", builder.rust_release())
@@ -475,7 +480,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Command) {
         cargo.env("CFG_DEFAULT_LINKER", s);
     }
     if builder.config.rustc_parallel {
-        cargo.env("RUSTC_PARALLEL_COMPILER", "1");
+        cargo.rustflag("--cfg=parallel_compiler");
     }
     if builder.config.rust_verify_llvm_ir {
         cargo.env("RUSTC_VERIFY_LLVM_IR", "1");
@@ -577,14 +582,11 @@ impl Step for CodegenBackend {
         rustc_cargo_env(builder, &mut cargo);
 
         let features = build_codegen_backend(&builder, &mut cargo, &compiler, target, backend);
+        cargo.arg("--features").arg(features);
 
         let tmp_stamp = out_dir.join(".tmp.stamp");
 
-        let files = run_cargo(builder,
-                              cargo.arg("--features").arg(features),
-                              vec![],
-                              &tmp_stamp,
-                              false);
+        let files = run_cargo(builder, cargo, vec![], &tmp_stamp, false);
         if builder.config.dry_run {
             return;
         }
@@ -609,7 +611,7 @@ impl Step for CodegenBackend {
 }
 
 pub fn build_codegen_backend(builder: &Builder<'_>,
-                             cargo: &mut Command,
+                             cargo: &mut Cargo,
                              compiler: &Compiler,
                              target: Interned<String>,
                              backend: Interned<String>) -> String {
@@ -949,7 +951,7 @@ pub fn add_to_sysroot(
 }
 
 pub fn run_cargo(builder: &Builder<'_>,
-                 cargo: &mut Command,
+                 cargo: Cargo,
                  tail_args: Vec<String>,
                  stamp: &Path,
                  is_check: bool)
@@ -1081,10 +1083,11 @@ pub fn run_cargo(builder: &Builder<'_>,
 
 pub fn stream_cargo(
     builder: &Builder<'_>,
-    cargo: &mut Command,
+    cargo: Cargo,
     tail_args: Vec<String>,
     cb: &mut dyn FnMut(CargoMessage<'_>),
 ) -> bool {
+    let mut cargo = Command::from(cargo);
     if builder.config.dry_run {
         return true;
     }
diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs
index 873a3c31d15..4ee8cd2485c 100644
--- a/src/bootstrap/doc.rs
+++ b/src/bootstrap/doc.rs
@@ -475,7 +475,7 @@ impl Step for Std {
                  .arg("--resource-suffix").arg(crate::channel::CFG_RELEASE_NUM)
                  .arg("--index-page").arg(&builder.src.join("src/doc/index.md"));
 
-            builder.run(&mut cargo);
+            builder.run(&mut cargo.into());
         };
         for krate in &["alloc", "core", "std", "proc_macro", "test"] {
             run_cargo_rustdoc_for(krate);
@@ -561,7 +561,7 @@ impl Step for Rustc {
             cargo.arg("-p").arg(krate);
         }
 
-        builder.run(&mut cargo);
+        builder.run(&mut cargo.into());
     }
 }
 
@@ -656,7 +656,7 @@ impl Step for Rustdoc {
         cargo.arg("-p").arg("rustdoc");
 
         cargo.env("RUSTDOCFLAGS", "--document-private-items");
-        builder.run(&mut cargo);
+        builder.run(&mut cargo.into());
     }
 }
 
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index 5d7581c8211..9203a558f64 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -1320,3 +1320,13 @@ impl Compiler {
         self.stage >= final_stage
     }
 }
+
+fn envify(s: &str) -> String {
+    s.chars()
+        .map(|c| match c {
+            '-' => '_',
+            c => c,
+        })
+        .flat_map(|c| c.to_uppercase())
+        .collect()
+}
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index 00d87f3841c..b7ce9c7b397 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -23,7 +23,7 @@ use crate::tool::{self, Tool, SourceType};
 use crate::toolstate::ToolState;
 use crate::util::{self, dylib_path, dylib_path_var};
 use crate::Crate as CargoCrate;
-use crate::{DocTests, Mode, GitRepo};
+use crate::{DocTests, Mode, GitRepo, envify};
 
 const ADB_TEST_DIR: &str = "/data/tmp/work";
 
@@ -233,10 +233,9 @@ impl Step for Cargo {
         // those features won't be able to land.
         cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1");
 
-        try_run(
-            builder,
-            cargo.env("PATH", &path_for_cargo(builder, compiler)),
-        );
+        cargo.env("PATH", &path_for_cargo(builder, compiler));
+
+        try_run(builder, &mut cargo.into());
     }
 }
 
@@ -290,7 +289,7 @@ impl Step for Rls {
         cargo.arg("--")
             .args(builder.config.cmd.test_args());
 
-        if try_run(builder, &mut cargo) {
+        if try_run(builder, &mut cargo.into()) {
             builder.save_toolstate("rls", ToolState::TestPass);
         }
     }
@@ -348,7 +347,7 @@ impl Step for Rustfmt {
 
         builder.add_rustc_lib_path(compiler, &mut cargo);
 
-        if try_run(builder, &mut cargo) {
+        if try_run(builder, &mut cargo.into()) {
             builder.save_toolstate("rustfmt", ToolState::TestPass);
         }
     }
@@ -418,6 +417,7 @@ impl Step for Miri {
             cargo.env("CARGO_INSTALL_ROOT", &builder.out); // cargo adds a `bin/`
             cargo.env("XARGO", builder.out.join("bin").join("xargo"));
 
+            let mut cargo = Command::from(cargo);
             if !try_run(builder, &mut cargo) {
                 return;
             }
@@ -467,7 +467,7 @@ impl Step for Miri {
 
             builder.add_rustc_lib_path(compiler, &mut cargo);
 
-            if !try_run(builder, &mut cargo) {
+            if !try_run(builder, &mut cargo.into()) {
                 return;
             }
 
@@ -502,16 +502,16 @@ impl Step for CompiletestTest {
         let host = self.host;
         let compiler = builder.compiler(0, host);
 
-        let mut cargo = tool::prepare_tool_cargo(builder,
-                                                 compiler,
-                                                 Mode::ToolBootstrap,
-                                                 host,
-                                                 "test",
-                                                 "src/tools/compiletest",
-                                                 SourceType::InTree,
-                                                 &[]);
+        let cargo = tool::prepare_tool_cargo(builder,
+                                             compiler,
+                                             Mode::ToolBootstrap,
+                                             host,
+                                             "test",
+                                             "src/tools/compiletest",
+                                             SourceType::InTree,
+                                             &[]);
 
-        try_run(builder, &mut cargo);
+        try_run(builder, &mut cargo.into());
     }
 }
 
@@ -571,7 +571,7 @@ impl Step for Clippy {
 
             builder.add_rustc_lib_path(compiler, &mut cargo);
 
-            if try_run(builder, &mut cargo) {
+            if try_run(builder, &mut cargo.into()) {
                 builder.save_toolstate("clippy-driver", ToolState::TestPass);
             }
         } else {
@@ -1814,10 +1814,6 @@ impl Step for Crate {
                     .expect("nodejs not configured"),
             );
         } else if target.starts_with("wasm32") {
-            // On the wasm32-unknown-unknown target we're using LTO which is
-            // incompatible with `-C prefer-dynamic`, so disable that here
-            cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
-
             let node = builder
                 .config
                 .nodejs
@@ -1841,7 +1837,7 @@ impl Step for Crate {
             test_kind, krate, compiler.stage, &compiler.host, target
         ));
         let _time = util::timeit(&builder);
-        try_run(builder, &mut cargo);
+        try_run(builder, &mut cargo.into());
     }
 }
 
@@ -1909,20 +1905,10 @@ impl Step for CrateRustdoc {
         ));
         let _time = util::timeit(&builder);
 
-        try_run(builder, &mut cargo);
+        try_run(builder, &mut cargo.into());
     }
 }
 
-fn envify(s: &str) -> String {
-    s.chars()
-        .map(|c| match c {
-            '-' => '_',
-            c => c,
-        })
-        .flat_map(|c| c.to_uppercase())
-        .collect()
-}
-
 /// Some test suites are run inside emulators or on remote devices, and most
 /// of our test binaries are linked dynamically which means we need to ship
 /// the standard library and such to the emulator ahead of time. This step
diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs
index 54fe26f18e7..f1baeafe26a 100644
--- a/src/bootstrap/tool.rs
+++ b/src/bootstrap/tool.rs
@@ -8,7 +8,7 @@ use build_helper::t;
 
 use crate::Mode;
 use crate::Compiler;
-use crate::builder::{Step, RunConfig, ShouldRun, Builder};
+use crate::builder::{Step, RunConfig, ShouldRun, Builder, Cargo as CargoCommand};
 use crate::util::{exe, add_lib_path, CiEnv};
 use crate::compile;
 use crate::channel::GitInfo;
@@ -63,7 +63,7 @@ impl Step for ToolBuild {
             _ => panic!("unexpected Mode for tool build")
         }
 
-        let mut cargo = prepare_tool_cargo(
+        let cargo = prepare_tool_cargo(
             builder,
             compiler,
             self.mode,
@@ -76,7 +76,7 @@ impl Step for ToolBuild {
 
         builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
         let mut duplicates = Vec::new();
-        let is_expected = compile::stream_cargo(builder, &mut cargo, vec![], &mut |msg| {
+        let is_expected = compile::stream_cargo(builder, cargo, vec![], &mut |msg| {
             // Only care about big things like the RLS/Cargo for now
             match tool {
                 | "rls"
@@ -229,15 +229,11 @@ pub fn prepare_tool_cargo(
     path: &'static str,
     source_type: SourceType,
     extra_features: &[String],
-) -> Command {
+) -> CargoCommand {
     let mut cargo = builder.cargo(compiler, mode, target, command);
     let dir = builder.src.join(path);
     cargo.arg("--manifest-path").arg(dir.join("Cargo.toml"));
 
-    // We don't want to build tools dynamically as they'll be running across
-    // stages and such and it's just easier if they're not dynamically linked.
-    cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
-
     if source_type == SourceType::Submodule {
         cargo.env("RUSTC_EXTERNAL_TOOL", "1");
     }
@@ -517,7 +513,7 @@ impl Step for Rustdoc {
         // libraries here. The intuition here is that If we've built a compiler, we should be able
         // to build rustdoc.
 
-        let mut cargo = prepare_tool_cargo(
+        let cargo = prepare_tool_cargo(
             builder,
             build_compiler,
             Mode::ToolRustc,
@@ -530,7 +526,7 @@ impl Step for Rustdoc {
 
         builder.info(&format!("Building rustdoc for stage{} ({})",
             target_compiler.stage, target_compiler.host));
-        builder.run(&mut cargo);
+        builder.run(&mut cargo.into());
 
         // Cargo adds a number of paths to the dylib search path on windows, which results in
         // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"