diff options
Diffstat (limited to 'src/bootstrap')
| -rw-r--r-- | src/bootstrap/Cargo.toml | 2 | ||||
| -rw-r--r-- | src/bootstrap/bin/rustdoc.rs | 11 | ||||
| -rw-r--r-- | src/bootstrap/bootstrap.py | 149 | ||||
| -rw-r--r-- | src/bootstrap/builder.rs | 116 | ||||
| -rw-r--r-- | src/bootstrap/builder/tests.rs | 226 | ||||
| -rw-r--r-- | src/bootstrap/check.rs | 4 | ||||
| -rw-r--r-- | src/bootstrap/compile.rs | 6 | ||||
| -rw-r--r-- | src/bootstrap/config.rs | 122 | ||||
| -rw-r--r-- | src/bootstrap/dist.rs | 8 | ||||
| -rw-r--r-- | src/bootstrap/doc.rs | 7 | ||||
| -rw-r--r-- | src/bootstrap/flags.rs | 146 | ||||
| -rw-r--r-- | src/bootstrap/lib.rs | 10 | ||||
| -rw-r--r-- | src/bootstrap/native.rs | 292 | ||||
| -rw-r--r-- | src/bootstrap/tarball.rs | 4 | ||||
| -rw-r--r-- | src/bootstrap/test.rs | 15 |
15 files changed, 673 insertions, 445 deletions
diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 4c32547f059..dea8d998bde 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -42,10 +42,12 @@ cc = "1.0.69" libc = "0.2" serde = { version = "1.0.8", features = ["derive"] } serde_json = "1.0.2" +tar = "0.4" toml = "0.5" ignore = "0.4.10" opener = "0.5" once_cell = "1.7.2" +xz2 = "0.1" [target.'cfg(windows)'.dependencies.winapi] version = "0.3" diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs index 0349277a36b..5f85fc5aa59 100644 --- a/src/bootstrap/bin/rustdoc.rs +++ b/src/bootstrap/bin/rustdoc.rs @@ -31,8 +31,8 @@ fn main() { let mut cmd = Command::new(rustdoc); - // I am not actually sure why it's necessary to pass the sysroot for `--test`, - // but `test --doc --stage 0` is broken without it :( + // cfg(bootstrap) + // NOTE: the `--test` special-casing can be removed when https://github.com/rust-lang/cargo/pull/10594 lands on beta. if target.is_some() || args.iter().any(|x| x == "--test") { // The stage0 compiler has a special sysroot distinct from what we // actually downloaded, so we just always pass the `--sysroot` option, @@ -65,13 +65,6 @@ fn main() { } } - // Needed to be able to run all rustdoc tests. - if let Some(ref x) = env::var_os("RUSTDOC_RESOURCE_SUFFIX") { - // This "unstable-options" can be removed when `--resource-suffix` is stabilized - cmd.arg("-Z").arg("unstable-options"); - cmd.arg("--resource-suffix").arg(x); - } - if verbose > 1 { eprintln!( "rustdoc command: {:?}={:?} {:?}", diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index ac1c47524fd..e38a574ca23 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -500,81 +500,6 @@ class RustBuild(object): with output(self.rustfmt_stamp()) as rustfmt_stamp: rustfmt_stamp.write(self.stage0_rustfmt.channel()) - # Avoid downloading LLVM twice (once for stage0 and once for the master rustc) - if self.downloading_llvm() and stage0: - # We want the most recent LLVM submodule update to avoid downloading - # LLVM more often than necessary. - # - # This git command finds that commit SHA, looking for bors-authored - # commits that modified src/llvm-project or other relevant version - # stamp files. - # - # This works even in a repository that has not yet initialized - # submodules. - top_level = subprocess.check_output([ - "git", "rev-parse", "--show-toplevel", - ]).decode(sys.getdefaultencoding()).strip() - llvm_sha = subprocess.check_output([ - "git", "rev-list", "--author=bors@rust-lang.org", "-n1", - "--first-parent", "HEAD", - "--", - "{}/src/llvm-project".format(top_level), - "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level), - # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` - "{}/src/version".format(top_level) - ]).decode(sys.getdefaultencoding()).strip() - llvm_assertions = self.get_toml('assertions', 'llvm') == 'true' - llvm_root = self.llvm_root() - llvm_lib = os.path.join(llvm_root, "lib") - if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)): - self._download_ci_llvm(llvm_sha, llvm_assertions) - for binary in ["llvm-config", "FileCheck"]: - self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary)) - for lib in os.listdir(llvm_lib): - if lib.endswith(".so"): - self.fix_bin_or_dylib(os.path.join(llvm_lib, lib)) - with output(self.llvm_stamp()) as llvm_stamp: - llvm_stamp.write(llvm_sha + str(llvm_assertions)) - - def downloading_llvm(self): - opt = self.get_toml('download-ci-llvm', 'llvm') - # This is currently all tier 1 targets and tier 2 targets with host tools - # (since others may not have CI artifacts) - # https://doc.rust-lang.org/rustc/platform-support.html#tier-1 - supported_platforms = [ - # tier 1 - "aarch64-unknown-linux-gnu", - "i686-pc-windows-gnu", - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-gnu", - "x86_64-pc-windows-msvc", - # tier 2 with host tools - "aarch64-apple-darwin", - "aarch64-pc-windows-msvc", - "aarch64-unknown-linux-musl", - "arm-unknown-linux-gnueabi", - "arm-unknown-linux-gnueabihf", - "armv7-unknown-linux-gnueabihf", - "mips-unknown-linux-gnu", - "mips64-unknown-linux-gnuabi64", - "mips64el-unknown-linux-gnuabi64", - "mipsel-unknown-linux-gnu", - "powerpc-unknown-linux-gnu", - "powerpc64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv64gc-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-unknown-freebsd", - "x86_64-unknown-illumos", - "x86_64-unknown-linux-musl", - "x86_64-unknown-netbsd", - ] - return opt == "true" \ - or (opt == "if-available" and self.build in supported_platforms) - def _download_component_helper( self, filename, pattern, tarball_suffix, stage0=True, key=None ): @@ -606,53 +531,6 @@ class RustBuild(object): ) unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose) - def _download_ci_llvm(self, llvm_sha, llvm_assertions): - if not llvm_sha: - print("error: could not find commit hash for downloading LLVM") - print("help: maybe your repository history is too shallow?") - print("help: consider disabling `download-ci-llvm`") - print("help: or fetch enough history to include one upstream commit") - exit(1) - cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions) - cache_dst = os.path.join(self.build_dir, "cache") - rustc_cache = os.path.join(cache_dst, cache_prefix) - if not os.path.exists(rustc_cache): - os.makedirs(rustc_cache) - - base = "https://ci-artifacts.rust-lang.org" - url = "rustc-builds/{}".format(llvm_sha) - if llvm_assertions: - url = url.replace('rustc-builds', 'rustc-builds-alt') - # ci-artifacts are only stored as .xz, not .gz - if not support_xz(): - print("error: XZ support is required to download LLVM") - print("help: consider disabling `download-ci-llvm` or using python3") - exit(1) - tarball_suffix = '.tar.xz' - filename = "rust-dev-nightly-" + self.build + tarball_suffix - tarball = os.path.join(rustc_cache, filename) - if not os.path.exists(tarball): - help_on_error = "error: failed to download llvm from ci" - help_on_error += "\nhelp: old builds get deleted after a certain time" - help_on_error += "\nhelp: if trying to compile an old commit of rustc," - help_on_error += " disable `download-ci-llvm` in config.toml:" - help_on_error += "\n" - help_on_error += "\n[llvm]" - help_on_error += "\ndownload-ci-llvm = false" - help_on_error += "\n" - get( - base, - "{}/{}".format(url, filename), - tarball, - self.checksums_sha256, - verbose=self.verbose, - do_verify=False, - help_on_error=help_on_error, - ) - unpack(tarball, tarball_suffix, self.llvm_root(), - match="rust-dev", - verbose=self.verbose) - def fix_bin_or_dylib(self, fname): """Modifies the interpreter section of 'fname' to fix the dynamic linker, or the RPATH section, to fix the dynamic library search path @@ -816,17 +694,6 @@ class RustBuild(object): """ return os.path.join(self.bin_root(True), '.rustfmt-stamp') - def llvm_stamp(self): - """Return the path for .llvm-stamp - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp") - True - """ - return os.path.join(self.llvm_root(), '.llvm-stamp') - - def program_out_of_date(self, stamp_path, key): """Check if the given program stamp is out of date""" if not os.path.exists(stamp_path) or self.clean: @@ -856,22 +723,6 @@ class RustBuild(object): subdir = "ci-rustc" return os.path.join(self.build_dir, self.build, subdir) - def llvm_root(self): - """Return the CI LLVM root directory - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.llvm_root() == os.path.join("build", "ci-llvm") - True - - When the 'build' property is given should be a nested directory: - - >>> rb.build = "devel" - >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm") - True - """ - return os.path.join(self.build_dir, self.build, "ci-llvm") - def get_toml(self, key, section=None): """Returns the value of the given key in config.toml, otherwise returns None diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index d688f798956..dd45bd3a213 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -1,9 +1,9 @@ -use std::any::Any; +use std::any::{type_name, Any}; use std::cell::{Cell, RefCell}; use std::collections::BTreeSet; use std::env; use std::ffi::OsStr; -use std::fmt::Debug; +use std::fmt::{Debug, Write}; use std::fs; use std::hash::Hash; use std::ops::Deref; @@ -12,9 +12,8 @@ use std::process::Command; use std::time::{Duration, Instant}; use crate::cache::{Cache, Interned, INTERNER}; -use crate::check; use crate::compile; -use crate::config::TargetSelection; +use crate::config::{SplitDebuginfo, TargetSelection}; use crate::dist; use crate::doc; use crate::flags::{Color, Subcommand}; @@ -25,6 +24,7 @@ use crate::test; use crate::tool::{self, SourceType}; use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; use crate::EXTRA_CHECK_CFGS; +use crate::{check, Config}; use crate::{Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; @@ -125,7 +125,8 @@ impl TaskPath { if found_kind.is_empty() { panic!("empty kind in task path {}", path.display()); } - kind = Some(Kind::parse(found_kind)); + kind = Kind::parse(found_kind); + assert!(kind.is_some()); path = Path::new(found_prefix).join(components.as_path()); } } @@ -388,11 +389,13 @@ impl<'a> ShouldRun<'a> { paths .iter() .map(|p| { - assert!( - self.builder.src.join(p).exists(), - "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", - p - ); + // FIXME(#96188): make sure this is actually a path. + // This currently breaks for paths within submodules. + //assert!( + // self.builder.src.join(p).exists(), + // "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", + // p + //); TaskPath { path: p.into(), kind: Some(self.kind) } }) .collect(), @@ -429,43 +432,53 @@ pub enum Kind { Check, Clippy, Fix, + Format, Test, Bench, - Dist, Doc, + Clean, + Dist, Install, Run, + Setup, } impl Kind { - fn parse(string: &str) -> Kind { - match string { - "build" => Kind::Build, - "check" => Kind::Check, + pub fn parse(string: &str) -> Option<Kind> { + // these strings, including the one-letter aliases, must match the x.py help text + Some(match string { + "build" | "b" => Kind::Build, + "check" | "c" => Kind::Check, "clippy" => Kind::Clippy, "fix" => Kind::Fix, - "test" => Kind::Test, + "fmt" => Kind::Format, + "test" | "t" => Kind::Test, "bench" => Kind::Bench, + "doc" | "d" => Kind::Doc, + "clean" => Kind::Clean, "dist" => Kind::Dist, - "doc" => Kind::Doc, "install" => Kind::Install, - "run" => Kind::Run, - other => panic!("unknown kind: {}", other), - } + "run" | "r" => Kind::Run, + "setup" => Kind::Setup, + _ => return None, + }) } - fn as_str(&self) -> &'static str { + pub fn as_str(&self) -> &'static str { match self { Kind::Build => "build", Kind::Check => "check", Kind::Clippy => "clippy", Kind::Fix => "fix", + Kind::Format => "fmt", Kind::Test => "test", Kind::Bench => "bench", - Kind::Dist => "dist", Kind::Doc => "doc", + Kind::Clean => "clean", + Kind::Dist => "dist", Kind::Install => "install", Kind::Run => "run", + Kind::Setup => "setup", } } } @@ -509,7 +522,7 @@ impl<'a> Builder<'a> { native::Lld, native::CrtBeginEnd ), - Kind::Check | Kind::Clippy { .. } | Kind::Fix => describe!( + Kind::Check => describe!( check::Std, check::Rustc, check::Rustdoc, @@ -639,32 +652,29 @@ impl<'a> Builder<'a> { install::Rustc ), Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest, run::BumpStage0), + // These commands either don't use paths, or they're special-cased in Build::build() + Kind::Clean | Kind::Clippy | Kind::Fix | Kind::Format | Kind::Setup => vec![], } } - pub fn get_help(build: &Build, subcommand: &str) -> Option<String> { - let kind = match subcommand { - "build" | "b" => Kind::Build, - "doc" | "d" => Kind::Doc, - "test" | "t" => Kind::Test, - "bench" => Kind::Bench, - "dist" => Kind::Dist, - "install" => Kind::Install, - _ => return None, - }; + pub fn get_help(build: &Build, kind: Kind) -> Option<String> { + let step_descriptions = Builder::get_step_descriptions(kind); + if step_descriptions.is_empty() { + return None; + } let builder = Self::new_internal(build, kind, vec![]); let builder = &builder; // The "build" kind here is just a placeholder, it will be replaced with something else in // the following statement. let mut should_run = ShouldRun::new(builder, Kind::Build); - for desc in Builder::get_step_descriptions(builder.kind) { + for desc in step_descriptions { should_run.kind = desc.kind; should_run = (desc.should_run)(should_run); } let mut help = String::from("Available paths:\n"); let mut add_path = |path: &Path| { - help.push_str(&format!(" ./x.py {} {}\n", subcommand, path.display())); + t!(write!(help, " ./x.py {} {}\n", kind.as_str(), path.display())); }; for pathset in should_run.paths { match pathset { @@ -950,6 +960,11 @@ impl<'a> Builder<'a> { None } + /// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`. + pub(crate) fn llvm_link_shared(&self) -> bool { + Config::llvm_link_shared(self) + } + /// Prepares an invocation of `cargo` to be run. /// /// This will create a `Command` that represents a pending execution of @@ -1388,17 +1403,17 @@ impl<'a> Builder<'a> { }, ); - // `dsymutil` adds time to builds on Apple platforms for no clear benefit, and also makes - // it more difficult for debuggers to find debug info. The compiler currently defaults to - // running `dsymutil` to preserve its historical default, but when compiling the compiler - // itself, we skip it by default since we know it's safe to do so in that case. - // See https://github.com/rust-lang/rust/issues/79361 for more info on this flag. - if target.contains("apple") { - if self.config.rust_run_dsymutil { - rustflags.arg("-Csplit-debuginfo=packed"); - } else { - rustflags.arg("-Csplit-debuginfo=unpacked"); + // FIXME(davidtwco): #[cfg(not(bootstrap))] - #95612 needs to be in the bootstrap compiler + // for this conditional to be removed. + if !target.contains("windows") || compiler.stage >= 1 { + if target.contains("linux") || target.contains("windows") { + rustflags.arg("-Zunstable-options"); } + match self.config.rust_split_debuginfo { + SplitDebuginfo::Packed => rustflags.arg("-Csplit-debuginfo=packed"), + SplitDebuginfo::Unpacked => rustflags.arg("-Csplit-debuginfo=unpacked"), + SplitDebuginfo::Off => rustflags.arg("-Csplit-debuginfo=off"), + }; } if self.config.cmd.bless() { @@ -1753,7 +1768,16 @@ impl<'a> Builder<'a> { }; if self.config.print_step_timings && !self.config.dry_run { - println!("[TIMING] {:?} -- {}.{:03}", step, dur.as_secs(), dur.subsec_millis()); + let step_string = format!("{:?}", step); + let brace_index = step_string.find("{").unwrap_or(0); + let type_string = type_name::<S>(); + println!( + "[TIMING] {} {} -- {}.{:03}", + &type_string.strip_prefix("bootstrap::").unwrap_or(type_string), + &step_string[brace_index..], + dur.as_secs(), + dur.subsec_millis() + ); } { diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index a59f72ed968..3b6cd7564f0 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -7,6 +7,7 @@ fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { // don't save toolstates config.save_toolstates = None; config.dry_run = true; + config.submodules = Some(false); config.ninja_in_file = false; // try to avoid spurious failures in dist where we create/delete each others file // HACK: rather than pull in `tempdir`, use the one that cargo has conveniently created for us @@ -25,36 +26,74 @@ fn first<A, B>(v: Vec<(A, B)>) -> Vec<A> { v.into_iter().map(|(a, _)| a).collect::<Vec<_>>() } +fn run_build(paths: &[PathBuf], config: Config) -> Cache { + let kind = config.cmd.kind(); + let build = Build::new(config); + let builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(kind), paths); + builder.cache +} + +#[test] +fn test_exclude() { + let mut config = configure("test", &["A"], &["A"]); + config.exclude = vec![TaskPath::parse("src/tools/tidy")]; + let cache = run_build(&[], config); + + // Ensure we have really excluded tidy + assert!(!cache.contains::<test::Tidy>()); + + // Ensure other tests are not affected. + assert!(cache.contains::<test::RustdocUi>()); +} + +#[test] +fn test_exclude_kind() { + let path = PathBuf::from("src/tools/cargotest"); + let exclude = TaskPath::parse("test::src/tools/cargotest"); + assert_eq!(exclude, TaskPath { kind: Some(Kind::Test), path: path.clone() }); + + let mut config = configure("test", &["A"], &["A"]); + // Ensure our test is valid, and `test::Cargotest` would be run without the exclude. + assert!(run_build(&[path.clone()], config.clone()).contains::<test::Cargotest>()); + // Ensure tests for cargotest are skipped. + config.exclude = vec![exclude.clone()]; + assert!(!run_build(&[path.clone()], config).contains::<test::Cargotest>()); + + // Ensure builds for cargotest are not skipped. + let mut config = configure("build", &["A"], &["A"]); + config.exclude = vec![exclude]; + assert!(run_build(&[path], config).contains::<tool::CargoTest>()); +} + mod defaults { - use super::{configure, first}; + use super::{configure, first, run_build}; use crate::builder::*; use crate::Config; use pretty_assertions::assert_eq; #[test] fn build_default() { - let build = Build::new(configure("build", &["A"], &["A"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + let mut cache = run_build(&[], configure("build", &["A"], &["A"])); let a = TargetSelection::from_user("A"); assert_eq!( - first(builder.cache.all::<compile::Std>()), + first(cache.all::<compile::Std>()), &[ compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, ] ); - assert!(!builder.cache.all::<compile::Assemble>().is_empty()); + assert!(!cache.all::<compile::Assemble>().is_empty()); // Make sure rustdoc is only built once. assert_eq!( - first(builder.cache.all::<tool::Rustdoc>()), + first(cache.all::<tool::Rustdoc>()), // Recall that rustdoc stages are off-by-one // - this is the compiler it's _linked_ to, not built with. &[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }], ); assert_eq!( - first(builder.cache.all::<compile::Rustc>()), + first(cache.all::<compile::Rustc>()), &[compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a },] ); } @@ -62,31 +101,27 @@ mod defaults { #[test] fn build_stage_0() { let config = Config { stage: 0, ..configure("build", &["A"], &["A"]) }; - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + let mut cache = run_build(&[], config); let a = TargetSelection::from_user("A"); assert_eq!( - first(builder.cache.all::<compile::Std>()), + first(cache.all::<compile::Std>()), &[compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },] ); - assert!(!builder.cache.all::<compile::Assemble>().is_empty()); + assert!(!cache.all::<compile::Assemble>().is_empty()); assert_eq!( - first(builder.cache.all::<tool::Rustdoc>()), + first(cache.all::<tool::Rustdoc>()), // This is the beta rustdoc. // Add an assert here to make sure this is the only rustdoc built. &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }], ); - assert!(builder.cache.all::<compile::Rustc>().is_empty()); + assert!(cache.all::<compile::Rustc>().is_empty()); } #[test] fn build_cross_compile() { let config = Config { stage: 1, ..configure("build", &["A", "B"], &["A", "B"]) }; - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + let mut cache = run_build(&[], config); let a = TargetSelection::from_user("A"); let b = TargetSelection::from_user("B"); @@ -97,7 +132,7 @@ mod defaults { // (since we're producing stage 1 libraries/binaries). But currently // rustbuild is just a bit buggy here; this should be fixed though. assert_eq!( - first(builder.cache.all::<compile::Std>()), + first(cache.all::<compile::Std>()), &[ compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, @@ -106,7 +141,7 @@ mod defaults { ] ); assert_eq!( - first(builder.cache.all::<compile::Assemble>()), + first(cache.all::<compile::Assemble>()), &[ compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, @@ -114,14 +149,14 @@ mod defaults { ] ); assert_eq!( - first(builder.cache.all::<tool::Rustdoc>()), + first(cache.all::<tool::Rustdoc>()), &[ tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } }, ], ); assert_eq!( - first(builder.cache.all::<compile::Rustc>()), + first(cache.all::<compile::Rustc>()), &[ compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: b }, @@ -134,33 +169,28 @@ mod defaults { let mut config = configure("doc", &["A"], &["A"]); config.compiler_docs = true; config.cmd = Subcommand::Doc { paths: Vec::new(), open: false }; - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); + let mut cache = run_build(&[], config); let a = TargetSelection::from_user("A"); // error_index_generator uses stage 0 to share rustdoc artifacts with the // rustdoc tool. + assert_eq!(first(cache.all::<doc::ErrorIndex>()), &[doc::ErrorIndex { target: a },]); assert_eq!( - first(builder.cache.all::<doc::ErrorIndex>()), - &[doc::ErrorIndex { target: a },] - ); - assert_eq!( - first(builder.cache.all::<tool::ErrorIndex>()), + first(cache.all::<tool::ErrorIndex>()), &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 0 } }] ); // docs should be built with the beta compiler, not with the stage0 artifacts. // recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to, // not the one it was built by. assert_eq!( - first(builder.cache.all::<tool::Rustdoc>()), + first(cache.all::<tool::Rustdoc>()), &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },] ); } } mod dist { - use super::{first, Config}; + use super::{first, run_build, Config}; use crate::builder::*; use pretty_assertions::assert_eq; @@ -170,94 +200,88 @@ mod dist { #[test] fn dist_baseline() { - let build = Build::new(configure(&["A"], &["A"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + let mut cache = run_build(&[], configure(&["A"], &["A"])); let a = TargetSelection::from_user("A"); - assert_eq!(first(builder.cache.all::<dist::Docs>()), &[dist::Docs { host: a },]); - assert_eq!(first(builder.cache.all::<dist::Mingw>()), &[dist::Mingw { host: a },]); + assert_eq!(first(cache.all::<dist::Docs>()), &[dist::Docs { host: a },]); + assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: a },]); assert_eq!( - first(builder.cache.all::<dist::Rustc>()), + first(cache.all::<dist::Rustc>()), &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] ); assert_eq!( - first(builder.cache.all::<dist::Std>()), + first(cache.all::<dist::Std>()), &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },] ); - assert_eq!(first(builder.cache.all::<dist::Src>()), &[dist::Src]); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); // Make sure rustdoc is only built once. assert_eq!( - first(builder.cache.all::<tool::Rustdoc>()), + first(cache.all::<tool::Rustdoc>()), &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] ); } #[test] fn dist_with_targets() { - let build = Build::new(configure(&["A"], &["A", "B"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + let mut cache = run_build(&[], configure(&["A"], &["A", "B"])); let a = TargetSelection::from_user("A"); let b = TargetSelection::from_user("B"); assert_eq!( - first(builder.cache.all::<dist::Docs>()), + first(cache.all::<dist::Docs>()), &[dist::Docs { host: a }, dist::Docs { host: b },] ); assert_eq!( - first(builder.cache.all::<dist::Mingw>()), + first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: a }, dist::Mingw { host: b },] ); assert_eq!( - first(builder.cache.all::<dist::Rustc>()), + first(cache.all::<dist::Rustc>()), &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] ); assert_eq!( - first(builder.cache.all::<dist::Std>()), + first(cache.all::<dist::Std>()), &[ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, ] ); - assert_eq!(first(builder.cache.all::<dist::Src>()), &[dist::Src]); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); } #[test] fn dist_with_hosts() { - let build = Build::new(configure(&["A", "B"], &["A", "B"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); let a = TargetSelection::from_user("A"); let b = TargetSelection::from_user("B"); assert_eq!( - first(builder.cache.all::<dist::Docs>()), + first(cache.all::<dist::Docs>()), &[dist::Docs { host: a }, dist::Docs { host: b },] ); assert_eq!( - first(builder.cache.all::<dist::Mingw>()), + first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: a }, dist::Mingw { host: b },] ); assert_eq!( - first(builder.cache.all::<dist::Rustc>()), + first(cache.all::<dist::Rustc>()), &[ dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, ] ); assert_eq!( - first(builder.cache.all::<dist::Std>()), + first(cache.all::<dist::Std>()), &[ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, ] ); assert_eq!( - first(builder.cache.all::<compile::Std>()), + first(cache.all::<compile::Std>()), &[ compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, @@ -266,26 +290,25 @@ mod dist { compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, ], ); - assert_eq!(first(builder.cache.all::<dist::Src>()), &[dist::Src]); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); } #[test] fn dist_only_cross_host() { let a = TargetSelection::from_user("A"); let b = TargetSelection::from_user("B"); - let mut build = Build::new(configure(&["A", "B"], &["A", "B"])); - build.config.docs = false; - build.config.extended = true; - build.hosts = vec![b]; - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + let mut config = configure(&["A", "B"], &["A", "B"]); + config.docs = false; + config.extended = true; + config.hosts = vec![b]; + let mut cache = run_build(&[], config); assert_eq!( - first(builder.cache.all::<dist::Rustc>()), + first(cache.all::<dist::Rustc>()), &[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },] ); assert_eq!( - first(builder.cache.all::<compile::Rustc>()), + first(cache.all::<compile::Rustc>()), &[ compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b }, @@ -295,92 +318,86 @@ mod dist { #[test] fn dist_with_targets_and_hosts() { - let build = Build::new(configure(&["A", "B"], &["A", "B", "C"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B", "C"])); let a = TargetSelection::from_user("A"); let b = TargetSelection::from_user("B"); let c = TargetSelection::from_user("C"); assert_eq!( - first(builder.cache.all::<dist::Docs>()), + first(cache.all::<dist::Docs>()), &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] ); assert_eq!( - first(builder.cache.all::<dist::Mingw>()), + first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] ); assert_eq!( - first(builder.cache.all::<dist::Rustc>()), + first(cache.all::<dist::Rustc>()), &[ dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, ] ); assert_eq!( - first(builder.cache.all::<dist::Std>()), + first(cache.all::<dist::Std>()), &[ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, ] ); - assert_eq!(first(builder.cache.all::<dist::Src>()), &[dist::Src]); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); } #[test] fn dist_with_empty_host() { let config = configure(&[], &["C"]); - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + let mut cache = run_build(&[], config); let a = TargetSelection::from_user("A"); let c = TargetSelection::from_user("C"); - assert_eq!(first(builder.cache.all::<dist::Docs>()), &[dist::Docs { host: c },]); - assert_eq!(first(builder.cache.all::<dist::Mingw>()), &[dist::Mingw { host: c },]); + assert_eq!(first(cache.all::<dist::Docs>()), &[dist::Docs { host: c },]); + assert_eq!(first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: c },]); assert_eq!( - first(builder.cache.all::<dist::Std>()), + first(cache.all::<dist::Std>()), &[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },] ); } #[test] fn dist_with_same_targets_and_hosts() { - let build = Build::new(configure(&["A", "B"], &["A", "B"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); let a = TargetSelection::from_user("A"); let b = TargetSelection::from_user("B"); assert_eq!( - first(builder.cache.all::<dist::Docs>()), + first(cache.all::<dist::Docs>()), &[dist::Docs { host: a }, dist::Docs { host: b },] ); assert_eq!( - first(builder.cache.all::<dist::Mingw>()), + first(cache.all::<dist::Mingw>()), &[dist::Mingw { host: a }, dist::Mingw { host: b },] ); assert_eq!( - first(builder.cache.all::<dist::Rustc>()), + first(cache.all::<dist::Rustc>()), &[ dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, ] ); assert_eq!( - first(builder.cache.all::<dist::Std>()), + first(cache.all::<dist::Std>()), &[ dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, ] ); - assert_eq!(first(builder.cache.all::<dist::Src>()), &[dist::Src]); + assert_eq!(first(cache.all::<dist::Src>()), &[dist::Src]); assert_eq!( - first(builder.cache.all::<compile::Std>()), + first(cache.all::<compile::Std>()), &[ compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, @@ -390,7 +407,7 @@ mod dist { ] ); assert_eq!( - first(builder.cache.all::<compile::Assemble>()), + first(cache.all::<compile::Assemble>()), &[ compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, @@ -515,35 +532,6 @@ mod dist { } #[test] - fn test_exclude() { - let mut config = configure(&["A"], &["A"]); - config.exclude = vec![TaskPath::parse("src/tools/tidy")]; - config.cmd = Subcommand::Test { - paths: Vec::new(), - test_args: Vec::new(), - rustc_args: Vec::new(), - fail_fast: true, - doc_tests: DocTests::No, - bless: false, - force_rerun: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - run: None, - }; - - let build = Build::new(config); - let builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); - - // Ensure we have really excluded tidy - assert!(!builder.cache.contains::<test::Tidy>()); - - // Ensure other tests are not affected. - assert!(builder.cache.contains::<test::RustdocUi>()); - } - - #[test] fn doc_ci() { let mut config = configure(&["A"], &["A"]); config.compiler_docs = true; diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 432a6c34ed5..731ebc41bb9 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -64,7 +64,7 @@ impl Step for Std { const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.all_krates("test") + run.all_krates("test").path("library") } fn make_run(run: RunConfig<'_>) { @@ -162,7 +162,7 @@ impl Step for Rustc { const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.all_krates("rustc-main") + run.all_krates("rustc-main").path("compiler") } fn make_run(run: RunConfig<'_>) { diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 45991381dc0..53933e4cd7d 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -43,7 +43,7 @@ impl Step for Std { // When downloading stage1, the standard library has already been copied to the sysroot, so // there's no need to rebuild it. let download_rustc = run.builder.config.download_rustc; - run.all_krates("test").default_condition(!download_rustc) + run.all_krates("test").path("library").default_condition(!download_rustc) } fn make_run(run: RunConfig<'_>) { @@ -737,7 +737,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS ); cargo.env("LLVM_STATIC_STDCPP", file); } - if builder.config.llvm_link_shared { + if builder.llvm_link_shared() { cargo.env("LLVM_LINK_SHARED", "1"); } if builder.config.llvm_use_libcxx { @@ -1047,7 +1047,7 @@ impl Step for Assemble { const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("compiler/rustc") + run.path("compiler/rustc").path("compiler") } fn make_run(run: RunConfig<'_>) { diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index d7c29f6900a..e39c9fa1c5a 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -3,6 +3,7 @@ //! This module implements parsing `config.toml` configuration files to tweak //! how the build runs. +use std::cell::Cell; use std::cmp; use std::collections::{HashMap, HashSet}; use std::env; @@ -11,7 +12,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::str::FromStr; -use crate::builder::TaskPath; +use crate::builder::{Builder, TaskPath}; use crate::cache::{Interned, INTERNER}; use crate::channel::GitInfo; pub use crate::flags::Subcommand; @@ -41,6 +42,7 @@ macro_rules! check_ci_llvm { /// each field, see the corresponding fields in /// `config.toml.example`. #[derive(Default)] +#[cfg_attr(test, derive(Clone))] pub struct Config { pub changelog_seen: Option<usize>, pub ccache: Option<String>, @@ -68,13 +70,14 @@ pub struct Config { pub test_compare_mode: bool, pub llvm_libunwind: LlvmLibunwind, pub color: Color, + pub patch_binaries_for_nix: bool, pub on_fail: Option<String>, pub stage: u32, pub keep_stage: Vec<u32>, pub keep_stage_std: Vec<u32>, pub src: PathBuf, - // defaults to `config.toml` + /// defaults to `config.toml` pub config: PathBuf, pub jobs: Option<u32>, pub cmd: Subcommand, @@ -95,7 +98,11 @@ pub struct Config { pub llvm_release_debuginfo: bool, pub llvm_version_check: bool, pub llvm_static_stdcpp: bool, - pub llvm_link_shared: bool, + /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. + #[cfg(not(test))] + llvm_link_shared: Cell<Option<bool>>, + #[cfg(test)] + pub llvm_link_shared: Cell<Option<bool>>, pub llvm_clang_cl: Option<String>, pub llvm_targets: Option<String>, pub llvm_experimental_targets: Option<String>, @@ -130,7 +137,7 @@ pub struct Config { pub rust_debuginfo_level_std: u32, pub rust_debuginfo_level_tools: u32, pub rust_debuginfo_level_tests: u32, - pub rust_run_dsymutil: bool, + pub rust_split_debuginfo: SplitDebuginfo, pub rust_rpath: bool, pub rustc_parallel: bool, pub rustc_default_linker: Option<String>, @@ -221,6 +228,46 @@ impl FromStr for LlvmLibunwind { } } +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SplitDebuginfo { + Packed, + Unpacked, + Off, +} + +impl Default for SplitDebuginfo { + fn default() -> Self { + SplitDebuginfo::Off + } +} + +impl std::str::FromStr for SplitDebuginfo { + type Err = (); + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s { + "packed" => Ok(SplitDebuginfo::Packed), + "unpacked" => Ok(SplitDebuginfo::Unpacked), + "off" => Ok(SplitDebuginfo::Off), + _ => Err(()), + } + } +} + +impl SplitDebuginfo { + /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for + /// `rust.split-debuginfo` in `config.toml.example`. + fn default_for_platform(target: &str) -> Self { + if target.contains("apple") { + SplitDebuginfo::Unpacked + } else if target.contains("windows") { + SplitDebuginfo::Packed + } else { + SplitDebuginfo::Off + } + } +} + #[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct TargetSelection { pub triple: Interned<String>, @@ -290,6 +337,7 @@ impl PartialEq<&str> for TargetSelection { /// Per-target configuration stored in the global configuration structure. #[derive(Default)] +#[cfg_attr(test, derive(Clone))] pub struct Target { /// Some(path to llvm-config) if using an external LLVM. pub llvm_config: Option<PathBuf>, @@ -586,6 +634,7 @@ define_config! { debuginfo_level_std: Option<u32> = "debuginfo-level-std", debuginfo_level_tools: Option<u32> = "debuginfo-level-tools", debuginfo_level_tests: Option<u32> = "debuginfo-level-tests", + split_debuginfo: Option<String> = "split-debuginfo", run_dsymutil: Option<bool> = "run-dsymutil", backtrace: Option<bool> = "backtrace", incremental: Option<bool> = "incremental", @@ -815,6 +864,7 @@ impl Config { set(&mut config.local_rebuild, build.local_rebuild); set(&mut config.print_step_timings, build.print_step_timings); set(&mut config.print_step_rusage, build.print_step_rusage); + set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix); config.verbose = cmp::max(config.verbose, flags.verbose); @@ -870,7 +920,9 @@ impl Config { set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo); set(&mut config.llvm_version_check, llvm.version_check); set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp); - set(&mut config.llvm_link_shared, llvm.link_shared); + if let Some(v) = llvm.link_shared { + config.llvm_link_shared.set(Some(v)); + } config.llvm_targets = llvm.targets.clone(); config.llvm_experimental_targets = llvm.experimental_targets.clone(); config.llvm_link_jobs = llvm.link_jobs; @@ -940,6 +992,7 @@ impl Config { check_ci_llvm!(llvm.optimize); check_ci_llvm!(llvm.thin_lto); check_ci_llvm!(llvm.release_debuginfo); + // CI-built LLVM can be either dynamic or static. We won't know until we download it. check_ci_llvm!(llvm.link_shared); check_ci_llvm!(llvm.static_libstdcpp); check_ci_llvm!(llvm.targets); @@ -957,26 +1010,14 @@ impl Config { check_ci_llvm!(llvm.clang); check_ci_llvm!(llvm.build_config); check_ci_llvm!(llvm.plugins); - - // CI-built LLVM can be either dynamic or static. - let ci_llvm = config.out.join(&*config.build.triple).join("ci-llvm"); - config.llvm_link_shared = if config.dry_run { - // just assume dynamic for now - true - } else { - let link_type = t!( - std::fs::read_to_string(ci_llvm.join("link-type.txt")), - format!("CI llvm missing: {}", ci_llvm.display()) - ); - link_type == "dynamic" - }; } + // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above. if config.llvm_thin_lto && llvm.link_shared.is_none() { // If we're building with ThinLTO on, by default we want to link // to LLVM shared, to avoid re-doing ThinLTO (which happens in // the link step) with each stage. - config.llvm_link_shared = true; + config.llvm_link_shared.set(Some(true)); } } @@ -992,7 +1033,12 @@ impl Config { debuginfo_level_std = rust.debuginfo_level_std; debuginfo_level_tools = rust.debuginfo_level_tools; debuginfo_level_tests = rust.debuginfo_level_tests; - config.rust_run_dsymutil = rust.run_dsymutil.unwrap_or(false); + config.rust_split_debuginfo = rust + .split_debuginfo + .as_deref() + .map(SplitDebuginfo::from_str) + .map(|v| v.expect("invalid value for rust.split_debuginfo")) + .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple)); optimize = rust.optimize; ignore_git = rust.ignore_git; config.rust_new_symbol_mangling = rust.new_symbol_mangling; @@ -1226,6 +1272,42 @@ impl Config { } } + /// The absolute path to the downloaded LLVM artifacts. + pub(crate) fn ci_llvm_root(&self) -> PathBuf { + assert!(self.llvm_from_ci); + self.out.join(&*self.build.triple).join("ci-llvm") + } + + /// Determine whether llvm should be linked dynamically. + /// + /// If `false`, llvm should be linked statically. + /// This is computed on demand since LLVM might have to first be downloaded from CI. + pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool { + let mut opt = builder.config.llvm_link_shared.get(); + if opt.is_none() && builder.config.dry_run { + // just assume static for now - dynamic linking isn't supported on all platforms + return false; + } + + let llvm_link_shared = *opt.get_or_insert_with(|| { + if builder.config.llvm_from_ci { + crate::native::maybe_download_ci_llvm(builder); + let ci_llvm = builder.config.ci_llvm_root(); + let link_type = t!( + std::fs::read_to_string(ci_llvm.join("link-type.txt")), + format!("CI llvm missing: {}", ci_llvm.display()) + ); + link_type == "dynamic" + } else { + // unclear how thought-through this default is, but it maintains compatibility with + // previous behavior + false + } + }); + builder.config.llvm_link_shared.set(opt); + llvm_link_shared + } + pub fn verbose(&self) -> bool { self.verbose > 0 } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e3287e35227..5d812e8b332 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1904,7 +1904,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir // clear why this is the case, though. llvm-config will emit the versioned // paths and we don't want those in the sysroot (as we're expecting // unversioned paths). - if target.contains("apple-darwin") && builder.config.llvm_link_shared { + if target.contains("apple-darwin") && builder.llvm_link_shared() { let src_libdir = builder.llvm_out(target).join("lib"); let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); if llvm_dylib_path.exists() { @@ -1939,7 +1939,7 @@ pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, // We do not need to copy LLVM files into the sysroot if it is not // dynamically linked; it is already included into librustc_llvm // statically. - if builder.config.llvm_link_shared { + if builder.llvm_link_shared() { maybe_install_llvm(builder, target, &dst_libdir); } } @@ -1951,7 +1951,7 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection // We do not need to copy LLVM files into the sysroot if it is not // dynamically linked; it is already included into librustc_llvm // statically. - if builder.config.llvm_link_shared { + if builder.llvm_link_shared() { maybe_install_llvm(builder, target, &dst_libdir); } } @@ -2077,7 +2077,7 @@ impl Step for RustDev { // compiler libraries. let dst_libdir = tarball.image_dir().join("lib"); maybe_install_llvm(builder, target, &dst_libdir); - let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" }; + let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" }; t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir); Some(tarball.generate()) diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index a2802f76008..fcef784d2d1 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -416,7 +416,7 @@ impl Step for Std { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { let builder = run.builder; - run.all_krates("test").default_condition(builder.config.docs) + run.all_krates("test").path("library").default_condition(builder.config.docs) } fn make_run(run: RunConfig<'_>) { @@ -477,11 +477,14 @@ impl Step for Std { .iter() .map(components_simplified) .filter_map(|path| { - if path.get(0) == Some(&"library") { + if path.len() >= 2 && path.get(0) == Some(&"library") { + // single crate Some(path[1].to_owned()) } else if !path.is_empty() { + // ?? Some(path[0].to_owned()) } else { + // all library crates None } }) diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 1a4e6a96888..58571ea129c 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -8,12 +8,13 @@ use std::process; use getopts::Options; -use crate::builder::Builder; +use crate::builder::{Builder, Kind}; use crate::config::{Config, TargetSelection}; use crate::setup::Profile; use crate::util::t; use crate::{Build, DocTests}; +#[derive(Copy, Clone)] pub enum Color { Always, Never, @@ -79,6 +80,7 @@ pub struct Flags { pub llvm_profile_generate: bool, } +#[cfg_attr(test, derive(Clone))] pub enum Subcommand { Build { paths: Vec<PathBuf>, @@ -243,27 +245,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", // the subcommand. Therefore we must manually identify the subcommand first, so that we can // complete the definition of the options. Then we can use the getopt::Matches object from // there on out. - let subcommand = args.iter().find(|&s| { - (s == "build") - || (s == "b") - || (s == "check") - || (s == "c") - || (s == "clippy") - || (s == "fix") - || (s == "fmt") - || (s == "test") - || (s == "t") - || (s == "bench") - || (s == "doc") - || (s == "d") - || (s == "clean") - || (s == "dist") - || (s == "install") - || (s == "run") - || (s == "r") - || (s == "setup") - }); - let subcommand = match subcommand { + let subcommand = match args.iter().find_map(|s| Kind::parse(&s)) { Some(s) => s, None => { // No or an invalid subcommand -- show the general usage and subcommand help @@ -276,8 +258,8 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", }; // Some subcommands get extra options - match subcommand.as_str() { - "test" | "t" => { + match subcommand { + Kind::Test => { opts.optflag("", "no-fail-fast", "Run all tests regardless of failure"); opts.optmulti( "", @@ -316,22 +298,22 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", `/<build_base>/rustfix_missing_coverage.txt`", ); } - "check" | "c" => { + Kind::Check => { opts.optflag("", "all-targets", "Check all targets"); } - "bench" => { + Kind::Bench => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); } - "clippy" => { + Kind::Clippy => { opts.optflag("", "fix", "automatically apply lint suggestions"); } - "doc" | "d" => { + Kind::Doc => { opts.optflag("", "open", "open the docs in a browser"); } - "clean" => { + Kind::Clean => { opts.optflag("", "all", "clean all build artifacts"); } - "fmt" => { + Kind::Format => { opts.optflag("", "check", "check formatting instead of applying."); } _ => {} @@ -339,25 +321,22 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", // fn usage() let usage = |exit_code: i32, opts: &Options, verbose: bool, subcommand_help: &str| -> ! { - let mut extra_help = String::new(); - - // All subcommands except `clean` can have an optional "Available paths" section - if verbose { - let config = Config::parse(&["build".to_string()]); - let build = Build::new(config); - - let maybe_rules_help = Builder::get_help(&build, subcommand.as_str()); - extra_help.push_str(maybe_rules_help.unwrap_or_default().as_str()); - } else if !(subcommand.as_str() == "clean" || subcommand.as_str() == "fmt") { - extra_help.push_str( - format!("Run `./x.py {} -h -v` to see a list of available paths.", subcommand) - .as_str(), - ); - } + let config = Config::parse(&["build".to_string()]); + let build = Build::new(config); + let paths = Builder::get_help(&build, subcommand); println!("{}", opts.usage(subcommand_help)); - if !extra_help.is_empty() { - println!("{}", extra_help); + if let Some(s) = paths { + if verbose { + println!("{}", s); + } else { + println!( + "Run `./x.py {} -h -v` to see a list of available paths.", + subcommand.as_str() + ); + } + } else if verbose { + panic!("No paths available for subcommand `{}`", subcommand.as_str()); } process::exit(exit_code); }; @@ -375,7 +354,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", // ^-- option ^ ^- actual subcommand // \_ arg to option could be mistaken as subcommand let mut pass_sanity_check = true; - match matches.free.get(0) { + match matches.free.get(0).and_then(|s| Kind::parse(&s)) { Some(check_subcommand) => { if check_subcommand != subcommand { pass_sanity_check = false; @@ -394,8 +373,8 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", process::exit(1); } // Extra help text for some commands - match subcommand.as_str() { - "build" | "b" => { + match subcommand { + Kind::Build => { subcommand_help.push_str( "\n Arguments: @@ -415,7 +394,7 @@ Arguments: ./x.py build ", ); } - "check" | "c" => { + Kind::Check => { subcommand_help.push_str( "\n Arguments: @@ -427,7 +406,7 @@ Arguments: If no arguments are passed then many artifacts are checked.", ); } - "clippy" => { + Kind::Clippy => { subcommand_help.push_str( "\n Arguments: @@ -438,7 +417,7 @@ Arguments: ./x.py clippy library/core library/proc_macro", ); } - "fix" => { + Kind::Fix => { subcommand_help.push_str( "\n Arguments: @@ -449,7 +428,7 @@ Arguments: ./x.py fix library/core library/proc_macro", ); } - "fmt" => { + Kind::Format => { subcommand_help.push_str( "\n Arguments: @@ -460,7 +439,7 @@ Arguments: ./x.py fmt --check", ); } - "test" | "t" => { + Kind::Test => { subcommand_help.push_str( "\n Arguments: @@ -488,7 +467,7 @@ Arguments: ./x.py test --stage 1", ); } - "doc" | "d" => { + Kind::Doc => { subcommand_help.push_str( "\n Arguments: @@ -506,7 +485,7 @@ Arguments: ./x.py doc --stage 1", ); } - "run" | "r" => { + Kind::Run => { subcommand_help.push_str( "\n Arguments: @@ -518,7 +497,7 @@ Arguments: At least a tool needs to be called.", ); } - "setup" => { + Kind::Setup => { subcommand_help.push_str(&format!( "\n x.py setup creates a `config.toml` which changes the defaults for x.py itself. @@ -535,7 +514,7 @@ Arguments: Profile::all_for_help(" ").trim_end() )); } - _ => {} + Kind::Bench | Kind::Clean | Kind::Dist | Kind::Install => {} }; // Get any optional paths which occur after the subcommand let mut paths = matches.free[1..].iter().map(|p| p.into()).collect::<Vec<PathBuf>>(); @@ -547,9 +526,9 @@ Arguments: usage(0, &opts, verbose, &subcommand_help); } - let cmd = match subcommand.as_str() { - "build" | "b" => Subcommand::Build { paths }, - "check" | "c" => { + let cmd = match subcommand { + Kind::Build => Subcommand::Build { paths }, + Kind::Check => { if matches.opt_present("all-targets") { eprintln!( "Warning: --all-targets is now on by default and does not need to be passed explicitly." @@ -557,9 +536,9 @@ Arguments: } Subcommand::Check { paths } } - "clippy" => Subcommand::Clippy { paths, fix: matches.opt_present("fix") }, - "fix" => Subcommand::Fix { paths }, - "test" | "t" => Subcommand::Test { + Kind::Clippy => Subcommand::Clippy { paths, fix: matches.opt_present("fix") }, + Kind::Fix => Subcommand::Fix { paths }, + Kind::Test => Subcommand::Test { paths, bless: matches.opt_present("bless"), force_rerun: matches.opt_present("force-rerun"), @@ -578,9 +557,9 @@ Arguments: DocTests::Yes }, }, - "bench" => Subcommand::Bench { paths, test_args: matches.opt_strs("test-args") }, - "doc" | "d" => Subcommand::Doc { paths, open: matches.opt_present("open") }, - "clean" => { + Kind::Bench => Subcommand::Bench { paths, test_args: matches.opt_strs("test-args") }, + Kind::Doc => Subcommand::Doc { paths, open: matches.opt_present("open") }, + Kind::Clean => { if !paths.is_empty() { println!("\nclean does not take a path argument\n"); usage(1, &opts, verbose, &subcommand_help); @@ -588,17 +567,17 @@ Arguments: Subcommand::Clean { all: matches.opt_present("all") } } - "fmt" => Subcommand::Format { check: matches.opt_present("check"), paths }, - "dist" => Subcommand::Dist { paths }, - "install" => Subcommand::Install { paths }, - "run" | "r" => { + Kind::Format => Subcommand::Format { check: matches.opt_present("check"), paths }, + Kind::Dist => Subcommand::Dist { paths }, + Kind::Install => Subcommand::Install { paths }, + Kind::Run => { if paths.is_empty() { println!("\nrun requires at least a path!\n"); usage(1, &opts, verbose, &subcommand_help); } Subcommand::Run { paths } } - "setup" => { + Kind::Setup => { let profile = if paths.len() > 1 { println!("\nat most one profile can be passed to setup\n"); usage(1, &opts, verbose, &subcommand_help) @@ -618,9 +597,6 @@ Arguments: }; Subcommand::Setup { profile } } - _ => { - usage(1, &opts, verbose, &subcommand_help); - } }; if let Subcommand::Check { .. } = &cmd { @@ -694,6 +670,24 @@ Arguments: } impl Subcommand { + pub fn kind(&self) -> Kind { + match self { + Subcommand::Bench { .. } => Kind::Bench, + Subcommand::Build { .. } => Kind::Build, + Subcommand::Check { .. } => Kind::Check, + Subcommand::Clippy { .. } => Kind::Clippy, + Subcommand::Doc { .. } => Kind::Doc, + Subcommand::Fix { .. } => Kind::Fix, + Subcommand::Format { .. } => Kind::Format, + Subcommand::Test { .. } => Kind::Test, + Subcommand::Clean { .. } => Kind::Clean, + Subcommand::Dist { .. } => Kind::Dist, + Subcommand::Install { .. } => Kind::Install, + Subcommand::Run { .. } => Kind::Run, + Subcommand::Setup { .. } => Kind::Setup, + } + } + pub fn test_args(&self) -> Vec<&str> { match *self { Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 59102ad9f50..b4b973b4247 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -1391,6 +1391,16 @@ impl Build { paths } + /// Create a temporary directory in `out` and return its path. + /// + /// NOTE: this temporary directory is shared between all steps; + /// if you need an empty directory, create a new subdirectory inside it. + fn tempdir(&self) -> PathBuf { + let tmp = self.out.join("tmp"); + t!(fs::create_dir_all(&tmp)); + tmp + } + /// Copies a file from `src` to `dst` pub fn copy(&self, src: &Path, dst: &Path) { if self.config.dry_run { diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 73fb2dad1e3..64e25f803b2 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -12,9 +12,12 @@ use std::env; use std::env::consts::EXE_EXTENSION; use std::ffi::{OsStr, OsString}; use std::fs::{self, File}; -use std::io; +use std::io::{self, BufRead, BufReader, ErrorKind}; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; + +use once_cell::sync::OnceCell; +use xz2::bufread::XzDecoder; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::config::TargetSelection; @@ -62,6 +65,8 @@ pub fn prebuilt_llvm_config( builder: &Builder<'_>, target: TargetSelection, ) -> Result<PathBuf, Meta> { + maybe_download_ci_llvm(builder); + // If we're using a custom LLVM bail out here, but we can only use a // custom LLVM for the build triple. if let Some(config) = builder.config.target_config.get(&target) { @@ -111,6 +116,285 @@ pub fn prebuilt_llvm_config( Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) } +pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { + let config = &builder.config; + if !config.llvm_from_ci { + return; + } + let mut rev_list = Command::new("git"); + rev_list.args(&[ + PathBuf::from("rev-list"), + "--author=bors@rust-lang.org".into(), + "-n1".into(), + "--first-parent".into(), + "HEAD".into(), + "--".into(), + builder.src.join("src/llvm-project"), + builder.src.join("src/bootstrap/download-ci-llvm-stamp"), + // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` + builder.src.join("src/version"), + ]); + let llvm_sha = output(&mut rev_list); + let llvm_sha = llvm_sha.trim(); + + if llvm_sha == "" { + println!("error: could not find commit hash for downloading LLVM"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `download-ci-llvm`"); + println!("help: or fetch enough history to include one upstream commit"); + panic!(); + } + + let llvm_root = config.ci_llvm_root(); + let llvm_stamp = llvm_root.join(".llvm-stamp"); + let key = format!("{}{}", llvm_sha, config.llvm_assertions); + if program_out_of_date(&llvm_stamp, &key) && !config.dry_run { + download_ci_llvm(builder, &llvm_sha); + for binary in ["llvm-config", "FileCheck"] { + fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary)); + } + let llvm_lib = llvm_root.join("lib"); + for entry in t!(fs::read_dir(&llvm_lib)) { + let lib = t!(entry).path(); + if lib.ends_with(".so") { + fix_bin_or_dylib(builder, &lib); + } + } + t!(fs::write(llvm_stamp, key)); + } +} + +fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { + let llvm_assertions = builder.config.llvm_assertions; + + let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); + let cache_dst = builder.out.join("cache"); + let rustc_cache = cache_dst.join(cache_prefix); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + let base = "https://ci-artifacts.rust-lang.org"; + let url = if llvm_assertions { + format!("rustc-builds-alt/{}", llvm_sha) + } else { + format!("rustc-builds/{}", llvm_sha) + }; + let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + download_component(builder, base, &format!("{}/{}", url, filename), &tarball); + } + let llvm_root = builder.config.ci_llvm_root(); + unpack(builder, &tarball, &llvm_root); +} + +/// Modifies the interpreter section of 'fname' to fix the dynamic linker, +/// or the RPATH section, to fix the dynamic library search path +/// +/// This is only required on NixOS and uses the PatchELF utility to +/// change the interpreter/RPATH of ELF executables. +/// +/// Please see https://nixos.org/patchelf.html for more information +fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) { + // FIXME: cache NixOS detection? + match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { + Err(_) => return, + Ok(output) if !output.status.success() => return, + Ok(output) => { + let mut s = output.stdout; + if s.last() == Some(&b'\n') { + s.pop(); + } + if s != b"Linux" { + return; + } + } + } + + // If the user has asked binaries to be patched for Nix, then + // don't check for NixOS or `/lib`, just continue to the patching. + // FIXME: shouldn't this take precedence over the `uname` check above? + if !builder.config.patch_binaries_for_nix { + // Use `/etc/os-release` instead of `/etc/NIXOS`. + // The latter one does not exist on NixOS when using tmpfs as root. + const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""]; + let os_release = match File::open("/etc/os-release") { + Err(e) if e.kind() == ErrorKind::NotFound => return, + Err(e) => panic!("failed to access /etc/os-release: {}", e), + Ok(f) => f, + }; + if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) { + return; + } + if Path::new("/lib").exists() { + return; + } + } + + // At this point we're pretty sure the user is running NixOS or using Nix + println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); + + // Only build `.nix-deps` once. + static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new(); + let mut nix_build_succeeded = true; + let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { + // Run `nix-build` to "build" each dependency (which will likely reuse + // the existing `/nix/store` copy, or at most download a pre-built copy). + // + // Importantly, we create a gc-root called `.nix-deps` in the `build/` + // directory, but still reference the actual `/nix/store` path in the rpath + // as it makes it significantly more robust against changes to the location of + // the `.nix-deps` location. + // + // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). + // zlib: Needed as a system dependency of `libLLVM-*.so`. + // patchelf: Needed for patching ELF binaries (see doc comment above). + let nix_deps_dir = builder.out.join(".nix-deps"); + const NIX_EXPR: &str = " + with (import <nixpkgs> {}); + symlinkJoin { + name = \"rust-stage0-dependencies\"; + paths = [ + zlib + patchelf + stdenv.cc.bintools + ]; + } + "; + nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[ + Path::new("-E"), + Path::new(NIX_EXPR), + Path::new("-o"), + &nix_deps_dir, + ])); + nix_deps_dir + }); + if !nix_build_succeeded { + return; + } + + let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); + let rpath_entries = { + // ORIGIN is a relative default, all binary and dynamic libraries we ship + // appear to have this (even when `../lib` is redundant). + // NOTE: there are only two paths here, delimited by a `:` + let mut entries = OsString::from("$ORIGIN/../lib:"); + entries.push(t!(fs::canonicalize(nix_deps_dir))); + entries.push("/lib"); + entries + }; + patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); + if !fname.ends_with(".so") { + // Finally, set the corret .interp for binaries + let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); + // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ... + let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); + patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); + } + + builder.try_run(patchelf.arg(fname)); +} + +fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) { + // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. + let tempfile = builder.tempdir().join(dest_path.file_name().unwrap()); + // FIXME: support `do_verify` (only really needed for nightly rustfmt) + // FIXME: support non-utf8 paths? + download_with_retries(builder, tempfile.to_str().unwrap(), &format!("{}/{}", base, url)); + t!(std::fs::rename(&tempfile, dest_path)); +} + +fn download_with_retries(builder: &Builder<'_>, tempfile: &str, url: &str) { + println!("downloading {}", url); + + // FIXME: check if curl is installed instead of skipping straight to powershell + if builder.build.build.contains("windows-msvc") { + for _ in 0..3 { + if builder.try_run(Command::new("PowerShell.exe").args(&[ + "/nologo", + "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempfile + ), + ])) { + return; + } + println!("\nspurious failure, trying again"); + } + } else { + builder.run(Command::new("curl").args(&[ + "-#", + "-y", + "30", + "-Y", + "10", // timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", + "30", // timeout if cannot connect within 30 seconds + "--retry", + "3", + "-Sf", + "-o", + tempfile, + url, + ])); + } +} + +fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) { + println!("extracting {} to {}", tarball.display(), dst.display()); + if !dst.exists() { + t!(fs::create_dir_all(dst)); + } + + // FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild + const MATCH: &str = "rust-dev"; + + // `tarball` ends with `.tar.xz`; strip that suffix + // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` + let uncompressed_filename = + Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); + let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); + + // decompress the file + let data = t!(File::open(tarball)); + let decompressor = XzDecoder::new(BufReader::new(data)); + + let mut tar = tar::Archive::new(decompressor); + for member in t!(tar.entries()) { + let mut member = t!(member); + let original_path = t!(member.path()).into_owned(); + // skip the top-level directory + if original_path == directory_prefix { + continue; + } + let mut short_path = t!(original_path.strip_prefix(directory_prefix)); + if !short_path.starts_with(MATCH) { + continue; + } + short_path = t!(short_path.strip_prefix(MATCH)); + let dst_path = dst.join(short_path); + builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); + if !t!(member.unpack_in(dst)) { + panic!("path traversal attack ??"); + } + let src_path = dst.join(original_path); + if src_path.is_dir() && dst_path.exists() { + continue; + } + t!(fs::rename(src_path, dst_path)); + } + t!(fs::remove_dir_all(dst.join(directory_prefix))); +} + +fn program_out_of_date(stamp: &Path, key: &str) -> bool { + if !stamp.exists() { + return true; + } + t!(fs::read_to_string(stamp)) != key +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Llvm { pub target: TargetSelection, @@ -153,7 +437,7 @@ impl Step for Llvm { }; builder.update_submodule(&Path::new("src").join("llvm-project")); - if builder.config.llvm_link_shared + if builder.llvm_link_shared() && (target.contains("windows") || target.contains("apple-darwin")) { panic!("shared linking to LLVM is not currently supported on {}", target.triple); @@ -255,7 +539,7 @@ impl Step for Llvm { // // If we're not linking rustc to a dynamic LLVM, though, then don't link // tools to it. - if builder.llvm_link_tools_dynamically(target) && builder.config.llvm_link_shared { + if builder.llvm_link_tools_dynamically(target) && builder.llvm_link_shared() { cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index c743c5188e7..689b4819cdd 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -262,11 +262,13 @@ impl<'a> Tarball<'a> { t!(std::fs::rename(&self.image_dir, &dest)); self.run(|this, cmd| { + let distdir = crate::dist::distdir(this.builder); + t!(std::fs::create_dir_all(&distdir)); cmd.arg("tarball") .arg("--input") .arg(&dest) .arg("--output") - .arg(crate::dist::distdir(this.builder).join(this.package_name())); + .arg(distdir.join(this.package_name())); }) } diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index da468909811..1193546992c 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1400,9 +1400,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the targetflags.extend(builder.lld_flags(target)); cmd.arg("--target-rustcflags").arg(targetflags.join(" ")); - cmd.arg("--docck-python").arg(builder.python()); - - cmd.arg("--lldb-python").arg(builder.python()); + cmd.arg("--python").arg(builder.python()); if let Some(ref gdb) = builder.config.gdb { cmd.arg("--gdb").arg(gdb); @@ -1577,9 +1575,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the cmd.env("RUSTC_PROFILER_SUPPORT", "1"); } - let tmp = builder.out.join("tmp"); - std::fs::create_dir_all(&tmp).unwrap(); - cmd.env("RUST_TEST_TMPDIR", tmp); + cmd.env("RUST_TEST_TMPDIR", builder.tempdir()); cmd.arg("--adb-path").arg("adb"); cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); @@ -2259,14 +2255,13 @@ impl Step for RemoteCopyLibs { builder.ensure(compile::Std { compiler, target }); builder.info(&format!("REMOTE copy libs to emulator ({})", target)); - t!(fs::create_dir_all(builder.out.join("tmp"))); let server = builder.ensure(tool::RemoteTestServer { compiler, target }); // Spawn the emulator and wait for it to come online let tool = builder.tool_exe(Tool::RemoteTestClient); let mut cmd = Command::new(&tool); - cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.out.join("tmp")); + cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); if let Some(rootfs) = builder.qemu_rootfs(target) { cmd.arg(rootfs); } @@ -2300,7 +2295,7 @@ impl Step for Distcheck { /// Runs "distcheck", a 'make check' from a tarball fn run(self, builder: &Builder<'_>) { builder.info("Distcheck"); - let dir = builder.out.join("tmp").join("distcheck"); + let dir = builder.tempdir().join("distcheck"); let _ = fs::remove_dir_all(&dir); t!(fs::create_dir_all(&dir)); @@ -2326,7 +2321,7 @@ impl Step for Distcheck { // Now make sure that rust-src has all of libstd's dependencies builder.info("Distcheck rust-src"); - let dir = builder.out.join("tmp").join("distcheck-src"); + let dir = builder.tempdir().join("distcheck-src"); let _ = fs::remove_dir_all(&dir); t!(fs::create_dir_all(&dir)); |