diff options
Diffstat (limited to 'src/bootstrap')
| -rw-r--r-- | src/bootstrap/CHANGELOG.md | 7 | ||||
| -rw-r--r-- | src/bootstrap/README.md | 1 | ||||
| -rw-r--r-- | src/bootstrap/bootstrap.py | 31 | ||||
| -rw-r--r-- | src/bootstrap/builder.rs | 17 | ||||
| -rw-r--r-- | src/bootstrap/compile.rs | 73 | ||||
| -rw-r--r-- | src/bootstrap/config.rs | 17 | ||||
| -rwxr-xr-x | src/bootstrap/configure.py | 4 | ||||
| -rw-r--r-- | src/bootstrap/dist.rs | 1073 | ||||
| -rw-r--r-- | src/bootstrap/doc.rs | 17 | ||||
| -rw-r--r-- | src/bootstrap/download-ci-llvm-stamp | 2 | ||||
| -rw-r--r-- | src/bootstrap/flags.rs | 7 | ||||
| -rw-r--r-- | src/bootstrap/install.rs | 120 | ||||
| -rw-r--r-- | src/bootstrap/lib.rs | 26 | ||||
| -rw-r--r-- | src/bootstrap/mk/Makefile.in | 4 | ||||
| -rw-r--r-- | src/bootstrap/sanity.rs | 44 | ||||
| -rw-r--r-- | src/bootstrap/tarball.rs | 333 | ||||
| -rw-r--r-- | src/bootstrap/test.rs | 21 | ||||
| -rw-r--r-- | src/bootstrap/tool.rs | 37 |
18 files changed, 815 insertions, 1019 deletions
diff --git a/src/bootstrap/CHANGELOG.md b/src/bootstrap/CHANGELOG.md index a103c9fb0b7..f899f21080e 100644 --- a/src/bootstrap/CHANGELOG.md +++ b/src/bootstrap/CHANGELOG.md @@ -4,7 +4,12 @@ All notable changes to bootstrap will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [Non-breaking changes since the last major version] + +## [Changes since the last major version] + +- `llvm-libunwind` now accepts `in-tree` (formerly true), `system` or `no` (formerly false) [#77703](https://github.com/rust-lang/rust/pull/77703) + +### Non-breaking changes - `x.py check` needs opt-in to check tests (--all-targets) [#77473](https://github.com/rust-lang/rust/pull/77473) - The default bootstrap profiles are now located at `bootstrap/defaults/config.$PROFILE.toml` (previously they were located at `bootstrap/defaults/config.toml.$PROFILE`) [#77558](https://github.com/rust-lang/rust/pull/77558) diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 84ed9446ae7..a2e596bf4e9 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -201,7 +201,6 @@ build/ # Output for all compiletest-based test suites test/ ui/ - compile-fail/ debuginfo/ ... diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index a819e1b6e2f..b8bae69d063 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -192,8 +192,10 @@ def default_build_triple(verbose): # If the user already has a host build triple with an existing `rustc` # install, use their preference. This fixes most issues with Windows builds # being detected as GNU instead of MSVC. + default_encoding = sys.getdefaultencoding() try: version = subprocess.check_output(["rustc", "--version", "--verbose"]) + version = version.decode(default_encoding) host = next(x for x in version.split('\n') if x.startswith("host: ")) triple = host.split("host: ")[1] if verbose: @@ -204,7 +206,6 @@ def default_build_triple(verbose): print("rustup not detected: {}".format(e)) print("falling back to auto-detect") - default_encoding = sys.getdefaultencoding() required = sys.platform != 'win32' ostype = require(["uname", "-s"], exit=required) cputype = require(['uname', '-m'], exit=required) @@ -350,11 +351,13 @@ def output(filepath): with open(tmp, 'w') as f: yield f try: - os.remove(filepath) # PermissionError/OSError on Win32 if in use - os.rename(tmp, filepath) + if os.path.exists(filepath): + os.remove(filepath) # PermissionError/OSError on Win32 if in use except OSError: shutil.copy2(tmp, filepath) os.remove(tmp) + return + os.rename(tmp, filepath) class RustBuild(object): @@ -794,7 +797,7 @@ class RustBuild(object): env.setdefault("RUSTFLAGS", "") env["RUSTFLAGS"] += " -Cdebuginfo=2" - build_section = "target.{}".format(self.build_triple()) + build_section = "target.{}".format(self.build) target_features = [] if self.get_toml("crt-static", build_section) == "true": target_features += ["+crt-static"] @@ -825,7 +828,11 @@ class RustBuild(object): run(args, env=env, verbose=self.verbose) def build_triple(self): - """Build triple as in LLVM""" + """Build triple as in LLVM + + Note that `default_build_triple` is moderately expensive, + so use `self.build` where possible. + """ config = self.get_toml('build') if config: return config @@ -892,13 +899,17 @@ class RustBuild(object): filtered_submodules = [] submodules_names = [] llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git")) + external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm() + llvm_needed = not self.get_toml('codegen-backends', 'rust') \ + or "llvm" in self.get_toml('codegen-backends', 'rust') for module in submodules: if module.endswith("llvm-project"): - # Don't sync the llvm-project submodule either if an external LLVM - # was provided, or if we are downloading LLVM. Also, if the - # submodule has been initialized already, sync it anyways so that - # it doesn't mess up contributor pull requests. - if self.get_toml('llvm-config') or self.downloading_llvm(): + # Don't sync the llvm-project submodule if an external LLVM was + # provided, if we are downloading LLVM or if the LLVM backend is + # not being built. Also, if the submodule has been initialized + # already, sync it anyways so that it doesn't mess up contributor + # pull requests. + if external_llvm_provided or not llvm_needed: if self.get_toml('lld') != 'true' and not llvm_checked_out: continue check = self.check_submodule(module, slow_submodules) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 6d97943548d..c2abb01fa8c 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -384,7 +384,6 @@ impl<'a> Builder<'a> { test::ExpandYamlAnchors, test::Tidy, test::Ui, - test::CompileFail, test::RunPassValgrind, test::MirOpt, test::Codegen, @@ -471,6 +470,7 @@ impl<'a> Builder<'a> { dist::RustDev, dist::Extended, dist::BuildManifest, + dist::ReproducibleArtifacts, ), Kind::Install => describe!( install::Docs, @@ -732,11 +732,11 @@ impl<'a> Builder<'a> { .env("CFG_RELEASE_CHANNEL", &self.config.channel) .env("RUSTDOC_REAL", self.rustdoc(compiler)) .env("RUSTC_BOOTSTRAP", "1") - .arg("-Znormalize_docs") .arg("-Winvalid_codeblock_attributes"); if self.config.deny_warnings { cmd.arg("-Dwarnings"); } + cmd.arg("-Znormalize-docs"); // Remove make-related flags that can cause jobserver problems. cmd.env_remove("MAKEFLAGS"); @@ -1123,6 +1123,19 @@ impl<'a> Builder<'a> { }, ); + // `dsymutil` adds time to builds on Apple platforms for no clear benefit, and also makes + // it more difficult for debuggers to find debug info. The compiler currently defaults to + // running `dsymutil` to preserve its historical default, but when compiling the compiler + // itself, we skip it by default since we know it's safe to do so in that case. + // See https://github.com/rust-lang/rust/issues/79361 for more info on this flag. + if target.contains("apple") { + if self.config.rust_run_dsymutil { + rustflags.arg("-Zrun-dsymutil=yes"); + } else { + rustflags.arg("-Zrun-dsymutil=no"); + } + } + if self.config.cmd.bless() { // Bless `expect!` tests. cargo.env("UPDATE_EXPECT", "1"); diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 3cfb2f2836d..9477a7cb354 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -7,6 +7,7 @@ //! goes along from the output of the previous stage. use std::borrow::Cow; +use std::collections::HashSet; use std::env; use std::fs; use std::io::prelude::*; @@ -519,6 +520,41 @@ impl Step for Rustc { let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "build"); rustc_cargo(builder, &mut cargo, target); + if builder.config.rust_profile_use.is_some() + && builder.config.rust_profile_generate.is_some() + { + panic!("Cannot use and generate PGO profiles at the same time"); + } + + let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-generate={}", path)); + // Apparently necessary to avoid overflowing the counters during + // a Cargo build profile + cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); + true + } else { + false + } + } else if let Some(path) = &builder.config.rust_profile_use { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-use={}", path)); + cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); + true + } else { + false + } + } else { + false + }; + if is_collecting { + // Ensure paths to Rust sources are relative, not absolute. + cargo.rustflag(&format!( + "-Cllvm-args=-static-func-strip-dirname-prefix={}", + builder.config.src.components().count() + )); + } + builder.info(&format!( "Building stage{} compiler artifacts ({} -> {})", compiler.stage, &compiler.host, target @@ -770,7 +806,7 @@ fn copy_codegen_backends_to_sysroot( // Here we're looking for the output dylib of the `CodegenBackend` step and // we're copying that into the `codegen-backends` folder. let dst = builder.sysroot_codegen_backends(target_compiler); - t!(fs::create_dir_all(&dst)); + t!(fs::create_dir_all(&dst), dst); if builder.config.dry_run { return; @@ -974,28 +1010,51 @@ impl Step for Assemble { builder.info(&format!("Assembling stage{} compiler ({})", stage, host)); // Link in all dylibs to the libdir + let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); + let proc_macros = builder + .read_stamp_file(&stamp) + .into_iter() + .filter_map(|(path, dependency_type)| { + if dependency_type == DependencyType::Host { + Some(path.file_name().unwrap().to_owned().into_string().unwrap()) + } else { + None + } + }) + .collect::<HashSet<_>>(); + let sysroot = builder.sysroot(target_compiler); let rustc_libdir = builder.rustc_libdir(target_compiler); t!(fs::create_dir_all(&rustc_libdir)); let src_libdir = builder.sysroot_libdir(build_compiler, host); for f in builder.read_dir(&src_libdir) { let filename = f.file_name().into_string().unwrap(); - if is_dylib(&filename) { + if is_dylib(&filename) && !proc_macros.contains(&filename) { builder.copy(&f.path(), &rustc_libdir.join(&filename)); } } copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); + // We prepend this bin directory to the user PATH when linking Rust binaries. To + // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. let libdir = builder.sysroot_libdir(target_compiler, target_compiler.host); + let libdir_bin = libdir.parent().unwrap().join("bin"); + t!(fs::create_dir_all(&libdir_bin)); + if let Some(lld_install) = lld_install { let src_exe = exe("lld", target_compiler.host); let dst_exe = exe("rust-lld", target_compiler.host); - // we prepend this bin directory to the user PATH when linking Rust binaries. To - // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. - let dst = libdir.parent().unwrap().join("bin"); - t!(fs::create_dir_all(&dst)); - builder.copy(&lld_install.join("bin").join(&src_exe), &dst.join(&dst_exe)); + builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); + } + + // Similarly, copy `llvm-dwp` into libdir for Split DWARF. + { + let src_exe = exe("llvm-dwp", target_compiler.host); + let dst_exe = exe("rust-llvm-dwp", target_compiler.host); + let llvm_config_bin = builder.ensure(native::Llvm { target: target_compiler.host }); + let llvm_bin_dir = llvm_config_bin.parent().unwrap(); + builder.copy(&llvm_bin_dir.join(&src_exe), &libdir_bin.join(&dst_exe)); } // Ensure that `libLLVM.so` ends up in the newly build compiler directory, diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index fb2c6d1f92a..f4d89a89c14 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -123,6 +123,7 @@ pub struct Config { pub rust_debuginfo_level_std: u32, pub rust_debuginfo_level_tools: u32, pub rust_debuginfo_level_tests: u32, + pub rust_run_dsymutil: bool, pub rust_rpath: bool, pub rustc_parallel: bool, pub rustc_default_linker: Option<String>, @@ -133,6 +134,8 @@ pub struct Config { pub rust_thin_lto_import_instr_limit: Option<u32>, pub rust_remap_debuginfo: bool, pub rust_new_symbol_mangling: bool, + pub rust_profile_use: Option<String>, + pub rust_profile_generate: Option<String>, pub build: TargetSelection, pub hosts: Vec<TargetSelection>, @@ -145,6 +148,7 @@ pub struct Config { pub dist_sign_folder: Option<PathBuf>, pub dist_upload_addr: Option<String>, pub dist_gpg_password_file: Option<PathBuf>, + pub dist_compression_formats: Option<Vec<String>>, // libstd features pub backtrace: bool, // support for RUST_BACKTRACE @@ -331,7 +335,7 @@ impl Merge for TomlConfig { *x = Some(new); } } - }; + } do_merge(&mut self.build, build); do_merge(&mut self.install, install); do_merge(&mut self.llvm, llvm); @@ -435,6 +439,7 @@ struct Dist { upload_addr: Option<String>, src_tarball: Option<bool>, missing_tools: Option<bool>, + compression_formats: Option<Vec<String>>, } #[derive(Deserialize)] @@ -466,6 +471,7 @@ struct Rust { debuginfo_level_std: Option<u32>, debuginfo_level_tools: Option<u32>, debuginfo_level_tests: Option<u32>, + run_dsymutil: Option<bool>, backtrace: Option<bool>, incremental: Option<bool>, parallel_compiler: Option<bool>, @@ -494,6 +500,8 @@ struct Rust { llvm_libunwind: Option<String>, control_flow_guard: Option<bool>, new_symbol_mangling: Option<bool>, + profile_generate: Option<String>, + profile_use: Option<String>, } /// TOML representation of how each build target is configured. @@ -830,6 +838,7 @@ impl Config { debuginfo_level_std = rust.debuginfo_level_std; debuginfo_level_tools = rust.debuginfo_level_tools; debuginfo_level_tests = rust.debuginfo_level_tests; + config.rust_run_dsymutil = rust.run_dsymutil.unwrap_or(false); optimize = rust.optimize; ignore_git = rust.ignore_git; set(&mut config.rust_new_symbol_mangling, rust.new_symbol_mangling); @@ -871,6 +880,11 @@ impl Config { config.rust_codegen_units = rust.codegen_units.map(threads_from_config); config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); + config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); + config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); + } else { + config.rust_profile_use = flags.rust_profile_use; + config.rust_profile_generate = flags.rust_profile_generate; } if let Some(t) = toml.target { @@ -924,6 +938,7 @@ impl Config { config.dist_sign_folder = t.sign_folder.map(PathBuf::from); config.dist_gpg_password_file = t.gpg_password_file.map(PathBuf::from); config.dist_upload_addr = t.upload_addr; + config.dist_compression_formats = t.compression_formats; set(&mut config.rust_dist_src, t.src_tarball); set(&mut config.missing_tools, t.missing_tools); } diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 42f00ce9621..2cabaee68ea 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -147,6 +147,8 @@ v("experimental-targets", "llvm.experimental-targets", "experimental LLVM targets to build") v("release-channel", "rust.channel", "the name of the release channel to build") v("release-description", "rust.description", "optional descriptive string for version output") +v("dist-compression-formats", None, + "comma-separated list of compression formats to use") # Used on systems where "cc" is unavailable v("default-linker", "rust.default-linker", "the default linker") @@ -349,6 +351,8 @@ for key in known_args: elif option.name == 'option-checking': # this was handled above pass + elif option.name == 'dist-compression-formats': + set('dist.compression-formats', value.split(',')) else: raise RuntimeError("unhandled option {}".format(option.name)) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 354be109cf2..daec1656b27 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -19,6 +19,7 @@ use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; use crate::compile; use crate::config::TargetSelection; +use crate::tarball::{GeneratedTarball, OverlayKind, Tarball}; use crate::tool::{self, Tool}; use crate::util::{exe, is_dylib, timeit}; use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; @@ -36,10 +37,6 @@ pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { builder.out.join("tmp/dist") } -fn rust_installer(builder: &Builder<'_>) -> Command { - builder.tool_cmd(Tool::RustInstaller) -} - fn missing_tool(tool_name: &str, skip: bool) { if skip { println!("Unable to build {}, skipping dist", tool_name) @@ -54,7 +51,7 @@ pub struct Docs { } impl Step for Docs { - type Output = PathBuf; + type Output = Option<GeneratedTarball>; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -66,48 +63,20 @@ impl Step for Docs { } /// Builds the `rust-docs` installer component. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let host = self.host; - - let name = pkgname(builder, "rust-docs"); - if !builder.config.docs { - return distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)); + return None; } - builder.default_doc(None); - builder.info(&format!("Dist docs ({})", host)); - let _time = timeit(builder); + let dest = "share/doc/rust/html"; - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - - let dst = image.join("share/doc/rust/html"); - t!(fs::create_dir_all(&dst)); - let src = builder.doc_out(host); - builder.cp_r(&src, &dst); - builder.install(&builder.src.join("src/doc/robots.txt"), &dst, 0o644); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust-Documentation") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-documentation-is-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rust-docs") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--bulk-dirs=share/doc/rust/html"); - builder.run(&mut cmd); - builder.remove_dir(&image); - - distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)) + let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); + tarball.set_product_name("Rust Documentation"); + tarball.add_dir(&builder.doc_out(host), dest); + tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); + Some(tarball.generate()) } } @@ -117,7 +86,7 @@ pub struct RustcDocs { } impl Step for RustcDocs { - type Output = PathBuf; + type Output = Option<GeneratedTarball>; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -129,47 +98,17 @@ impl Step for RustcDocs { } /// Builds the `rustc-docs` installer component. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let host = self.host; - - let name = pkgname(builder, "rustc-docs"); - if !builder.config.compiler_docs { - return distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)); + return None; } - builder.default_doc(None); - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - - let dst = image.join("share/doc/rust/html/rustc"); - t!(fs::create_dir_all(&dst)); - let src = builder.compiler_doc_out(host); - builder.cp_r(&src, &dst); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rustc-Documentation") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rustc-documentation-is-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rustc-docs") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--bulk-dirs=share/doc/rust/html/rustc"); - - builder.info(&format!("Dist compiler docs ({})", host)); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - - distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)) + let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); + tarball.set_product_name("Rustc Documentation"); + tarball.add_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); + Some(tarball.generate()) } } @@ -328,7 +267,7 @@ pub struct Mingw { } impl Step for Mingw { - type Output = Option<PathBuf>; + type Output = Option<GeneratedTarball>; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -343,43 +282,22 @@ impl Step for Mingw { /// /// This contains all the bits and pieces to run the MinGW Windows targets /// without any extra installed software (e.g., we bundle gcc, libraries, etc). - fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let host = self.host; - if !host.contains("pc-windows-gnu") { return None; } - builder.info(&format!("Dist mingw ({})", host)); - let _time = timeit(builder); - let name = pkgname(builder, "rust-mingw"); - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - t!(fs::create_dir_all(&image)); + let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); + tarball.set_product_name("Rust MinGW"); // The first argument is a "temporary directory" which is just // thrown away (this contains the runtime DLLs included in the rustc package // above) and the second argument is where to place all the MinGW components // (which is what we want). - make_win_dist(&tmpdir(builder), &image, host, &builder); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust-MinGW") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-MinGW-is-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rust-mingw") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - builder.run(&mut cmd); - t!(fs::remove_dir_all(&image)); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple))) + make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder); + + Some(tarball.generate()) } } @@ -389,7 +307,7 @@ pub struct Rustc { } impl Step for Rustc { - type Output = PathBuf; + type Output = GeneratedTarball; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -403,34 +321,14 @@ impl Step for Rustc { } /// Creates the `rustc` installer component. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { let compiler = self.compiler; let host = self.compiler.host; - let name = pkgname(builder, "rustc"); - let image = tmpdir(builder).join(format!("{}-{}-image", name, host.triple)); - let _ = fs::remove_dir_all(&image); - let overlay = tmpdir(builder).join(format!("{}-{}-overlay", name, host.triple)); - let _ = fs::remove_dir_all(&overlay); + let tarball = Tarball::new(builder, "rustc", &host.triple); // Prepare the rustc "image", what will actually end up getting installed - prepare_image(builder, compiler, &image); - - // Prepare the overlay which is part of the tarball but won't actually be - // installed - let cp = |file: &str| { - builder.install(&builder.src.join(file), &overlay, 0o644); - }; - cp("COPYRIGHT"); - cp("LICENSE-APACHE"); - cp("LICENSE-MIT"); - cp("README.md"); - // tiny morsel of metadata is used by rust-packaging - let version = builder.rust_version(); - builder.create(&overlay.join("version"), &version); - if let Some(sha) = builder.rust_sha() { - builder.create(&overlay.join("git-commit-hash"), &sha); - } + prepare_image(builder, compiler, tarball.image_dir()); // On MinGW we've got a few runtime DLL dependencies that we need to // include. The first argument to this script is where to put these DLLs @@ -443,38 +341,11 @@ impl Step for Rustc { // install will *also* include the rust-mingw package, which also needs // licenses, so to be safe we just include it here in all MinGW packages. if host.contains("pc-windows-gnu") { - make_win_dist(&image, &tmpdir(builder), host, builder); - - let dst = image.join("share/doc"); - t!(fs::create_dir_all(&dst)); - builder.cp_r(&builder.src.join("src/etc/third-party"), &dst); + make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); + tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); } - // Finally, wrap everything up in a nice tarball! - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-roll.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, host.triple)) - .arg("--component-name=rustc") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info(&format!("Dist rustc stage{} ({})", compiler.stage, host.triple)); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - builder.remove_dir(&overlay); - - return distdir(builder).join(format!("{}-{}.tar.gz", name, host.triple)); + return tarball.generate(); fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { let host = compiler.host; @@ -523,17 +394,20 @@ impl Step for Rustc { // component for now. maybe_install_llvm_runtime(builder, host, image); + let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin"); + let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin"); + t!(fs::create_dir_all(&dst_dir)); + // Copy over lld if it's there if builder.config.lld_enabled { let exe = exe("rust-lld", compiler.host); - let src = - builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin").join(&exe); - // for the rationale about this rename check `compile::copy_lld_to_sysroot` - let dst = image.join("lib/rustlib").join(&*host.triple).join("bin").join(&exe); - t!(fs::create_dir_all(&dst.parent().unwrap())); - builder.copy(&src, &dst); + builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe)); } + // Copy over llvm-dwp if it's there + let exe = exe("rust-llvm-dwp", compiler.host); + builder.copy(&src_dir.join(&exe), &dst_dir.join(&exe)); + // Man pages t!(fs::create_dir_all(image.join("share/man/man1"))); let man_src = builder.src.join("src/doc/man"); @@ -681,7 +555,7 @@ pub struct Std { } impl Step for Std { - type Output = PathBuf; + type Output = Option<GeneratedTarball>; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -699,46 +573,24 @@ impl Step for Std { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let compiler = self.compiler; let target = self.target; - let name = pkgname(builder, "rust-std"); - let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)); if skip_host_target_lib(builder, compiler) { - return archive; + return None; } builder.ensure(compile::Std { compiler, target }); - let image = tmpdir(builder).join(format!("{}-{}-image", name, target.triple)); - let _ = fs::remove_dir_all(&image); + let mut tarball = Tarball::new(builder, "rust-std", &target.triple); + tarball.include_target_in_component_name(true); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); let stamp = compile::libstd_stamp(builder, compiler_to_use, target); - copy_target_libs(builder, target, &image, &stamp); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=std-is-standing-at-the-ready.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg(format!("--component-name=rust-std-{}", target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder - .info(&format!("Dist std stage{} ({} -> {})", compiler.stage, &compiler.host, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - archive + copy_target_libs(builder, target, &tarball.image_dir(), &stamp); + + Some(tarball.generate()) } } @@ -749,7 +601,7 @@ pub struct RustcDev { } impl Step for RustcDev { - type Output = PathBuf; + type Output = Option<GeneratedTarball>; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -768,60 +620,36 @@ impl Step for RustcDev { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let compiler = self.compiler; let target = self.target; - - let name = pkgname(builder, "rustc-dev"); - let archive = distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)); if skip_host_target_lib(builder, compiler) { - return archive; + return None; } builder.ensure(compile::Rustc { compiler, target }); - let image = tmpdir(builder).join(format!("{}-{}-image", name, target.triple)); - let _ = fs::remove_dir_all(&image); + let tarball = Tarball::new(builder, "rustc-dev", &target.triple); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); let stamp = compile::librustc_stamp(builder, compiler_to_use, target); - copy_target_libs(builder, target, &image, &stamp); + copy_target_libs(builder, target, tarball.image_dir(), &stamp); - // Copy compiler sources. - let dst_src = image.join("lib/rustlib/rustc-src/rust"); - t!(fs::create_dir_all(&dst_src)); - - let src_files = ["Cargo.lock"]; + let src_files = &["Cargo.lock"]; // This is the reduced set of paths which will become the rustc-dev component // (essentially the compiler crates and all of their path dependencies). - copy_src_dirs(builder, &builder.src, &["compiler"], &[], &dst_src); - for file in src_files.iter() { - builder.copy(&builder.src.join(file), &dst_src.join(file)); + copy_src_dirs( + builder, + &builder.src, + &["compiler"], + &[], + &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), + ); + for file in src_files { + tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); } - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-develop.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg(format!("--component-name=rustc-dev-{}", target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info(&format!( - "Dist rustc-dev stage{} ({} -> {})", - compiler.stage, &compiler.host, target - )); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - archive + Some(tarball.generate()) } } @@ -832,7 +660,7 @@ pub struct Analysis { } impl Step for Analysis { - type Output = PathBuf; + type Output = Option<GeneratedTarball>; const DEFAULT: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -855,52 +683,26 @@ impl Step for Analysis { } /// Creates a tarball of save-analysis metadata, if available. - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); - let name = pkgname(builder, "rust-analysis"); - if compiler.host != builder.config.build { - return distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)); + return None; } builder.ensure(compile::Std { compiler, target }); - - let image = tmpdir(builder).join(format!("{}-{}-image", name, target.triple)); - let src = builder .stage_out(compiler, Mode::Std) .join(target.triple) .join(builder.cargo_dir()) - .join("deps"); + .join("deps") + .join("save-analysis"); - let image_src = src.join("save-analysis"); - let dst = image.join("lib/rustlib").join(target.triple).join("analysis"); - t!(fs::create_dir_all(&dst)); - builder.info(&format!("image_src: {:?}, dst: {:?}", image_src, dst)); - builder.cp_r(&image_src, &dst); - - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=save-analysis-saved.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg(format!("--component-name=rust-analysis-{}", target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info("Dist analysis"); - let _time = timeit(builder); - builder.run(&mut cmd); - builder.remove_dir(&image); - distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)) + let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); + tarball.include_target_in_component_name(true); + tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); + Some(tarball.generate()) } } @@ -994,7 +796,7 @@ pub struct Src; impl Step for Src { /// The output path of the src installer tarball - type Output = PathBuf; + type Output = GeneratedTarball; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -1007,10 +809,8 @@ impl Step for Src { } /// Creates the `rust-src` installer component - fn run(self, builder: &Builder<'_>) -> PathBuf { - let name = pkgname(builder, "rust-src"); - let image = tmpdir(builder).join(format!("{}-image", name)); - let _ = fs::remove_dir_all(&image); + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let tarball = Tarball::new_targetless(builder, "rust-src"); // A lot of tools expect the rust-src component to be entirely in this directory, so if you // change that (e.g. by adding another directory `lib/rustlib/src/foo` or @@ -1019,8 +819,7 @@ impl Step for Src { // // NOTE: if you update the paths here, you also should update the "virtual" path // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` - let dst_src = image.join("lib/rustlib/src/rust"); - t!(fs::create_dir_all(&dst_src)); + let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); let src_files = ["Cargo.lock"]; // This is the reduced set of paths which will become the rust-src component @@ -1040,50 +839,7 @@ impl Step for Src { builder.copy(&builder.src.join(file), &dst_src.join(file)); } - // libtest includes std and everything else, so vendoring it - // creates exactly what's needed for `cargo -Zbuild-std` or any - // other analysis of the stdlib's source. Cargo also needs help - // finding the lock, so we copy it to libtest temporarily. - // - // Note that this requires std to only have one version of each - // crate. e.g. two versions of getopts won't be patchable. - let dst_libtest = dst_src.join("library/test"); - let dst_vendor = dst_src.join("vendor"); - let root_lock = dst_src.join("Cargo.lock"); - let temp_lock = dst_libtest.join("Cargo.lock"); - - // `cargo vendor` will delete everything from the lockfile that - // isn't used by libtest, so we need to not use any links! - builder.really_copy(&root_lock, &temp_lock); - - let mut cmd = Command::new(&builder.initial_cargo); - cmd.arg("vendor").arg(dst_vendor).current_dir(&dst_libtest); - builder.info("Dist src"); - let _time = timeit(builder); - builder.run(&mut cmd); - - builder.remove(&temp_lock); - - // Create source tarball in rust-installer format - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Awesome-Source.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}", name)) - .arg("--component-name=rust-src") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.run(&mut cmd); - - builder.remove_dir(&image); - distdir(builder).join(&format!("{}.tar.gz", name)) + tarball.generate() } } @@ -1092,7 +848,7 @@ pub struct PlainSourceTarball; impl Step for PlainSourceTarball { /// Produces the location of the tarball generated - type Output = PathBuf; + type Output = GeneratedTarball; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -1106,12 +862,9 @@ impl Step for PlainSourceTarball { } /// Creates the plain source tarball - fn run(self, builder: &Builder<'_>) -> PathBuf { - // Make sure that the root folder of tarball has the correct name - let plain_name = format!("{}-src", pkgname(builder, "rustc")); - let plain_dst_src = tmpdir(builder).join(&plain_name); - let _ = fs::remove_dir_all(&plain_dst_src); - t!(fs::create_dir_all(&plain_dst_src)); + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let tarball = Tarball::new(builder, "rustc", "src"); + let plain_dst_src = tarball.image_dir(); // This is the set of root paths which will become part of the source package let src_files = [ @@ -1154,28 +907,7 @@ impl Step for PlainSourceTarball { builder.run(&mut cmd); } - // Create plain source tarball - let plain_name = format!("rustc-{}-src", builder.rust_package_vers()); - let mut tarball = distdir(builder).join(&format!("{}.tar.gz", plain_name)); - tarball.set_extension(""); // strip .gz - tarball.set_extension(""); // strip .tar - if let Some(dir) = tarball.parent() { - builder.create_dir(&dir); - } - builder.info("running installer"); - let mut cmd = rust_installer(builder); - cmd.arg("tarball") - .arg("--input") - .arg(&plain_name) - .arg("--output") - .arg(&tarball) - .arg("--work-dir=.") - .current_dir(tmpdir(builder)); - - builder.info("Create plain source tarball"); - let _time = timeit(builder); - builder.run(&mut cmd); - distdir(builder).join(&format!("{}.tar.gz", plain_name)) + tarball.bare() } } @@ -1186,7 +918,7 @@ pub fn sanitize_sh(path: &Path) -> String { return change_drive(unc_to_lfs(&path)).unwrap_or(path); fn unc_to_lfs(s: &str) -> &str { - if s.starts_with("//?/") { &s[4..] } else { s } + s.strip_prefix("//?/").unwrap_or(s) } fn change_drive(s: &str) -> Option<String> { @@ -1209,7 +941,7 @@ pub struct Cargo { } impl Step for Cargo { - type Output = PathBuf; + type Output = GeneratedTarball; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1227,70 +959,32 @@ impl Step for Cargo { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { let compiler = self.compiler; let target = self.target; + let cargo = builder.ensure(tool::Cargo { compiler, target }); let src = builder.src.join("src/tools/cargo"); let etc = src.join("src/etc"); - let release_num = builder.release_num("cargo"); - let name = pkgname(builder, "cargo"); - let version = builder.cargo_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("cargo-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); // Prepare the image directory - builder.create_dir(&image.join("share/zsh/site-functions")); - builder.create_dir(&image.join("etc/bash_completion.d")); - let cargo = builder.ensure(tool::Cargo { compiler, target }); - builder.install(&cargo, &image.join("bin"), 0o755); - for man in t!(etc.join("man").read_dir()) { - let man = t!(man); - builder.install(&man.path(), &image.join("share/man/man1"), 0o644); + let mut tarball = Tarball::new(builder, "cargo", &target.triple); + tarball.set_overlay(OverlayKind::Cargo); + + tarball.add_file(&cargo, "bin", 0o755); + tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); + tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); + tarball.add_dir(etc.join("man"), "share/man/man1"); + tarball.add_legal_and_readme_to("share/doc/cargo"); + + for dirent in fs::read_dir(cargo.parent().unwrap()).expect("read_dir") { + let dirent = dirent.expect("read dir entry"); + if dirent.file_name().to_str().expect("utf8").starts_with("cargo-credential-") { + tarball.add_file(&dirent.path(), "libexec", 0o755); + } } - builder.install(&etc.join("_cargo"), &image.join("share/zsh/site-functions"), 0o644); - builder.copy(&etc.join("cargo.bashcomp.sh"), &image.join("etc/bash_completion.d/cargo")); - let doc = image.join("share/doc/cargo"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-THIRD-PARTY"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("cargo-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.install(&src.join("LICENSE-THIRD-PARTY"), &overlay, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-roll.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--component-name=cargo") - .arg("--legacy-manifest-dirs=rustlib,cargo"); - - builder.info(&format!("Dist cargo stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)) + + tarball.generate() } } @@ -1301,7 +995,7 @@ pub struct Rls { } impl Step for Rls { - type Output = Option<PathBuf>; + type Output = Option<GeneratedTarball>; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1319,24 +1013,11 @@ impl Step for Rls { }); } - fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); - let src = builder.src.join("src/tools/rls"); - let release_num = builder.release_num("rls"); - let name = pkgname(builder, "rls"); - let version = builder.rls_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("rls-image"); - drop(fs::remove_dir_all(&image)); - t!(fs::create_dir_all(&image)); - - // Prepare the image directory - // We expect RLS to build, because we've exited this step above if tool - // state for RLS isn't testing. let rls = builder .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) .or_else(|| { @@ -1344,43 +1025,12 @@ impl Step for Rls { None })?; - builder.install(&rls, &image.join("bin"), 0o755); - let doc = image.join("share/doc/rls"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("rls-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=RLS-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rls-preview"); - - builder.info(&format!("Dist RLS stage{} ({})", compiler.stage, target.triple)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "rls", &target.triple); + tarball.set_overlay(OverlayKind::RLS); + tarball.is_preview(true); + tarball.add_file(rls, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rls"); + Some(tarball.generate()) } } @@ -1391,7 +1041,7 @@ pub struct RustAnalyzer { } impl Step for RustAnalyzer { - type Output = Option<PathBuf>; + type Output = Option<GeneratedTarball>; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1409,7 +1059,7 @@ impl Step for RustAnalyzer { }); } - fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); @@ -1420,60 +1070,16 @@ impl Step for RustAnalyzer { return None; } - let src = builder.src.join("src/tools/rust-analyzer"); - let release_num = builder.release_num("rust-analyzer/crates/rust-analyzer"); - let name = pkgname(builder, "rust-analyzer"); - let version = builder.rust_analyzer_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("rust-analyzer-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - - // Prepare the image directory - // We expect rust-analyer to always build, as it doesn't depend on rustc internals - // and doesn't have associated toolstate. let rust_analyzer = builder .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() }) .expect("rust-analyzer always builds"); - builder.install(&rust_analyzer, &image.join("bin"), 0o755); - let doc = image.join("share/doc/rust-analyzer"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("rust-analyzer-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=rust-analyzer-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rust-analyzer-preview"); - - builder.info(&format!("Dist rust-analyzer stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); + tarball.set_overlay(OverlayKind::RustAnalyzer); + tarball.is_preview(true); + tarball.add_file(rust_analyzer, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); + Some(tarball.generate()) } } @@ -1484,7 +1090,7 @@ pub struct Clippy { } impl Step for Clippy { - type Output = PathBuf; + type Output = GeneratedTarball; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1502,21 +1108,11 @@ impl Step for Clippy { }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); - let src = builder.src.join("src/tools/clippy"); - let release_num = builder.release_num("clippy"); - let name = pkgname(builder, "clippy"); - let version = builder.clippy_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("clippy-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - // Prepare the image directory // We expect clippy to build, because we've exited this step above if tool // state for clippy isn't testing. @@ -1527,44 +1123,13 @@ impl Step for Clippy { .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) .expect("clippy expected to build - essential tool"); - builder.install(&clippy, &image.join("bin"), 0o755); - builder.install(&cargoclippy, &image.join("bin"), 0o755); - let doc = image.join("share/doc/clippy"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("clippy-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=clippy-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=clippy-preview"); - - builder.info(&format!("Dist clippy stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)) + let mut tarball = Tarball::new(builder, "clippy", &target.triple); + tarball.set_overlay(OverlayKind::Clippy); + tarball.is_preview(true); + tarball.add_file(clippy, "bin", 0o755); + tarball.add_file(cargoclippy, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/clippy"); + tarball.generate() } } @@ -1575,7 +1140,7 @@ pub struct Miri { } impl Step for Miri { - type Output = Option<PathBuf>; + type Output = Option<GeneratedTarball>; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1593,24 +1158,11 @@ impl Step for Miri { }); } - fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let compiler = self.compiler; let target = self.target; assert!(builder.config.extended); - let src = builder.src.join("src/tools/miri"); - let release_num = builder.release_num("miri"); - let name = pkgname(builder, "miri"); - let version = builder.miri_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("miri-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - - // Prepare the image directory - // We expect miri to build, because we've exited this step above if tool - // state for miri isn't testing. let miri = builder .ensure(tool::Miri { compiler, target, extra_features: Vec::new() }) .or_else(|| { @@ -1624,44 +1176,13 @@ impl Step for Miri { None })?; - builder.install(&miri, &image.join("bin"), 0o755); - builder.install(&cargomiri, &image.join("bin"), 0o755); - let doc = image.join("share/doc/miri"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("miri-overlay"); - drop(fs::remove_dir_all(&overlay)); - t!(fs::create_dir_all(&overlay)); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=miri-ready-to-serve.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=miri-preview"); - - builder.info(&format!("Dist miri stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "miri", &target.triple); + tarball.set_overlay(OverlayKind::Miri); + tarball.is_preview(true); + tarball.add_file(miri, "bin", 0o755); + tarball.add_file(cargomiri, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/miri"); + Some(tarball.generate()) } } @@ -1672,7 +1193,7 @@ pub struct Rustfmt { } impl Step for Rustfmt { - type Output = Option<PathBuf>; + type Output = Option<GeneratedTarball>; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -1690,21 +1211,10 @@ impl Step for Rustfmt { }); } - fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let compiler = self.compiler; let target = self.target; - let src = builder.src.join("src/tools/rustfmt"); - let release_num = builder.release_num("rustfmt"); - let name = pkgname(builder, "rustfmt"); - let version = builder.rustfmt_info.version(builder, &release_num); - - let tmp = tmpdir(builder); - let image = tmp.join("rustfmt-image"); - drop(fs::remove_dir_all(&image)); - builder.create_dir(&image); - - // Prepare the image directory let rustfmt = builder .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }) .or_else(|| { @@ -1718,44 +1228,13 @@ impl Step for Rustfmt { None })?; - builder.install(&rustfmt, &image.join("bin"), 0o755); - builder.install(&cargofmt, &image.join("bin"), 0o755); - let doc = image.join("share/doc/rustfmt"); - builder.install(&src.join("README.md"), &doc, 0o644); - builder.install(&src.join("LICENSE-MIT"), &doc, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &doc, 0o644); - - // Prepare the overlay - let overlay = tmp.join("rustfmt-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.md"), &overlay, 0o644); - builder.install(&src.join("LICENSE-MIT"), &overlay, 0o644); - builder.install(&src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.create(&overlay.join("version"), &version); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=rustfmt-ready-to-fmt.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rustfmt-preview"); - - builder.info(&format!("Dist Rustfmt stage{} ({})", compiler.stage, target)); - let _time = timeit(builder); - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); + tarball.set_overlay(OverlayKind::Rustfmt); + tarball.is_preview(true); + tarball.add_file(rustfmt, "bin", 0o755); + tarball.add_file(cargofmt, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rustfmt"); + Some(tarball.generate()) } } @@ -1804,24 +1283,14 @@ impl Step for Extended { let analysis_installer = builder.ensure(Analysis { compiler, target }); let docs_installer = builder.ensure(Docs { host: target }); - let std_installer = - builder.ensure(Std { compiler: builder.compiler(stage, target), target }); + let std_installer = builder.ensure(Std { compiler, target }); - let tmp = tmpdir(builder); - let overlay = tmp.join("extended-overlay"); let etc = builder.src.join("src/etc/installer"); - let work = tmp.join("work"); - - let _ = fs::remove_dir_all(&overlay); - builder.install(&builder.src.join("COPYRIGHT"), &overlay, 0o644); - builder.install(&builder.src.join("LICENSE-APACHE"), &overlay, 0o644); - builder.install(&builder.src.join("LICENSE-MIT"), &overlay, 0o644); - let version = builder.rust_version(); - builder.create(&overlay.join("version"), &version); - if let Some(sha) = builder.rust_sha() { - builder.create(&overlay.join("git-commit-hash"), &sha); + + // Avoid producing tarballs during a dry run. + if builder.config.dry_run { + return; } - builder.install(&etc.join("README.md"), &overlay, 0o644); // When rust-std package split from rustc, we needed to ensure that during // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering @@ -1836,39 +1305,22 @@ impl Step for Extended { tarballs.extend(miri_installer.clone()); tarballs.extend(rustfmt_installer.clone()); tarballs.extend(llvm_tools_installer); - tarballs.push(analysis_installer); - tarballs.push(std_installer); - if builder.config.docs { + if let Some(analysis_installer) = analysis_installer { + tarballs.push(analysis_installer); + } + tarballs.push(std_installer.expect("missing std")); + if let Some(docs_installer) = docs_installer { tarballs.push(docs_installer); } if target.contains("pc-windows-gnu") { tarballs.push(mingw_installer.unwrap()); } - let mut input_tarballs = tarballs[0].as_os_str().to_owned(); - for tarball in &tarballs[1..] { - input_tarballs.push(","); - input_tarballs.push(tarball); - } - builder.info("building combined installer"); - let mut cmd = rust_installer(builder); - cmd.arg("combine") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=Rust-is-ready-to-roll.") - .arg("--work-dir") - .arg(&work) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg(format!("--package-name={}-{}", pkgname(builder, "rust"), target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--input-tarballs") - .arg(input_tarballs) - .arg("--non-installed-overlay") - .arg(&overlay); - let time = timeit(&builder); - builder.run(&mut cmd); - drop(time); + let tarball = Tarball::new(builder, "rust", &target.triple); + let generated = tarball.combine(&tarballs); + + let tmp = tmpdir(builder).join("combined-tarball"); + let work = generated.work_dir(); let mut license = String::new(); license += &builder.read(&builder.src.join("COPYRIGHT")); @@ -2418,7 +1870,7 @@ pub struct LlvmTools { } impl Step for LlvmTools { - type Output = Option<PathBuf>; + type Output = Option<GeneratedTarball>; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { @@ -2429,7 +1881,7 @@ impl Step for LlvmTools { run.builder.ensure(LlvmTools { target: run.target }); } - fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let target = self.target; assert!(builder.config.extended); @@ -2441,58 +1893,25 @@ impl Step for LlvmTools { } } - builder.info(&format!("Dist LlvmTools ({})", target)); - let _time = timeit(builder); - let src = builder.src.join("src/llvm-project/llvm"); - let name = pkgname(builder, "llvm-tools"); - - let tmp = tmpdir(builder); - let image = tmp.join("llvm-tools-image"); - drop(fs::remove_dir_all(&image)); + let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); + tarball.is_preview(true); // Prepare the image directory let src_bindir = builder.llvm_out(target).join("bin"); - let dst_bindir = image.join("lib/rustlib").join(&*target.triple).join("bin"); - t!(fs::create_dir_all(&dst_bindir)); + let dst_bindir = format!("lib/rustlib/{}/bin", target.triple); for tool in LLVM_TOOLS { let exe = src_bindir.join(exe(tool, target)); - builder.install(&exe, &dst_bindir, 0o755); + tarball.add_file(&exe, &dst_bindir, 0o755); } // Copy libLLVM.so to the target lib dir as well, so the RPATH like // `$ORIGIN/../lib` can find it. It may also be used as a dependency // of `rustc-dev` to support the inherited `-lLLVM` when using the // compiler libraries. - maybe_install_llvm_target(builder, target, &image); - - // Prepare the overlay - let overlay = tmp.join("llvm-tools-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.txt"), &overlay, 0o644); - builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644); - builder.create(&overlay.join("version"), &builder.llvm_tools_vers()); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=llvm-tools-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=llvm-tools-preview"); - - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + maybe_install_llvm_target(builder, target, tarball.image_dir()); + + Some(tarball.generate()) } } @@ -2505,7 +1924,7 @@ pub struct RustDev { } impl Step for RustDev { - type Output = Option<PathBuf>; + type Output = Option<GeneratedTarball>; const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; @@ -2517,7 +1936,7 @@ impl Step for RustDev { run.builder.ensure(RustDev { target: run.target }); } - fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { let target = self.target; /* run only if llvm-config isn't used */ @@ -2528,69 +1947,35 @@ impl Step for RustDev { } } - builder.info(&format!("Dist RustDev ({})", target)); - let _time = timeit(builder); - let src = builder.src.join("src/llvm-project/llvm"); - let name = pkgname(builder, "rust-dev"); - - let tmp = tmpdir(builder); - let image = tmp.join("rust-dev-image"); - drop(fs::remove_dir_all(&image)); - - // Prepare the image directory - let dst_bindir = image.join("bin"); - t!(fs::create_dir_all(&dst_bindir)); + let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); let src_bindir = builder.llvm_out(target).join("bin"); - let install_bin = - |name| builder.install(&src_bindir.join(exe(name, target)), &dst_bindir, 0o755); - install_bin("llvm-config"); - install_bin("llvm-ar"); - install_bin("llvm-objdump"); - install_bin("llvm-profdata"); - install_bin("llvm-bcanalyzer"); - install_bin("llvm-cov"); - builder.install(&builder.llvm_filecheck(target), &dst_bindir, 0o755); + for bin in &[ + "llvm-config", + "llvm-ar", + "llvm-objdump", + "llvm-profdata", + "llvm-bcanalyzer", + "llvm-cov", + "llvm-dwp", + ] { + tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755); + } + tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755); // Copy the include directory as well; needed mostly to build // librustc_llvm properly (e.g., llvm-config.h is in here). But also // just broadly useful to be able to link against the bundled LLVM. - builder.cp_r(&builder.llvm_out(target).join("include"), &image.join("include")); + tarball.add_dir(&builder.llvm_out(target).join("include"), "include"); // Copy libLLVM.so to the target lib dir as well, so the RPATH like // `$ORIGIN/../lib` can find it. It may also be used as a dependency // of `rustc-dev` to support the inherited `-lLLVM` when using the // compiler libraries. - maybe_install_llvm(builder, target, &image.join("lib")); - - // Prepare the overlay - let overlay = tmp.join("rust-dev-overlay"); - drop(fs::remove_dir_all(&overlay)); - builder.create_dir(&overlay); - builder.install(&src.join("README.txt"), &overlay, 0o644); - builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644); - builder.create(&overlay.join("version"), &builder.rust_version()); - - // Generate the installer tarball - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=rust-dev-installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=rust-dev"); - - builder.run(&mut cmd); - Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + maybe_install_llvm(builder, target, &tarball.image_dir().join("lib")); + + Some(tarball.generate()) } } @@ -2604,7 +1989,7 @@ pub struct BuildManifest { } impl Step for BuildManifest { - type Output = PathBuf; + type Output = GeneratedTarball; const DEFAULT: bool = false; const ONLY_HOSTS: bool = true; @@ -2616,47 +2001,43 @@ impl Step for BuildManifest { run.builder.ensure(BuildManifest { target: run.target }); } - fn run(self, builder: &Builder<'_>) -> PathBuf { + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { let build_manifest = builder.tool_exe(Tool::BuildManifest); - let name = pkgname(builder, "build-manifest"); - let tmp = tmpdir(builder); - - // Prepare the image. - let image = tmp.join("build-manifest-image"); - let image_bin = image.join("bin"); - let _ = fs::remove_dir_all(&image); - t!(fs::create_dir_all(&image_bin)); - builder.install(&build_manifest, &image_bin, 0o755); - - // Prepare the overlay. - let overlay = tmp.join("build-manifest-overlay"); - let _ = fs::remove_dir_all(&overlay); - builder.create_dir(&overlay); - builder.create(&overlay.join("version"), &builder.rust_version()); - for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] { - builder.install(&builder.src.join(file), &overlay, 0o644); - } + let tarball = Tarball::new(builder, "build-manifest", &self.target.triple); + tarball.add_file(&build_manifest, "bin", 0o755); + tarball.generate() + } +} + +/// Tarball containing artifacts necessary to reproduce the build of rustc. +/// +/// Currently this is the PGO profile data. +/// +/// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct ReproducibleArtifacts { + pub target: TargetSelection, +} + +impl Step for ReproducibleArtifacts { + type Output = Option<GeneratedTarball>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("reproducible") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ReproducibleArtifacts { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let path = builder.config.rust_profile_use.as_ref()?; - // Create the final tarball. - let mut cmd = rust_installer(builder); - cmd.arg("generate") - .arg("--product-name=Rust") - .arg("--rel-manifest-dir=rustlib") - .arg("--success-message=build-manifest installed.") - .arg("--image-dir") - .arg(&image) - .arg("--work-dir") - .arg(&tmpdir(builder)) - .arg("--output-dir") - .arg(&distdir(builder)) - .arg("--non-installed-overlay") - .arg(&overlay) - .arg(format!("--package-name={}-{}", name, self.target.triple)) - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--component-name=build-manifest"); - - builder.run(&mut cmd); - distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple)) + let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple); + tarball.add_file(path, ".", 0o644); + Some(tarball.generate()) } } diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index a296a1fe3f4..8c849846676 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -500,18 +500,17 @@ impl Step for Rustc { let target = self.target; builder.info(&format!("Documenting stage{} compiler ({})", stage, target)); - // This is the intended out directory for compiler documentation. - let out = builder.compiler_doc_out(target); - t!(fs::create_dir_all(&out)); - - let compiler = builder.compiler(stage, builder.config.build); - if !builder.config.compiler_docs { builder.info("\tskipping - compiler/librustdoc docs disabled"); return; } + // This is the intended out directory for compiler documentation. + let out = builder.compiler_doc_out(target); + t!(fs::create_dir_all(&out)); + // Build rustc. + let compiler = builder.compiler(stage, builder.config.build); builder.ensure(compile::Rustc { compiler, target }); // This uses a shared directory so that librustdoc documentation gets @@ -521,6 +520,10 @@ impl Step for Rustc { // merging the search index, or generating local (relative) links. let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); t!(symlink_dir_force(&builder.config, &out, &out_dir)); + // Cargo puts proc macros in `target/doc` even if you pass `--target` + // explicitly (https://github.com/rust-lang/cargo/issues/7677). + let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); + t!(symlink_dir_force(&builder.config, &out, &proc_macro_out_dir)); // Build cargo command. let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); @@ -625,6 +628,8 @@ impl Step for Rustdoc { cargo.arg("-p").arg("rustdoc"); cargo.rustdocflag("--document-private-items"); + cargo.rustdocflag("--enable-index-page"); + cargo.rustdocflag("-Zunstable-options"); builder.run(&mut cargo.into()); } } diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp index d857618eefa..b29ecd65401 100644 --- a/src/bootstrap/download-ci-llvm-stamp +++ b/src/bootstrap/download-ci-llvm-stamp @@ -1,4 +1,4 @@ Change this file to make users of the `download-ci-llvm` configuration download a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. -Last change is for: https://github.com/rust-lang/rust/pull/78131 +Last change is for: https://github.com/rust-lang/rust/pull/80087 diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 5a8096674c6..d6a45f1c170 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -68,6 +68,9 @@ pub struct Flags { pub deny_warnings: Option<bool>, pub llvm_skip_rebuild: Option<bool>, + + pub rust_profile_use: Option<String>, + pub rust_profile_generate: Option<String>, } pub enum Subcommand { @@ -219,6 +222,8 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", VALUE overrides the skip-rebuild option in config.toml.", "VALUE", ); + opts.optopt("", "rust-profile-generate", "rustc error format", "FORMAT"); + opts.optopt("", "rust-profile-use", "rustc error format", "FORMAT"); // We can't use getopt to parse the options until we have completed specifying which // options are valid, but under the current implementation, some options are conditional on @@ -674,6 +679,8 @@ Arguments: color: matches .opt_get_default("color", Color::Auto) .expect("`color` should be `always`, `never`, or `auto`"), + rust_profile_use: matches.opt_str("rust-profile-use"), + rust_profile_generate: matches.opt_str("rust-profile-generate"), } } } diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index 074f5cd73f3..96164947943 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -10,60 +10,19 @@ use std::process::Command; use build_helper::t; -use crate::dist::{self, pkgname, sanitize_sh, tmpdir}; +use crate::dist::{self, sanitize_sh}; +use crate::tarball::GeneratedTarball; use crate::Compiler; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::config::{Config, TargetSelection}; -pub fn install_docs(builder: &Builder<'_>, stage: u32, host: TargetSelection) { - install_sh(builder, "docs", "rust-docs", stage, Some(host)); -} - -pub fn install_std(builder: &Builder<'_>, stage: u32, target: TargetSelection) { - install_sh(builder, "std", "rust-std", stage, Some(target)); -} - -pub fn install_cargo(builder: &Builder<'_>, stage: u32, host: TargetSelection) { - install_sh(builder, "cargo", "cargo", stage, Some(host)); -} - -pub fn install_rls(builder: &Builder<'_>, stage: u32, host: TargetSelection) { - install_sh(builder, "rls", "rls", stage, Some(host)); -} - -pub fn install_rust_analyzer(builder: &Builder<'_>, stage: u32, host: TargetSelection) { - install_sh(builder, "rust-analyzer", "rust-analyzer", stage, Some(host)); -} - -pub fn install_clippy(builder: &Builder<'_>, stage: u32, host: TargetSelection) { - install_sh(builder, "clippy", "clippy", stage, Some(host)); -} -pub fn install_miri(builder: &Builder<'_>, stage: u32, host: TargetSelection) { - install_sh(builder, "miri", "miri", stage, Some(host)); -} - -pub fn install_rustfmt(builder: &Builder<'_>, stage: u32, host: TargetSelection) { - install_sh(builder, "rustfmt", "rustfmt", stage, Some(host)); -} - -pub fn install_analysis(builder: &Builder<'_>, stage: u32, host: TargetSelection) { - install_sh(builder, "analysis", "rust-analysis", stage, Some(host)); -} - -pub fn install_src(builder: &Builder<'_>, stage: u32) { - install_sh(builder, "src", "rust-src", stage, None); -} -pub fn install_rustc(builder: &Builder<'_>, stage: u32, host: TargetSelection) { - install_sh(builder, "rustc", "rustc", stage, Some(host)); -} - fn install_sh( builder: &Builder<'_>, package: &str, - name: &str, stage: u32, host: Option<TargetSelection>, + tarball: &GeneratedTarball, ) { builder.info(&format!("Install {} stage{} ({:?})", package, stage, host)); @@ -73,12 +32,7 @@ fn install_sh( let docdir_default = datadir_default.join("doc/rust"); let libdir_default = PathBuf::from("lib"); let mandir_default = datadir_default.join("man"); - let prefix = builder.config.prefix.as_ref().map_or(prefix_default, |p| { - fs::create_dir_all(p) - .unwrap_or_else(|err| panic!("could not create {}: {}", p.display(), err)); - fs::canonicalize(p) - .unwrap_or_else(|err| panic!("could not canonicalize {}: {}", p.display(), err)) - }); + let prefix = builder.config.prefix.as_ref().unwrap_or(&prefix_default); let sysconfdir = builder.config.sysconfdir.as_ref().unwrap_or(&sysconfdir_default); let datadir = builder.config.datadir.as_ref().unwrap_or(&datadir_default); let docdir = builder.config.docdir.as_ref().unwrap_or(&docdir_default); @@ -103,18 +57,20 @@ fn install_sh( let libdir = add_destdir(&libdir, &destdir); let mandir = add_destdir(&mandir, &destdir); + let prefix = { + fs::create_dir_all(&prefix) + .unwrap_or_else(|err| panic!("could not create {}: {}", prefix.display(), err)); + fs::canonicalize(&prefix) + .unwrap_or_else(|err| panic!("could not canonicalize {}: {}", prefix.display(), err)) + }; + let empty_dir = builder.out.join("tmp/empty_dir"); t!(fs::create_dir_all(&empty_dir)); - let package_name = if let Some(host) = host { - format!("{}-{}", pkgname(builder, name), host.triple) - } else { - pkgname(builder, name) - }; let mut cmd = Command::new("sh"); cmd.current_dir(&empty_dir) - .arg(sanitize_sh(&tmpdir(builder).join(&package_name).join("install.sh"))) + .arg(sanitize_sh(&tarball.decompressed_output().join("install.sh"))) .arg(format!("--prefix={}", sanitize_sh(&prefix))) .arg(format!("--sysconfdir={}", sanitize_sh(&sysconfdir))) .arg(format!("--datadir={}", sanitize_sh(&datadir))) @@ -189,25 +145,25 @@ macro_rules! install { install!((self, builder, _config), Docs, "src/doc", _config.docs, only_hosts: false, { - builder.ensure(dist::Docs { host: self.target }); - install_docs(builder, self.compiler.stage, self.target); + let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs"); + install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); }; Std, "library/std", true, only_hosts: false, { for target in &builder.targets { - builder.ensure(dist::Std { + let tarball = builder.ensure(dist::Std { compiler: self.compiler, target: *target - }); - install_std(builder, self.compiler.stage, *target); + }).expect("missing std"); + install_sh(builder, "std", self.compiler.stage, Some(*target), &tarball); } }; Cargo, "cargo", Self::should_build(_config), only_hosts: true, { - builder.ensure(dist::Cargo { compiler: self.compiler, target: self.target }); - install_cargo(builder, self.compiler.stage, self.target); + let tarball = builder.ensure(dist::Cargo { compiler: self.compiler, target: self.target }); + install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball); }; Rls, "rls", Self::should_build(_config), only_hosts: true, { - if builder.ensure(dist::Rls { compiler: self.compiler, target: self.target }).is_some() { - install_rls(builder, self.compiler.stage, self.target); + if let Some(tarball) = builder.ensure(dist::Rls { compiler: self.compiler, target: self.target }) { + install_sh(builder, "rls", self.compiler.stage, Some(self.target), &tarball); } else { builder.info( &format!("skipping Install RLS stage{} ({})", self.compiler.stage, self.target), @@ -215,16 +171,18 @@ install!((self, builder, _config), } }; RustAnalyzer, "rust-analyzer", Self::should_build(_config), only_hosts: true, { - builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }); - install_rust_analyzer(builder, self.compiler.stage, self.target); + let tarball = builder + .ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }) + .expect("missing rust-analyzer"); + install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball); }; Clippy, "clippy", Self::should_build(_config), only_hosts: true, { - builder.ensure(dist::Clippy { compiler: self.compiler, target: self.target }); - install_clippy(builder, self.compiler.stage, self.target); + let tarball = builder.ensure(dist::Clippy { compiler: self.compiler, target: self.target }); + install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball); }; Miri, "miri", Self::should_build(_config), only_hosts: true, { - if builder.ensure(dist::Miri { compiler: self.compiler, target: self.target }).is_some() { - install_miri(builder, self.compiler.stage, self.target); + if let Some(tarball) = builder.ensure(dist::Miri { compiler: self.compiler, target: self.target }) { + install_sh(builder, "miri", self.compiler.stage, Some(self.target), &tarball); } else { builder.info( &format!("skipping Install miri stage{} ({})", self.compiler.stage, self.target), @@ -232,11 +190,11 @@ install!((self, builder, _config), } }; Rustfmt, "rustfmt", Self::should_build(_config), only_hosts: true, { - if builder.ensure(dist::Rustfmt { + if let Some(tarball) = builder.ensure(dist::Rustfmt { compiler: self.compiler, target: self.target - }).is_some() { - install_rustfmt(builder, self.compiler.stage, self.target); + }) { + install_sh(builder, "rustfmt", self.compiler.stage, Some(self.target), &tarball); } else { builder.info( &format!("skipping Install Rustfmt stage{} ({})", self.compiler.stage, self.target), @@ -244,20 +202,20 @@ install!((self, builder, _config), } }; Analysis, "analysis", Self::should_build(_config), only_hosts: false, { - builder.ensure(dist::Analysis { + let tarball = builder.ensure(dist::Analysis { // Find the actual compiler (handling the full bootstrap option) which // produced the save-analysis data because that data isn't copied // through the sysroot uplifting. compiler: builder.compiler_for(builder.top_stage, builder.config.build, self.target), target: self.target - }); - install_analysis(builder, self.compiler.stage, self.target); + }).expect("missing analysis"); + install_sh(builder, "analysis", self.compiler.stage, Some(self.target), &tarball); }; Rustc, "src/librustc", true, only_hosts: true, { - builder.ensure(dist::Rustc { + let tarball = builder.ensure(dist::Rustc { compiler: builder.compiler(builder.top_stage, self.target), }); - install_rustc(builder, self.compiler.stage, self.target); + install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball); }; ); @@ -282,7 +240,7 @@ impl Step for Src { } fn run(self, builder: &Builder<'_>) { - builder.ensure(dist::Src); - install_src(builder, self.stage); + let tarball = builder.ensure(dist::Src); + install_sh(builder, "src", self.stage, None, &tarball); } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 06ccd72186d..a47ddfbcc1f 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -142,6 +142,7 @@ mod native; mod run; mod sanity; mod setup; +mod tarball; mod test; mod tool; mod toolstate; @@ -1068,10 +1069,6 @@ impl Build { self.package_vers(&self.version) } - fn llvm_tools_vers(&self) -> String { - self.rust_version() - } - fn llvm_link_tools_dynamically(&self, target: TargetSelection) -> bool { target.contains("linux-gnu") || target.contains("apple-darwin") } @@ -1182,27 +1179,6 @@ impl Build { paths } - /// Copies a file from `src` to `dst` and doesn't use links, so - /// that the copy can be modified without affecting the original. - pub fn really_copy(&self, src: &Path, dst: &Path) { - if self.config.dry_run { - return; - } - self.verbose_than(1, &format!("Copy {:?} to {:?}", src, dst)); - if src == dst { - return; - } - let _ = fs::remove_file(&dst); - let metadata = t!(src.symlink_metadata()); - if let Err(e) = fs::copy(src, dst) { - panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e) - } - t!(fs::set_permissions(dst, metadata.permissions())); - let atime = FileTime::from_last_access_time(&metadata); - let mtime = FileTime::from_last_modification_time(&metadata); - t!(filetime::set_file_times(dst, atime, mtime)); - } - /// Copies a file from `src` to `dst` pub fn copy(&self, src: &Path, dst: &Path) { if self.config.dry_run { diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 1564cfb0619..fd39944e176 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -66,7 +66,6 @@ check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu: TESTS_IN_2 := \ src/test/ui \ - src/test/compile-fail \ src/tools/linkchecker ci-subset-1: @@ -75,8 +74,7 @@ ci-subset-2: $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_2) TESTS_IN_MINGW_2 := \ - src/test/ui \ - src/test/compile-fail + src/test/ui ci-mingw-subset-1: $(Q)$(BOOTSTRAP) test --stage 2 $(TESTS_IN_MINGW_2:%=--exclude %) diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 4cfcf6ca407..acb941d9540 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -17,6 +17,7 @@ use std::process::Command; use build_helper::{output, t}; +use crate::cache::INTERNER; use crate::config::Target; use crate::Build; @@ -79,18 +80,19 @@ pub fn check(build: &mut Build) { } // We need cmake, but only if we're actually building LLVM or sanitizers. - let building_llvm = build - .hosts - .iter() - .map(|host| { - build - .config - .target_config - .get(host) - .map(|config| config.llvm_config.is_none()) - .unwrap_or(true) - }) - .any(|build_llvm_ourselves| build_llvm_ourselves); + let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) + && build + .hosts + .iter() + .map(|host| { + build + .config + .target_config + .get(host) + .map(|config| config.llvm_config.is_none()) + .unwrap_or(true) + }) + .any(|build_llvm_ourselves| build_llvm_ourselves); if building_llvm || build.config.any_sanitizers_enabled() { cmd_finder.must_have("cmake"); } @@ -147,10 +149,12 @@ pub fn check(build: &mut Build) { } } - // Externally configured LLVM requires FileCheck to exist - let filecheck = build.llvm_filecheck(build.build); - if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { - panic!("FileCheck executable {:?} does not exist", filecheck); + if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { + // Externally configured LLVM requires FileCheck to exist + let filecheck = build.llvm_filecheck(build.build); + if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { + panic!("FileCheck executable {:?} does not exist", filecheck); + } } for target in &build.targets { @@ -159,11 +163,7 @@ pub fn check(build: &mut Build) { panic!("the iOS target is only supported on macOS"); } - build - .config - .target_config - .entry(target.clone()) - .or_insert(Target::from_triple(&target.triple)); + build.config.target_config.entry(*target).or_insert(Target::from_triple(&target.triple)); if target.contains("-none-") || target.contains("nvptx") { if build.no_std(*target) == Some(false) { @@ -176,7 +176,7 @@ pub fn check(build: &mut Build) { // If this is a native target (host is also musl) and no musl-root is given, // fall back to the system toolchain in /usr before giving up if build.musl_root(*target).is_none() && build.config.build == *target { - let target = build.config.target_config.entry(target.clone()).or_default(); + let target = build.config.target_config.entry(*target).or_default(); target.musl_root = Some("/usr".into()); } match build.musl_libdir(*target) { diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs new file mode 100644 index 00000000000..7fb03056f1b --- /dev/null +++ b/src/bootstrap/tarball.rs @@ -0,0 +1,333 @@ +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +use build_helper::t; + +use crate::builder::Builder; + +#[derive(Copy, Clone)] +pub(crate) enum OverlayKind { + Rust, + LLVM, + Cargo, + Clippy, + Miri, + Rustfmt, + RLS, + RustAnalyzer, +} + +impl OverlayKind { + fn legal_and_readme(&self) -> &[&str] { + match self { + OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], + OverlayKind::LLVM => { + &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] + } + OverlayKind::Cargo => &[ + "src/tools/cargo/README.md", + "src/tools/cargo/LICENSE-MIT", + "src/tools/cargo/LICENSE-APACHE", + "src/tools/cargo/LICENSE-THIRD-PARTY", + ], + OverlayKind::Clippy => &[ + "src/tools/clippy/README.md", + "src/tools/clippy/LICENSE-APACHE", + "src/tools/clippy/LICENSE-MIT", + ], + OverlayKind::Miri => &[ + "src/tools/miri/README.md", + "src/tools/miri/LICENSE-APACHE", + "src/tools/miri/LICENSE-MIT", + ], + OverlayKind::Rustfmt => &[ + "src/tools/rustfmt/README.md", + "src/tools/rustfmt/LICENSE-APACHE", + "src/tools/rustfmt/LICENSE-MIT", + ], + OverlayKind::RLS => &[ + "src/tools/rls/README.md", + "src/tools/rls/LICENSE-APACHE", + "src/tools/rls/LICENSE-MIT", + ], + OverlayKind::RustAnalyzer => &[ + "src/tools/rust-analyzer/README.md", + "src/tools/rust-analyzer/LICENSE-APACHE", + "src/tools/rust-analyzer/LICENSE-MIT", + ], + } + } + + fn version(&self, builder: &Builder<'_>) -> String { + match self { + OverlayKind::Rust => builder.rust_version(), + OverlayKind::LLVM => builder.rust_version(), + OverlayKind::Cargo => { + builder.cargo_info.version(builder, &builder.release_num("cargo")) + } + OverlayKind::Clippy => { + builder.clippy_info.version(builder, &builder.release_num("clippy")) + } + OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), + OverlayKind::Rustfmt => { + builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) + } + OverlayKind::RLS => builder.rls_info.version(builder, &builder.release_num("rls")), + OverlayKind::RustAnalyzer => builder + .rust_analyzer_info + .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), + } + } +} + +pub(crate) struct Tarball<'a> { + builder: &'a Builder<'a>, + + pkgname: String, + component: String, + target: Option<String>, + product_name: String, + overlay: OverlayKind, + + temp_dir: PathBuf, + image_dir: PathBuf, + overlay_dir: PathBuf, + + include_target_in_component_name: bool, + is_preview: bool, +} + +impl<'a> Tarball<'a> { + pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { + Self::new_inner(builder, component, Some(target.into())) + } + + pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self { + Self::new_inner(builder, component, None) + } + + fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option<String>) -> Self { + let pkgname = crate::dist::pkgname(builder, component); + + let mut temp_dir = builder.out.join("tmp").join("tarball").join(component); + if let Some(target) = &target { + temp_dir = temp_dir.join(target); + } + let _ = std::fs::remove_dir_all(&temp_dir); + + let image_dir = temp_dir.join("image"); + let overlay_dir = temp_dir.join("overlay"); + + Self { + builder, + + pkgname, + component: component.into(), + target, + product_name: "Rust".into(), + overlay: OverlayKind::Rust, + + temp_dir, + image_dir, + overlay_dir, + + include_target_in_component_name: false, + is_preview: false, + } + } + + pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) { + self.overlay = overlay; + } + + pub(crate) fn set_product_name(&mut self, name: &str) { + self.product_name = name.into(); + } + + pub(crate) fn include_target_in_component_name(&mut self, include: bool) { + self.include_target_in_component_name = include; + } + + pub(crate) fn is_preview(&mut self, is: bool) { + self.is_preview = is; + } + + pub(crate) fn image_dir(&self) -> &Path { + t!(std::fs::create_dir_all(&self.image_dir)); + &self.image_dir + } + + pub(crate) fn add_file(&self, src: impl AsRef<Path>, destdir: impl AsRef<Path>, perms: u32) { + // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply + // uses the base directory as the destination directory. + let destdir = if destdir.as_ref() == Path::new(".") { + self.image_dir.clone() + } else { + self.image_dir.join(destdir.as_ref()) + }; + + t!(std::fs::create_dir_all(&destdir)); + self.builder.install(src.as_ref(), &destdir, perms); + } + + pub(crate) fn add_renamed_file( + &self, + src: impl AsRef<Path>, + destdir: impl AsRef<Path>, + new_name: &str, + ) { + let destdir = self.image_dir.join(destdir.as_ref()); + t!(std::fs::create_dir_all(&destdir)); + self.builder.copy(src.as_ref(), &destdir.join(new_name)); + } + + pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef<Path>) { + for file in self.overlay.legal_and_readme() { + self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644); + } + } + + pub(crate) fn add_dir(&self, src: impl AsRef<Path>, dest: impl AsRef<Path>) { + let dest = self.image_dir.join(dest.as_ref()); + + t!(std::fs::create_dir_all(&dest)); + self.builder.cp_r(src.as_ref(), &dest); + } + + pub(crate) fn generate(self) -> GeneratedTarball { + let mut component_name = self.component.clone(); + if self.is_preview { + component_name.push_str("-preview"); + } + if self.include_target_in_component_name { + component_name.push('-'); + component_name.push_str( + &self + .target + .as_ref() + .expect("include_target_in_component_name used in a targetless tarball"), + ); + } + + self.run(|this, cmd| { + cmd.arg("generate") + .arg("--image-dir") + .arg(&this.image_dir) + .arg(format!("--component-name={}", &component_name)); + this.non_bare_args(cmd); + }) + } + + pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball { + let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string(); + for tarball in &tarballs[1..] { + input_tarballs.push(","); + input_tarballs.push(&tarball.path); + } + + self.run(|this, cmd| { + cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs); + this.non_bare_args(cmd); + }) + } + + pub(crate) fn bare(self) -> GeneratedTarball { + // Bare tarballs should have the top level directory match the package + // name, not "image". We rename the image directory just before passing + // into rust-installer. + let dest = self.temp_dir.join(self.package_name()); + t!(std::fs::rename(&self.image_dir, &dest)); + + self.run(|this, cmd| { + cmd.arg("tarball") + .arg("--input") + .arg(&dest) + .arg("--output") + .arg(crate::dist::distdir(this.builder).join(this.package_name())); + }) + } + + fn package_name(&self) -> String { + if let Some(target) = &self.target { + format!("{}-{}", self.pkgname, target) + } else { + self.pkgname.clone() + } + } + + fn non_bare_args(&self, cmd: &mut Command) { + cmd.arg("--rel-manifest-dir=rustlib") + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg(format!("--product-name={}", self.product_name)) + .arg(format!("--success-message={} installed.", self.component)) + .arg(format!("--package-name={}", self.package_name())) + .arg("--non-installed-overlay") + .arg(&self.overlay_dir) + .arg("--output-dir") + .arg(crate::dist::distdir(self.builder)); + } + + fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball { + t!(std::fs::create_dir_all(&self.overlay_dir)); + self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); + if let Some(sha) = self.builder.rust_sha() { + self.builder.create(&self.overlay_dir.join("git-commit-hash"), &sha); + } + for file in self.overlay.legal_and_readme() { + self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); + } + + let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); + + let package_name = self.package_name(); + self.builder.info(&format!("Dist {}", package_name)); + let _time = crate::util::timeit(self.builder); + + build_cli(&self, &mut cmd); + cmd.arg("--work-dir").arg(&self.temp_dir); + if let Some(formats) = &self.builder.config.dist_compression_formats { + assert!(!formats.is_empty(), "dist.compression-formats can't be empty"); + cmd.arg("--compression-formats").arg(formats.join(",")); + } + self.builder.run(&mut cmd); + + // Use either the first compression format defined, or "gz" as the default. + let ext = self + .builder + .config + .dist_compression_formats + .as_ref() + .and_then(|formats| formats.get(0)) + .map(|s| s.as_str()) + .unwrap_or("gz"); + + GeneratedTarball { + path: crate::dist::distdir(self.builder).join(format!("{}.tar.{}", package_name, ext)), + decompressed_output: self.temp_dir.join(package_name), + work: self.temp_dir, + } + } +} + +#[derive(Debug, Clone)] +pub struct GeneratedTarball { + path: PathBuf, + decompressed_output: PathBuf, + work: PathBuf, +} + +impl GeneratedTarball { + pub(crate) fn tarball(&self) -> &Path { + &self.path + } + + pub(crate) fn decompressed_output(&self) -> &Path { + &self.decompressed_output + } + + pub(crate) fn work_dir(&self) -> &Path { + &self.work + } +} diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 78b5de7897d..33e252a63c9 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -869,12 +869,6 @@ default_test_with_compare_mode!(Ui { compare_mode: "nll" }); -default_test!(CompileFail { - path: "src/test/compile-fail", - mode: "compile-fail", - suite: "compile-fail" -}); - default_test!(RunPassValgrind { path: "src/test/run-pass-valgrind", mode: "run-pass-valgrind", @@ -897,7 +891,12 @@ default_test!(Incremental { suite: "incremental" }); -default_test!(Debuginfo { path: "src/test/debuginfo", mode: "debuginfo", suite: "debuginfo" }); +default_test_with_compare_mode!(Debuginfo { + path: "src/test/debuginfo", + mode: "debuginfo", + suite: "debuginfo", + compare_mode: "split-dwarf" +}); host_test!(UiFullDeps { path: "src/test/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); @@ -1963,8 +1962,8 @@ impl Step for Distcheck { builder.ensure(dist::Src); let mut cmd = Command::new("tar"); - cmd.arg("-xzf") - .arg(builder.ensure(dist::PlainSourceTarball)) + cmd.arg("-xf") + .arg(builder.ensure(dist::PlainSourceTarball).tarball()) .arg("--strip-components=1") .current_dir(&dir); builder.run(&mut cmd); @@ -1987,8 +1986,8 @@ impl Step for Distcheck { t!(fs::create_dir_all(&dir)); let mut cmd = Command::new("tar"); - cmd.arg("-xzf") - .arg(builder.ensure(dist::Src)) + cmd.arg("-xf") + .arg(builder.ensure(dist::Src).tarball()) .arg("--strip-components=1") .current_dir(&dir); builder.run(&mut cmd); diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 290e3744852..dc786249d99 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -563,7 +563,7 @@ impl Step for Cargo { } fn run(self, builder: &Builder<'_>) -> PathBuf { - builder + let cargo_bin_path = builder .ensure(ToolBuild { compiler: self.compiler, target: self.target, @@ -574,7 +574,40 @@ impl Step for Cargo { source_type: SourceType::Submodule, extra_features: Vec::new(), }) - .expect("expected to build -- essential tool") + .expect("expected to build -- essential tool"); + + let build_cred = |name, path| { + // These credential helpers are currently experimental. + // Any build failures will be ignored. + let _ = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: name, + mode: Mode::ToolRustc, + path, + is_optional_tool: true, + source_type: SourceType::Submodule, + extra_features: Vec::new(), + }); + }; + + if self.target.contains("windows") { + build_cred( + "cargo-credential-wincred", + "src/tools/cargo/crates/credential/cargo-credential-wincred", + ); + } + if self.target.contains("apple-darwin") { + build_cred( + "cargo-credential-macos-keychain", + "src/tools/cargo/crates/credential/cargo-credential-macos-keychain", + ); + } + build_cred( + "cargo-credential-1password", + "src/tools/cargo/crates/credential/cargo-credential-1password", + ); + cargo_bin_path } } |