diff options
| -rw-r--r-- | Cargo.lock | 3 | ||||
| -rw-r--r-- | src/bootstrap/Cargo.toml | 3 | ||||
| -rw-r--r-- | src/bootstrap/bootstrap.py | 149 | ||||
| -rw-r--r-- | src/bootstrap/builder.rs | 7 | ||||
| -rw-r--r-- | src/bootstrap/compile.rs | 2 | ||||
| -rw-r--r-- | src/bootstrap/config.rs | 70 | ||||
| -rw-r--r-- | src/bootstrap/dist.rs | 8 | ||||
| -rw-r--r-- | src/bootstrap/lib.rs | 17 | ||||
| -rw-r--r-- | src/bootstrap/native.rs | 293 |
9 files changed, 375 insertions, 177 deletions
diff --git a/Cargo.lock b/Cargo.lock index 30175ae3561..aa2fd230d0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,8 +225,11 @@ dependencies = [ "pretty_assertions", "serde", "serde_json", + "tar", + "tempfile", "toml", "winapi", + "xz2", ] [[package]] diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 4c32547f059..19cd0928395 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -42,10 +42,13 @@ cc = "1.0.69" libc = "0.2" serde = { version = "1.0.8", features = ["derive"] } serde_json = "1.0.2" +tar = "0.4" +tempfile = "3" toml = "0.5" ignore = "0.4.10" opener = "0.5" once_cell = "1.7.2" +xz2 = "0.1" [target.'cfg(windows)'.dependencies.winapi] version = "0.3" diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index ac1c47524fd..e38a574ca23 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -500,81 +500,6 @@ class RustBuild(object): with output(self.rustfmt_stamp()) as rustfmt_stamp: rustfmt_stamp.write(self.stage0_rustfmt.channel()) - # Avoid downloading LLVM twice (once for stage0 and once for the master rustc) - if self.downloading_llvm() and stage0: - # We want the most recent LLVM submodule update to avoid downloading - # LLVM more often than necessary. - # - # This git command finds that commit SHA, looking for bors-authored - # commits that modified src/llvm-project or other relevant version - # stamp files. - # - # This works even in a repository that has not yet initialized - # submodules. - top_level = subprocess.check_output([ - "git", "rev-parse", "--show-toplevel", - ]).decode(sys.getdefaultencoding()).strip() - llvm_sha = subprocess.check_output([ - "git", "rev-list", "--author=bors@rust-lang.org", "-n1", - "--first-parent", "HEAD", - "--", - "{}/src/llvm-project".format(top_level), - "{}/src/bootstrap/download-ci-llvm-stamp".format(top_level), - # the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` - "{}/src/version".format(top_level) - ]).decode(sys.getdefaultencoding()).strip() - llvm_assertions = self.get_toml('assertions', 'llvm') == 'true' - llvm_root = self.llvm_root() - llvm_lib = os.path.join(llvm_root, "lib") - if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)): - self._download_ci_llvm(llvm_sha, llvm_assertions) - for binary in ["llvm-config", "FileCheck"]: - self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary)) - for lib in os.listdir(llvm_lib): - if lib.endswith(".so"): - self.fix_bin_or_dylib(os.path.join(llvm_lib, lib)) - with output(self.llvm_stamp()) as llvm_stamp: - llvm_stamp.write(llvm_sha + str(llvm_assertions)) - - def downloading_llvm(self): - opt = self.get_toml('download-ci-llvm', 'llvm') - # This is currently all tier 1 targets and tier 2 targets with host tools - # (since others may not have CI artifacts) - # https://doc.rust-lang.org/rustc/platform-support.html#tier-1 - supported_platforms = [ - # tier 1 - "aarch64-unknown-linux-gnu", - "i686-pc-windows-gnu", - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-gnu", - "x86_64-pc-windows-msvc", - # tier 2 with host tools - "aarch64-apple-darwin", - "aarch64-pc-windows-msvc", - "aarch64-unknown-linux-musl", - "arm-unknown-linux-gnueabi", - "arm-unknown-linux-gnueabihf", - "armv7-unknown-linux-gnueabihf", - "mips-unknown-linux-gnu", - "mips64-unknown-linux-gnuabi64", - "mips64el-unknown-linux-gnuabi64", - "mipsel-unknown-linux-gnu", - "powerpc-unknown-linux-gnu", - "powerpc64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv64gc-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-unknown-freebsd", - "x86_64-unknown-illumos", - "x86_64-unknown-linux-musl", - "x86_64-unknown-netbsd", - ] - return opt == "true" \ - or (opt == "if-available" and self.build in supported_platforms) - def _download_component_helper( self, filename, pattern, tarball_suffix, stage0=True, key=None ): @@ -606,53 +531,6 @@ class RustBuild(object): ) unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose) - def _download_ci_llvm(self, llvm_sha, llvm_assertions): - if not llvm_sha: - print("error: could not find commit hash for downloading LLVM") - print("help: maybe your repository history is too shallow?") - print("help: consider disabling `download-ci-llvm`") - print("help: or fetch enough history to include one upstream commit") - exit(1) - cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions) - cache_dst = os.path.join(self.build_dir, "cache") - rustc_cache = os.path.join(cache_dst, cache_prefix) - if not os.path.exists(rustc_cache): - os.makedirs(rustc_cache) - - base = "https://ci-artifacts.rust-lang.org" - url = "rustc-builds/{}".format(llvm_sha) - if llvm_assertions: - url = url.replace('rustc-builds', 'rustc-builds-alt') - # ci-artifacts are only stored as .xz, not .gz - if not support_xz(): - print("error: XZ support is required to download LLVM") - print("help: consider disabling `download-ci-llvm` or using python3") - exit(1) - tarball_suffix = '.tar.xz' - filename = "rust-dev-nightly-" + self.build + tarball_suffix - tarball = os.path.join(rustc_cache, filename) - if not os.path.exists(tarball): - help_on_error = "error: failed to download llvm from ci" - help_on_error += "\nhelp: old builds get deleted after a certain time" - help_on_error += "\nhelp: if trying to compile an old commit of rustc," - help_on_error += " disable `download-ci-llvm` in config.toml:" - help_on_error += "\n" - help_on_error += "\n[llvm]" - help_on_error += "\ndownload-ci-llvm = false" - help_on_error += "\n" - get( - base, - "{}/{}".format(url, filename), - tarball, - self.checksums_sha256, - verbose=self.verbose, - do_verify=False, - help_on_error=help_on_error, - ) - unpack(tarball, tarball_suffix, self.llvm_root(), - match="rust-dev", - verbose=self.verbose) - def fix_bin_or_dylib(self, fname): """Modifies the interpreter section of 'fname' to fix the dynamic linker, or the RPATH section, to fix the dynamic library search path @@ -816,17 +694,6 @@ class RustBuild(object): """ return os.path.join(self.bin_root(True), '.rustfmt-stamp') - def llvm_stamp(self): - """Return the path for .llvm-stamp - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp") - True - """ - return os.path.join(self.llvm_root(), '.llvm-stamp') - - def program_out_of_date(self, stamp_path, key): """Check if the given program stamp is out of date""" if not os.path.exists(stamp_path) or self.clean: @@ -856,22 +723,6 @@ class RustBuild(object): subdir = "ci-rustc" return os.path.join(self.build_dir, self.build, subdir) - def llvm_root(self): - """Return the CI LLVM root directory - - >>> rb = RustBuild() - >>> rb.build_dir = "build" - >>> rb.llvm_root() == os.path.join("build", "ci-llvm") - True - - When the 'build' property is given should be a nested directory: - - >>> rb.build = "devel" - >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm") - True - """ - return os.path.join(self.build_dir, self.build, "ci-llvm") - def get_toml(self, key, section=None): """Returns the value of the given key in config.toml, otherwise returns None diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 0c4f3265dbf..2270a0f492d 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -12,7 +12,6 @@ use std::process::Command; use std::time::{Duration, Instant}; use crate::cache::{Cache, Interned, INTERNER}; -use crate::check; use crate::compile; use crate::config::{SplitDebuginfo, TargetSelection}; use crate::dist; @@ -25,6 +24,7 @@ use crate::test; use crate::tool::{self, SourceType}; use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; use crate::EXTRA_CHECK_CFGS; +use crate::{check, Config}; use crate::{Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; @@ -960,6 +960,11 @@ impl<'a> Builder<'a> { None } + /// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`. + pub(crate) fn llvm_link_shared(&self) -> bool { + Config::llvm_link_shared(self) + } + /// Prepares an invocation of `cargo` to be run. /// /// This will create a `Command` that represents a pending execution of diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 45991381dc0..9fbb3a26f2b 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -737,7 +737,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS ); cargo.env("LLVM_STATIC_STDCPP", file); } - if builder.config.llvm_link_shared { + if builder.llvm_link_shared() { cargo.env("LLVM_LINK_SHARED", "1"); } if builder.config.llvm_use_libcxx { diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index f273fb42215..1cffcb29a15 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -3,6 +3,7 @@ //! This module implements parsing `config.toml` configuration files to tweak //! how the build runs. +use std::cell::Cell; use std::cmp; use std::collections::{HashMap, HashSet}; use std::env; @@ -11,7 +12,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::str::FromStr; -use crate::builder::TaskPath; +use crate::builder::{Builder, TaskPath}; use crate::cache::{Interned, INTERNER}; use crate::channel::GitInfo; pub use crate::flags::Subcommand; @@ -68,13 +69,14 @@ pub struct Config { pub test_compare_mode: bool, pub llvm_libunwind: LlvmLibunwind, pub color: Color, + pub patch_binaries_for_nix: bool, pub on_fail: Option<String>, pub stage: u32, pub keep_stage: Vec<u32>, pub keep_stage_std: Vec<u32>, pub src: PathBuf, - // defaults to `config.toml` + /// defaults to `config.toml` pub config: PathBuf, pub jobs: Option<u32>, pub cmd: Subcommand, @@ -95,7 +97,11 @@ pub struct Config { pub llvm_release_debuginfo: bool, pub llvm_version_check: bool, pub llvm_static_stdcpp: bool, - pub llvm_link_shared: bool, + /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. + #[cfg(not(test))] + llvm_link_shared: Cell<Option<bool>>, + #[cfg(test)] + pub llvm_link_shared: Cell<Option<bool>>, pub llvm_clang_cl: Option<String>, pub llvm_targets: Option<String>, pub llvm_experimental_targets: Option<String>, @@ -856,6 +862,7 @@ impl Config { set(&mut config.local_rebuild, build.local_rebuild); set(&mut config.print_step_timings, build.print_step_timings); set(&mut config.print_step_rusage, build.print_step_rusage); + set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix); config.verbose = cmp::max(config.verbose, flags.verbose); @@ -911,7 +918,9 @@ impl Config { set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo); set(&mut config.llvm_version_check, llvm.version_check); set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp); - set(&mut config.llvm_link_shared, llvm.link_shared); + if let Some(v) = llvm.link_shared { + config.llvm_link_shared.set(Some(v)); + } config.llvm_targets = llvm.targets.clone(); config.llvm_experimental_targets = llvm.experimental_targets.clone(); config.llvm_link_jobs = llvm.link_jobs; @@ -981,6 +990,7 @@ impl Config { check_ci_llvm!(llvm.optimize); check_ci_llvm!(llvm.thin_lto); check_ci_llvm!(llvm.release_debuginfo); + // CI-built LLVM can be either dynamic or static. We won't know until we download it. check_ci_llvm!(llvm.link_shared); check_ci_llvm!(llvm.static_libstdcpp); check_ci_llvm!(llvm.targets); @@ -998,26 +1008,14 @@ impl Config { check_ci_llvm!(llvm.clang); check_ci_llvm!(llvm.build_config); check_ci_llvm!(llvm.plugins); - - // CI-built LLVM can be either dynamic or static. - let ci_llvm = config.out.join(&*config.build.triple).join("ci-llvm"); - config.llvm_link_shared = if config.dry_run { - // just assume dynamic for now - true - } else { - let link_type = t!( - std::fs::read_to_string(ci_llvm.join("link-type.txt")), - format!("CI llvm missing: {}", ci_llvm.display()) - ); - link_type == "dynamic" - }; } + // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above. if config.llvm_thin_lto && llvm.link_shared.is_none() { // If we're building with ThinLTO on, by default we want to link // to LLVM shared, to avoid re-doing ThinLTO (which happens in // the link step) with each stage. - config.llvm_link_shared = true; + config.llvm_link_shared.set(Some(true)); } } @@ -1272,6 +1270,42 @@ impl Config { } } + /// The absolute path to the downloaded LLVM artifacts. + pub(crate) fn ci_llvm_root(&self) -> PathBuf { + assert!(self.llvm_from_ci); + self.out.join(&*self.build.triple).join("ci-llvm") + } + + /// Determine whether llvm should be linked dynamically. + /// + /// If `false`, llvm should be linked statically. + /// This is computed on demand since LLVM might have to first be downloaded from CI. + pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool { + let mut opt = builder.config.llvm_link_shared.get(); + if opt.is_none() && builder.config.dry_run { + // just assume static for now - dynamic linking isn't supported on all platforms + return false; + } + + let llvm_link_shared = *opt.get_or_insert_with(|| { + if builder.config.llvm_from_ci { + crate::native::maybe_download_ci_llvm(builder); + let ci_llvm = builder.config.ci_llvm_root(); + let link_type = t!( + std::fs::read_to_string(ci_llvm.join("link-type.txt")), + format!("CI llvm missing: {}", ci_llvm.display()) + ); + link_type == "dynamic" + } else { + // unclear how thought-through this default is, but it maintains compatibility with + // previous behavior + false + } + }); + builder.config.llvm_link_shared.set(opt); + llvm_link_shared + } + pub fn verbose(&self) -> bool { self.verbose > 0 } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e3287e35227..5d812e8b332 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1904,7 +1904,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir // clear why this is the case, though. llvm-config will emit the versioned // paths and we don't want those in the sysroot (as we're expecting // unversioned paths). - if target.contains("apple-darwin") && builder.config.llvm_link_shared { + if target.contains("apple-darwin") && builder.llvm_link_shared() { let src_libdir = builder.llvm_out(target).join("lib"); let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); if llvm_dylib_path.exists() { @@ -1939,7 +1939,7 @@ pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, // We do not need to copy LLVM files into the sysroot if it is not // dynamically linked; it is already included into librustc_llvm // statically. - if builder.config.llvm_link_shared { + if builder.llvm_link_shared() { maybe_install_llvm(builder, target, &dst_libdir); } } @@ -1951,7 +1951,7 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection // We do not need to copy LLVM files into the sysroot if it is not // dynamically linked; it is already included into librustc_llvm // statically. - if builder.config.llvm_link_shared { + if builder.llvm_link_shared() { maybe_install_llvm(builder, target, &dst_libdir); } } @@ -2077,7 +2077,7 @@ impl Step for RustDev { // compiler libraries. let dst_libdir = tarball.image_dir().join("lib"); maybe_install_llvm(builder, target, &dst_libdir); - let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" }; + let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" }; t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir); Some(tarball.generate()) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 59102ad9f50..11be33ed4f0 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -1391,6 +1391,23 @@ impl Build { paths } + pub fn rename(&self, src: &Path, dst: &Path) { + if self.config.dry_run { + return; + } + self.verbose_than(1, &format!("Move {:?} to {:?}", src, dst)); + if src == dst { + return; + } + if let Err(e) = fs::rename(src, dst) { + if e.raw_os_error() == Some(libc::EXDEV) { + self.copy(src, dst); + return; + } + panic!("failed to rename `{}` to `{}`: {}", src.display(), dst.display(), e); + } + } + /// Copies a file from `src` to `dst` pub fn copy(&self, src: &Path, dst: &Path) { if self.config.dry_run { diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 73fb2dad1e3..42b7ef2b7a0 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -12,9 +12,12 @@ use std::env; use std::env::consts::EXE_EXTENSION; use std::ffi::{OsStr, OsString}; use std::fs::{self, File}; -use std::io; +use std::io::{self, BufRead, BufReader, ErrorKind}; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; + +use once_cell::sync::OnceCell; +use xz2::bufread::XzDecoder; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::config::TargetSelection; @@ -62,6 +65,8 @@ pub fn prebuilt_llvm_config( builder: &Builder<'_>, target: TargetSelection, ) -> Result<PathBuf, Meta> { + maybe_download_ci_llvm(builder); + // If we're using a custom LLVM bail out here, but we can only use a // custom LLVM for the build triple. if let Some(config) = builder.config.target_config.get(&target) { @@ -111,6 +116,286 @@ pub fn prebuilt_llvm_config( Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) } +pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { + let config = &builder.config; + if !config.llvm_from_ci { + return; + } + let mut rev_list = Command::new("git"); + rev_list.args(&[ + PathBuf::from("rev-list"), + "--author=bors@rust-lang.org".into(), + "-n1".into(), + "--first-parent".into(), + "HEAD".into(), + "--".into(), + builder.src.join("src/llvm-project"), + builder.src.join("src/bootstrap/download-ci-llvm-stamp"), + // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` + builder.src.join("src/version"), + ]); + let llvm_sha = output(&mut rev_list); + let llvm_sha = llvm_sha.trim(); + + if llvm_sha == "" { + println!("error: could not find commit hash for downloading LLVM"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `download-ci-llvm`"); + println!("help: or fetch enough history to include one upstream commit"); + panic!(); + } + + let llvm_root = config.ci_llvm_root(); + let llvm_stamp = llvm_root.join(".llvm-stamp"); + let key = format!("{}{}", llvm_sha, config.llvm_assertions); + if program_out_of_date(&llvm_stamp, &key) && !config.dry_run { + download_ci_llvm(builder, &llvm_sha); + for binary in ["llvm-config", "FileCheck"] { + fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary)); + } + let llvm_lib = llvm_root.join("lib"); + for entry in t!(fs::read_dir(&llvm_lib)) { + let lib = t!(entry).path(); + if lib.ends_with(".so") { + fix_bin_or_dylib(builder, &lib); + } + } + t!(fs::write(llvm_stamp, key)); + } +} + +fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { + let llvm_assertions = builder.config.llvm_assertions; + + let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); + let cache_dst = builder.out.join("cache"); + let rustc_cache = cache_dst.join(cache_prefix); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + let base = "https://ci-artifacts.rust-lang.org"; + let url = if llvm_assertions { + format!("rustc-builds-alt/{}", llvm_sha) + } else { + format!("rustc-builds/{}", llvm_sha) + }; + let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + download_component(builder, base, &format!("{}/{}", url, filename), &tarball); + } + let llvm_root = builder.config.ci_llvm_root(); + unpack(builder, &tarball, &llvm_root); +} + +/// Modifies the interpreter section of 'fname' to fix the dynamic linker, +/// or the RPATH section, to fix the dynamic library search path +/// +/// This is only required on NixOS and uses the PatchELF utility to +/// change the interpreter/RPATH of ELF executables. +/// +/// Please see https://nixos.org/patchelf.html for more information +fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) { + // FIXME: cache NixOS detection? + match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { + Err(_) => return, + Ok(output) if !output.status.success() => return, + Ok(output) => { + let mut s = output.stdout; + if s.last() == Some(&b'\n') { + s.pop(); + } + if s != b"Linux" { + return; + } + } + } + + // If the user has asked binaries to be patched for Nix, then + // don't check for NixOS or `/lib`, just continue to the patching. + // FIXME: shouldn't this take precedence over the `uname` check above? + if !builder.config.patch_binaries_for_nix { + // Use `/etc/os-release` instead of `/etc/NIXOS`. + // The latter one does not exist on NixOS when using tmpfs as root. + let os_release = match File::open("/etc/os-release") { + Err(e) if e.kind() == ErrorKind::NotFound => return, + Err(e) => panic!("failed to access /etc/os-release: {}", e), + Ok(f) => f, + }; + if !BufReader::new(os_release).lines().any(|l| t!(l).trim() == "ID=nixos") { + return; + } + if Path::new("/lib").exists() { + return; + } + } + + // At this point we're pretty sure the user is running NixOS or using Nix + println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); + + // Only build `.nix-deps` once. + static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new(); + let mut nix_build_succeeded = true; + let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { + // Run `nix-build` to "build" each dependency (which will likely reuse + // the existing `/nix/store` copy, or at most download a pre-built copy). + // + // Importantly, we create a gc-root called `.nix-deps` in the `build/` + // directory, but still reference the actual `/nix/store` path in the rpath + // as it makes it significantly more robust against changes to the location of + // the `.nix-deps` location. + // + // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). + // zlib: Needed as a system dependency of `libLLVM-*.so`. + // patchelf: Needed for patching ELF binaries (see doc comment above). + let nix_deps_dir = builder.out.join(".nix-deps"); + const NIX_EXPR: &str = " + with (import <nixpkgs> {}); + symlinkJoin { + name = \"rust-stage0-dependencies\"; + paths = [ + zlib + patchelf + stdenv.cc.bintools + ]; + } + "; + nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[ + Path::new("-E"), + Path::new(NIX_EXPR), + Path::new("-o"), + &nix_deps_dir, + ])); + nix_deps_dir + }); + if !nix_build_succeeded { + return; + } + + let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); + let rpath_entries = { + // ORIGIN is a relative default, all binary and dynamic libraries we ship + // appear to have this (even when `../lib` is redundant). + // NOTE: there are only two paths here, delimited by a `:` + let mut entries = OsString::from("$ORIGIN/../lib:"); + entries.push(t!(fs::canonicalize(nix_deps_dir))); + entries.push("/lib"); + entries + }; + patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); + if !fname.ends_with(".so") { + // Finally, set the corret .interp for binaries + let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); + // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ... + let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); + patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); + } + + builder.try_run(patchelf.arg(fname)); +} + +fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) { + // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. + let tempfile = t!(tempfile::NamedTempFile::new()); + let temppath = tempfile.path().to_owned(); + drop(tempfile); + let tempfile_str = temppath.to_str().expect("tempdir must be valid unicode"); + // FIXME: support `do_verify` (only really needed for nightly rustfmt) + download_with_retries(builder, tempfile_str, &format!("{}/{}", base, url)); + builder.rename(&temppath, dest_path); +} + +fn download_with_retries(builder: &Builder<'_>, tempdir: &str, url: &str) { + println!("downloading {}", url); + + // FIXME: check if curl is installed instead of skipping straight to powershell + if builder.build.build.contains("windows-msvc") { + for _ in 0..3 { + if builder.try_run(Command::new("PowerShell.exe").args(&[ + "/nologo", + "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempdir + ), + ])) { + return; + } + println!("\nspurious failure, trying again"); + } + } else { + builder.run(Command::new("curl").args(&[ + "-#", + "-y", + "30", + "-Y", + "10", // timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", + "30", // timeout if cannot connect within 30 seconds + "--retry", + "3", + "-Sf", + "-o", + tempdir, + url, + ])); + } +} + +fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) { + println!("extracting {} to {}", tarball.display(), dst.display()); + if !dst.exists() { + t!(fs::create_dir_all(dst)); + } + + // FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild + const MATCH: &str = "rust-dev"; + + // `tarball` ends with `.tar.xz`; strip that suffix + // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` + let uncompressed_filename = + Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); + let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); + + // decompress the file + let data = t!(File::open(tarball)); + let decompressor = XzDecoder::new(BufReader::new(data)); + + let mut tar = tar::Archive::new(decompressor); + for member in t!(tar.entries()) { + let mut member = t!(member); + let original_path = t!(member.path()).into_owned(); + // skip the top-level directory + if original_path == directory_prefix { + continue; + } + let mut short_path = t!(original_path.strip_prefix(directory_prefix)); + if !short_path.starts_with(MATCH) { + continue; + } + short_path = t!(short_path.strip_prefix(MATCH)); + let dst_path = dst.join(short_path); + builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); + if !t!(member.unpack_in(dst)) { + panic!("path traversal attack ??"); + } + let src_path = dst.join(original_path); + if src_path.is_dir() && dst_path.exists() { + continue; + } + t!(fs::rename(src_path, dst_path)); + } + t!(fs::remove_dir_all(dst.join(directory_prefix))); +} + +fn program_out_of_date(stamp: &Path, key: &str) -> bool { + if !stamp.exists() { + return true; + } + t!(fs::read_to_string(stamp)) != key +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Llvm { pub target: TargetSelection, @@ -153,7 +438,7 @@ impl Step for Llvm { }; builder.update_submodule(&Path::new("src").join("llvm-project")); - if builder.config.llvm_link_shared + if builder.llvm_link_shared() && (target.contains("windows") || target.contains("apple-darwin")) { panic!("shared linking to LLVM is not currently supported on {}", target.triple); @@ -255,7 +540,7 @@ impl Step for Llvm { // // If we're not linking rustc to a dynamic LLVM, though, then don't link // tools to it. - if builder.llvm_link_tools_dynamically(target) && builder.config.llvm_link_shared { + if builder.llvm_link_tools_dynamically(target) && builder.llvm_link_shared() { cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); } |