diff options
| -rw-r--r-- | src/bootstrap/builder.rs | 211 | ||||
| -rw-r--r-- | src/bootstrap/native.rs | 225 | ||||
| -rw-r--r-- | src/bootstrap/util.rs | 8 |
3 files changed, 222 insertions, 222 deletions
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 761bd66b505..a0ce200883d 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -2,13 +2,14 @@ use std::any::{type_name, Any}; use std::cell::{Cell, RefCell}; use std::collections::BTreeSet; use std::env; -use std::ffi::OsStr; +use std::ffi::{OsStr, OsString}; use std::fmt::{Debug, Write}; -use std::fs; +use std::fs::{self, File}; use std::hash::Hash; +use std::io::{BufRead, BufReader, ErrorKind}; use std::ops::Deref; use std::path::{Component, Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; use crate::cache::{Cache, Interned, INTERNER}; @@ -29,7 +30,8 @@ use crate::{Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; // FIXME: replace with std::lazy after it gets stabilized and reaches beta -use once_cell::sync::Lazy; +use once_cell::sync::{Lazy, OnceCell}; +use xz2::bufread::XzDecoder; pub struct Builder<'a> { pub build: &'a Build, @@ -758,6 +760,207 @@ impl<'a> Builder<'a> { StepDescription::run(v, self, paths); } + /// Modifies the interpreter section of 'fname' to fix the dynamic linker, + /// or the RPATH section, to fix the dynamic library search path + /// + /// This is only required on NixOS and uses the PatchELF utility to + /// change the interpreter/RPATH of ELF executables. + /// + /// Please see https://nixos.org/patchelf.html for more information + pub(crate) fn fix_bin_or_dylib(&self, fname: &Path) { + // FIXME: cache NixOS detection? + match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { + Err(_) => return, + Ok(output) if !output.status.success() => return, + Ok(output) => { + let mut s = output.stdout; + if s.last() == Some(&b'\n') { + s.pop(); + } + if s != b"Linux" { + return; + } + } + } + + // If the user has asked binaries to be patched for Nix, then + // don't check for NixOS or `/lib`, just continue to the patching. + // FIXME: shouldn't this take precedence over the `uname` check above? + if !self.config.patch_binaries_for_nix { + // Use `/etc/os-release` instead of `/etc/NIXOS`. + // The latter one does not exist on NixOS when using tmpfs as root. + const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""]; + let os_release = match File::open("/etc/os-release") { + Err(e) if e.kind() == ErrorKind::NotFound => return, + Err(e) => panic!("failed to access /etc/os-release: {}", e), + Ok(f) => f, + }; + if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) { + return; + } + if Path::new("/lib").exists() { + return; + } + } + + // At this point we're pretty sure the user is running NixOS or using Nix + println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); + + // Only build `.nix-deps` once. + static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new(); + let mut nix_build_succeeded = true; + let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { + // Run `nix-build` to "build" each dependency (which will likely reuse + // the existing `/nix/store` copy, or at most download a pre-built copy). + // + // Importantly, we create a gc-root called `.nix-deps` in the `build/` + // directory, but still reference the actual `/nix/store` path in the rpath + // as it makes it significantly more robust against changes to the location of + // the `.nix-deps` location. + // + // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). + // zlib: Needed as a system dependency of `libLLVM-*.so`. + // patchelf: Needed for patching ELF binaries (see doc comment above). + let nix_deps_dir = self.out.join(".nix-deps"); + const NIX_EXPR: &str = " + with (import <nixpkgs> {}); + symlinkJoin { + name = \"rust-stage0-dependencies\"; + paths = [ + zlib + patchelf + stdenv.cc.bintools + ]; + } + "; + nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[ + Path::new("-E"), + Path::new(NIX_EXPR), + Path::new("-o"), + &nix_deps_dir, + ])); + nix_deps_dir + }); + if !nix_build_succeeded { + return; + } + + let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); + let rpath_entries = { + // ORIGIN is a relative default, all binary and dynamic libraries we ship + // appear to have this (even when `../lib` is redundant). + // NOTE: there are only two paths here, delimited by a `:` + let mut entries = OsString::from("$ORIGIN/../lib:"); + entries.push(t!(fs::canonicalize(nix_deps_dir))); + entries.push("/lib"); + entries + }; + patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); + if !fname.extension().map_or(false, |ext| ext == "so") { + // Finally, set the corret .interp for binaries + let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); + // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ... + let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); + patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); + } + + self.try_run(patchelf.arg(fname)); + } + + pub(crate) fn download_component(&self, base: &str, url: &str, dest_path: &Path) { + // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. + let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); + // FIXME: support `do_verify` (only really needed for nightly rustfmt) + // FIXME: support non-utf8 paths? + self.download_with_retries(tempfile.to_str().unwrap(), &format!("{}/{}", base, url)); + t!(std::fs::rename(&tempfile, dest_path)); + } + + fn download_with_retries(&self, tempfile: &str, url: &str) { + println!("downloading {}", url); + // Try curl. If that fails and we are on windows, fallback to PowerShell. + if !self.check_run(Command::new("curl").args(&[ + "-#", + "-y", + "30", + "-Y", + "10", // timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", + "30", // timeout if cannot connect within 30 seconds + "--retry", + "3", + "-Sf", + "-o", + tempfile, + url, + ])) { + if self.build.build.contains("windows-msvc") { + println!("Fallback to PowerShell"); + for _ in 0..3 { + if self.try_run(Command::new("PowerShell.exe").args(&[ + "/nologo", + "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempfile + ), + ])) { + return; + } + println!("\nspurious failure, trying again"); + } + } + std::process::exit(1); + } + } + + pub(crate) fn unpack(&self, tarball: &Path, dst: &Path) { + println!("extracting {} to {}", tarball.display(), dst.display()); + if !dst.exists() { + t!(fs::create_dir_all(dst)); + } + + // FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild + const MATCH: &str = "rust-dev"; + + // `tarball` ends with `.tar.xz`; strip that suffix + // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` + let uncompressed_filename = + Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); + let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); + + // decompress the file + let data = t!(File::open(tarball)); + let decompressor = XzDecoder::new(BufReader::new(data)); + + let mut tar = tar::Archive::new(decompressor); + for member in t!(tar.entries()) { + let mut member = t!(member); + let original_path = t!(member.path()).into_owned(); + // skip the top-level directory + if original_path == directory_prefix { + continue; + } + let mut short_path = t!(original_path.strip_prefix(directory_prefix)); + if !short_path.starts_with(MATCH) { + continue; + } + short_path = t!(short_path.strip_prefix(MATCH)); + let dst_path = dst.join(short_path); + self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); + if !t!(member.unpack_in(dst)) { + panic!("path traversal attack ??"); + } + let src_path = dst.join(original_path); + if src_path.is_dir() && dst_path.exists() { + continue; + } + t!(fs::rename(src_path, dst_path)); + } + t!(fs::remove_dir_all(dst.join(directory_prefix))); + } + /// Obtain a compiler at a given stage and for a given host. Explicitly does /// not take `Compiler` since all `Compiler` instances are meant to be /// obtained through this function, since it ensures that they are valid diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 09b8cbe9014..5bb94f297c8 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -12,16 +12,13 @@ use std::env; use std::env::consts::EXE_EXTENSION; use std::ffi::{OsStr, OsString}; use std::fs::{self, File}; -use std::io::{self, BufRead, BufReader, ErrorKind}; +use std::io; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; - -use once_cell::sync::OnceCell; -use xz2::bufread::XzDecoder; +use std::process::Command; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::config::TargetSelection; -use crate::util::{self, exe, output, t, up_to_date}; +use crate::util::{self, exe, output, program_out_of_date, t, up_to_date}; use crate::{CLang, GitRepo}; pub struct Meta { @@ -151,13 +148,13 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { if program_out_of_date(&llvm_stamp, &key) && !config.dry_run { download_ci_llvm(builder, &llvm_sha); for binary in ["llvm-config", "FileCheck"] { - fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary)); + builder.fix_bin_or_dylib(&llvm_root.join("bin").join(binary)); } let llvm_lib = llvm_root.join("lib"); for entry in t!(fs::read_dir(&llvm_lib)) { let lib = t!(entry).path(); if lib.extension().map_or(false, |ext| ext == "so") { - fix_bin_or_dylib(builder, &lib); + builder.fix_bin_or_dylib(&lib); } } t!(fs::write(llvm_stamp, key)); @@ -182,218 +179,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple); let tarball = rustc_cache.join(&filename); if !tarball.exists() { - download_component(builder, base, &format!("{}/{}", url, filename), &tarball); + builder.download_component(base, &format!("{}/{}", url, filename), &tarball); } let llvm_root = builder.config.ci_llvm_root(); - unpack(builder, &tarball, &llvm_root); -} - -/// Modifies the interpreter section of 'fname' to fix the dynamic linker, -/// or the RPATH section, to fix the dynamic library search path -/// -/// This is only required on NixOS and uses the PatchELF utility to -/// change the interpreter/RPATH of ELF executables. -/// -/// Please see https://nixos.org/patchelf.html for more information -fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) { - // FIXME: cache NixOS detection? - match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { - Err(_) => return, - Ok(output) if !output.status.success() => return, - Ok(output) => { - let mut s = output.stdout; - if s.last() == Some(&b'\n') { - s.pop(); - } - if s != b"Linux" { - return; - } - } - } - - // If the user has asked binaries to be patched for Nix, then - // don't check for NixOS or `/lib`, just continue to the patching. - // FIXME: shouldn't this take precedence over the `uname` check above? - if !builder.config.patch_binaries_for_nix { - // Use `/etc/os-release` instead of `/etc/NIXOS`. - // The latter one does not exist on NixOS when using tmpfs as root. - const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""]; - let os_release = match File::open("/etc/os-release") { - Err(e) if e.kind() == ErrorKind::NotFound => return, - Err(e) => panic!("failed to access /etc/os-release: {}", e), - Ok(f) => f, - }; - if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) { - return; - } - if Path::new("/lib").exists() { - return; - } - } - - // At this point we're pretty sure the user is running NixOS or using Nix - println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); - - // Only build `.nix-deps` once. - static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new(); - let mut nix_build_succeeded = true; - let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { - // Run `nix-build` to "build" each dependency (which will likely reuse - // the existing `/nix/store` copy, or at most download a pre-built copy). - // - // Importantly, we create a gc-root called `.nix-deps` in the `build/` - // directory, but still reference the actual `/nix/store` path in the rpath - // as it makes it significantly more robust against changes to the location of - // the `.nix-deps` location. - // - // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). - // zlib: Needed as a system dependency of `libLLVM-*.so`. - // patchelf: Needed for patching ELF binaries (see doc comment above). - let nix_deps_dir = builder.out.join(".nix-deps"); - const NIX_EXPR: &str = " - with (import <nixpkgs> {}); - symlinkJoin { - name = \"rust-stage0-dependencies\"; - paths = [ - zlib - patchelf - stdenv.cc.bintools - ]; - } - "; - nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[ - Path::new("-E"), - Path::new(NIX_EXPR), - Path::new("-o"), - &nix_deps_dir, - ])); - nix_deps_dir - }); - if !nix_build_succeeded { - return; - } - - let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); - let rpath_entries = { - // ORIGIN is a relative default, all binary and dynamic libraries we ship - // appear to have this (even when `../lib` is redundant). - // NOTE: there are only two paths here, delimited by a `:` - let mut entries = OsString::from("$ORIGIN/../lib:"); - entries.push(t!(fs::canonicalize(nix_deps_dir))); - entries.push("/lib"); - entries - }; - patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); - if !fname.extension().map_or(false, |ext| ext == "so") { - // Finally, set the corret .interp for binaries - let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); - // FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ... - let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); - patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); - } - - builder.try_run(patchelf.arg(fname)); -} - -fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) { - // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. - let tempfile = builder.tempdir().join(dest_path.file_name().unwrap()); - // FIXME: support `do_verify` (only really needed for nightly rustfmt) - // FIXME: support non-utf8 paths? - download_with_retries(builder, tempfile.to_str().unwrap(), &format!("{}/{}", base, url)); - t!(std::fs::rename(&tempfile, dest_path)); -} - -fn download_with_retries(builder: &Builder<'_>, tempfile: &str, url: &str) { - println!("downloading {}", url); - // Try curl. If that fails and we are on windows, fallback to PowerShell. - if !builder.check_run(Command::new("curl").args(&[ - "-#", - "-y", - "30", - "-Y", - "10", // timeout if speed is < 10 bytes/sec for > 30 seconds - "--connect-timeout", - "30", // timeout if cannot connect within 30 seconds - "--retry", - "3", - "-Sf", - "-o", - tempfile, - url, - ])) { - if builder.build.build.contains("windows-msvc") { - println!("Fallback to PowerShell"); - for _ in 0..3 { - if builder.try_run(Command::new("PowerShell.exe").args(&[ - "/nologo", - "-Command", - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - &format!( - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, tempfile - ), - ])) { - return; - } - println!("\nspurious failure, trying again"); - } - } - std::process::exit(1); - } -} - -fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) { - println!("extracting {} to {}", tarball.display(), dst.display()); - if !dst.exists() { - t!(fs::create_dir_all(dst)); - } - - // FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild - const MATCH: &str = "rust-dev"; - - // `tarball` ends with `.tar.xz`; strip that suffix - // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` - let uncompressed_filename = - Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); - let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); - - // decompress the file - let data = t!(File::open(tarball)); - let decompressor = XzDecoder::new(BufReader::new(data)); - - let mut tar = tar::Archive::new(decompressor); - for member in t!(tar.entries()) { - let mut member = t!(member); - let original_path = t!(member.path()).into_owned(); - // skip the top-level directory - if original_path == directory_prefix { - continue; - } - let mut short_path = t!(original_path.strip_prefix(directory_prefix)); - if !short_path.starts_with(MATCH) { - continue; - } - short_path = t!(short_path.strip_prefix(MATCH)); - let dst_path = dst.join(short_path); - builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); - if !t!(member.unpack_in(dst)) { - panic!("path traversal attack ??"); - } - let src_path = dst.join(original_path); - if src_path.is_dir() && dst_path.exists() { - continue; - } - t!(fs::rename(src_path, dst_path)); - } - t!(fs::remove_dir_all(dst.join(directory_prefix))); -} - -fn program_out_of_date(stamp: &Path, key: &str) -> bool { - if !stamp.exists() { - return true; - } - t!(fs::read_to_string(stamp)) != key + builder.unpack(&tarball, &llvm_root); } #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index 710b3588b84..b4691840618 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -115,6 +115,14 @@ impl Drop for TimeIt { } } +/// Used for download caching +pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { + if !stamp.exists() { + return true; + } + t!(fs::read_to_string(stamp)) != key +} + /// Symlinks two directories, using junctions on Windows and normal symlinks on /// Unix. pub fn symlink_dir(config: &Config, src: &Path, dest: &Path) -> io::Result<()> { |