From 282b1e476883fbfd8924535185512199b4eaff3c Mon Sep 17 00:00:00 2001 From: Tuna Date: Wed, 14 Sep 2022 18:17:05 -0500 Subject: Distribute bootstrap in CI artifacts - Add a new `bootstrap` component Originally, we planned to combine this with the `rust-dev` component. However, I realized that would force LLVM to be redownloaded whenever bootstrap is modified. LLVM is a much larger download, so split this to get better caching. - Build bootstrap for all tier 1 and 2 targets --- src/bootstrap/builder.rs | 1 + src/bootstrap/dist.rs | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) (limited to 'src/bootstrap') diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 14e8ebd6876..8ec64657f47 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -723,6 +723,7 @@ impl<'a> Builder<'a> { dist::Miri, dist::LlvmTools, dist::RustDev, + dist::Bootstrap, dist::Extended, // It seems that PlainSourceTarball somehow changes how some of the tools // perceive their dependencies (see #93033) which would invalidate fingerprints diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 1a59b3958f1..679f017c770 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2050,6 +2050,41 @@ impl Step for RustDev { } } +// Tarball intended for internal consumption to ease rustc/std development. +// +// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct Bootstrap { + pub target: TargetSelection, +} + +impl Step for Bootstrap { + type Output = Option; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("bootstrap") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Bootstrap { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + + let tarball = Tarball::new(builder, "bootstrap", &target.triple); + + let bootstrap_outdir = &builder.bootstrap_out; + for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] { + tarball.add_file(bootstrap_outdir.join(file), "bootstrap/bin", 0o755); + } + + Some(tarball.generate()) + } +} + /// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the /// release process to avoid cloning the monorepo and building stuff. /// -- cgit 1.4.1-3-g733a5 From 63f6289db2d90be05acdf615f8039383c52c4ddc Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 14 Sep 2022 18:18:57 -0500 Subject: Fix `--dry-run` for `dist::RustDev` --- src/bootstrap/dist.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 679f017c770..3d03a056c15 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1873,7 +1873,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir let mut cmd = Command::new(llvm_config); cmd.arg("--libfiles"); builder.verbose(&format!("running {:?}", cmd)); - let files = output(&mut cmd); + let files = if builder.config.dry_run { "".into() } else { output(&mut cmd) }; let build_llvm_out = &builder.llvm_out(builder.config.build); let target_llvm_out = &builder.llvm_out(target); for file in files.trim_end().split(' ') { -- cgit 1.4.1-3-g733a5 From 55c040e52964e4ad81b4b2af3c91068a80b92f11 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 14 Sep 2022 18:19:28 -0500 Subject: Make it possible to run bootstrap on a different machine than the one it was built - Default to trying git rev-parse for the root directory CARGO_MANIFEST_DIR is a path on the build machine, not the running machine. Don't require this to succeed, to allow building from a tarball; in that case fall back to CARGO_MANIFEST_DIR. - Set `initial_rustc` to a path based on the path of the running executable, not CARGO_MANIFEST_DIR. We only reset `initial_rustc` if we're sure this isn't the working tree bootstrap was originally built in, since I'm paranoid that setting this in other cases will cause things to break; it's not clear to me when $RUSTC differs from `build/$TARGET/stage0/bin/rustc` (maybe never? but better to be sure). Instead, only set this when a) We are not using a custom rustc. If someone has specified a custom rustc we should respect their wishes. b) We are in a checkout of rust-lang/rust other than the one bootstrap was built in. --- src/bootstrap/config.rs | 48 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 7c062460c4f..b8e776485e6 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -772,21 +772,20 @@ impl Config { // set by build.rs config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); // Undo `src/bootstrap` config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); config.out = PathBuf::from("build"); - config.initial_cargo = PathBuf::from(env!("CARGO")); - config.initial_rustc = PathBuf::from(env!("RUSTC")); - config } pub fn parse(args: &[String]) -> Config { let flags = Flags::parse(&args); - let mut config = Config::default_opts(); + + // Set flags. config.exclude = flags.exclude.into_iter().map(|path| TaskPath::parse(path)).collect(); config.include_default_paths = flags.include_default_paths; config.rustc_error_format = flags.rustc_error_format; @@ -805,7 +804,41 @@ impl Config { config.llvm_profile_use = flags.llvm_profile_use; config.llvm_profile_generate = flags.llvm_profile_generate; + // Infer the rest of the configuration. + + // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, + // running on a completely machine from where it was compiled. + let mut cmd = Command::new("git"); + // NOTE: we cannot support running from outside the repository because the only path we have available + // is set at compile time, which can be wrong if bootstrap was downloaded from source. + // We still support running outside the repository if we find we aren't in a git directory. + cmd.arg("rev-parse").arg("--show-toplevel"); + // Discard stderr because we expect this to fail when building from a tarball. + let output = cmd + .stderr(std::process::Stdio::null()) + .output() + .ok() + .and_then(|output| if output.status.success() { Some(output) } else { None }); + if let Some(output) = output { + let git_root = String::from_utf8(output.stdout).unwrap(); + config.src = PathBuf::from(git_root.trim().to_owned()) + } else { + // We're building from a tarball, not git sources. + // We don't support pre-downloaded bootstrap in this case. + } + + if cfg!(test) { + // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. + config.out = Path::new( + &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), + ) + .parent() + .unwrap() + .to_path_buf(); + } + let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); + config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); #[cfg(test)] @@ -860,6 +893,7 @@ impl Config { config.config = toml_path; let build = toml.build.unwrap_or_default(); + let has_custom_rustc = build.rustc.is_some(); set(&mut config.initial_rustc, build.rustc.map(PathBuf::from)); set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from))); @@ -870,6 +904,12 @@ impl Config { config.out = crate::util::absolute(&config.out); } + if !has_custom_rustc && !config.initial_rustc.starts_with(&config.out) { + config.initial_rustc = config.out.join(config.build.triple).join("stage0/bin/rustc"); + config.initial_cargo = config.out.join(config.build.triple).join("stage0/bin/cargo"); + } + + // NOTE: it's important this comes *after* we set `initial_rustc` just above. if config.dry_run { let dir = config.out.join("tmp-dry-run"); t!(fs::create_dir_all(&dir)); -- cgit 1.4.1-3-g733a5 From b31628ddacda76253b707e230293f13d95b88369 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 14 Sep 2022 18:09:33 -0500 Subject: Don't hardcode the path to `bootstrap_out` The `rust-dev` dist component puts binaries in `bootstrap/bin`, but we expected them to be in `bootstrap/debug` to match cargo's behavior. Rather than making the dist component inconsistent with other components, make bootstrap slightly smarter and allow using any path as long as all the binaries are in the same directory. As a bonus, this greatly simplifies the logic, and makes it possible for the shell scripts to start avoiding python. Co-authored-by: Joshua Nelson --- src/bootstrap/lib.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index cc0cf12bd18..2f2b3aed98e 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -456,19 +456,15 @@ impl Build { .expect("failed to read src/version"); let version = version.trim(); - let bootstrap_out = if std::env::var("BOOTSTRAP_PYTHON").is_ok() { - out.join("bootstrap").join("debug") - } else { - let workspace_target_dir = std::env::var("CARGO_TARGET_DIR") - .map(PathBuf::from) - .unwrap_or_else(|_| src.join("target")); - let bootstrap_out = workspace_target_dir.join("debug"); - if !bootstrap_out.join("rustc").exists() && !cfg!(test) { - // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented - panic!("run `cargo build --bins` before `cargo run`") - } - bootstrap_out - }; + let bootstrap_out = std::env::current_exe() + .expect("could not determine path to running process") + .parent() + .unwrap() + .to_path_buf(); + if !bootstrap_out.join("rustc").exists() && !cfg!(test) { + // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented + panic!("run `cargo build --bins` before `cargo run`") + } let mut build = Build { initial_rustc: config.initial_rustc.clone(), -- cgit 1.4.1-3-g733a5 From 0a1fde953319bee0060268aec946afd64d7845da Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 14 Sep 2022 18:01:18 -0500 Subject: Fix pre-existing bug in exe check --- src/bootstrap/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 2f2b3aed98e..4795ae2f956 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -461,7 +461,7 @@ impl Build { .parent() .unwrap() .to_path_buf(); - if !bootstrap_out.join("rustc").exists() && !cfg!(test) { + if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented panic!("run `cargo build --bins` before `cargo run`") } -- cgit 1.4.1-3-g733a5 From 0bb4e25ec432beb3629280ea65d1d5475b4f3ee1 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 11 Sep 2022 15:14:03 -0500 Subject: Give a better error messages when the rustc shim is missing --- src/bootstrap/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 4795ae2f956..dc53e4b545b 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -463,7 +463,10 @@ impl Build { .to_path_buf(); if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented - panic!("run `cargo build --bins` before `cargo run`") + panic!( + "`rustc` not found in {}, run `cargo build --bins` before `cargo run`", + bootstrap_out.display() + ) } let mut build = Build { -- cgit 1.4.1-3-g733a5 From e248523d2aaff6c74919baada18934a19e997509 Mon Sep 17 00:00:00 2001 From: Tuna Date: Tue, 20 Sep 2022 06:29:34 +0300 Subject: Update src/bootstrap/config.rs Co-authored-by: Bruno Kolenbrander <59372212+mejrs@users.noreply.github.com> --- src/bootstrap/config.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index b8e776485e6..8b657788a68 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -821,7 +821,14 @@ impl Config { .and_then(|output| if output.status.success() { Some(output) } else { None }); if let Some(output) = output { let git_root = String::from_utf8(output.stdout).unwrap(); - config.src = PathBuf::from(git_root.trim().to_owned()) + let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); + let s = git_root.to_str().unwrap(); + + // Bootstrap is quite bad at handling /? in front of paths + config.src = match s.strip_prefix("\\\\?\\") { + Some(p) => PathBuf::from(p), + None => PathBuf::from(git_root), + }; } else { // We're building from a tarball, not git sources. // We don't support pre-downloaded bootstrap in this case. -- cgit 1.4.1-3-g733a5 From 790d9d4dd0d13e254631c17b045a7c6a9587a2ca Mon Sep 17 00:00:00 2001 From: Tuna Date: Tue, 20 Sep 2022 06:39:05 +0300 Subject: Update src/bootstrap/config.rs Co-authored-by: Joshua Nelson --- src/bootstrap/config.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'src/bootstrap') diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 8b657788a68..a6333976f2a 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -821,6 +821,7 @@ impl Config { .and_then(|output| if output.status.success() { Some(output) } else { None }); if let Some(output) = output { let git_root = String::from_utf8(output.stdout).unwrap(); + // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); let s = git_root.to_str().unwrap(); -- cgit 1.4.1-3-g733a5 From 2ef3d172c4c574bd58abca0e8bbe7548c313c60b Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 24 Sep 2022 13:32:19 -0500 Subject: Copy `bootstrap.exe` on Windows, not `bootstrap` --- src/bootstrap/dist.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 3d03a056c15..e02031d09bc 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2078,7 +2078,7 @@ impl Step for Bootstrap { let bootstrap_outdir = &builder.bootstrap_out; for file in &["bootstrap", "llvm-config-wrapper", "rustc", "rustdoc", "sccache-plus-cl"] { - tarball.add_file(bootstrap_outdir.join(file), "bootstrap/bin", 0o755); + tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755); } Some(tarball.generate()) -- cgit 1.4.1-3-g733a5