diff options
| author | Laurențiu Nicola <lnicola@dend.ro> | 2024-06-20 08:03:36 +0300 |
|---|---|---|
| committer | Laurențiu Nicola <lnicola@dend.ro> | 2024-06-20 08:03:36 +0300 |
| commit | 9d2bb7f40ff91c36b590e743a6e9a92153159d1c (patch) | |
| tree | 0343d8f8c519d3e76f27f77fbae414eb88ef2738 /src/bootstrap | |
| parent | 35d0bcd89fa7454093abcf2c7a8624782c269375 (diff) | |
| parent | 3d5d7a24f76006b391d8a53d903ae64c1b4a52d2 (diff) | |
| download | rust-9d2bb7f40ff91c36b590e743a6e9a92153159d1c.tar.gz rust-9d2bb7f40ff91c36b590e743a6e9a92153159d1c.zip | |
Merge from rust-lang/rust
Diffstat (limited to 'src/bootstrap')
37 files changed, 713 insertions, 753 deletions
diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index 4a5d768961f..9c24742cff1 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -18,12 +18,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -181,34 +175,28 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.17" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if", -] +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crypto-common" @@ -331,19 +319,19 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "junction" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca39ef0d69b18e6a2fd14c2f0a1d593200f4a4ed949b240b5917ab51fac754cb" +checksum = "1c9c415a9b7b1e86cd5738f39d34c9e78c765da7fb1756dbd7d31b3b0d2e7afa" dependencies = [ "scopeguard", - "winapi", + "windows-sys", ] [[package]] name = "libc" -version = "0.2.151" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "linux-raw-sys" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index ca0d1fa5bd0..32dd3efa7a6 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -7,6 +7,7 @@ default-run = "bootstrap" [features] build-metrics = ["sysinfo"] +bootstrap-self-test = [] # enabled in the bootstrap unit tests [lib] path = "src/lib.rs" diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 077db44ae54..fb3c8627043 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -6,7 +6,7 @@ and some of the technical details of the build system. Note that this README only covers internal information, not how to use the tool. Please check [bootstrapping dev guide][bootstrapping-dev-guide] for further information. -[bootstrapping-dev-guide]: https://rustc-dev-guide.rust-lang.org/building/bootstrapping.html +[bootstrapping-dev-guide]: https://rustc-dev-guide.rust-lang.org/building/bootstrapping/intro.html ## Introduction diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index e60e8f0aa1f..9861121aac0 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -599,6 +599,12 @@ class RustBuild(object): print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs') p = Pool(pool_size) try: + # FIXME: A cheap workaround for https://github.com/rust-lang/rust/issues/125578, + # remove this once the issue is closed. + bootstrap_out = self.bootstrap_out() + if os.path.exists(bootstrap_out): + shutil.rmtree(bootstrap_out) + p.map(unpack_component, tarballs_download_info) finally: p.close() @@ -864,6 +870,16 @@ class RustBuild(object): return line[start + 1:end] return None + def bootstrap_out(self): + """Return the path of the bootstrap build artifacts + + >>> rb = RustBuild() + >>> rb.build_dir = "build" + >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap") + True + """ + return os.path.join(self.build_dir, "bootstrap") + def bootstrap_binary(self): """Return the path of the bootstrap binary @@ -873,7 +889,7 @@ class RustBuild(object): ... "debug", "bootstrap") True """ - return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") + return os.path.join(self.bootstrap_out(), "debug", "bootstrap") def build_bootstrap(self): """Build bootstrap""" diff --git a/src/bootstrap/defaults/config.compiler.toml b/src/bootstrap/defaults/config.compiler.toml index fd2da246990..789586b58f7 100644 --- a/src/bootstrap/defaults/config.compiler.toml +++ b/src/bootstrap/defaults/config.compiler.toml @@ -19,8 +19,6 @@ lto = "off" # Forces frame pointers to be used with `-Cforce-frame-pointers`. # This can be helpful for profiling at a small performance cost. frame-pointers = true -# Build the llvm-bitcode-linker as it is required for running nvptx tests -llvm-bitcode-linker = true [llvm] # Having this set to true disrupts compiler development workflows for people who use `llvm.download-ci-llvm = true` diff --git a/src/bootstrap/defaults/config.dist.toml b/src/bootstrap/defaults/config.dist.toml index f3c6ffc9bd5..d06930f2b9d 100644 --- a/src/bootstrap/defaults/config.dist.toml +++ b/src/bootstrap/defaults/config.dist.toml @@ -16,7 +16,7 @@ download-ci-llvm = false # Make sure they don't get set when installing from source. channel = "nightly" download-rustc = false -# Build the llvm-bitcode-linker as it is required for running nvptx tests +# Build the llvm-bitcode-linker llvm-bitcode-linker = true [dist] diff --git a/src/bootstrap/defaults/config.library.toml b/src/bootstrap/defaults/config.library.toml index 4a1a49b7275..087544397f5 100644 --- a/src/bootstrap/defaults/config.library.toml +++ b/src/bootstrap/defaults/config.library.toml @@ -10,8 +10,6 @@ bench-stage = 0 incremental = true # Make the compiler and standard library faster to build, at the expense of a ~20% runtime slowdown. lto = "off" -# Build the llvm-bitcode-linker as it is required for running nvptx tests -llvm-bitcode-linker = true [llvm] # Will download LLVM from CI if available on your platform. diff --git a/src/bootstrap/defaults/config.tools.toml b/src/bootstrap/defaults/config.tools.toml index 94c8b724cbf..6e6c3660027 100644 --- a/src/bootstrap/defaults/config.tools.toml +++ b/src/bootstrap/defaults/config.tools.toml @@ -12,8 +12,6 @@ incremental = true # Using these defaults will download the stage2 compiler (see `download-rustc` # setting) and the stage2 toolchain should therefore be used for these defaults. download-rustc = "if-unchanged" -# Build the llvm-bitcode-linker as it is required for running nvptx tests -llvm-bitcode-linker = true [build] # Document with the in-tree rustdoc by default, since `download-rustc` makes it quick to compile. diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp index bd1f9699c3c..e4dbd3a13fe 100644 --- a/src/bootstrap/download-ci-llvm-stamp +++ b/src/bootstrap/download-ci-llvm-stamp @@ -1,4 +1,4 @@ Change this file to make users of the `download-ci-llvm` configuration download a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. -Last change is for: https://github.com/rust-lang/rust/pull/120761 +Last change is for: https://github.com/rust-lang/rust/pull/125141 diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index fc433bc5843..cab37e0da47 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -59,17 +59,18 @@ check-aux: library/alloc \ --no-doc # Some doctests have intentional memory leaks. + # Some use file system operations to demonstrate dealing with `Result`. $(Q)MIRIFLAGS="-Zmiri-ignore-leaks -Zmiri-disable-isolation" \ $(BOOTSTRAP) miri --stage 2 \ library/core \ library/alloc \ --doc - # In `std` we cannot test everything. - $(Q)MIRIFLAGS="-Zmiri-disable-isolation" BOOTSTRAP_SKIP_TARGET_SANITY=1 \ + # In `std` we cannot test everything, so we skip some modules. + $(Q)MIRIFLAGS="-Zmiri-disable-isolation" \ $(BOOTSTRAP) miri --stage 2 library/std \ --no-doc -- \ --skip fs:: --skip net:: --skip process:: --skip sys::pal:: - $(Q)MIRIFLAGS="-Zmiri-ignore-leaks -Zmiri-disable-isolation" BOOTSTRAP_SKIP_TARGET_SANITY=1 \ + $(Q)MIRIFLAGS="-Zmiri-ignore-leaks -Zmiri-disable-isolation" \ $(BOOTSTRAP) miri --stage 2 library/std \ --doc -- \ --skip fs:: --skip net:: --skip process:: --skip sys::pal:: diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs index 01b5e99116f..40a2112b192 100644 --- a/src/bootstrap/src/core/build_steps/clippy.rs +++ b/src/bootstrap/src/core/build_steps/clippy.rs @@ -322,7 +322,6 @@ lint_any!( RemoteTestServer, "src/tools/remote-test-server", "remote-test-server"; Rls, "src/tools/rls", "rls"; RustAnalyzer, "src/tools/rust-analyzer", "rust-analyzer"; - RustDemangler, "src/tools/rust-demangler", "rust-demangler"; Rustdoc, "src/tools/rustdoc", "clippy"; Rustfmt, "src/tools/rustfmt", "rustfmt"; RustInstaller, "src/tools/rust-installer", "rust-installer"; diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 66692a2a2cb..cc5b0073213 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1008,10 +1008,7 @@ pub fn rustc_cargo( // If the rustc output is piped to e.g. `head -n1` we want the process to be // killed, rather than having an error bubble up and cause a panic. - // FIXME: Synthetic #[cfg(bootstrap)]. Remove when the bootstrap compiler supports it. - if compiler.stage != 0 { - cargo.rustflag("-Zon-broken-pipe=kill"); - } + cargo.rustflag("-Zon-broken-pipe=kill"); // We currently don't support cross-crate LTO in stage0. This also isn't hugely necessary // and may just be a time sink. @@ -1137,7 +1134,9 @@ pub fn rustc_cargo_env( } // Enable rustc's env var for `rust-lld` when requested. - if builder.config.lld_enabled { + if builder.config.lld_enabled + && (builder.config.channel == "dev" || builder.config.channel == "nightly") + { cargo.env("CFG_USE_SELF_CONTAINED_LINKER", "1"); } diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 1f006e1453f..54136d2aebd 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -78,7 +78,7 @@ impl Step for Docs { let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); tarball.set_product_name("Rust Documentation"); - tarball.add_bulk_dir(&builder.doc_out(host), dest); + tarball.add_bulk_dir(builder.doc_out(host), dest); tarball.add_file(builder.src.join("src/doc/robots.txt"), dest, 0o644); Some(tarball.generate()) } @@ -117,7 +117,7 @@ impl Step for JsonDocs { let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple); tarball.set_product_name("Rust Documentation In JSON Format"); tarball.is_preview(true); - tarball.add_bulk_dir(&builder.json_doc_out(host), dest); + tarball.add_bulk_dir(builder.json_doc_out(host), dest); Some(tarball.generate()) } } @@ -148,7 +148,7 @@ impl Step for RustcDocs { let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); tarball.set_product_name("Rustc Documentation"); - tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); + tarball.add_bulk_dir(builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); Some(tarball.generate()) } } @@ -1010,6 +1010,9 @@ impl Step for PlainSourceTarball { if builder.rust_info().is_managed_git_subrepository() || builder.rust_info().is_from_tarball() { + // FIXME: This code looks _very_ similar to what we have in `src/core/build_steps/vendor.rs` + // perhaps it should be removed in favor of making `dist` perform the `vendor` step? + // Ensure we have all submodules from src and other directories checked out. for submodule in builder.get_all_submodules() { builder.update_submodule(Path::new(submodule)); @@ -1029,11 +1032,30 @@ impl Step for PlainSourceTarball { .arg(builder.src.join("./compiler/rustc_codegen_gcc/Cargo.toml")) .arg("--sync") .arg(builder.src.join("./src/bootstrap/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./src/tools/opt-dist/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./src/tools/rustc-perf/Cargo.toml")) // Will read the libstd Cargo.toml // which uses the unstable `public-dependency` feature. .env("RUSTC_BOOTSTRAP", "1") .current_dir(plain_dst_src); + // Vendor packages that are required by opt-dist to collect PGO profiles. + let pkgs_for_pgo_training = build_helper::LLVM_PGO_CRATES + .iter() + .chain(build_helper::RUSTC_PGO_CRATES) + .map(|pkg| { + let mut manifest_path = + builder.src.join("./src/tools/rustc-perf/collector/compile-benchmarks"); + manifest_path.push(pkg); + manifest_path.push("Cargo.toml"); + manifest_path + }); + for manifest_path in pkgs_for_pgo_training { + cmd.arg("--sync").arg(manifest_path); + } + let config = if !builder.config.dry_run() { t!(String::from_utf8(t!(cmd.output()).stdout)) } else { @@ -1438,62 +1460,6 @@ impl Step for Rustfmt { } #[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] -pub struct RustDemangler { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustDemangler { - type Output = Option<GeneratedTarball>; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // While other tools use `should_build_extended_tool` to decide whether to be run by - // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like - // the other ones or if `profiler = true`. Because we don't know the target at this stage - // we run the step by default when only `extended = true`, and decide whether to actually - // run it or not later. - let default = run.builder.config.extended; - run.alias("rust-demangler").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustDemangler { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> { - let compiler = self.compiler; - let target = self.target; - - // Only build this extended tool if explicitly included in `tools`, or if `profiler = true` - let condition = should_build_extended_tool(builder, "rust-demangler") - || builder.config.profiler_enabled(target); - if builder.config.extended && !condition { - return None; - } - - let rust_demangler = - builder.ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }); - - // Prepare the image directory - let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple); - tarball.set_overlay(OverlayKind::RustDemangler); - tarball.is_preview(true); - tarball.add_file(rust_demangler, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rust-demangler"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Clone, Hash, PartialEq, Eq)] pub struct Extended { stage: u32, host: TargetSelection, @@ -1550,7 +1516,6 @@ impl Step for Extended { add_component!("rust-docs" => Docs { host: target }); add_component!("rust-json-docs" => JsonDocs { host: target }); - add_component!("rust-demangler"=> RustDemangler { compiler, target }); add_component!("cargo" => Cargo { compiler, target }); add_component!("rustfmt" => Rustfmt { compiler, target }); add_component!("rls" => Rls { compiler, target }); @@ -1614,7 +1579,7 @@ impl Step for Extended { let xform = |p: &Path| { let mut contents = t!(fs::read_to_string(p)); - for tool in &["rust-demangler", "miri", "rust-docs"] { + for tool in &["miri", "rust-docs"] { if !built_tools.contains(tool) { contents = filter(&contents, tool); } @@ -1655,7 +1620,7 @@ impl Step for Extended { prepare("rust-analysis"); prepare("clippy"); prepare("rust-analyzer"); - for tool in &["rust-docs", "rust-demangler", "miri", "rustc-codegen-cranelift"] { + for tool in &["rust-docs", "miri", "rustc-codegen-cranelift"] { if built_tools.contains(tool) { prepare(tool); } @@ -1695,8 +1660,6 @@ impl Step for Extended { "rust-analyzer-preview".to_string() } else if name == "clippy" { "clippy-preview".to_string() - } else if name == "rust-demangler" { - "rust-demangler-preview".to_string() } else if name == "miri" { "miri-preview".to_string() } else if name == "rustc-codegen-cranelift" { @@ -1716,7 +1679,7 @@ impl Step for Extended { prepare("cargo"); prepare("rust-analysis"); prepare("rust-std"); - for tool in &["clippy", "rust-analyzer", "rust-docs", "rust-demangler", "miri"] { + for tool in &["clippy", "rust-analyzer", "rust-docs", "miri"] { if built_tools.contains(tool) { prepare(tool); } @@ -1840,25 +1803,6 @@ impl Step for Extended { .arg(etc.join("msi/remove-duplicates.xsl")), ); } - if built_tools.contains("rust-demangler") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-demangler") - .args(heat_flags) - .arg("-cg") - .arg("RustDemanglerGroup") - .arg("-dr") - .arg("RustDemangler") - .arg("-var") - .arg("var.RustDemanglerDir") - .arg("-out") - .arg(exe.join("RustDemanglerGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - } if built_tools.contains("miri") { builder.run( Command::new(&heat) @@ -1936,9 +1880,6 @@ impl Step for Extended { if built_tools.contains("rust-docs") { cmd.arg("-dDocsDir=rust-docs"); } - if built_tools.contains("rust-demangler") { - cmd.arg("-dRustDemanglerDir=rust-demangler"); - } if built_tools.contains("rust-analyzer") { cmd.arg("-dRustAnalyzerDir=rust-analyzer"); } @@ -1965,9 +1906,6 @@ impl Step for Extended { if built_tools.contains("miri") { candle("MiriGroup.wxs".as_ref()); } - if built_tools.contains("rust-demangler") { - candle("RustDemanglerGroup.wxs".as_ref()); - } if built_tools.contains("rust-analyzer") { candle("RustAnalyzerGroup.wxs".as_ref()); } @@ -2009,9 +1947,6 @@ impl Step for Extended { if built_tools.contains("rust-analyzer") { cmd.arg("RustAnalyzerGroup.wixobj"); } - if built_tools.contains("rust-demangler") { - cmd.arg("RustDemanglerGroup.wixobj"); - } if built_tools.contains("rust-docs") { cmd.arg("DocsGroup.wixobj"); } diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 38c48bd9570..6748625f132 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -29,7 +29,7 @@ macro_rules! submodule_helper { } macro_rules! book { - ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => { + ($($name:ident, $path:expr, $book_name:expr, $lang:expr $(, submodule $(= $submodule:literal)? )? ;)+) => { $( #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct $name { @@ -61,6 +61,7 @@ macro_rules! book { name: $book_name.to_owned(), src: builder.src.join($path), parent: Some(self), + languages: $lang.into(), }) } } @@ -74,15 +75,15 @@ macro_rules! book { // FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules // and checking against it?). book!( - CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo"; - ClippyBook, "src/tools/clippy/book", "clippy"; - EditionGuide, "src/doc/edition-guide", "edition-guide", submodule; - EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule; - Nomicon, "src/doc/nomicon", "nomicon", submodule; - Reference, "src/doc/reference", "reference", submodule; - RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule; - RustdocBook, "src/doc/rustdoc", "rustdoc"; - StyleGuide, "src/doc/style-guide", "style-guide"; + CargoBook, "src/tools/cargo/src/doc", "cargo", &[], submodule = "src/tools/cargo"; + ClippyBook, "src/tools/clippy/book", "clippy", &[]; + EditionGuide, "src/doc/edition-guide", "edition-guide", &[], submodule; + EmbeddedBook, "src/doc/embedded-book", "embedded-book", &[], submodule; + Nomicon, "src/doc/nomicon", "nomicon", &[], submodule; + Reference, "src/doc/reference", "reference", &[], submodule; + RustByExample, "src/doc/rust-by-example", "rust-by-example", &["ja"], submodule; + RustdocBook, "src/doc/rustdoc", "rustdoc", &[]; + StyleGuide, "src/doc/style-guide", "style-guide", &[]; ); #[derive(Debug, Clone, Hash, PartialEq, Eq)] @@ -110,6 +111,7 @@ impl Step for UnstableBook { name: "unstable-book".to_owned(), src: builder.md_doc_out(self.target).join("unstable-book"), parent: Some(self), + languages: vec![], }) } } @@ -120,6 +122,7 @@ struct RustbookSrc<P: Step> { name: String, src: PathBuf, parent: Option<P>, + languages: Vec<&'static str>, } impl<P: Step> Step for RustbookSrc<P> { @@ -151,7 +154,19 @@ impl<P: Step> Step for RustbookSrc<P> { builder.info(&format!("Rustbook ({target}) - {name}")); let _ = fs::remove_dir_all(&out); - builder.run(rustbook_cmd.arg("build").arg(src).arg("-d").arg(out)); + builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(&out)); + + for lang in &self.languages { + let out = out.join(lang); + + builder.info(&format!("Rustbook ({target}) - {name} - {lang}")); + let _ = fs::remove_dir_all(&out); + + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + builder.run( + rustbook_cmd.arg("build").arg(&src).arg("-d").arg(&out).arg("-l").arg(lang), + ); + } } if self.parent.is_some() { @@ -214,6 +229,7 @@ impl Step for TheBook { name: "book".to_owned(), src: absolute_path.clone(), parent: Some(self), + languages: vec![], }); // building older edition redirects @@ -225,6 +241,7 @@ impl Step for TheBook { // There should only be one book that is marked as the parent for each target, so // treat the other editions as not having a parent. parent: Option::<Self>::None, + languages: vec![], }); } @@ -361,7 +378,7 @@ impl Step for Standalone { .arg(&favicon) .arg("--markdown-no-toc") .arg("--index-page") - .arg(&builder.src.join("src/doc/index.md")) + .arg(builder.src.join("src/doc/index.md")) .arg("--markdown-playground-url") .arg("https://play.rust-lang.org/") .arg("-o") @@ -465,7 +482,7 @@ impl Step for Releases { .arg("--markdown-css") .arg("rust.css") .arg("--index-page") - .arg(&builder.src.join("src/doc/index.md")) + .arg(builder.src.join("src/doc/index.md")) .arg("--markdown-playground-url") .arg("https://play.rust-lang.org/") .arg("-o") @@ -1028,6 +1045,14 @@ tool_doc!( is_library = true, crates = ["bootstrap"] ); +tool_doc!( + RunMakeSupport, + "run_make_support", + "src/tools/run-make-support", + rustc_tool = false, + is_library = true, + crates = ["run_make_support"] +); #[derive(Ord, PartialOrd, Debug, Clone, Hash, PartialEq, Eq)] pub struct ErrorIndex { @@ -1200,6 +1225,7 @@ impl Step for RustcBook { name: "rustc".to_owned(), src: out_base, parent: Some(self), + languages: vec![], }); } } diff --git a/src/bootstrap/src/core/build_steps/format.rs b/src/bootstrap/src/core/build_steps/format.rs index d9dc34c0137..f5e34672686 100644 --- a/src/bootstrap/src/core/build_steps/format.rs +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -1,7 +1,7 @@ //! Runs rustfmt on the repository. use crate::core::builder::Builder; -use crate::utils::helpers::{output, program_out_of_date, t}; +use crate::utils::helpers::{self, output, program_out_of_date, t}; use build_helper::ci::CiEnv; use build_helper::git::get_git_modified_files; use ignore::WalkBuilder; @@ -9,12 +9,13 @@ use std::collections::VecDeque; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::mpsc::SyncSender; +use std::sync::Mutex; fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut(bool) -> bool { let mut cmd = Command::new(rustfmt); - // avoid the submodule config paths from coming into play, - // we only allow a single global config for the workspace for now - cmd.arg("--config-path").arg(&src.canonicalize().unwrap()); + // Avoid the submodule config paths from coming into play. We only allow a single global config + // for the workspace for now. + cmd.arg("--config-path").arg(src.canonicalize().unwrap()); cmd.arg("--edition").arg("2021"); cmd.arg("--unstable-features"); cmd.arg("--skip-children"); @@ -24,20 +25,23 @@ fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl F cmd.args(paths); let cmd_debug = format!("{cmd:?}"); let mut cmd = cmd.spawn().expect("running rustfmt"); - // poor man's async: return a closure that'll wait for rustfmt's completion + // Poor man's async: return a closure that might wait for rustfmt's completion (depending on + // the value of the `block` argument). move |block: bool| -> bool { - if !block { + let status = if !block { match cmd.try_wait() { - Ok(Some(_)) => {} - _ => return false, + Ok(Some(status)) => Ok(status), + Ok(None) => return false, + Err(err) => Err(err), } - } - let status = cmd.wait().unwrap(); - if !status.success() { + } else { + cmd.wait() + }; + if !status.unwrap().success() { eprintln!( - "Running `{}` failed.\nIf you're running `tidy`, \ - try again with `--bless`. Or, if you just want to format \ - code, run `./x.py fmt` instead.", + "fmt error: Running `{}` failed.\nIf you're running `tidy`, \ + try again with `--bless`. Or, if you just want to format \ + code, run `./x.py fmt` instead.", cmd_debug, ); crate::exit!(1); @@ -72,7 +76,7 @@ fn verify_rustfmt_version(build: &Builder<'_>) -> bool { !program_out_of_date(&stamp_file, &version) } -/// Updates the last rustfmt version used +/// Updates the last rustfmt version used. fn update_rustfmt_version(build: &Builder<'_>) { let Some((version, stamp_file)) = get_rustfmt_version(build) else { return; @@ -89,7 +93,7 @@ fn get_modified_rs_files(build: &Builder<'_>) -> Result<Option<Vec<String>>, Str return Ok(None); } - get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &vec!["rs"]) + get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"]) } #[derive(serde_derive::Deserialize)] @@ -97,31 +101,66 @@ struct RustfmtConfig { ignore: Vec<String>, } -pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { +// Prints output describing a collection of paths, with lines such as "formatted modified file +// foo/bar/baz" or "skipped 20 untracked files". +fn print_paths(verb: &str, adjective: Option<&str>, paths: &[String]) { + let len = paths.len(); + let adjective = + if let Some(adjective) = adjective { format!("{adjective} ") } else { String::new() }; + if len <= 10 { + for path in paths { + println!("fmt: {verb} {adjective}file {path}"); + } + } else { + println!("fmt: {verb} {len} {adjective}files"); + } +} + +pub fn format(build: &Builder<'_>, check: bool, all: bool, paths: &[PathBuf]) { + if !paths.is_empty() { + eprintln!( + "fmt error: path arguments are no longer accepted; use `--all` to format everything" + ); + crate::exit!(1); + }; if build.config.dry_run() { return; } + + // By default, we only check modified files locally to speed up runtime. Exceptions are if + // `--all` is specified or we are in CI. We check all files in CI to avoid bugs in + // `get_modified_rs_files` letting regressions slip through; we also care about CI time less + // since this is still very fast compared to building the compiler. + let all = all || CiEnv::is_ci(); + let mut builder = ignore::types::TypesBuilder::new(); builder.add_defaults(); builder.select("rust"); let matcher = builder.build().unwrap(); let rustfmt_config = build.src.join("rustfmt.toml"); if !rustfmt_config.exists() { - eprintln!("Not running formatting checks; rustfmt.toml does not exist."); - eprintln!("This may happen in distributed tarballs."); + eprintln!("fmt error: Not running formatting checks; rustfmt.toml does not exist."); + eprintln!("fmt error: This may happen in distributed tarballs."); return; } let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config)); let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config)); - let mut fmt_override = ignore::overrides::OverrideBuilder::new(&build.src); + let mut override_builder = ignore::overrides::OverrideBuilder::new(&build.src); for ignore in rustfmt_config.ignore { - if let Some(ignore) = ignore.strip_prefix('!') { - fmt_override.add(ignore).expect(ignore); + if ignore.starts_with('!') { + // A `!`-prefixed entry could be added as a whitelisted entry in `override_builder`, + // i.e. strip the `!` prefix. But as soon as whitelisted entries are added, an + // `OverrideBuilder` will only traverse those whitelisted entries, and won't traverse + // any files that aren't explicitly mentioned. No bueno! Maybe there's a way to combine + // explicit whitelisted entries and traversal of unmentioned files, but for now just + // forbid such entries. + eprintln!("fmt error: `!`-prefixed entries are not supported in rustfmt.toml, sorry"); + crate::exit!(1); } else { - fmt_override.add(&format!("!{ignore}")).expect(&ignore); + override_builder.add(&format!("!{ignore}")).expect(&ignore); } } - let git_available = match Command::new("git") + let git_available = match helpers::git(None) .arg("--version") .stdout(Stdio::null()) .stderr(Stdio::null()) @@ -131,10 +170,9 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { Err(_) => false, }; + let mut adjective = None; if git_available { - let in_working_tree = match build - .config - .git() + let in_working_tree = match helpers::git(Some(&build.src)) .arg("rev-parse") .arg("--is-inside-work-tree") .stdout(Stdio::null()) @@ -146,150 +184,80 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { }; if in_working_tree { let untracked_paths_output = output( - build - .config - .git() + helpers::git(Some(&build.src)) .arg("status") .arg("--porcelain") .arg("-z") .arg("--untracked-files=normal"), ); - let untracked_paths = untracked_paths_output.split_terminator('\0').filter_map( - |entry| entry.strip_prefix("?? "), // returns None if the prefix doesn't match - ); - let mut untracked_count = 0; + let untracked_paths: Vec<_> = untracked_paths_output + .split_terminator('\0') + .filter_map( + |entry| entry.strip_prefix("?? "), // returns None if the prefix doesn't match + ) + .map(|x| x.to_string()) + .collect(); + print_paths("skipped", Some("untracked"), &untracked_paths); + for untracked_path in untracked_paths { - println!("skip untracked path {untracked_path} during rustfmt invocations"); // The leading `/` makes it an exact match against the // repository root, rather than a glob. Without that, if you // have `foo.rs` in the repository root it will also match // against anything like `compiler/rustc_foo/src/foo.rs`, // preventing the latter from being formatted. - untracked_count += 1; - fmt_override.add(&format!("!/{untracked_path}")).expect(untracked_path); + override_builder.add(&format!("!/{untracked_path}")).expect(&untracked_path); } - // Only check modified files locally to speed up runtime. - // We still check all files in CI to avoid bugs in `get_modified_rs_files` letting regressions slip through; - // we also care about CI time less since this is still very fast compared to building the compiler. - if !CiEnv::is_ci() && paths.is_empty() { + if !all { + adjective = Some("modified"); match get_modified_rs_files(build) { Ok(Some(files)) => { - if files.len() <= 10 { - for file in &files { - println!("formatting modified file {file}"); - } - } else { - let pluralized = |count| if count > 1 { "files" } else { "file" }; - let untracked_msg = if untracked_count == 0 { - "".to_string() - } else { - format!( - ", skipped {} untracked {}", - untracked_count, - pluralized(untracked_count), - ) - }; - println!( - "formatting {} modified {}{}", - files.len(), - pluralized(files.len()), - untracked_msg - ); - } for file in files { - fmt_override.add(&format!("/{file}")).expect(&file); + override_builder.add(&format!("/{file}")).expect(&file); } } Ok(None) => {} Err(err) => { - println!( - "WARN: Something went wrong when running git commands:\n{err}\n\ - Falling back to formatting all files." - ); + eprintln!("fmt warning: Something went wrong running git commands:"); + eprintln!("fmt warning: {err}"); + eprintln!("fmt warning: Falling back to formatting all files."); } } } } else { - println!("Not in git tree. Skipping git-aware format checks"); + eprintln!("fmt: warning: Not in git tree. Skipping git-aware format checks"); } } else { - println!("Could not find usable git. Skipping git-aware format checks"); + eprintln!("fmt: warning: Could not find usable git. Skipping git-aware format checks"); } - let fmt_override = fmt_override.build().unwrap(); + let override_ = override_builder.build().unwrap(); // `override` is a reserved keyword let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { - eprintln!("./x.py fmt is not supported on this channel"); + eprintln!("fmt error: `x fmt` is not supported on this channel"); crate::exit!(1); }); assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); let src = build.src.clone(); let (tx, rx): (SyncSender<PathBuf>, _) = std::sync::mpsc::sync_channel(128); - let walker = match paths.first() { - Some(first) => { - let find_shortcut_candidates = |p: &PathBuf| { - let mut candidates = Vec::new(); - for entry in - WalkBuilder::new(src.clone()).max_depth(Some(3)).build().map_while(Result::ok) - { - if let Some(dir_name) = p.file_name() { - if entry.path().is_dir() && entry.file_name() == dir_name { - candidates.push(entry.into_path()); - } - } - } - candidates - }; - - // Only try to look for shortcut candidates for single component paths like - // `std` and not for e.g. relative paths like `../library/std`. - let should_look_for_shortcut_dir = |p: &PathBuf| p.components().count() == 1; + let walker = WalkBuilder::new(src.clone()).types(matcher).overrides(override_).build_parallel(); - let mut walker = if should_look_for_shortcut_dir(first) { - if let [single_candidate] = &find_shortcut_candidates(first)[..] { - WalkBuilder::new(single_candidate) - } else { - WalkBuilder::new(first) - } - } else { - WalkBuilder::new(src.join(first)) - }; - - for path in &paths[1..] { - if should_look_for_shortcut_dir(path) { - if let [single_candidate] = &find_shortcut_candidates(path)[..] { - walker.add(single_candidate); - } else { - walker.add(path); - } - } else { - walker.add(src.join(path)); - } - } - - walker - } - None => WalkBuilder::new(src.clone()), - } - .types(matcher) - .overrides(fmt_override) - .build_parallel(); - - // there is a lot of blocking involved in spawning a child process and reading files to format. - // spawn more processes than available concurrency to keep the CPU busy + // There is a lot of blocking involved in spawning a child process and reading files to format. + // Spawn more processes than available concurrency to keep the CPU busy. let max_processes = build.jobs() as usize * 2; - // spawn child processes on a separate thread so we can batch entries we have received from ignore + // Spawn child processes on a separate thread so we can batch entries we have received from + // ignore. let thread = std::thread::spawn(move || { let mut children = VecDeque::new(); while let Ok(path) = rx.recv() { - // try getting more paths from the channel to amortize the overhead of spawning processes + // Try getting more paths from the channel to amortize the overhead of spawning + // processes. let paths: Vec<_> = rx.try_iter().take(63).chain(std::iter::once(path)).collect(); let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check); children.push_back(child); - // poll completion before waiting + // Poll completion before waiting. for i in (0..children.len()).rev() { if children[i](false) { children.swap_remove_back(i); @@ -298,27 +266,44 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { } if children.len() >= max_processes { - // await oldest child + // Await oldest child. children.pop_front().unwrap()(true); } } - // await remaining children + // Await remaining children. for mut child in children { child(true); } }); + let formatted_paths = Mutex::new(Vec::new()); + let formatted_paths_ref = &formatted_paths; walker.run(|| { let tx = tx.clone(); Box::new(move |entry| { + let cwd = std::env::current_dir(); let entry = t!(entry); if entry.file_type().map_or(false, |t| t.is_file()) { + formatted_paths_ref.lock().unwrap().push({ + // `into_path` produces an absolute path. Try to strip `cwd` to get a shorter + // relative path. + let mut path = entry.clone().into_path(); + if let Ok(cwd) = cwd { + if let Ok(path2) = path.strip_prefix(cwd) { + path = path2.to_path_buf(); + } + } + path.display().to_string() + }); t!(tx.send(entry.into_path())); } ignore::WalkState::Continue }) }); + let mut paths = formatted_paths.into_inner().unwrap(); + paths.sort(); + print_paths(if check { "checked" } else { "formatted" }, adjective, &paths); drop(tx); diff --git a/src/bootstrap/src/core/build_steps/install.rs b/src/bootstrap/src/core/build_steps/install.rs index 6a75f35c93a..c47233ca42a 100644 --- a/src/bootstrap/src/core/build_steps/install.rs +++ b/src/bootstrap/src/core/build_steps/install.rs @@ -265,22 +265,6 @@ install!((self, builder, _config), ); } }; - RustDemangler, alias = "rust-demangler", Self::should_build(_config), only_hosts: true, { - // NOTE: Even though `should_build` may return true for `extended` default tools, - // dist::RustDemangler may still return None, unless the target-dependent `profiler` config - // is also true, or the `tools` array explicitly includes "rust-demangler". - if let Some(tarball) = builder.ensure(dist::RustDemangler { - compiler: self.compiler, - target: self.target - }) { - install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install RustDemangler stage{} ({})", - self.compiler.stage, self.target), - ); - } - }; Rustc, path = "compiler/rustc", true, only_hosts: true, { let tarball = builder.ensure(dist::Rustc { compiler: builder.compiler(builder.top_stage, self.target), diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 3af1a05caa8..f90403d2ec4 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -159,7 +159,7 @@ pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { // in that case. let closest_upstream = get_git_merge_base(&config.git_config(), Some(&config.src)) .unwrap_or_else(|_| "HEAD".into()); - let mut rev_list = config.git(); + let mut rev_list = helpers::git(Some(&config.src)); rev_list.args(&[ PathBuf::from("rev-list"), format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(), @@ -252,7 +252,7 @@ pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool { // We assume we have access to git, so it's okay to unconditionally pass // `true` here. let llvm_sha = detect_llvm_sha(config, true); - let head_sha = output(config.git().arg("rev-parse").arg("HEAD")); + let head_sha = output(helpers::git(Some(&config.src)).arg("rev-parse").arg("HEAD")); let head_sha = head_sha.trim(); llvm_sha == head_sha } @@ -330,7 +330,7 @@ impl Step for Llvm { let llvm_exp_targets = match builder.config.llvm_experimental_targets { Some(ref s) => s, - None => "AVR;M68k;CSKY", + None => "AVR;M68k;CSKY;Xtensa", }; let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" }; @@ -508,7 +508,7 @@ impl Step for Llvm { cfg.define("LLVM_VERSION_SUFFIX", suffix); } - configure_cmake(builder, target, &mut cfg, true, ldflags, &[], &[]); + configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); configure_llvm(builder, target, &mut cfg); for (key, val) in &builder.config.llvm_build_config { @@ -597,7 +597,6 @@ fn configure_cmake( cfg: &mut cmake::Config, use_compiler_launcher: bool, mut ldflags: LdFlags, - extra_compiler_flags: &[&str], suppressed_compiler_flag_prefixes: &[&str], ) { // Do not print installation messages for up-to-date files. @@ -751,9 +750,6 @@ fn configure_cmake( if builder.config.llvm_clang_cl.is_some() { cflags.push(&format!(" --target={target}")); } - for flag in extra_compiler_flags { - cflags.push(&format!(" {flag}")); - } cfg.define("CMAKE_C_FLAGS", cflags); let mut cxxflags: OsString = builder .cflags(target, GitRepo::Llvm, CLang::Cxx) @@ -773,9 +769,6 @@ fn configure_cmake( if builder.config.llvm_clang_cl.is_some() { cxxflags.push(&format!(" --target={target}")); } - for flag in extra_compiler_flags { - cxxflags.push(&format!(" {flag}")); - } cfg.define("CMAKE_CXX_FLAGS", cxxflags); if let Some(ar) = builder.ar(target) { if ar.is_absolute() { @@ -944,7 +937,7 @@ impl Step for Lld { ldflags.push_all("-Wl,-rpath,'$ORIGIN/../../../'"); } - configure_cmake(builder, target, &mut cfg, true, ldflags, &[], &[]); + configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); configure_llvm(builder, target, &mut cfg); // Re-use the same flags as llvm to control the level of debug information @@ -1043,8 +1036,6 @@ impl Step for Sanitizers { // Unfortunately sccache currently lacks support to build them successfully. // Disable compiler launcher on Darwin targets to avoid potential issues. let use_compiler_launcher = !self.target.contains("apple-darwin"); - let extra_compiler_flags: &[&str] = - if self.target.contains("apple") { &["-fembed-bitcode=off"] } else { &[] }; // Since v1.0.86, the cc crate adds -mmacosx-version-min to the default // flags on MacOS. A long-standing bug in the CMake rules for compiler-rt // causes architecture detection to be skipped when this flag is present, @@ -1057,7 +1048,6 @@ impl Step for Sanitizers { &mut cfg, use_compiler_launcher, LdFlags::default(), - extra_compiler_flags, suppressed_compiler_flag_prefixes, ); diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 354eb2b6003..9268b335db7 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -149,12 +149,14 @@ impl Step for Miri { &[], ); miri.add_rustc_lib_path(builder); - // Forward arguments. miri.arg("--").arg("--target").arg(target.rustc_target_arg()); - miri.args(builder.config.args()); // miri tests need to know about the stage sysroot - miri.env("MIRI_SYSROOT", &miri_sysroot); + miri.arg("--sysroot").arg(miri_sysroot); + + // Forward arguments. This may contain further arguments to the program + // after another --, so this must be at the end. + miri.args(builder.config.args()); let mut miri = Command::from(miri); builder.run(&mut miri); diff --git a/src/bootstrap/src/core/build_steps/setup.rs b/src/bootstrap/src/core/build_steps/setup.rs index df38d6166eb..947c74f32c9 100644 --- a/src/bootstrap/src/core/build_steps/setup.rs +++ b/src/bootstrap/src/core/build_steps/setup.rs @@ -8,7 +8,7 @@ use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::t; use crate::utils::change_tracker::CONFIG_CHANGE_HISTORY; -use crate::utils::helpers::hex_encode; +use crate::utils::helpers::{self, hex_encode}; use crate::Config; use sha2::Digest; use std::env::consts::EXE_SUFFIX; @@ -482,10 +482,13 @@ impl Step for Hook { // install a git hook to automatically run tidy, if they want fn install_git_hook_maybe(config: &Config) -> io::Result<()> { - let git = config.git().args(["rev-parse", "--git-common-dir"]).output().map(|output| { - assert!(output.status.success(), "failed to run `git`"); - PathBuf::from(t!(String::from_utf8(output.stdout)).trim()) - })?; + let git = helpers::git(Some(&config.src)) + .args(["rev-parse", "--git-common-dir"]) + .output() + .map(|output| { + assert!(output.status.success(), "failed to run `git`"); + PathBuf::from(t!(String::from_utf8(output.stdout)).trim()) + })?; let hooks_dir = git.join("hooks"); let dst = hooks_dir.join("pre-push"); if dst.exists() { diff --git a/src/bootstrap/src/core/build_steps/synthetic_targets.rs b/src/bootstrap/src/core/build_steps/synthetic_targets.rs index 89d50b5ffff..281a9b093b9 100644 --- a/src/bootstrap/src/core/build_steps/synthetic_targets.rs +++ b/src/bootstrap/src/core/build_steps/synthetic_targets.rs @@ -80,8 +80,5 @@ fn create_synthetic_target( customize(spec_map); std::fs::write(&path, serde_json::to_vec_pretty(&spec).unwrap()).unwrap(); - let target = TargetSelection::create_synthetic(&name, path.to_str().unwrap()); - crate::utils::cc_detect::find_target(builder, target); - - target + TargetSelection::create_synthetic(&name, path.to_str().unwrap()) } diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 360bd3840d4..445096e9786 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -308,7 +308,7 @@ impl Step for Cargo { // Forcibly disable tests using nightly features since any changes to // those features won't be able to land. cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1"); - cargo.env("PATH", &path_for_cargo(builder, compiler)); + cargo.env("PATH", path_for_cargo(builder, compiler)); #[cfg(feature = "build-metrics")] builder.metrics.begin_test_suite( @@ -433,65 +433,6 @@ impl Step for Rustfmt { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct RustDemangler { - stage: u32, - host: TargetSelection, -} - -impl Step for RustDemangler { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rust-demangler") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for rust-demangler. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - let rust_demangler = builder.ensure(tool::RustDemangler { - compiler, - target: self.host, - extra_features: Vec::new(), - }); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/rust-demangler", - SourceType::InTree, - &[], - ); - - let dir = testdir(builder, compiler.host); - t!(fs::create_dir_all(dir)); - - cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler); - cargo.add_rustc_lib_path(builder); - - run_cargo_test( - cargo, - &[], - &[], - "rust-demangler", - "rust-demangler", - compiler, - host, - builder, - ); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Miri { target: TargetSelection, } @@ -1101,6 +1042,8 @@ impl Step for Tidy { /// Once tidy passes, this step also runs `fmt --check` if tests are being run /// for the `dev` or `nightly` channels. fn run(self, builder: &Builder<'_>) { + builder.build.update_submodule(Path::new("src/tools/rustc-perf")); + let mut cmd = builder.tool_cmd(Tool::Tidy); cmd.arg(&builder.src); cmd.arg(&builder.initial_cargo); @@ -1140,7 +1083,13 @@ HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to ); crate::exit!(1); } - crate::core::build_steps::format::format(builder, !builder.config.cmd.bless(), &[]); + let all = false; + crate::core::build_steps::format::format( + builder, + !builder.config.cmd.bless(), + all, + &[], + ); } builder.info("tidy check"); @@ -1481,12 +1430,6 @@ impl Step for RunMake { } } -host_test!(RunMakeFullDeps { - path: "tests/run-make-fulldeps", - mode: "run-make", - suite: "run-make-fulldeps" -}); - default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); /// Coverage tests are a bit more complicated than other test suites, because @@ -1775,25 +1718,11 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); } - if mode == "coverage-map" { - let coverage_dump = builder.ensure(tool::CoverageDump { - compiler: compiler.with_stage(0), - target: compiler.host, - }); + if matches!(mode, "coverage-map" | "coverage-run") { + let coverage_dump = builder.tool_exe(Tool::CoverageDump); cmd.arg("--coverage-dump-path").arg(coverage_dump); } - if mode == "coverage-run" { - // The demangler doesn't need the current compiler, so we can avoid - // unnecessary rebuilds by using the bootstrap compiler instead. - let rust_demangler = builder.ensure(tool::RustDemangler { - compiler: compiler.with_stage(0), - target: compiler.host, - extra_features: Vec::new(), - }); - cmd.arg("--rust-demangler-path").arg(rust_demangler); - } - cmd.arg("--src-base").arg(builder.src.join("tests").join(suite)); cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); @@ -1981,9 +1910,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd); } - if !builder.config.dry_run() - && (matches!(suite, "run-make" | "run-make-fulldeps") || mode == "coverage-run") - { + if !builder.config.dry_run() && matches!(mode, "run-make" | "coverage-run") { // The llvm/bin directory contains many useful cross-platform // tools. Pass the path to run-make tests so they can use them. // (The coverage-run tests also need these tools to process @@ -1995,7 +1922,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the cmd.arg("--llvm-bin-dir").arg(llvm_bin_path); } - if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { + if !builder.config.dry_run() && mode == "run-make" { // If LLD is available, add it to the PATH if builder.config.lld_enabled { let lld_install_root = @@ -2015,7 +1942,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the // Only pass correct values for these flags for the `run-make` suite as it // requires that a C++ compiler was configured which isn't always the case. - if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { + if !builder.config.dry_run() && mode == "run-make" { cmd.arg("--cc") .arg(builder.cc(target)) .arg("--cxx") @@ -2573,7 +2500,7 @@ fn prepare_cargo_test( cargo.arg("-p").arg(krate); } - cargo.arg("--").args(&builder.config.test_args()).args(libtest_args); + cargo.arg("--").args(builder.config.test_args()).args(libtest_args); if !builder.config.verbose_tests { cargo.arg("--quiet"); } @@ -3061,6 +2988,7 @@ impl Step for Bootstrap { let mut cmd = Command::new(&builder.initial_cargo); cmd.arg("test") + .args(["--features", "bootstrap-self-test"]) .current_dir(builder.src.join("src/bootstrap")) .env("RUSTFLAGS", "-Cdebuginfo=2") .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) @@ -3121,7 +3049,7 @@ impl Step for TierCheck { &[], ); cargo.arg(builder.src.join("src/doc/rustc/src/platform-support.md")); - cargo.arg(&builder.rustc(self.compiler)); + cargo.arg(builder.rustc(self.compiler)); if builder.is_verbose() { cargo.arg("--verbose"); } diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 21344a4224e..613484788b6 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -1,6 +1,6 @@ use std::env; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; use crate::core::build_steps::compile; @@ -9,7 +9,7 @@ use crate::core::builder; use crate::core::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; use crate::utils::channel::GitInfo; -use crate::utils::exec::BootstrapCommand; +use crate::utils::helpers::output; use crate::utils::helpers::{add_dylib_path, exe, t}; use crate::Compiler; use crate::Mode; @@ -109,9 +109,8 @@ impl Step for ToolBuild { &self.target, ); - let mut cargo = Command::from(cargo); // we check this below - let build_success = builder.run_cmd(BootstrapCommand::from(&mut cargo).allow_failure()); + let build_success = compile::stream_cargo(builder, cargo, vec![], &mut |_| {}); builder.save_toolstate( tool, @@ -200,7 +199,7 @@ pub fn prepare_tool_cargo( cargo.env("CFG_COMMIT_DATE", date); } if !features.is_empty() { - cargo.arg("--features").arg(&features.join(", ")); + cargo.arg("--features").arg(features.join(", ")); } // Enable internal lints for clippy and rustdoc @@ -313,10 +312,47 @@ bootstrap_tool!( SuggestTests, "src/tools/suggest-tests", "suggest-tests"; GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys"; RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; - OptimizedDist, "src/tools/opt-dist", "opt-dist"; CoverageDump, "src/tools/coverage-dump", "coverage-dump"; ); +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct OptimizedDist { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for OptimizedDist { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/opt-dist") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(OptimizedDist { + compiler: run.builder.compiler(0, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + // We need to ensure the rustc-perf submodule is initialized when building opt-dist since + // the tool requires it to be in place to run. + builder.update_submodule(Path::new("src/tools/rustc-perf")); + + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "opt-dist", + mode: Mode::ToolBootstrap, + path: "src/tools/opt-dist", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) + } +} + #[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] pub struct ErrorIndex { pub compiler: Compiler, @@ -485,10 +521,7 @@ impl Step for Rustdoc { // If the rustdoc output is piped to e.g. `head -n1` we want the process // to be killed, rather than having an error bubble up and cause a // panic. - // FIXME: Synthetic #[cfg(bootstrap)]. Remove when the bootstrap compiler supports it. - if build_compiler.stage > 0 { - cargo.rustflag("-Zon-broken-pipe=kill"); - } + cargo.rustflag("-Zon-broken-pipe=kill"); let _guard = builder.msg_tool( Kind::Build, @@ -767,6 +800,69 @@ impl Step for LlvmBitcodeLinker { } } +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct LibcxxVersionTool { + pub target: TargetSelection, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub enum LibcxxVersion { + Gnu(usize), + Llvm(usize), +} + +impl Step for LibcxxVersionTool { + type Output = LibcxxVersion; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> LibcxxVersion { + let out_dir = builder.out.join(self.target.to_string()).join("libcxx-version"); + let executable = out_dir.join(exe("libcxx-version", self.target)); + + // This is a sanity-check specific step, which means it is frequently called (when using + // CI LLVM), and compiling `src/tools/libcxx-version/main.cpp` at the beginning of the bootstrap + // invocation adds a fair amount of overhead to the process (see https://github.com/rust-lang/rust/issues/126423). + // Therefore, we want to avoid recompiling this file unnecessarily. + if !executable.exists() { + if !out_dir.exists() { + t!(fs::create_dir_all(&out_dir)); + } + + let compiler = builder.cxx(self.target).unwrap(); + let mut cmd = Command::new(compiler); + + cmd.arg("-o") + .arg(&executable) + .arg(builder.src.join("src/tools/libcxx-version/main.cpp")); + + builder.run_cmd(&mut cmd); + + if !executable.exists() { + panic!("Something went wrong. {} is not present", executable.display()); + } + } + + let version_output = output(&mut Command::new(executable)); + + let version_str = version_output.split_once("version:").unwrap().1; + let version = version_str.trim().parse::<usize>().unwrap(); + + if version_output.starts_with("libstdc++") { + LibcxxVersion::Gnu(version) + } else if version_output.starts_with("libc++") { + LibcxxVersion::Llvm(version) + } else { + panic!("Coudln't recognize the standard library version."); + } + } +} + macro_rules! tool_extended { (($sel:ident, $builder:ident), $($name:ident, @@ -868,7 +964,6 @@ tool_extended!((self, builder), // But `builder.cargo` doesn't know how to handle ToolBootstrap in stages other than 0, // and this is close enough for now. Rls, "src/tools/rls", "rls", stable=true, tool_std=true; - RustDemangler, "src/tools/rust-demangler", "rust-demangler", stable=false, tool_std=true; Rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, add_bins_to_sysroot = ["rustfmt", "cargo-fmt"]; ); diff --git a/src/bootstrap/src/core/build_steps/toolstate.rs b/src/bootstrap/src/core/build_steps/toolstate.rs index ca3756df4d7..9e6d03349b5 100644 --- a/src/bootstrap/src/core/build_steps/toolstate.rs +++ b/src/bootstrap/src/core/build_steps/toolstate.rs @@ -5,7 +5,7 @@ //! [Toolstate]: https://forge.rust-lang.org/infra/toolstate.html use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::utils::helpers::t; +use crate::utils::helpers::{self, t}; use serde_derive::{Deserialize, Serialize}; use std::collections::HashMap; use std::env; @@ -13,7 +13,6 @@ use std::fmt; use std::fs; use std::io::{Seek, SeekFrom}; use std::path::{Path, PathBuf}; -use std::process::Command; use std::time; // Each cycle is 42 days long (6 weeks); the last week is 35..=42 then. @@ -102,12 +101,8 @@ fn print_error(tool: &str, submodule: &str) { fn check_changed_files(toolstates: &HashMap<Box<str>, ToolState>) { // Changed files - let output = std::process::Command::new("git") - .arg("diff") - .arg("--name-status") - .arg("HEAD") - .arg("HEAD^") - .output(); + let output = + helpers::git(None).arg("diff").arg("--name-status").arg("HEAD").arg("HEAD^").output(); let output = match output { Ok(o) => o, Err(e) => { @@ -324,7 +319,7 @@ fn checkout_toolstate_repo() { t!(fs::remove_dir_all(TOOLSTATE_DIR)); } - let status = Command::new("git") + let status = helpers::git(None) .arg("clone") .arg("--depth=1") .arg(toolstate_repo()) @@ -342,7 +337,7 @@ fn checkout_toolstate_repo() { /// Sets up config and authentication for modifying the toolstate repo. fn prepare_toolstate_config(token: &str) { fn git_config(key: &str, value: &str) { - let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status(); + let status = helpers::git(None).arg("config").arg("--global").arg(key).arg(value).status(); let success = match status { Ok(s) => s.success(), Err(_) => false, @@ -406,8 +401,7 @@ fn commit_toolstate_change(current_toolstate: &ToolstateData) { publish_test_results(current_toolstate); // `git commit` failing means nothing to commit. - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) + let status = t!(helpers::git(Some(Path::new(TOOLSTATE_DIR))) .arg("commit") .arg("-a") .arg("-m") @@ -418,8 +412,7 @@ fn commit_toolstate_change(current_toolstate: &ToolstateData) { break; } - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) + let status = t!(helpers::git(Some(Path::new(TOOLSTATE_DIR))) .arg("push") .arg("origin") .arg("master") @@ -431,15 +424,13 @@ fn commit_toolstate_change(current_toolstate: &ToolstateData) { } eprintln!("Sleeping for 3 seconds before retrying push"); std::thread::sleep(std::time::Duration::from_secs(3)); - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) + let status = t!(helpers::git(Some(Path::new(TOOLSTATE_DIR))) .arg("fetch") .arg("origin") .arg("master") .status()); assert!(status.success()); - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) + let status = t!(helpers::git(Some(Path::new(TOOLSTATE_DIR))) .arg("reset") .arg("--hard") .arg("origin/master") @@ -458,7 +449,7 @@ fn commit_toolstate_change(current_toolstate: &ToolstateData) { /// `publish_toolstate.py` script if the PR passes all tests and is merged to /// master. fn publish_test_results(current_toolstate: &ToolstateData) { - let commit = t!(std::process::Command::new("git").arg("rev-parse").arg("HEAD").output()); + let commit = t!(helpers::git(None).arg("rev-parse").arg("HEAD").output()); let commit = t!(String::from_utf8(commit.stdout)); let toolstate_serialized = t!(serde_json::to_string(¤t_toolstate)); diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 045cde56f41..3c4806a1311 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -332,7 +332,6 @@ const PATH_REMAP: &[(&str, &[&str])] = &[ "tests/mir-opt", "tests/pretty", "tests/run-make", - "tests/run-make-fulldeps", "tests/run-pass-valgrind", "tests/rustdoc", "tests/rustdoc-gui", @@ -737,7 +736,6 @@ impl<'a> Builder<'a> { tool::Rls, tool::RustAnalyzer, tool::RustAnalyzerProcMacroSrv, - tool::RustDemangler, tool::Rustdoc, tool::Clippy, tool::CargoClippy, @@ -775,7 +773,6 @@ impl<'a> Builder<'a> { clippy::RemoteTestServer, clippy::Rls, clippy::RustAnalyzer, - clippy::RustDemangler, clippy::Rustdoc, clippy::Rustfmt, clippy::RustInstaller, @@ -828,7 +825,6 @@ impl<'a> Builder<'a> { test::RustAnalyzer, test::ErrorIndex, test::Distcheck, - test::RunMakeFullDeps, test::Nomicon, test::Reference, test::RustdocBook, @@ -844,7 +840,6 @@ impl<'a> Builder<'a> { test::Miri, test::CargoMiri, test::Clippy, - test::RustDemangler, test::CompiletestTest, test::CrateRunMakeSupport, test::RustdocJSStd, @@ -888,6 +883,7 @@ impl<'a> Builder<'a> { doc::Tidy, doc::Bootstrap, doc::Releases, + doc::RunMakeSupport, ), Kind::Dist => describe!( dist::Docs, @@ -904,7 +900,6 @@ impl<'a> Builder<'a> { dist::Rls, dist::RustAnalyzer, dist::Rustfmt, - dist::RustDemangler, dist::Clippy, dist::Miri, dist::LlvmTools, @@ -931,7 +926,6 @@ impl<'a> Builder<'a> { install::Cargo, install::RustAnalyzer, install::Rustfmt, - install::RustDemangler, install::Clippy, install::Miri, install::LlvmTools, @@ -1044,7 +1038,8 @@ impl<'a> Builder<'a> { // custom build of rustdoc maybe? link to the latest stable docs just in case _ => "stable", }; - "https://doc.rust-lang.org/".to_owned() + channel + + format!("https://doc.rust-lang.org/{channel}") } fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) { @@ -1555,7 +1550,6 @@ impl<'a> Builder<'a> { // features but cargo isn't involved in the #[path] process and so cannot pass the // complete list of features, so for that reason we don't enable checking of // features for std crates. - cargo.arg("-Zcheck-cfg"); if mode == Mode::Std { rustflags.arg("--check-cfg=cfg(feature,values(any()))"); } diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index 9710365ef11..276fd0b11d6 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -60,7 +60,14 @@ fn check_cli<const N: usize>(paths: [&str; N]) { macro_rules! std { ($host:ident => $target:ident, stage = $stage:literal) => { compile::Std::new( - Compiler { host: TargetSelection::from_user(concat!(stringify!($host), "-", stringify!($host))), stage: $stage }, + Compiler { + host: TargetSelection::from_user(concat!( + stringify!($host), + "-", + stringify!($host) + )), + stage: $stage, + }, TargetSelection::from_user(concat!(stringify!($target), "-", stringify!($target))), ) }; @@ -83,7 +90,14 @@ macro_rules! doc_std { macro_rules! rustc { ($host:ident => $target:ident, stage = $stage:literal) => { compile::Rustc::new( - Compiler { host: TargetSelection::from_user(concat!(stringify!($host), "-", stringify!($host))), stage: $stage }, + Compiler { + host: TargetSelection::from_user(concat!( + stringify!($host), + "-", + stringify!($host) + )), + stage: $stage, + }, TargetSelection::from_user(concat!(stringify!($target), "-", stringify!($target))), ) }; @@ -141,10 +155,14 @@ fn check_missing_paths_for_x_test_tests() { // Skip if not a test directory. if path.ends_with("tests/auxiliary") || !path.is_dir() { - continue + continue; } - assert!(tests_remap_paths.iter().any(|item| path.ends_with(*item)), "{} is missing in PATH_REMAP tests list.", path.display()); + assert!( + tests_remap_paths.iter().any(|item| path.ends_with(*item)), + "{} is missing in PATH_REMAP tests list.", + path.display() + ); } } @@ -185,7 +203,8 @@ fn alias_and_path_for_library() { &[std!(A => A, stage = 0), std!(A => A, stage = 1)] ); - let mut cache = run_build(&["library".into(), "core".into()], configure("doc", &["A-A"], &["A-A"])); + let mut cache = + run_build(&["library".into(), "core".into()], configure("doc", &["A-A"], &["A-A"])); assert_eq!(first(cache.all::<doc::Std>()), &[doc_std!(A => A, stage = 0)]); } diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 19119a073c5..0438dee7241 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -10,7 +10,7 @@ use std::env; use std::fmt::{self, Display}; use std::fs; use std::io::IsTerminal; -use std::path::{Path, PathBuf}; +use std::path::{absolute, Path, PathBuf}; use std::process::Command; use std::str::FromStr; use std::sync::OnceLock; @@ -20,10 +20,8 @@ use crate::core::build_steps::llvm; use crate::core::config::flags::{Color, Flags, Warnings}; use crate::utils::cache::{Interned, INTERNER}; use crate::utils::channel::{self, GitInfo}; -use crate::utils::helpers::{exe, output, t}; +use crate::utils::helpers::{self, exe, output, t}; use build_helper::exit; -use build_helper::util::fail; -use semver::Version; use serde::{Deserialize, Deserializer}; use serde_derive::Deserialize; @@ -314,7 +312,6 @@ pub struct Config { pub save_toolstates: Option<PathBuf>, pub print_step_timings: bool, pub print_step_rusage: bool, - pub missing_tools: bool, // FIXME: Deprecated field. Remove it at 2024. // Fallback musl-root for all targets pub musl_root: Option<PathBuf>, @@ -360,7 +357,7 @@ pub enum RustfmtState { LazyEvaluated, } -#[derive(Debug, Default, Clone, Copy, PartialEq)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub enum LlvmLibunwind { #[default] No, @@ -381,7 +378,7 @@ impl FromStr for LlvmLibunwind { } } -#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum SplitDebuginfo { Packed, Unpacked, @@ -542,7 +539,7 @@ impl PartialEq<&str> for TargetSelection { } /// Per-target configuration stored in the global configuration structure. -#[derive(Default, Clone)] +#[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct Target { /// Some(path to llvm-config) if using an external LLVM. pub llvm_config: Option<PathBuf>, @@ -644,7 +641,20 @@ impl Merge for TomlConfig { do_merge(&mut self.llvm, llvm, replace); do_merge(&mut self.rust, rust, replace); do_merge(&mut self.dist, dist, replace); - assert!(target.is_none(), "merging target-specific config is not currently supported"); + + match (self.target.as_mut(), target) { + (_, None) => {} + (None, Some(target)) => self.target = Some(target), + (Some(original_target), Some(new_target)) => { + for (triple, new) in new_target { + if let Some(original) = original_target.get_mut(&triple) { + original.merge(new, replace); + } else { + original_target.insert(triple, new); + } + } + } + } } } @@ -892,14 +902,13 @@ define_config! { sign_folder: Option<String> = "sign-folder", upload_addr: Option<String> = "upload-addr", src_tarball: Option<bool> = "src-tarball", - missing_tools: Option<bool> = "missing-tools", compression_formats: Option<Vec<String>> = "compression-formats", compression_profile: Option<String> = "compression-profile", include_mingw_linker: Option<bool> = "include-mingw-linker", } } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] #[serde(untagged)] pub enum StringOrBool { String(String), @@ -1239,7 +1248,7 @@ impl Config { // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, // running on a completely different machine from where it was compiled. - let mut cmd = Command::new("git"); + let mut cmd = helpers::git(None); // NOTE: we cannot support running from outside the repository because the only other path we have available // is set at compile time, which can be wrong if bootstrap was downloaded rather than compiled locally. // We still support running outside the repository if we find we aren't in a git directory. @@ -1428,7 +1437,7 @@ impl Config { // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. if !config.out.is_absolute() { // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. - config.out = crate::utils::helpers::absolute(&config.out); + config.out = absolute(&config.out).expect("can't make empty path absolute"); } config.initial_rustc = if let Some(rustc) = rustc { @@ -1599,19 +1608,8 @@ impl Config { set(&mut config.channel, channel); config.download_rustc_commit = config.download_ci_rustc_commit(download_rustc); - // This list is incomplete, please help by expanding it! - if config.download_rustc_commit.is_some() { - // We need the channel used by the downloaded compiler to match the one we set for rustdoc; - // otherwise rustdoc-ui tests break. - if config.channel != ci_channel - && !(config.channel == "dev" && ci_channel == "nightly") - { - panic!( - "setting rust.channel={} is incompatible with download-rustc", - config.channel - ); - } - } + + // FIXME: handle download-rustc incompatible options. debug = debug_toml; debug_assertions = debug_assertions_toml; @@ -1720,7 +1718,23 @@ impl Config { config.omit_git_hash = omit_git_hash.unwrap_or(default); config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); - if config.rust_info.is_from_tarball() && !is_user_configured_rust_channel { + // We need to override `rust.channel` if it's manually specified when using the CI rustc. + // This is because if the compiler uses a different channel than the one specified in config.toml, + // tests may fail due to using a different channel than the one used by the compiler during tests. + if let Some(commit) = &config.download_rustc_commit { + if is_user_configured_rust_channel { + println!( + "WARNING: `rust.download-rustc` is enabled. The `rust.channel` option will be overridden by the CI rustc's channel." + ); + + let channel = config + .read_file_by_commit(&PathBuf::from("src/ci/channel"), commit) + .trim() + .to_owned(); + + config.channel = channel; + } + } else if config.rust_info.is_from_tarball() && !is_user_configured_rust_channel { ci_channel.clone_into(&mut config.channel); } @@ -1923,7 +1937,6 @@ impl Config { sign_folder, upload_addr, src_tarball, - missing_tools, compression_formats, compression_profile, include_mingw_linker, @@ -1933,7 +1946,6 @@ impl Config { config.dist_compression_formats = compression_formats; set(&mut config.dist_compression_profile, compression_profile); set(&mut config.rust_dist_src, src_tarball); - set(&mut config.missing_tools, missing_tools); set(&mut config.dist_include_mingw_linker, include_mingw_linker) } @@ -2094,15 +2106,6 @@ impl Config { build_helper::util::try_run(cmd, self.is_verbose()) } - /// A git invocation which runs inside the source directory. - /// - /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. - pub(crate) fn git(&self) -> Command { - let mut git = Command::new("git"); - git.current_dir(&self.src); - git - } - pub(crate) fn test_args(&self) -> Vec<&str> { let mut test_args = match self.cmd { Subcommand::Test { ref test_args, .. } @@ -2127,17 +2130,29 @@ impl Config { args } + /// Returns the content of the given file at a specific commit. + pub(crate) fn read_file_by_commit(&self, file: &Path, commit: &str) -> String { + assert!( + self.rust_info.is_managed_git_subrepository(), + "`Config::read_file_by_commit` is not supported in non-git sources." + ); + + let mut git = helpers::git(Some(&self.src)); + git.arg("show").arg(format!("{commit}:{}", file.to_str().unwrap())); + output(&mut git) + } + /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. /// Return the version it would have used for the given commit. pub(crate) fn artifact_version_part(&self, commit: &str) -> String { let (channel, version) = if self.rust_info.is_managed_git_subrepository() { - let mut channel = self.git(); - channel.arg("show").arg(format!("{commit}:src/ci/channel")); - let channel = output(&mut channel); - let mut version = self.git(); - version.arg("show").arg(format!("{commit}:src/version")); - let version = output(&mut version); - (channel.trim().to_owned(), version.trim().to_owned()) + let channel = self + .read_file_by_commit(&PathBuf::from("src/ci/channel"), commit) + .trim() + .to_owned(); + let version = + self.read_file_by_commit(&PathBuf::from("src/version"), commit).trim().to_owned(); + (channel, version) } else { let channel = fs::read_to_string(self.src.join("src/ci/channel")); let version = fs::read_to_string(self.src.join("src/version")); @@ -2373,8 +2388,14 @@ impl Config { } } - // check rustc/cargo version is same or lower with 1 apart from the building one + #[cfg(feature = "bootstrap-self-test")] + pub fn check_stage0_version(&self, _program_path: &Path, _component_name: &'static str) {} + + /// check rustc/cargo version is same or lower with 1 apart from the building one + #[cfg(not(feature = "bootstrap-self-test"))] pub fn check_stage0_version(&self, program_path: &Path, component_name: &'static str) { + use build_helper::util::fail; + if self.dry_run() { return; } @@ -2391,11 +2412,12 @@ impl Config { } let stage0_version = - Version::parse(stage0_output.next().unwrap().split('-').next().unwrap().trim()) - .unwrap(); - let source_version = - Version::parse(fs::read_to_string(self.src.join("src/version")).unwrap().trim()) + semver::Version::parse(stage0_output.next().unwrap().split('-').next().unwrap().trim()) .unwrap(); + let source_version = semver::Version::parse( + fs::read_to_string(self.src.join("src/version")).unwrap().trim(), + ) + .unwrap(); if !(source_version == stage0_version || (source_version.major == stage0_version.major && (source_version.minor == stage0_version.minor @@ -2421,7 +2443,8 @@ impl Config { }; // Handle running from a directory other than the top level - let top_level = output(self.git().args(["rev-parse", "--show-toplevel"])); + let top_level = + output(helpers::git(Some(&self.src)).args(["rev-parse", "--show-toplevel"])); let top_level = top_level.trim_end(); let compiler = format!("{top_level}/compiler/"); let library = format!("{top_level}/library/"); @@ -2429,7 +2452,7 @@ impl Config { // Look for a version to compare to based on the current commit. // Only commits merged by bors will have CI artifacts. let merge_base = output( - self.git() + helpers::git(Some(&self.src)) .arg("rev-list") .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) .args(["-n1", "--first-parent", "HEAD"]), @@ -2444,8 +2467,7 @@ impl Config { } // Warn if there were changes to the compiler or standard library since the ancestor commit. - let has_changes = !t!(self - .git() + let has_changes = !t!(helpers::git(Some(&self.src)) .args(["diff-index", "--quiet", commit, "--", &compiler, &library]) .status()) .success(); @@ -2518,13 +2540,14 @@ impl Config { if_unchanged: bool, ) -> Option<String> { // Handle running from a directory other than the top level - let top_level = output(self.git().args(["rev-parse", "--show-toplevel"])); + let top_level = + output(helpers::git(Some(&self.src)).args(["rev-parse", "--show-toplevel"])); let top_level = top_level.trim_end(); // Look for a version to compare to based on the current commit. // Only commits merged by bors will have CI artifacts. let merge_base = output( - self.git() + helpers::git(Some(&self.src)) .arg("rev-list") .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) .args(["-n1", "--first-parent", "HEAD"]), @@ -2539,7 +2562,7 @@ impl Config { } // Warn if there were changes to the compiler or standard library since the ancestor commit. - let mut git = self.git(); + let mut git = helpers::git(Some(&self.src)); git.args(["diff-index", "--quiet", commit, "--"]); for path in modified_paths { diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs index f4ed7e76fba..83def0c6df0 100644 --- a/src/bootstrap/src/core/config/flags.rs +++ b/src/bootstrap/src/core/config/flags.rs @@ -284,8 +284,8 @@ pub enum Subcommand { name = "fmt", long_about = "\n Arguments: - This subcommand optionally accepts a `--check` flag which succeeds if formatting is correct and - fails if it is not. For example: + This subcommand optionally accepts a `--check` flag which succeeds if + formatting is correct and fails if it is not. For example: ./x.py fmt ./x.py fmt --check" )] @@ -294,6 +294,10 @@ pub enum Subcommand { /// check formatting instead of applying #[arg(long)] check: bool, + + /// apply to all appropriate files, not just those that have been modified + #[arg(long)] + all: bool, }, #[command(aliases = ["d"], long_about = "\n Arguments: diff --git a/src/bootstrap/src/core/config/tests.rs b/src/bootstrap/src/core/config/tests.rs index 59e16b65427..bfb2c02860d 100644 --- a/src/bootstrap/src/core/config/tests.rs +++ b/src/bootstrap/src/core/config/tests.rs @@ -1,5 +1,7 @@ use super::{flags::Flags, ChangeIdWrapper, Config}; use crate::core::build_steps::clippy::get_clippy_rules_in_order; +use crate::core::config::Target; +use crate::core::config::TargetSelection; use crate::core::config::{LldMode, TomlConfig}; use clap::CommandFactory; @@ -12,16 +14,9 @@ use std::{ }; fn parse(config: &str) -> Config { - Config::parse_inner( - &[ - "check".to_string(), - "--set=build.rustc=/does/not/exist".to_string(), - "--set=build.cargo=/does/not/exist".to_string(), - "--config=/does/not/exist".to_string(), - "--skip-stage0-validation".to_string(), - ], - |&_| toml::from_str(&config).unwrap(), - ) + Config::parse_inner(&["check".to_string(), "--config=/does/not/exist".to_string()], |&_| { + toml::from_str(&config).unwrap() + }) } #[test] @@ -124,6 +119,10 @@ fn override_toml() { "--set=build.gdb=\"bar\"".to_owned(), "--set=build.tools=[\"cargo\"]".to_owned(), "--set=llvm.build-config={\"foo\" = \"bar\"}".to_owned(), + "--set=target.x86_64-unknown-linux-gnu.runner=bar".to_owned(), + "--set=target.x86_64-unknown-linux-gnu.rpath=false".to_owned(), + "--set=target.aarch64-unknown-linux-gnu.sanitizers=false".to_owned(), + "--set=target.aarch64-apple-darwin.runner=apple".to_owned(), ], |&_| { toml::from_str( @@ -140,6 +139,17 @@ tools = [] [llvm] download-ci-llvm = false build-config = {} + +[target.aarch64-unknown-linux-gnu] +sanitizers = true +rpath = true +runner = "aarch64-runner" + +[target.x86_64-unknown-linux-gnu] +sanitizers = true +rpath = true +runner = "x86_64-runner" + "#, ) .unwrap() @@ -163,6 +173,30 @@ build-config = {} [("foo".to_string(), "bar".to_string())].into_iter().collect(), "setting dictionary value" ); + + let x86_64 = TargetSelection::from_user("x86_64-unknown-linux-gnu"); + let x86_64_values = Target { + sanitizers: Some(true), + rpath: Some(false), + runner: Some("bar".into()), + ..Default::default() + }; + let aarch64 = TargetSelection::from_user("aarch64-unknown-linux-gnu"); + let aarch64_values = Target { + sanitizers: Some(false), + rpath: Some(true), + runner: Some("aarch64-runner".into()), + ..Default::default() + }; + let darwin = TargetSelection::from_user("aarch64-apple-darwin"); + let darwin_values = Target { runner: Some("apple".into()), ..Default::default() }; + assert_eq!( + config.target_config, + [(x86_64, x86_64_values), (aarch64, aarch64_values), (darwin, darwin_values)] + .into_iter() + .collect(), + "setting dictionary value" + ); } #[test] @@ -171,10 +205,7 @@ fn override_toml_duplicate() { Config::parse_inner( &[ "check".to_owned(), - "--set=build.rustc=/does/not/exist".to_string(), - "--set=build.cargo=/does/not/exist".to_string(), - "--config=/does/not/exist".to_owned(), - "--skip-stage0-validation".to_owned(), + "--config=/does/not/exist".to_string(), "--set=change-id=1".to_owned(), "--set=change-id=2".to_owned(), ], @@ -197,15 +228,7 @@ fn profile_user_dist() { .and_then(|table: toml::Value| TomlConfig::deserialize(table)) .unwrap() } - Config::parse_inner( - &[ - "check".to_owned(), - "--set=build.rustc=/does/not/exist".to_string(), - "--set=build.cargo=/does/not/exist".to_string(), - "--skip-stage0-validation".to_string(), - ], - get_toml, - ); + Config::parse_inner(&["check".to_owned()], get_toml); } #[test] diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index 60f48c5923e..2b11b8c3d4f 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -9,11 +9,10 @@ use std::{ }; use build_helper::ci::CiEnv; -use build_helper::stage0_parser::VersionMetadata; use xz2::bufread::XzDecoder; +use crate::utils::helpers::hex_encode; use crate::utils::helpers::{check_run, exe, move_file, program_out_of_date}; -use crate::{core::build_steps::llvm::detect_llvm_sha, utils::helpers::hex_encode}; use crate::{t, Config}; static SHOULD_FIX_BINS_AND_DYLIBS: OnceLock<bool> = OnceLock::new(); @@ -405,9 +404,17 @@ impl Config { cargo_clippy } + #[cfg(feature = "bootstrap-self-test")] + pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> { + None + } + /// NOTE: rustfmt is a completely different toolchain than the bootstrap compiler, so it can't /// reuse target directories or artifacts + #[cfg(not(feature = "bootstrap-self-test"))] pub(crate) fn maybe_download_rustfmt(&self) -> Option<PathBuf> { + use build_helper::stage0_parser::VersionMetadata; + let VersionMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; let channel = format!("{version}-{date}"); @@ -487,6 +494,10 @@ impl Config { ); } + #[cfg(feature = "bootstrap-self-test")] + pub(crate) fn download_beta_toolchain(&self) {} + + #[cfg(not(feature = "bootstrap-self-test"))] pub(crate) fn download_beta_toolchain(&self) { self.verbose(|| println!("downloading stage0 beta artifacts")); @@ -665,7 +676,13 @@ download-rustc = false self.unpack(&tarball, &bin_root, prefix); } + #[cfg(feature = "bootstrap-self-test")] + pub(crate) fn maybe_download_ci_llvm(&self) {} + + #[cfg(not(feature = "bootstrap-self-test"))] pub(crate) fn maybe_download_ci_llvm(&self) { + use crate::core::build_steps::llvm::detect_llvm_sha; + if !self.llvm_from_ci { return; } @@ -707,6 +724,7 @@ download-rustc = false } } + #[cfg(not(feature = "bootstrap-self-test"))] fn download_ci_llvm(&self, llvm_sha: &str) { let llvm_assertions = self.llvm_assertions; diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs index 493ad99cc70..5a0be2948a1 100644 --- a/src/bootstrap/src/core/sanity.rs +++ b/src/bootstrap/src/core/sanity.rs @@ -14,7 +14,13 @@ use std::ffi::{OsStr, OsString}; use std::fs; use std::path::PathBuf; use std::process::Command; -use walkdir::WalkDir; + +#[cfg(not(feature = "bootstrap-self-test"))] +use crate::builder::Builder; +#[cfg(not(feature = "bootstrap-self-test"))] +use crate::core::build_steps::tool; +#[cfg(not(feature = "bootstrap-self-test"))] +use std::collections::HashSet; use crate::builder::Kind; use crate::core::config::Target; @@ -31,12 +37,16 @@ pub struct Finder { // it might not yet be included in stage0. In such cases, we handle the targets missing from stage0 in this list. // // Targets can be removed from this list once they are present in the stage0 compiler (usually by updating the beta compiler of the bootstrap). +#[cfg(not(feature = "bootstrap-self-test"))] const STAGE0_MISSING_TARGETS: &[&str] = &[ // just a dummy comment so the list doesn't get onelined - "aarch64-apple-visionos", - "aarch64-apple-visionos-sim", ]; +/// Minimum version threshold for libstdc++ required when using prebuilt LLVM +/// from CI (with`llvm.download-ci-llvm` option). +#[cfg(not(feature = "bootstrap-self-test"))] +const LIBSTDCXX_MIN_VERSION_THRESHOLD: usize = 8; + impl Finder { pub fn new() -> Self { Self { cache: HashMap::new(), path: env::var_os("PATH").unwrap_or_default() } @@ -101,20 +111,49 @@ pub fn check(build: &mut Build) { cmd_finder.must_have("git"); } + // Ensure that a compatible version of libstdc++ is available on the system when using `llvm.download-ci-llvm`. + #[cfg(not(feature = "bootstrap-self-test"))] + if !build.config.dry_run() && !build.build.is_msvc() && build.config.llvm_from_ci { + let builder = Builder::new(build); + let libcxx_version = builder.ensure(tool::LibcxxVersionTool { target: build.build }); + + match libcxx_version { + tool::LibcxxVersion::Gnu(version) => { + if LIBSTDCXX_MIN_VERSION_THRESHOLD > version { + eprintln!( + "\nYour system's libstdc++ version is too old for the `llvm.download-ci-llvm` option." + ); + eprintln!("Current version detected: '{}'", version); + eprintln!("Minimum required version: '{}'", LIBSTDCXX_MIN_VERSION_THRESHOLD); + eprintln!( + "Consider upgrading libstdc++ or disabling the `llvm.download-ci-llvm` option." + ); + eprintln!( + "If you choose to upgrade libstdc++, run `x clean` or delete `build/host/libcxx-version` manually after the upgrade." + ); + } + } + tool::LibcxxVersion::Llvm(_) => { + // FIXME: Handle libc++ version check. + } + } + } + // We need cmake, but only if we're actually building LLVM or sanitizers. - let building_llvm = build - .hosts - .iter() - .map(|host| { - build.config.llvm_enabled(*host) - && build - .config - .target_config - .get(host) - .map(|config| config.llvm_config.is_none()) - .unwrap_or(true) - }) - .any(|build_llvm_ourselves| build_llvm_ourselves); + let building_llvm = !build.config.llvm_from_ci + && build + .hosts + .iter() + .map(|host| { + build.config.llvm_enabled(*host) + && build + .config + .target_config + .get(host) + .map(|config| config.llvm_config.is_none()) + .unwrap_or(true) + }) + .any(|build_llvm_ourselves| build_llvm_ourselves); let need_cmake = building_llvm || build.config.any_sanitizers_to_build(); if need_cmake && cmd_finder.maybe_have("cmake").is_none() { @@ -169,6 +208,13 @@ than building it. .map(|p| cmd_finder.must_have(p)) .or_else(|| cmd_finder.maybe_have("reuse")); + #[cfg(not(feature = "bootstrap-self-test"))] + let stage0_supported_target_list: HashSet<String> = + output(Command::new(&build.config.initial_rustc).args(["--print", "target-list"])) + .lines() + .map(|s| s.to_string()) + .collect(); + // We're gonna build some custom C code here and there, host triples // also build some C++ shims for LLVM so we need a C++ compiler. for target in &build.targets { @@ -189,17 +235,29 @@ than building it. continue; } - let target_str = target.to_string(); - // Ignore fake targets that are only used for unit tests in bootstrap. - if !["A-A", "B-B", "C-C"].contains(&target_str.as_str()) { + #[cfg(not(feature = "bootstrap-self-test"))] + { let mut has_target = false; + let target_str = target.to_string(); - let supported_target_list = - output(Command::new(&build.config.initial_rustc).args(["--print", "target-list"])); + let missing_targets_hashset: HashSet<_> = + STAGE0_MISSING_TARGETS.iter().map(|t| t.to_string()).collect(); + let duplicated_targets: Vec<_> = + stage0_supported_target_list.intersection(&missing_targets_hashset).collect(); + + if !duplicated_targets.is_empty() { + println!( + "Following targets supported from the stage0 compiler, please remove them from STAGE0_MISSING_TARGETS list." + ); + for duplicated_target in duplicated_targets { + println!(" {duplicated_target}"); + } + std::process::exit(1); + } // Check if it's a built-in target. - has_target |= supported_target_list.contains(&target_str); + has_target |= stage0_supported_target_list.contains(&target_str); has_target |= STAGE0_MISSING_TARGETS.contains(&target_str.as_str()); if !has_target { @@ -210,7 +268,7 @@ than building it. target_filename.push(".json"); // Recursively traverse through nested directories. - let walker = WalkDir::new(custom_target_path).into_iter(); + let walker = walkdir::WalkDir::new(custom_target_path).into_iter(); for entry in walker.filter_map(|e| e.ok()) { has_target |= entry.file_name() == target_filename; } diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 698a576effa..449d8c128ec 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -84,13 +84,14 @@ const EXTRA_CHECK_CFGS: &[(Option<Mode>, &str, Option<&[&'static str]>)] = &[ (Some(Mode::ToolRustc), "rust_analyzer", None), (Some(Mode::ToolStd), "rust_analyzer", None), (Some(Mode::Codegen), "parallel_compiler", None), + // NOTE: consider updating `check-cfg` entries in `std/Cargo.toml` too. + // cfg(bootstrap) remove these once the bootstrap compiler supports + // `lints.rust.unexpected_cfgs.check-cfg` (Some(Mode::Std), "stdarch_intel_sde", None), (Some(Mode::Std), "no_fp_fmt_parse", None), (Some(Mode::Std), "no_global_oom_handling", None), (Some(Mode::Std), "no_rc", None), (Some(Mode::Std), "no_sync", None), - (Some(Mode::Std), "netbsd10", None), - (Some(Mode::Std), "backtrace_in_libstd", None), /* Extra values not defined in the built-in targets yet, but used in std */ (Some(Mode::Std), "target_env", Some(&["libnx", "p2"])), (Some(Mode::Std), "target_os", Some(&["visionos"])), @@ -468,7 +469,8 @@ impl Build { // Make sure we update these before gathering metadata so we don't get an error about missing // Cargo.toml files. - let rust_submodules = ["src/tools/cargo", "library/backtrace", "library/stdarch"]; + let rust_submodules = + ["src/tools/cargo", "src/doc/book", "library/backtrace", "library/stdarch"]; for s in rust_submodules { build.update_submodule(Path::new(s)); } @@ -518,33 +520,27 @@ impl Build { return; } - // check_submodule - let checked_out_hash = - output(Command::new("git").args(["rev-parse", "HEAD"]).current_dir(&absolute_path)); - // update_submodules - let recorded = output( - Command::new("git") - .args(["ls-tree", "HEAD"]) - .arg(relative_path) - .current_dir(&self.config.src), - ); + let submodule_git = || helpers::git(Some(&absolute_path)); + + // Determine commit checked out in submodule. + let checked_out_hash = output(submodule_git().args(["rev-parse", "HEAD"])); + let checked_out_hash = checked_out_hash.trim_end(); + // Determine commit that the submodule *should* have. + let recorded = + output(helpers::git(Some(&self.src)).args(["ls-tree", "HEAD"]).arg(relative_path)); let actual_hash = recorded .split_whitespace() .nth(2) .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); - // update_submodule - if actual_hash == checked_out_hash.trim_end() { + if actual_hash == checked_out_hash { // already checked out return; } println!("Updating submodule {}", relative_path.display()); self.run( - Command::new("git") - .args(["submodule", "-q", "sync"]) - .arg(relative_path) - .current_dir(&self.config.src), + helpers::git(Some(&self.src)).args(["submodule", "-q", "sync"]).arg(relative_path), ); // Try passing `--progress` to start, then run git again without if that fails. @@ -552,9 +548,7 @@ impl Build { // Git is buggy and will try to fetch submodules from the tracking branch for *this* repository, // even though that has no relation to the upstream for the submodule. let current_branch = { - let output = self - .config - .git() + let output = helpers::git(Some(&self.src)) .args(["symbolic-ref", "--short", "HEAD"]) .stderr(Stdio::inherit()) .output(); @@ -566,7 +560,7 @@ impl Build { } }; - let mut git = self.config.git(); + let mut git = helpers::git(Some(&self.src)); if let Some(branch) = current_branch { // If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name. // This syntax isn't accepted by `branch.{branch}`. Strip it. @@ -588,26 +582,22 @@ impl Build { // Save any local changes, but avoid running `git stash pop` if there are none (since it will exit with an error). // diff-index reports the modifications through the exit status let has_local_modifications = !self.run_cmd( - BootstrapCommand::from( - Command::new("git") - .args(["diff-index", "--quiet", "HEAD"]) - .current_dir(&absolute_path), - ) - .allow_failure() - .output_mode(match self.is_verbose() { - true => OutputMode::PrintAll, - false => OutputMode::PrintOutput, - }), + BootstrapCommand::from(submodule_git().args(["diff-index", "--quiet", "HEAD"])) + .allow_failure() + .output_mode(match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }), ); if has_local_modifications { - self.run(Command::new("git").args(["stash", "push"]).current_dir(&absolute_path)); + self.run(submodule_git().args(["stash", "push"])); } - self.run(Command::new("git").args(["reset", "-q", "--hard"]).current_dir(&absolute_path)); - self.run(Command::new("git").args(["clean", "-qdfx"]).current_dir(&absolute_path)); + self.run(submodule_git().args(["reset", "-q", "--hard"])); + self.run(submodule_git().args(["clean", "-qdfx"])); if has_local_modifications { - self.run(Command::new("git").args(["stash", "pop"]).current_dir(absolute_path)); + self.run(submodule_git().args(["stash", "pop"])); } } @@ -619,10 +609,9 @@ impl Build { return; } let output = output( - self.config - .git() + helpers::git(Some(&self.src)) .args(["config", "--file"]) - .arg(&self.config.src.join(".gitmodules")) + .arg(self.config.src.join(".gitmodules")) .args(["--get-regexp", "path"]), ); for line in output.lines() { @@ -659,10 +648,11 @@ impl Build { // hardcoded subcommands match &self.config.cmd { - Subcommand::Format { check } => { + Subcommand::Format { check, all } => { return core::build_steps::format::format( &builder::Builder::new(self), *check, + *all, &self.config.paths, ); } @@ -1560,10 +1550,14 @@ impl Build { // Figure out how many merge commits happened since we branched off master. // That's our beta number! // (Note that we use a `..` range, not the `...` symmetric difference.) - output(self.config.git().arg("rev-list").arg("--count").arg("--merges").arg(format!( - "refs/remotes/origin/{}..HEAD", - self.config.stage0_metadata.config.nightly_branch - ))) + output( + helpers::git(Some(&self.src)).arg("rev-list").arg("--count").arg("--merges").arg( + format!( + "refs/remotes/origin/{}..HEAD", + self.config.stage0_metadata.config.nightly_branch + ), + ), + ) }); let n = count.trim().parse().unwrap(); self.prerelease_version.set(Some(n)); @@ -1981,15 +1975,13 @@ fn envify(s: &str) -> String { /// In case of errors during `git` command execution (e.g., in tarball sources), default values /// are used to prevent panics. pub fn generate_smart_stamp_hash(dir: &Path, additional_input: &str) -> String { - let diff = Command::new("git") - .current_dir(dir) + let diff = helpers::git(Some(dir)) .arg("diff") .output() .map(|o| String::from_utf8(o.stdout).unwrap_or_default()) .unwrap_or_default(); - let status = Command::new("git") - .current_dir(dir) + let status = helpers::git(Some(dir)) .arg("status") .arg("--porcelain") .arg("-z") diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 2f9eaf51c34..bfe3622e40d 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -190,4 +190,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Warning, summary: "`rust.lld` has a new default value of `true` on `x86_64-unknown-linux-gnu`. Starting at stage1, `rust-lld` will thus be this target's default linker. No config changes should be necessary.", }, + ChangeInfo { + change_id: 125535, + severity: ChangeSeverity::Warning, + summary: "Removed `dist.missing-tools` configuration as it was deprecated long time ago.", + }, ]; diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs index 88988c33916..ce82c52f049 100644 --- a/src/bootstrap/src/utils/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -7,11 +7,12 @@ use std::fs; use std::path::Path; -use std::process::Command; use crate::utils::helpers::{output, t}; use crate::Build; +use super::helpers; + #[derive(Clone, Default)] pub enum GitInfo { /// This is not a git repository. @@ -44,7 +45,7 @@ impl GitInfo { } // Make sure git commands work - match Command::new("git").arg("rev-parse").current_dir(dir).output() { + match helpers::git(Some(dir)).arg("rev-parse").output() { Ok(ref out) if out.status.success() => {} _ => return GitInfo::Absent, } @@ -57,17 +58,15 @@ impl GitInfo { // Ok, let's scrape some info let ver_date = output( - Command::new("git") - .current_dir(dir) + helpers::git(Some(dir)) .arg("log") .arg("-1") .arg("--date=short") .arg("--pretty=format:%cd"), ); - let ver_hash = output(Command::new("git").current_dir(dir).arg("rev-parse").arg("HEAD")); - let short_ver_hash = output( - Command::new("git").current_dir(dir).arg("rev-parse").arg("--short=9").arg("HEAD"), - ); + let ver_hash = output(helpers::git(Some(dir)).arg("rev-parse").arg("HEAD")); + let short_ver_hash = + output(helpers::git(Some(dir)).arg("rev-parse").arg("--short=9").arg("HEAD")); GitInfo::Present(Some(Info { commit_date: ver_date.trim().to_string(), sha: ver_hash.trim().to_string(), diff --git a/src/bootstrap/src/utils/dylib.rs b/src/bootstrap/src/utils/dylib.rs index b6e7aec1756..90bcff59a64 100644 --- a/src/bootstrap/src/utils/dylib.rs +++ b/src/bootstrap/src/utils/dylib.rs @@ -5,7 +5,7 @@ pub fn dylib_path_var() -> &'static str { if cfg!(target_os = "windows") { "PATH" - } else if cfg!(target_os = "macos") { + } else if cfg!(target_vendor = "apple") { "DYLD_LIBRARY_PATH" } else if cfg!(target_os = "haiku") { "LIBRARY_PATH" diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index 278359cb08e..4b6dc45b436 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -331,115 +331,6 @@ fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool { }) } -/// Copied from `std::path::absolute` until it stabilizes. -/// -/// FIXME: this shouldn't exist. -pub(crate) fn absolute(path: &Path) -> PathBuf { - if path.as_os_str().is_empty() { - panic!("can't make empty path absolute"); - } - #[cfg(unix)] - { - t!(absolute_unix(path), format!("could not make path absolute: {}", path.display())) - } - #[cfg(windows)] - { - t!(absolute_windows(path), format!("could not make path absolute: {}", path.display())) - } - #[cfg(not(any(unix, windows)))] - { - println!("WARNING: bootstrap is not supported on non-unix platforms"); - t!(std::fs::canonicalize(t!(std::env::current_dir()))).join(path) - } -} - -#[cfg(unix)] -/// Make a POSIX path absolute without changing its semantics. -fn absolute_unix(path: &Path) -> io::Result<PathBuf> { - // This is mostly a wrapper around collecting `Path::components`, with - // exceptions made where this conflicts with the POSIX specification. - // See 4.13 Pathname Resolution, IEEE Std 1003.1-2017 - // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 - - use std::os::unix::prelude::OsStrExt; - let mut components = path.components(); - let path_os = path.as_os_str().as_bytes(); - - let mut normalized = if path.is_absolute() { - // "If a pathname begins with two successive <slash> characters, the - // first component following the leading <slash> characters may be - // interpreted in an implementation-defined manner, although more than - // two leading <slash> characters shall be treated as a single <slash> - // character." - if path_os.starts_with(b"//") && !path_os.starts_with(b"///") { - components.next(); - PathBuf::from("//") - } else { - PathBuf::new() - } - } else { - env::current_dir()? - }; - normalized.extend(components); - - // "Interfaces using pathname resolution may specify additional constraints - // when a pathname that does not name an existing directory contains at - // least one non- <slash> character and contains one or more trailing - // <slash> characters". - // A trailing <slash> is also meaningful if "a symbolic link is - // encountered during pathname resolution". - - if path_os.ends_with(b"/") { - normalized.push(""); - } - - Ok(normalized) -} - -#[cfg(windows)] -fn absolute_windows(path: &std::path::Path) -> std::io::Result<std::path::PathBuf> { - use std::ffi::OsString; - use std::io::Error; - use std::os::windows::ffi::{OsStrExt, OsStringExt}; - use std::ptr::null_mut; - #[link(name = "kernel32")] - extern "system" { - fn GetFullPathNameW( - lpFileName: *const u16, - nBufferLength: u32, - lpBuffer: *mut u16, - lpFilePart: *mut *const u16, - ) -> u32; - } - - unsafe { - // encode the path as UTF-16 - let path: Vec<u16> = path.as_os_str().encode_wide().chain([0]).collect(); - let mut buffer = Vec::new(); - // Loop until either success or failure. - loop { - // Try to get the absolute path - let len = GetFullPathNameW( - path.as_ptr(), - buffer.len().try_into().unwrap(), - buffer.as_mut_ptr(), - null_mut(), - ); - match len as usize { - // Failure - 0 => return Err(Error::last_os_error()), - // Buffer is too small, resize. - len if len > buffer.len() => buffer.resize(len, 0), - // Success! - len => { - buffer.truncate(len); - return Ok(OsString::from_wide(&buffer).into()); - } - } - } - } -} - /// Adapted from <https://github.com/llvm/llvm-project/blob/782e91224601e461c019e0a4573bbccc6094fbcd/llvm/cmake/modules/HandleLLVMOptions.cmake#L1058-L1079> /// /// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource @@ -598,3 +489,29 @@ pub fn check_cfg_arg(name: &str, values: Option<&[&str]>) -> String { }; format!("--check-cfg=cfg({name}{next})") } + +/// Prepares `Command` that runs git inside the source directory if given. +/// +/// Whenever a git invocation is needed, this function should be preferred over +/// manually building a git `Command`. This approach allows us to manage bootstrap-specific +/// needs/hacks from a single source, rather than applying them on next to every `Command::new("git")`, +/// which is painful to ensure that the required change is applied on each one of them correctly. +pub fn git(source_dir: Option<&Path>) -> Command { + let mut git = Command::new("git"); + + if let Some(source_dir) = source_dir { + git.current_dir(source_dir); + // If we are running inside git (e.g. via a hook), `GIT_DIR` is set and takes precedence + // over the current dir. Un-set it to make the current dir matter. + git.env_remove("GIT_DIR"); + // Also un-set some other variables, to be on the safe side (based on cargo's + // `fetch_with_cli`). In particular un-setting `GIT_INDEX_FILE` is required to fix some odd + // misbehavior. + git.env_remove("GIT_WORK_TREE") + .env_remove("GIT_INDEX_FILE") + .env_remove("GIT_OBJECT_DIRECTORY") + .env_remove("GIT_ALTERNATE_OBJECT_DIRECTORIES"); + } + + git +} diff --git a/src/bootstrap/src/utils/helpers/tests.rs b/src/bootstrap/src/utils/helpers/tests.rs index 9cfaa3eb67b..2ab3952ae5a 100644 --- a/src/bootstrap/src/utils/helpers/tests.rs +++ b/src/bootstrap/src/utils/helpers/tests.rs @@ -25,27 +25,6 @@ fn test_make() { } } -#[cfg(unix)] -#[test] -fn test_absolute_unix() { - use crate::utils::helpers::absolute_unix; - - // Test an absolute path - let path = PathBuf::from("/home/user/file.txt"); - assert_eq!(absolute_unix(&path).unwrap(), PathBuf::from("/home/user/file.txt")); - - // Test an absolute path with double leading slashes - let path = PathBuf::from("//root//file.txt"); - assert_eq!(absolute_unix(&path).unwrap(), PathBuf::from("//root/file.txt")); - - // Test a relative path - let path = PathBuf::from("relative/path"); - assert_eq!( - absolute_unix(&path).unwrap(), - std::env::current_dir().unwrap().join("relative/path") - ); -} - #[test] fn test_beta_rev_parsing() { // single digit revision diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs index 57cdf7473a1..fd934f18de2 100644 --- a/src/bootstrap/src/utils/tarball.rs +++ b/src/bootstrap/src/utils/tarball.rs @@ -23,7 +23,6 @@ pub(crate) enum OverlayKind { Clippy, Miri, Rustfmt, - RustDemangler, Rls, RustAnalyzer, RustcCodegenCranelift, @@ -58,9 +57,6 @@ impl OverlayKind { "src/tools/rustfmt/LICENSE-APACHE", "src/tools/rustfmt/LICENSE-MIT", ], - OverlayKind::RustDemangler => { - &["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"] - } OverlayKind::Rls => &["src/tools/rls/README.md", "LICENSE-APACHE", "LICENSE-MIT"], OverlayKind::RustAnalyzer => &[ "src/tools/rust-analyzer/README.md", @@ -85,7 +81,6 @@ impl OverlayKind { match self { OverlayKind::Rust => builder.rust_version(), OverlayKind::Llvm => builder.rust_version(), - OverlayKind::RustDemangler => builder.release_num("rust-demangler"), OverlayKind::Cargo => { builder.cargo_info.version(builder, &builder.release_num("cargo")) } |