diff options
| author | Cassandra Fridkin <cass@swag.lgbt> | 2020-10-05 18:49:51 -0400 |
|---|---|---|
| committer | Cassandra Fridkin <cass@swag.lgbt> | 2020-10-05 18:49:51 -0400 |
| commit | 44af74f6ddf9102b358f271b371697c4b4e6dd2f (patch) | |
| tree | fd20034f9565a59750cdd1781a6a6e5f20d15a94 /src/bootstrap | |
| parent | a009e2838b25df2761093d727d322a59f69d8f68 (diff) | |
| parent | a1dfd2490a6cb456b92e469fa550dc217e20ad6d (diff) | |
| download | rust-44af74f6ddf9102b358f271b371697c4b4e6dd2f.tar.gz rust-44af74f6ddf9102b358f271b371697c4b4e6dd2f.zip | |
Merge branch 'master' into hooks
Diffstat (limited to 'src/bootstrap')
31 files changed, 1345 insertions, 471 deletions
diff --git a/src/bootstrap/CHANGELOG.md b/src/bootstrap/CHANGELOG.md new file mode 100644 index 00000000000..d8c704f451b --- /dev/null +++ b/src/bootstrap/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog + +All notable changes to bootstrap will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## [Non-breaking changes since the last major version] + +- `x.py check` needs opt-in to check tests (--all-targets) [#77473](https://github.com/rust-lang/rust/pull/77473) + +## [Version 2] - 2020-09-25 + +- `host` now defaults to the value of `build` in all cases + + Previously `host` defaulted to an empty list when `target` was overridden, and to `build` otherwise + +### Non-breaking changes + +- Add `x.py setup` [#76631](https://github.com/rust-lang/rust/pull/76631) +- Add a changelog for x.py [#76626](https://github.com/rust-lang/rust/pull/76626) +- Optionally, download LLVM from CI on Linux and NixOS + + [#76439](https://github.com/rust-lang/rust/pull/76349) + + [#76667](https://github.com/rust-lang/rust/pull/76667) + + [#76708](https://github.com/rust-lang/rust/pull/76708) +- Distribute rustc sources as part of `rustc-dev` [#76856](https://github.com/rust-lang/rust/pull/76856) +- Make the default stage for x.py configurable [#76625](https://github.com/rust-lang/rust/pull/76625) +- Add a dedicated debug-logging option [#76588](https://github.com/rust-lang/rust/pull/76588) +- Add sample defaults for x.py [#76628](https://github.com/rust-lang/rust/pull/76628) +- Add `--keep-stage-std`, which behaves like `keep-stage` but allows the stage + 0 compiler artifacts (i.e., stage1/bin/rustc) to be rebuilt if changed + [#77120](https://github.com/rust-lang/rust/pull/77120). + + +## [Version 1] - 2020-09-11 + +This is the first changelog entry, and it does not attempt to be an exhaustive list of features in x.py. +Instead, this documents the changes to bootstrap in the past 2 months. + +- Improve defaults in `x.py` [#73964](https://github.com/rust-lang/rust/pull/73964) + (see [blog post] for details) +- Set `ninja = true` by default [#74922](https://github.com/rust-lang/rust/pull/74922) +- Avoid trying to inversely cross-compile for build triple from host triples [#76415](https://github.com/rust-lang/rust/pull/76415) +- Allow blessing expect-tests in tools [#75975](https://github.com/rust-lang/rust/pull/75975) +- `x.py check` checks tests/examples/benches [#76258](https://github.com/rust-lang/rust/pull/76258) +- Fix `rust.use-lld` when linker is not set [#76326](https://github.com/rust-lang/rust/pull/76326) +- Build tests with LLD if `use-lld = true` was passed [#76378](https://github.com/rust-lang/rust/pull/76378) + +[blog post]: https://blog.rust-lang.org/inside-rust/2020/08/30/changes-to-x-py-defaults.html diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index faec2c53742..e04128d1b0b 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -3,6 +3,7 @@ authors = ["The Rust Project Developers"] name = "bootstrap" version = "0.0.0" edition = "2018" +build = "build.rs" [lib] path = "lib.rs" @@ -48,6 +49,7 @@ lazy_static = "1.3.0" time = "0.1" ignore = "0.4.10" opener = "0.4" +merge = "0.1.0" [target.'cfg(windows)'.dependencies.winapi] version = "0.3" diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 975b8be02c8..bc8bae14b21 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -26,10 +26,10 @@ The script accepts commands, flags, and arguments to determine what to do: ``` # build the whole compiler - ./x.py build + ./x.py build --stage 2 # build the stage1 compiler - ./x.py build --stage 1 + ./x.py build # build stage0 libstd ./x.py build --stage 0 library/std @@ -43,8 +43,8 @@ The script accepts commands, flags, and arguments to determine what to do: that belong to stage n or earlier: ``` - # keep old build products for stage 0 and build stage 1 - ./x.py build --keep-stage 0 --stage 1 + # build stage 1, keeping old build products for stage 0 + ./x.py build --keep-stage 0 ``` * `test` - a command for executing unit tests. Like the `build` command this @@ -123,24 +123,8 @@ that (b) leverage Rust as much as possible! ## Incremental builds -You can configure rustbuild to use incremental compilation. Because -incremental is new and evolving rapidly, if you want to use it, it is -recommended that you replace the snapshot with a locally installed -nightly build of rustc. You will want to keep this up to date. - -To follow this course of action, first thing you will want to do is to -install a nightly, presumably using `rustup`. You will then want to -configure your directory to use this build, like so: - -```sh -# configure to use local rust instead of downloading a beta. -# `--local-rust-root` is optional here. If elided, we will -# use whatever rustc we find on your PATH. -$ ./configure --local-rust-root=~/.cargo/ --enable-local-rebuild -``` - -After that, you can use the `--incremental` flag to actually do -incremental builds: +You can configure rustbuild to use incremental compilation with the +`--incremental` flag: ```sh $ ./x.py build --incremental @@ -150,9 +134,7 @@ The `--incremental` flag will store incremental compilation artifacts in `build/<host>/stage0-incremental`. Note that we only use incremental compilation for the stage0 -> stage1 compilation -- this is because the stage1 compiler is changing, and we don't try to cache and reuse -incremental artifacts across different versions of the compiler. For -this reason, `--incremental` defaults to `--stage 1` (though you can -manually select a higher stage, if you prefer). +incremental artifacts across different versions of the compiler. You can always drop the `--incremental` to build as normal (but you will still be using the local nightly as your bootstrap). @@ -331,8 +313,22 @@ are: `Config` struct. * Adding a sanity check? Take a look at `bootstrap/sanity.rs`. -If you have any questions feel free to reach out on `#infra` channel in the -[Rust Discord server][rust-discord] or ask on internals.rust-lang.org. When +If you make a major change, please remember to: + ++ Update `VERSION` in `src/bootstrap/main.rs`. +* Update `changelog-seen = N` in `config.toml.example`. +* Add an entry in `src/bootstrap/CHANGELOG.md`. + +A 'major change' includes + +* A new option or +* A change in the default options. + +Changes that do not affect contributors to the compiler or users +building rustc from source don't need an update to `VERSION`. + +If you have any questions feel free to reach out on the `#t-infra` channel in +the [Rust Zulip server][rust-zulip] or ask on internals.rust-lang.org. When you encounter bugs, please file issues on the rust-lang/rust issue tracker. -[rust-discord]: https://discord.gg/rust-lang +[rust-zulip]: https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs index b67486c9628..d31f95ee5e9 100644 --- a/src/bootstrap/bin/main.rs +++ b/src/bootstrap/bin/main.rs @@ -7,10 +7,58 @@ use std::env; -use bootstrap::{Build, Config}; +use bootstrap::{Build, Config, Subcommand}; fn main() { let args = env::args().skip(1).collect::<Vec<_>>(); let config = Config::parse(&args); + + let changelog_suggestion = check_version(&config); + + // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the + // changelog warning, not the `x.py setup` message. + let suggest_setup = !config.config.exists() && !matches!(config.cmd, Subcommand::Setup { .. }); + if suggest_setup { + println!("warning: you have not made a `config.toml`"); + println!("help: consider running `x.py setup` or copying `config.toml.example`"); + } else if let Some(suggestion) = &changelog_suggestion { + println!("{}", suggestion); + } + Build::new(config).build(); + + if suggest_setup { + println!("warning: you have not made a `config.toml`"); + println!("help: consider running `x.py setup` or copying `config.toml.example`"); + } else if let Some(suggestion) = &changelog_suggestion { + println!("{}", suggestion); + } + + if suggest_setup || changelog_suggestion.is_some() { + println!("note: this message was printed twice to make it more likely to be seen"); + } +} + +fn check_version(config: &Config) -> Option<String> { + const VERSION: usize = 2; + + let mut msg = String::new(); + + let suggestion = if let Some(seen) = config.changelog_seen { + if seen != VERSION { + msg.push_str("warning: there have been changes to x.py since you last updated.\n"); + format!("update `config.toml` to use `changelog-seen = {}` instead", VERSION) + } else { + return None; + } + } else { + msg.push_str("warning: x.py has made several changes recently you may want to look at\n"); + format!("add `changelog-seen = {}` at the top of `config.toml`", VERSION) + }; + + msg.push_str("help: consider looking at the changes in `src/bootstrap/CHANGELOG.md`\n"); + msg.push_str("note: to silence this warning, "); + msg.push_str(&suggestion); + + Some(msg) } diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index 4dd71ebade1..3694bdbf670 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -112,6 +112,9 @@ fn main() { if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") { cmd.arg(format!("-Clinker={}", host_linker)); } + if env::var_os("RUSTC_HOST_FUSE_LD_LLD").is_some() { + cmd.arg("-Clink-args=-fuse-ld=lld"); + } if let Ok(s) = env::var("RUSTC_HOST_CRT_STATIC") { if s == "true" { diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs index ab846adf942..cba17c8e608 100644 --- a/src/bootstrap/bin/rustdoc.rs +++ b/src/bootstrap/bin/rustdoc.rs @@ -11,7 +11,6 @@ fn main() { let args = env::args_os().skip(1).collect::<Vec<_>>(); let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set"); let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set"); - let stage = env::var("RUSTC_STAGE").expect("RUSTC_STAGE was not set"); let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); use std::str::FromStr; @@ -24,14 +23,8 @@ fn main() { let mut dylib_path = bootstrap::util::dylib_path(); dylib_path.insert(0, PathBuf::from(libdir.clone())); - //FIXME(misdreavus): once stdsimd uses cfg(doc) instead of cfg(dox), remove the `--cfg dox` - //arguments here let mut cmd = Command::new(rustdoc); cmd.args(&args) - .arg("--cfg") - .arg(format!("stage{}", stage)) - .arg("--cfg") - .arg("dox") .arg("--sysroot") .arg(&sysroot) .env(bootstrap::util::dylib_path_var(), env::join_paths(&dylib_path).unwrap()); @@ -42,11 +35,14 @@ fn main() { if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { cmd.arg("-Z").arg("force-unstable-if-unmarked"); } - if let Some(linker) = env::var_os("RUSTC_TARGET_LINKER") { + if let Some(linker) = env::var_os("RUSTDOC_LINKER") { let mut arg = OsString::from("-Clinker="); arg.push(&linker); cmd.arg(arg); } + if env::var_os("RUSTDOC_FUSE_LD_LLD").is_some() { + cmd.arg("-Clink-args=-fuse-ld=lld"); + } // Needed to be able to run all rustdoc tests. if let Some(ref x) = env::var_os("RUSTDOC_RESOURCE_SUFFIX") { diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index c3f1bac177d..5c9184f4506 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -14,8 +14,17 @@ import tempfile from time import time - -def get(url, path, verbose=False): +def support_xz(): + try: + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + temp_path = temp_file.name + with tarfile.open(temp_path, "w:xz"): + pass + return True + except tarfile.CompressionError: + return False + +def get(url, path, verbose=False, do_verify=True): suffix = '.sha256' sha_url = url + suffix with tempfile.NamedTemporaryFile(delete=False) as temp_file: @@ -24,19 +33,20 @@ def get(url, path, verbose=False): sha_path = sha_file.name try: - download(sha_path, sha_url, False, verbose) - if os.path.exists(path): - if verify(path, sha_path, False): - if verbose: - print("using already-download file", path) - return - else: - if verbose: - print("ignoring already-download file", - path, "due to failed verification") - os.unlink(path) + if do_verify: + download(sha_path, sha_url, False, verbose) + if os.path.exists(path): + if verify(path, sha_path, False): + if verbose: + print("using already-download file", path) + return + else: + if verbose: + print("ignoring already-download file", + path, "due to failed verification") + os.unlink(path) download(temp_path, url, True, verbose) - if not verify(temp_path, sha_path, verbose): + if do_verify and not verify(temp_path, sha_path, verbose): raise RuntimeError("failed verification") if verbose: print("moving {} to {}".format(temp_path, path)) @@ -365,16 +375,6 @@ class RustBuild(object): cargo_channel = self.cargo_channel rustfmt_channel = self.rustfmt_channel - def support_xz(): - try: - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_path = temp_file.name - with tarfile.open(temp_path, "w:xz"): - pass - return True - except tarfile.CompressionError: - return False - if self.rustc().startswith(self.bin_root()) and \ (not os.path.exists(self.rustc()) or self.program_out_of_date(self.rustc_stamp())): @@ -398,14 +398,6 @@ class RustBuild(object): with output(self.rustc_stamp()) as rust_stamp: rust_stamp.write(self.date) - # This is required so that we don't mix incompatible MinGW - # libraries/binaries that are included in rust-std with - # the system MinGW ones. - if "pc-windows-gnu" in self.build: - filename = "rust-mingw-{}-{}{}".format( - rustc_channel, self.build, tarball_suffix) - self._download_stage0_helper(filename, "rust-mingw", tarball_suffix) - if self.cargo().startswith(self.bin_root()) and \ (not os.path.exists(self.cargo()) or self.program_out_of_date(self.cargo_stamp())): @@ -431,6 +423,32 @@ class RustBuild(object): with output(self.rustfmt_stamp()) as rustfmt_stamp: rustfmt_stamp.write(self.date + self.rustfmt_channel) + if self.downloading_llvm(): + # We want the most recent LLVM submodule update to avoid downloading + # LLVM more often than necessary. + # + # This git command finds that commit SHA, looking for bors-authored + # merges that modified src/llvm-project. + # + # This works even in a repository that has not yet initialized + # submodules. + llvm_sha = subprocess.check_output([ + "git", "log", "--author=bors", "--format=%H", "-n1", + "-m", "--first-parent", + "--", "src/llvm-project" + ]).decode(sys.getdefaultencoding()).strip() + llvm_assertions = self.get_toml('assertions', 'llvm') == 'true' + if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)): + self._download_ci_llvm(llvm_sha, llvm_assertions) + for binary in ["llvm-config", "FileCheck"]: + self.fix_bin_or_dylib("{}/bin/{}".format(self.llvm_root(), binary)) + with output(self.llvm_stamp()) as llvm_stamp: + llvm_stamp.write(self.date + llvm_sha + str(llvm_assertions)) + + def downloading_llvm(self): + opt = self.get_toml('download-ci-llvm', 'llvm') + return opt == "true" + def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None): if date is None: date = self.date @@ -445,6 +463,25 @@ class RustBuild(object): get("{}/{}".format(url, filename), tarball, verbose=self.verbose) unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) + def _download_ci_llvm(self, llvm_sha, llvm_assertions): + cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions) + cache_dst = os.path.join(self.build_dir, "cache") + rustc_cache = os.path.join(cache_dst, cache_prefix) + if not os.path.exists(rustc_cache): + os.makedirs(rustc_cache) + + url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(llvm_sha) + if llvm_assertions: + url = url.replace('rustc-builds', 'rustc-builds-alt') + tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' + filename = "rust-dev-nightly-" + self.build + tarball_suffix + tarball = os.path.join(rustc_cache, filename) + if not os.path.exists(tarball): + get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=False) + unpack(tarball, tarball_suffix, self.llvm_root(), + match="rust-dev", + verbose=self.verbose) + def fix_bin_or_dylib(self, fname): """Modifies the interpreter section of 'fname' to fix the dynamic linker, or the RPATH section, to fix the dynamic library search path @@ -566,6 +603,17 @@ class RustBuild(object): """ return os.path.join(self.bin_root(), '.rustfmt-stamp') + def llvm_stamp(self): + """Return the path for .rustfmt-stamp + + >>> rb = RustBuild() + >>> rb.build_dir = "build" + >>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp") + True + """ + return os.path.join(self.llvm_root(), '.llvm-stamp') + + def program_out_of_date(self, stamp_path, extra=""): """Check if the given program stamp is out of date""" if not os.path.exists(stamp_path) or self.clean: @@ -589,6 +637,22 @@ class RustBuild(object): """ return os.path.join(self.build_dir, self.build, "stage0") + def llvm_root(self): + """Return the CI LLVM root directory + + >>> rb = RustBuild() + >>> rb.build_dir = "build" + >>> rb.llvm_root() == os.path.join("build", "ci-llvm") + True + + When the 'build' property is given should be a nested directory: + + >>> rb.build = "devel" + >>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm") + True + """ + return os.path.join(self.build_dir, self.build, "ci-llvm") + def get_toml(self, key, section=None): """Returns the value of the given key in config.toml, otherwise returns None @@ -714,7 +778,6 @@ class RustBuild(object): # See also: <https://github.com/rust-lang/rust/issues/70208>. if "CARGO_BUILD_TARGET" in env: del env["CARGO_BUILD_TARGET"] - env["RUSTC_BOOTSTRAP"] = '1' env["CARGO_TARGET_DIR"] = build_dir env["RUSTC"] = self.rustc() env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ @@ -829,8 +892,9 @@ class RustBuild(object): submodules_names = [] for module in submodules: if module.endswith("llvm-project"): - if self.get_toml('llvm-config') and self.get_toml('lld') != 'true': - continue + if self.get_toml('llvm-config') or self.get_toml('download-ci-llvm') == 'true': + if self.get_toml('lld') != 'true': + continue check = self.check_submodule(module, slow_submodules) filtered_submodules.append((module, check)) submodules_names.append(module) @@ -914,7 +978,6 @@ def bootstrap(help_triggered): parser = argparse.ArgumentParser(description='Build rust') parser.add_argument('--config') parser.add_argument('--build') - parser.add_argument('--src') parser.add_argument('--clean', action='store_true') parser.add_argument('-v', '--verbose', action='count', default=0) @@ -923,7 +986,7 @@ def bootstrap(help_triggered): # Configure initial bootstrap build = RustBuild() - build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..')) + build.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) build.verbose = args.verbose build.clean = args.clean @@ -980,18 +1043,12 @@ def bootstrap(help_triggered): args = [build.bootstrap_binary()] args.extend(sys.argv[1:]) env = os.environ.copy() - env["BUILD"] = build.build - env["SRC"] = build.rust_root env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) env["BOOTSTRAP_PYTHON"] = sys.executable env["BUILD_DIR"] = build.build_dir env["RUSTC_BOOTSTRAP"] = '1' - env["CARGO"] = build.cargo() - env["RUSTC"] = build.rustc() if toml_path: env["BOOTSTRAP_CONFIG"] = toml_path - if build.rustfmt(): - env["RUSTFMT"] = build.rustfmt() run(args, env=env, verbose=build.verbose) diff --git a/src/bootstrap/build.rs b/src/bootstrap/build.rs new file mode 100644 index 00000000000..d40b924e0ff --- /dev/null +++ b/src/bootstrap/build.rs @@ -0,0 +1,26 @@ +use std::env; +use std::path::PathBuf; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rustc-env=BUILD_TRIPLE={}", env::var("HOST").unwrap()); + + // This may not be a canonicalized path. + let mut rustc = PathBuf::from(env::var_os("RUSTC").unwrap()); + + if rustc.is_relative() { + for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { + let absolute = dir.join(&rustc); + if absolute.exists() { + rustc = absolute; + break; + } + } + } + assert!(rustc.is_absolute()); + + // FIXME: if the path is not utf-8, this is going to break. Unfortunately + // Cargo doesn't have a way for us to specify non-utf-8 paths easily, so + // we'll need to invent some encoding scheme if this becomes a problem. + println!("cargo:rustc-env=RUSTC={}", rustc.to_str().unwrap()); +} diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 0a7df6e595c..1359b162853 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -87,11 +87,16 @@ pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { pub struct RunConfig<'a> { pub builder: &'a Builder<'a>, - pub host: TargetSelection, pub target: TargetSelection, pub path: PathBuf, } +impl RunConfig<'_> { + pub fn build_triple(&self) -> TargetSelection { + self.builder.build.build + } +} + struct StepDescription { default: bool, only_hosts: bool, @@ -165,29 +170,13 @@ impl StepDescription { pathset, self.name, builder.config.exclude ); } - let hosts = &builder.hosts; // Determine the targets participating in this rule. - let targets = if self.only_hosts { - if builder.config.skip_only_host_steps { - return; // don't run anything - } else { - &builder.hosts - } - } else { - &builder.targets - }; + let targets = if self.only_hosts { &builder.hosts } else { &builder.targets }; - for host in hosts { - for target in targets { - let run = RunConfig { - builder, - path: pathset.path(builder), - host: *host, - target: *target, - }; - (self.make_run)(run); - } + for target in targets { + let run = RunConfig { builder, path: pathset.path(builder), target: *target }; + (self.make_run)(run); } } @@ -382,7 +371,7 @@ impl<'a> Builder<'a> { native::Lld ), Kind::Check | Kind::Clippy | Kind::Fix | Kind::Format => { - describe!(check::Std, check::Rustc, check::Rustdoc, check::Clippy) + describe!(check::Std, check::Rustc, check::Rustdoc, check::Clippy, check::Bootstrap) } Kind::Test => describe!( crate::toolstate::ToolStateCheck, @@ -471,6 +460,7 @@ impl<'a> Builder<'a> { dist::Clippy, dist::Miri, dist::LlvmTools, + dist::RustDev, dist::Extended, dist::HashSign ), @@ -487,7 +477,7 @@ impl<'a> Builder<'a> { install::Src, install::Rustc ), - Kind::Run => describe!(run::ExpandYamlAnchors, run::InstallGitHook), + Kind::Run => describe!(run::ExpandYamlAnchors, run::BuildManifest, run::InstallGitHook), } } @@ -528,23 +518,9 @@ impl<'a> Builder<'a> { } fn new_internal(build: &Build, kind: Kind, paths: Vec<PathBuf>) -> Builder<'_> { - let top_stage = if let Some(explicit_stage) = build.config.stage { - explicit_stage - } else { - // See https://github.com/rust-lang/compiler-team/issues/326 - match kind { - Kind::Doc => 0, - Kind::Build | Kind::Test => 1, - Kind::Bench | Kind::Dist | Kind::Install => 2, - // These are all bootstrap tools, which don't depend on the compiler. - // The stage we pass shouldn't matter, but use 0 just in case. - Kind::Check | Kind::Clippy | Kind::Fix | Kind::Run | Kind::Format => 0, - } - }; - Builder { build, - top_stage, + top_stage: build.config.stage, kind, cache: Cache::new(), stack: RefCell::new(Vec::new()), @@ -556,7 +532,7 @@ impl<'a> Builder<'a> { pub fn new(build: &Build) -> Builder<'_> { let (kind, paths) = match build.config.cmd { Subcommand::Build { ref paths } => (Kind::Build, &paths[..]), - Subcommand::Check { ref paths } => (Kind::Check, &paths[..]), + Subcommand::Check { ref paths, all_targets: _ } => (Kind::Check, &paths[..]), Subcommand::Clippy { ref paths } => (Kind::Clippy, &paths[..]), Subcommand::Fix { ref paths } => (Kind::Fix, &paths[..]), Subcommand::Doc { ref paths, .. } => (Kind::Doc, &paths[..]), @@ -565,23 +541,12 @@ impl<'a> Builder<'a> { Subcommand::Dist { ref paths } => (Kind::Dist, &paths[..]), Subcommand::Install { ref paths } => (Kind::Install, &paths[..]), Subcommand::Run { ref paths } => (Kind::Run, &paths[..]), - Subcommand::Format { .. } | Subcommand::Clean { .. } => panic!(), - }; - - let this = Self::new_internal(build, kind, paths.to_owned()); - - // CI should always run stage 2 builds, unless it specifically states otherwise - #[cfg(not(test))] - if build.config.stage.is_none() && build.ci_env != crate::CiEnv::None { - match kind { - Kind::Test | Kind::Doc | Kind::Build | Kind::Bench | Kind::Dist | Kind::Install => { - assert_eq!(this.top_stage, 2) - } - Kind::Check | Kind::Clippy | Kind::Fix | Kind::Run | Kind::Format => {} + Subcommand::Format { .. } | Subcommand::Clean { .. } | Subcommand::Setup { .. } => { + panic!() } - } + }; - this + Self::new_internal(build, kind, paths.to_owned()) } pub fn execute_cli(&self) { @@ -712,7 +677,7 @@ impl<'a> Builder<'a> { /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic /// library lookup path. - pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Cargo) { + pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Command) { // Windows doesn't need dylib path munging because the dlls for the // compiler live next to the compiler and the system will find them // automatically. @@ -720,7 +685,7 @@ impl<'a> Builder<'a> { return; } - add_dylib_path(vec![self.rustc_libdir(compiler)], &mut cmd.command); + add_dylib_path(vec![self.rustc_libdir(compiler)], cmd); } /// Gets a path to the compiler specified. @@ -755,8 +720,11 @@ impl<'a> Builder<'a> { cmd.env_remove("MAKEFLAGS"); cmd.env_remove("MFLAGS"); - if let Some(linker) = self.linker(compiler.host, true) { - cmd.env("RUSTC_TARGET_LINKER", linker); + if let Some(linker) = self.linker(compiler.host) { + cmd.env("RUSTDOC_LINKER", linker); + } + if self.is_fuse_ld_lld(compiler.host) { + cmd.env("RUSTDOC_FUSE_LD_LLD", "1"); } cmd } @@ -796,7 +764,7 @@ impl<'a> Builder<'a> { if cmd == "doc" || cmd == "rustdoc" { let my_out = match mode { // This is the intended out directory for compiler documentation. - Mode::Rustc | Mode::ToolRustc | Mode::Codegen => self.compiler_doc_out(target), + Mode::Rustc | Mode::ToolRustc => self.compiler_doc_out(target), Mode::Std => out_dir.join(target.triple).join("doc"), _ => panic!("doc mode {:?} not expected", mode), }; @@ -811,7 +779,7 @@ impl<'a> Builder<'a> { format!("CARGO_PROFILE_{}_{}", profile, name) }; - // See comment in librustc_llvm/build.rs for why this is necessary, largely llvm-config + // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config // needs to not accidentally link to libLLVM in stage0/lib. cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var()); if let Some(e) = env::var_os(util::dylib_path_var()) { @@ -828,9 +796,9 @@ impl<'a> Builder<'a> { // scripts can do less work (i.e. not building/requiring LLVM). if cmd == "check" || cmd == "clippy" || cmd == "fix" { // If we've not yet built LLVM, or it's stale, then bust - // the librustc_llvm cache. That will always work, even though it + // the rustc_llvm cache. That will always work, even though it // may mean that on the next non-check build we'll need to rebuild - // librustc_llvm. But if LLVM is stale, that'll be a tiny amount + // rustc_llvm. But if LLVM is stale, that'll be a tiny amount // of work comparitively, and we'd likely need to rebuild it anyway, // so that's okay. if crate::native::prebuilt_llvm_config(self, target).is_err() { @@ -874,7 +842,7 @@ impl<'a> Builder<'a> { match mode { Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {} - Mode::Rustc | Mode::Codegen | Mode::ToolRustc => { + Mode::Rustc | Mode::ToolRustc => { // Build proc macros both for the host and the target if target != compiler.host && cmd != "check" { cargo.arg("-Zdual-proc-macros"); @@ -1041,16 +1009,18 @@ impl<'a> Builder<'a> { } } - if let Some(host_linker) = self.linker(compiler.host, true) { + if let Some(host_linker) = self.linker(compiler.host) { cargo.env("RUSTC_HOST_LINKER", host_linker); } + if self.is_fuse_ld_lld(compiler.host) { + cargo.env("RUSTC_HOST_FUSE_LD_LLD", "1"); + } - if let Some(target_linker) = self.linker(target, true) { + if let Some(target_linker) = self.linker(target) { let target = crate::envify(&target.triple); cargo.env(&format!("CARGO_TARGET_{}_LINKER", target), target_linker); } - - if self.config.use_lld && !target.contains("msvc") { + if self.is_fuse_ld_lld(target) { rustflags.arg("-Clink-args=-fuse-ld=lld"); } @@ -1059,7 +1029,7 @@ impl<'a> Builder<'a> { } let debuginfo_level = match mode { - Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc, + Mode::Rustc => self.config.rust_debuginfo_level_rustc, Mode::Std => self.config.rust_debuginfo_level_std, Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => { self.config.rust_debuginfo_level_tools @@ -1196,7 +1166,7 @@ impl<'a> Builder<'a> { rustdocflags.arg("-Winvalid_codeblock_attributes"); } - if let Mode::Rustc | Mode::Codegen = mode { + if mode == Mode::Rustc { rustflags.arg("-Zunstable-options"); rustflags.arg("-Wrustc::internal"); } @@ -1233,7 +1203,7 @@ impl<'a> Builder<'a> { cargo.env(format!("CC_{}", target.triple), &cc); let cflags = self.cflags(target, GitRepo::Rustc).join(" "); - cargo.env(format!("CFLAGS_{}", target.triple), cflags.clone()); + cargo.env(format!("CFLAGS_{}", target.triple), &cflags); if let Some(ar) = self.ar(target) { let ranlib = format!("{} s", ar.display()); @@ -1359,7 +1329,7 @@ impl<'a> Builder<'a> { // When we build Rust dylibs they're all intended for intermediate // usage, so make sure we pass the -Cprefer-dynamic flag instead of // linking all deps statically into the dylib. - if let Mode::Std | Mode::Rustc | Mode::Codegen = mode { + if matches!(mode, Mode::Std | Mode::Rustc) { rustflags.arg("-Cprefer-dynamic"); } @@ -1418,7 +1388,7 @@ impl<'a> Builder<'a> { (out, dur - deps) }; - if self.config.print_step_timings && dur > Duration::from_millis(100) { + if self.config.print_step_timings && !self.config.dry_run { println!("[TIMING] {:?} -- {}.{:03}", step, dur.as_secs(), dur.subsec_millis()); } @@ -1512,6 +1482,10 @@ impl Cargo { self.command.env(key.as_ref(), value.as_ref()); self } + + pub fn add_rustc_lib_path(&mut self, builder: &Builder<'_>, compiler: Compiler) { + builder.add_rustc_lib_path(compiler, &mut self.command); + } } impl From<Cargo> for Command { diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs index aeb0d713ef0..a367aa53496 100644 --- a/src/bootstrap/builder/tests.rs +++ b/src/bootstrap/builder/tests.rs @@ -2,14 +2,16 @@ use super::*; use crate::config::{Config, TargetSelection}; use std::thread; -fn configure(host: &[&str], target: &[&str]) -> Config { - let mut config = Config::default_opts(); +fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { + let mut config = Config::parse(&[cmd.to_owned()]); // don't save toolstates config.save_toolstates = None; - config.skip_only_host_steps = false; config.dry_run = true; config.ninja_in_file = false; // try to avoid spurious failures in dist where we create/delete each others file + config.out = PathBuf::from(env::var_os("BOOTSTRAP_OUTPUT_DIRECTORY").unwrap()); + config.initial_rustc = PathBuf::from(env::var_os("RUSTC").unwrap()); + config.initial_cargo = PathBuf::from(env::var_os("BOOTSTRAP_INITIAL_CARGO").unwrap()); let dir = config .out .join("tmp-rustbuild-tests") @@ -17,16 +19,8 @@ fn configure(host: &[&str], target: &[&str]) -> Config { t!(fs::create_dir_all(&dir)); config.out = dir; config.build = TargetSelection::from_user("A"); - config.hosts = vec![config.build] - .into_iter() - .chain(host.iter().map(|s| TargetSelection::from_user(s))) - .collect::<Vec<_>>(); - config.targets = config - .hosts - .clone() - .into_iter() - .chain(target.iter().map(|s| TargetSelection::from_user(s))) - .collect::<Vec<_>>(); + config.hosts = host.iter().map(|s| TargetSelection::from_user(s)).collect(); + config.targets = target.iter().map(|s| TargetSelection::from_user(s)).collect(); config } @@ -42,7 +36,7 @@ mod defaults { #[test] fn build_default() { - let build = Build::new(configure(&[], &[])); + let build = Build::new(configure("build", &["A"], &["A"])); let mut builder = Builder::new(&build); builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); @@ -70,7 +64,7 @@ mod defaults { #[test] fn build_stage_0() { - let config = Config { stage: Some(0), ..configure(&[], &[]) }; + let config = Config { stage: 0, ..configure("build", &["A"], &["A"]) }; let build = Build::new(config); let mut builder = Builder::new(&build); builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); @@ -91,8 +85,56 @@ mod defaults { } #[test] + fn build_cross_compile() { + let config = Config { stage: 1, ..configure("build", &["A", "B"], &["A", "B"]) }; + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + // Ideally, this build wouldn't actually have `target: a` + // rustdoc/rustcc/std here (the user only requested a host=B build, so + // there's not really a need for us to build for target A in this case + // (since we're producing stage 1 libraries/binaries). But currently + // rustbuild is just a bit buggy here; this should be fixed though. + assert_eq!( + first(builder.cache.all::<compile::Std>()), + &[ + compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 0 }, target: b }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + assert_eq!( + first(builder.cache.all::<compile::Assemble>()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } }, + ] + ); + assert_eq!( + first(builder.cache.all::<tool::Rustdoc>()), + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } }, + ], + ); + assert_eq!( + first(builder.cache.all::<compile::Rustc>()), + &[ + compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: b }, + ] + ); + } + + #[test] fn doc_default() { - let mut config = configure(&[], &[]); + let mut config = configure("doc", &["A"], &["A"]); config.compiler_docs = true; config.cmd = Subcommand::Doc { paths: Vec::new(), open: false }; let build = Build::new(config); @@ -126,12 +168,12 @@ mod dist { use pretty_assertions::assert_eq; fn configure(host: &[&str], target: &[&str]) -> Config { - Config { stage: Some(2), ..super::configure(host, target) } + Config { stage: 2, ..super::configure("dist", host, target) } } #[test] fn dist_baseline() { - let build = Build::new(configure(&[], &[])); + let build = Build::new(configure(&["A"], &["A"])); let mut builder = Builder::new(&build); builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); @@ -157,7 +199,7 @@ mod dist { #[test] fn dist_with_targets() { - let build = Build::new(configure(&[], &["B"])); + let build = Build::new(configure(&["A"], &["A", "B"])); let mut builder = Builder::new(&build); builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); @@ -188,7 +230,7 @@ mod dist { #[test] fn dist_with_hosts() { - let build = Build::new(configure(&["B"], &[])); + let build = Build::new(configure(&["A", "B"], &["A", "B"])); let mut builder = Builder::new(&build); builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); @@ -217,6 +259,16 @@ mod dist { dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, ] ); + assert_eq!( + first(builder.cache.all::<compile::Std>()), + &[ + compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a }, + compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, + ], + ); assert_eq!(first(builder.cache.all::<dist::Src>()), &[dist::Src]); } @@ -224,7 +276,7 @@ mod dist { fn dist_only_cross_host() { let a = TargetSelection::from_user("A"); let b = TargetSelection::from_user("B"); - let mut build = Build::new(configure(&["B"], &[])); + let mut build = Build::new(configure(&["A", "B"], &["A", "B"])); build.config.docs = false; build.config.extended = true; build.hosts = vec![b]; @@ -246,7 +298,7 @@ mod dist { #[test] fn dist_with_targets_and_hosts() { - let build = Build::new(configure(&["B"], &["C"])); + let build = Build::new(configure(&["A", "B"], &["A", "B", "C"])); let mut builder = Builder::new(&build); builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); @@ -281,40 +333,26 @@ mod dist { } #[test] - fn dist_with_target_flag() { - let mut config = configure(&["B"], &["C"]); - config.skip_only_host_steps = true; // as-if --target=C was passed + fn dist_with_empty_host() { + let config = configure(&[], &["C"]); let build = Build::new(config); let mut builder = Builder::new(&build); builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); let c = TargetSelection::from_user("C"); - assert_eq!( - first(builder.cache.all::<dist::Docs>()), - &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] - ); - assert_eq!( - first(builder.cache.all::<dist::Mingw>()), - &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] - ); - assert_eq!(first(builder.cache.all::<dist::Rustc>()), &[]); + assert_eq!(first(builder.cache.all::<dist::Docs>()), &[dist::Docs { host: c },]); + assert_eq!(first(builder.cache.all::<dist::Mingw>()), &[dist::Mingw { host: c },]); assert_eq!( first(builder.cache.all::<dist::Std>()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - ] + &[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },] ); - assert_eq!(first(builder.cache.all::<dist::Src>()), &[]); } #[test] fn dist_with_same_targets_and_hosts() { - let build = Build::new(configure(&["B"], &["B"])); + let build = Build::new(configure(&["A", "B"], &["A", "B"])); let mut builder = Builder::new(&build); builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]); @@ -367,7 +405,7 @@ mod dist { #[test] fn build_all() { - let build = Build::new(configure(&["B"], &["C"])); + let build = Build::new(configure(&["A", "B"], &["A", "B", "C"])); let mut builder = Builder::new(&build); builder.run_step_descriptions( &Builder::get_step_descriptions(Kind::Build), @@ -384,12 +422,9 @@ mod dist { compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: a }, compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: b }, compile::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: c }, ] ); assert!(!builder.cache.all::<compile::Assemble>().is_empty()); @@ -399,24 +434,20 @@ mod dist { compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: a }, compile::Rustc { compiler: Compiler { host: a, stage: 2 }, target: a }, - compile::Rustc { compiler: Compiler { host: b, stage: 2 }, target: a }, compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b }, compile::Rustc { compiler: Compiler { host: a, stage: 2 }, target: b }, - compile::Rustc { compiler: Compiler { host: b, stage: 2 }, target: b }, ] ); } #[test] - fn build_with_target_flag() { - let mut config = configure(&["B"], &["C"]); - config.skip_only_host_steps = true; + fn build_with_empty_host() { + let config = configure(&[], &["C"]); let build = Build::new(config); let mut builder = Builder::new(&build); builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); let c = TargetSelection::from_user("C"); assert_eq!( @@ -424,13 +455,7 @@ mod dist { &[ compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: a }, - compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: b }, compile::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - compile::Std { compiler: Compiler { host: b, stage: 2 }, target: c }, ] ); assert_eq!( @@ -439,7 +464,6 @@ mod dist { compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, - compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, ] ); assert_eq!( @@ -447,15 +471,14 @@ mod dist { &[ compile::Rustc { compiler: Compiler { host: a, stage: 0 }, target: a }, compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: a }, - compile::Rustc { compiler: Compiler { host: a, stage: 1 }, target: b }, ] ); } #[test] fn test_with_no_doc_stage0() { - let mut config = configure(&[], &[]); - config.stage = Some(0); + let mut config = configure(&["A"], &["A"]); + config.stage = 0; config.cmd = Subcommand::Test { paths: vec!["library/std".into()], test_args: vec![], @@ -494,7 +517,7 @@ mod dist { #[test] fn test_exclude() { - let mut config = configure(&[], &[]); + let mut config = configure(&["A"], &["A"]); config.exclude = vec!["src/tools/tidy".into()]; config.cmd = Subcommand::Test { paths: Vec::new(), @@ -521,7 +544,7 @@ mod dist { #[test] fn doc_ci() { - let mut config = configure(&[], &[]); + let mut config = configure(&["A"], &["A"]); config.compiler_docs = true; config.cmd = Subcommand::Doc { paths: Vec::new(), open: false }; let build = Build::new(config); @@ -550,7 +573,7 @@ mod dist { #[test] fn test_docs() { // Behavior of `x.py test` doing various documentation tests. - let mut config = configure(&[], &[]); + let mut config = configure(&["A"], &["A"]); config.cmd = Subcommand::Test { paths: vec![], test_args: vec![], diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs index 2a461170b5c..2b82f6c30b2 100644 --- a/src/bootstrap/channel.rs +++ b/src/bootstrap/channel.rs @@ -12,9 +12,6 @@ use build_helper::output; use crate::Build; -// The version number -pub const CFG_RELEASE_NUM: &str = "1.48.0"; - pub struct GitInfo { inner: Option<Info>, } diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 9f34bb4e6cc..371631154f7 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -1,9 +1,12 @@ //! Implementation of compiling the compiler and standard library, in "check"-based modes. -use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, std_cargo}; use crate::config::TargetSelection; use crate::tool::{prepare_tool_cargo, SourceType}; +use crate::{ + builder::{Builder, Kind, RunConfig, ShouldRun, Step}, + Subcommand, +}; use crate::{Compiler, Mode}; use std::path::PathBuf; @@ -66,6 +69,45 @@ impl Step for Std { let libdir = builder.sysroot_libdir(compiler, target); let hostdir = builder.sysroot_libdir(compiler, compiler.host); add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + + // Then run cargo again, once we've put the rmeta files for the library + // crates into the sysroot. This is needed because e.g., core's tests + // depend on `libtest` -- Cargo presumes it will exist, but it doesn't + // since we initialize with an empty sysroot. + // + // Currently only the "libtest" tree of crates does this. + + if let Subcommand::Check { all_targets: true, .. } = builder.config.cmd { + let mut cargo = builder.cargo( + compiler, + Mode::Std, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + std_cargo(builder, target, compiler.stage, &mut cargo); + cargo.arg("--all-targets"); + + // Explicitly pass -p for all dependencies krates -- this will force cargo + // to also check the tests/benches/examples for these crates, rather + // than just the leaf crate. + for krate in builder.in_tree_crates("test") { + cargo.arg("-p").arg(krate.name); + } + + builder.info(&format!( + "Checking std test/bench/example targets ({} -> {})", + &compiler.host, target + )); + run_cargo( + builder, + cargo, + args(builder.kind), + &libstd_test_stamp(builder, compiler, target), + vec![], + true, + ); + } } } @@ -106,6 +148,16 @@ impl Step for Rustc { cargo_subcommand(builder.kind), ); rustc_cargo(builder, &mut cargo, target); + if let Subcommand::Check { all_targets: true, .. } = builder.config.cmd { + cargo.arg("--all-targets"); + } + + // Explicitly pass -p for all compiler krates -- this will force cargo + // to also check the tests/benches/examples for these crates, rather + // than just the leaf crate. + for krate in builder.in_tree_crates("rustc-main") { + cargo.arg("-p").arg(krate.name); + } builder.info(&format!("Checking compiler artifacts ({} -> {})", &compiler.host, target)); run_cargo( @@ -149,7 +201,7 @@ macro_rules! tool_check_step { builder.ensure(Rustc { target }); - let cargo = prepare_tool_cargo( + let mut cargo = prepare_tool_cargo( builder, compiler, Mode::ToolRustc, @@ -160,12 +212,16 @@ macro_rules! tool_check_step { &[], ); - println!( + if let Subcommand::Check { all_targets: true, .. } = builder.config.cmd { + cargo.arg("--all-targets"); + } + + builder.info(&format!( "Checking {} artifacts ({} -> {})", stringify!($name).to_lowercase(), &compiler.host.triple, target.triple - ); + )); run_cargo( builder, cargo, @@ -202,12 +258,24 @@ tool_check_step!(Rustdoc, "src/tools/rustdoc", SourceType::InTree); // rejected. tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree); +tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree); + /// Cargo's output path for the standard library in a given stage, compiled /// by a particular compiler for the specified target. fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp") } +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +fn libstd_test_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp") +} + /// Cargo's output path for librustc in a given stage, compiled by a particular /// compiler for the specified target. fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 7814ca8e5bb..40bf6c48296 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -45,7 +45,7 @@ impl Step for Std { fn make_run(run: RunConfig<'_>) { run.builder.ensure(Std { - compiler: run.builder.compiler(run.builder.top_stage, run.host), + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), target: run.target, }); } @@ -59,7 +59,9 @@ impl Step for Std { let target = self.target; let compiler = self.compiler; - if builder.config.keep_stage.contains(&compiler.stage) { + if builder.config.keep_stage.contains(&compiler.stage) + || builder.config.keep_stage_std.contains(&compiler.stage) + { builder.info("Warning: Using a potentially old libstd. This may not behave well."); builder.ensure(StdLink { compiler, target_compiler: compiler, target }); return; @@ -385,7 +387,7 @@ impl Step for StartupObjects { fn make_run(run: RunConfig<'_>) { run.builder.ensure(StartupObjects { - compiler: run.builder.compiler(run.builder.top_stage, run.host), + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), target: run.target, }); } @@ -454,7 +456,7 @@ impl Step for Rustc { fn make_run(run: RunConfig<'_>) { run.builder.ensure(Rustc { - compiler: run.builder.compiler(run.builder.top_stage, run.host), + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), target: run.target, }); } @@ -472,6 +474,7 @@ impl Step for Rustc { if builder.config.keep_stage.contains(&compiler.stage) { builder.info("Warning: Using a potentially old librustc. This may not behave well."); + builder.info("Warning: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); builder.ensure(RustcLink { compiler, target_compiler: compiler, target }); return; } @@ -560,7 +563,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS } // Pass down configuration from the LLVM build into the build of - // librustc_llvm and librustc_codegen_llvm. + // rustc_llvm and rustc_codegen_llvm. // // Note that this is disabled if LLVM itself is disabled or we're in a check // build. If we are in a check build we still go ahead here presuming we've @@ -579,7 +582,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { cargo.env("CFG_LLVM_ROOT", s); } - // Some LLVM linker flags (-L and -l) may be needed to link librustc_llvm. + // Some LLVM linker flags (-L and -l) may be needed to link rustc_llvm. if let Some(ref s) = builder.config.llvm_ldflags { cargo.env("LLVM_LINKER_FLAGS", s); } @@ -593,7 +596,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS let file = compiler_file(builder, builder.cxx(target).unwrap(), target, "libstdc++.a"); cargo.env("LLVM_STATIC_STDCPP", file); } - if builder.config.llvm_link_shared || builder.config.llvm_thin_lto { + if builder.config.llvm_link_shared { cargo.env("LLVM_LINK_SHARED", "1"); } if builder.config.llvm_use_libcxx { diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index ad2f4877867..5314398ce9a 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -10,14 +10,25 @@ use std::ffi::OsString; use std::fmt; use std::fs; use std::path::{Path, PathBuf}; -use std::process; use crate::cache::{Interned, INTERNER}; use crate::flags::Flags; pub use crate::flags::Subcommand; +use crate::util::exe; use build_helper::t; +use merge::Merge; use serde::Deserialize; +macro_rules! check_ci_llvm { + ($name:expr) => { + assert!( + $name.is_none(), + "setting {} is incompatible with download-ci-llvm.", + stringify!($name) + ); + }; +} + /// Global configuration for the entire build and/or bootstrap. /// /// This structure is derived from a combination of both `config.toml` and @@ -31,6 +42,7 @@ use serde::Deserialize; /// `config.toml.example`. #[derive(Default)] pub struct Config { + pub changelog_seen: Option<usize>, pub ccache: Option<String>, /// Call Build::ninja() instead of this. pub ninja_in_file: bool, @@ -54,12 +66,13 @@ pub struct Config { pub test_compare_mode: bool, pub llvm_libunwind: bool, - pub skip_only_host_steps: bool, - pub on_fail: Option<String>, - pub stage: Option<u32>, + pub stage: u32, pub keep_stage: Vec<u32>, + pub keep_stage_std: Vec<u32>, pub src: PathBuf, + // defaults to `config.toml` + pub config: PathBuf, pub jobs: Option<u32>, pub cmd: Subcommand, pub incremental: bool, @@ -84,6 +97,7 @@ pub struct Config { pub llvm_version_suffix: Option<String>, pub llvm_use_linker: Option<String>, pub llvm_allow_old_toolchain: Option<bool>, + pub llvm_from_ci: bool, pub use_lld: bool, pub lld_enabled: bool, @@ -100,6 +114,7 @@ pub struct Config { pub rust_codegen_units_std: Option<u32>, pub rust_debug_assertions: bool, pub rust_debug_assertions_std: bool, + pub rust_debug_logging: bool, pub rust_debuginfo_level_rustc: u32, pub rust_debuginfo_level_std: u32, pub rust_debuginfo_level_tools: u32, @@ -260,28 +275,51 @@ impl Target { #[derive(Deserialize, Default)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct TomlConfig { + changelog_seen: Option<usize>, build: Option<Build>, install: Option<Install>, llvm: Option<Llvm>, rust: Option<Rust>, target: Option<HashMap<String, TomlTarget>>, dist: Option<Dist>, + profile: Option<String>, +} + +impl Merge for TomlConfig { + fn merge( + &mut self, + TomlConfig { build, install, llvm, rust, dist, target, profile: _, changelog_seen: _ }: Self, + ) { + fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>) { + if let Some(new) = y { + if let Some(original) = x { + original.merge(new); + } else { + *x = Some(new); + } + } + }; + do_merge(&mut self.build, build); + do_merge(&mut self.install, install); + do_merge(&mut self.llvm, llvm); + do_merge(&mut self.rust, rust); + do_merge(&mut self.dist, dist); + assert!(target.is_none(), "merging target-specific config is not currently supported"); + } } /// TOML representation of various global build decisions. -#[derive(Deserialize, Default, Clone)] +#[derive(Deserialize, Default, Clone, Merge)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Build { build: Option<String>, - #[serde(default)] - host: Vec<String>, - #[serde(default)] - target: Vec<String>, + host: Option<Vec<String>>, + target: Option<Vec<String>>, // This is ignored, the rust code always gets the build directory from the `BUILD_DIR` env variable build_dir: Option<String>, cargo: Option<String>, rustc: Option<String>, - rustfmt: Option<String>, /* allow bootstrap.py to use rustfmt key */ + rustfmt: Option<PathBuf>, docs: Option<bool>, compiler_docs: Option<bool>, submodules: Option<bool>, @@ -302,10 +340,16 @@ struct Build { configure_args: Option<Vec<String>>, local_rebuild: Option<bool>, print_step_timings: Option<bool>, + doc_stage: Option<u32>, + build_stage: Option<u32>, + test_stage: Option<u32>, + install_stage: Option<u32>, + dist_stage: Option<u32>, + bench_stage: Option<u32>, } /// TOML representation of various global install decisions. -#[derive(Deserialize, Default, Clone)] +#[derive(Deserialize, Default, Clone, Merge)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Install { prefix: Option<String>, @@ -322,7 +366,7 @@ struct Install { } /// TOML representation of how the LLVM build is configured. -#[derive(Deserialize, Default)] +#[derive(Deserialize, Default, Merge)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Llvm { skip_rebuild: Option<bool>, @@ -346,9 +390,10 @@ struct Llvm { use_libcxx: Option<bool>, use_linker: Option<String>, allow_old_toolchain: Option<bool>, + download_ci_llvm: Option<bool>, } -#[derive(Deserialize, Default, Clone)] +#[derive(Deserialize, Default, Clone, Merge)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Dist { sign_folder: Option<String>, @@ -372,7 +417,7 @@ impl Default for StringOrBool { } /// TOML representation of how the Rust build is configured. -#[derive(Deserialize, Default)] +#[derive(Deserialize, Default, Merge)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct Rust { optimize: Option<bool>, @@ -381,6 +426,7 @@ struct Rust { codegen_units_std: Option<u32>, debug_assertions: Option<bool>, debug_assertions_std: Option<bool>, + debug_logging: Option<bool>, debuginfo_level: Option<u32>, debuginfo_level_rustc: Option<u32>, debuginfo_level_std: Option<u32>, @@ -416,7 +462,7 @@ struct Rust { } /// TOML representation of how each build target is configured. -#[derive(Deserialize, Default)] +#[derive(Deserialize, Default, Merge)] #[serde(deny_unknown_fields, rename_all = "kebab-case")] struct TomlTarget { cc: Option<String>, @@ -469,31 +515,32 @@ impl Config { config.missing_tools = false; // set by bootstrap.py - config.build = TargetSelection::from_user(&env::var("BUILD").expect("'BUILD' to be set")); - config.src = Config::path_from_python("SRC"); + config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); config.out = Config::path_from_python("BUILD_DIR"); - config.initial_rustc = Config::path_from_python("RUSTC"); - config.initial_cargo = Config::path_from_python("CARGO"); - config.initial_rustfmt = env::var_os("RUSTFMT").map(Config::normalize_python_path); + config.initial_cargo = PathBuf::from(env!("CARGO")); + config.initial_rustc = PathBuf::from(env!("RUSTC")); config } pub fn parse(args: &[String]) -> Config { let flags = Flags::parse(&args); - let file = flags.config.clone(); + let mut config = Config::default_opts(); config.exclude = flags.exclude; config.rustc_error_format = flags.rustc_error_format; config.json_output = flags.json_output; config.on_fail = flags.on_fail; - config.stage = flags.stage; config.jobs = flags.jobs.map(threads_from_config); config.cmd = flags.cmd; config.incremental = flags.incremental; config.dry_run = flags.dry_run; config.keep_stage = flags.keep_stage; + config.keep_stage_std = flags.keep_stage_std; config.bindir = "bin".into(); // default if let Some(value) = flags.deny_warnings { config.deny_warnings = value; @@ -505,48 +552,56 @@ impl Config { config.out = dir; } - // If --target was specified but --host wasn't specified, don't run any host-only tests. - let has_hosts = !flags.host.is_empty(); - let has_targets = !flags.target.is_empty(); - config.skip_only_host_steps = !has_hosts && has_targets; - - let toml = file - .map(|file| { - let contents = t!(fs::read_to_string(&file)); - match toml::from_str(&contents) { - Ok(table) => table, - Err(err) => { - println!( - "failed to parse TOML configuration '{}': {}", - file.display(), - err - ); - process::exit(2); - } + #[cfg(test)] + let get_toml = |_| TomlConfig::default(); + #[cfg(not(test))] + let get_toml = |file: &Path| { + use std::process; + + let contents = t!(fs::read_to_string(file), "`include` config not found"); + match toml::from_str(&contents) { + Ok(table) => table, + Err(err) => { + println!("failed to parse TOML configuration '{}': {}", file.display(), err); + process::exit(2); } - }) - .unwrap_or_else(TomlConfig::default); - - let build = toml.build.clone().unwrap_or_default(); - // set by bootstrap.py - config.hosts.push(config.build); - for host in build.host.iter().map(|h| TargetSelection::from_user(h)) { - if !config.hosts.contains(&host) { - config.hosts.push(host); } + }; + + let mut toml = flags.config.as_deref().map(get_toml).unwrap_or_else(TomlConfig::default); + if let Some(include) = &toml.profile { + let mut include_path = config.src.clone(); + include_path.push("src"); + include_path.push("bootstrap"); + include_path.push("defaults"); + include_path.push(format!("config.toml.{}", include)); + let included_toml = get_toml(&include_path); + toml.merge(included_toml); } - for target in config - .hosts - .iter() - .copied() - .chain(build.target.iter().map(|h| TargetSelection::from_user(h))) - { - if !config.targets.contains(&target) { - config.targets.push(target); - } + + config.changelog_seen = toml.changelog_seen; + if let Some(cfg) = flags.config { + config.config = cfg; } - config.hosts = if !flags.host.is_empty() { flags.host } else { config.hosts }; - config.targets = if !flags.target.is_empty() { flags.target } else { config.targets }; + + let build = toml.build.unwrap_or_default(); + + config.hosts = if let Some(arg_host) = flags.host { + arg_host + } else if let Some(file_host) = build.host { + file_host.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + vec![config.build] + }; + config.targets = if let Some(arg_target) = flags.target { + arg_target + } else if let Some(file_target) = build.target { + file_target.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + // If target is *not* configured, then default to the host + // toolchains. + config.hosts.clone() + }; config.nodejs = build.nodejs.map(PathBuf::from); config.gdb = build.gdb.map(PathBuf::from); @@ -561,6 +616,9 @@ impl Config { set(&mut config.full_bootstrap, build.full_bootstrap); set(&mut config.extended, build.extended); config.tools = build.tools; + if build.rustfmt.is_some() { + config.initial_rustfmt = build.rustfmt; + } set(&mut config.verbose, build.verbose); set(&mut config.sanitizers, build.sanitizers); set(&mut config.profiler, build.profiler); @@ -568,16 +626,62 @@ impl Config { set(&mut config.configure_args, build.configure_args); set(&mut config.local_rebuild, build.local_rebuild); set(&mut config.print_step_timings, build.print_step_timings); + + // See https://github.com/rust-lang/compiler-team/issues/326 + config.stage = match config.cmd { + Subcommand::Doc { .. } => flags.stage.or(build.doc_stage).unwrap_or(0), + Subcommand::Build { .. } => flags.stage.or(build.build_stage).unwrap_or(1), + Subcommand::Test { .. } => flags.stage.or(build.test_stage).unwrap_or(1), + Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2), + Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2), + Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2), + // These are all bootstrap tools, which don't depend on the compiler. + // The stage we pass shouldn't matter, but use 0 just in case. + Subcommand::Clean { .. } + | Subcommand::Check { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } => flags.stage.unwrap_or(0), + }; + + // CI should always run stage 2 builds, unless it specifically states otherwise + #[cfg(not(test))] + if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None { + match config.cmd { + Subcommand::Test { .. } + | Subcommand::Doc { .. } + | Subcommand::Build { .. } + | Subcommand::Bench { .. } + | Subcommand::Dist { .. } + | Subcommand::Install { .. } => { + assert_eq!( + config.stage, 2, + "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", + config.stage, + ); + } + Subcommand::Clean { .. } + | Subcommand::Check { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } => {} + } + } + config.verbose = cmp::max(config.verbose, flags.verbose); - if let Some(ref install) = toml.install { - config.prefix = install.prefix.clone().map(PathBuf::from); - config.sysconfdir = install.sysconfdir.clone().map(PathBuf::from); - config.datadir = install.datadir.clone().map(PathBuf::from); - config.docdir = install.docdir.clone().map(PathBuf::from); - set(&mut config.bindir, install.bindir.clone().map(PathBuf::from)); - config.libdir = install.libdir.clone().map(PathBuf::from); - config.mandir = install.mandir.clone().map(PathBuf::from); + if let Some(install) = toml.install { + config.prefix = install.prefix.map(PathBuf::from); + config.sysconfdir = install.sysconfdir.map(PathBuf::from); + config.datadir = install.datadir.map(PathBuf::from); + config.docdir = install.docdir.map(PathBuf::from); + set(&mut config.bindir, install.bindir.map(PathBuf::from)); + config.libdir = install.libdir.map(PathBuf::from); + config.mandir = install.mandir.map(PathBuf::from); } // We want the llvm-skip-rebuild flag to take precedence over the @@ -591,6 +695,7 @@ impl Config { let mut debug = None; let mut debug_assertions = None; let mut debug_assertions_std = None; + let mut debug_logging = None; let mut debuginfo_level = None; let mut debuginfo_level_rustc = None; let mut debuginfo_level_std = None; @@ -599,7 +704,7 @@ impl Config { let mut optimize = None; let mut ignore_git = None; - if let Some(ref llvm) = toml.llvm { + if let Some(llvm) = toml.llvm { match llvm.ccache { Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), Some(StringOrBool::Bool(true)) => { @@ -628,12 +733,50 @@ impl Config { set(&mut config.llvm_use_libcxx, llvm.use_libcxx); config.llvm_use_linker = llvm.use_linker.clone(); config.llvm_allow_old_toolchain = llvm.allow_old_toolchain; + config.llvm_from_ci = llvm.download_ci_llvm.unwrap_or(false); + + if config.llvm_from_ci { + // None of the LLVM options, except assertions, are supported + // when using downloaded LLVM. We could just ignore these but + // that's potentially confusing, so force them to not be + // explicitly set. The defaults and CI defaults don't + // necessarily match but forcing people to match (somewhat + // arbitrary) CI configuration locally seems bad/hard. + check_ci_llvm!(llvm.optimize); + check_ci_llvm!(llvm.thin_lto); + check_ci_llvm!(llvm.release_debuginfo); + check_ci_llvm!(llvm.link_shared); + check_ci_llvm!(llvm.static_libstdcpp); + check_ci_llvm!(llvm.targets); + check_ci_llvm!(llvm.experimental_targets); + check_ci_llvm!(llvm.link_jobs); + check_ci_llvm!(llvm.link_shared); + check_ci_llvm!(llvm.clang_cl); + check_ci_llvm!(llvm.version_suffix); + check_ci_llvm!(llvm.cflags); + check_ci_llvm!(llvm.cxxflags); + check_ci_llvm!(llvm.ldflags); + check_ci_llvm!(llvm.use_libcxx); + check_ci_llvm!(llvm.use_linker); + check_ci_llvm!(llvm.allow_old_toolchain); + + // CI-built LLVM is shared + config.llvm_link_shared = true; + } + + if config.llvm_thin_lto { + // If we're building with ThinLTO on, we want to link to LLVM + // shared, to avoid re-doing ThinLTO (which happens in the link + // step) with each stage. + config.llvm_link_shared = true; + } } - if let Some(ref rust) = toml.rust { + if let Some(rust) = toml.rust { debug = rust.debug; debug_assertions = rust.debug_assertions; debug_assertions_std = rust.debug_assertions_std; + debug_logging = rust.debug_logging; debuginfo_level = rust.debuginfo_level; debuginfo_level_rustc = rust.debuginfo_level_rustc; debuginfo_level_std = rust.debuginfo_level_std; @@ -649,7 +792,7 @@ impl Config { set(&mut config.test_compare_mode, rust.test_compare_mode); set(&mut config.llvm_libunwind, rust.llvm_libunwind); set(&mut config.backtrace, rust.backtrace); - set(&mut config.channel, rust.channel.clone()); + set(&mut config.channel, rust.channel); set(&mut config.rust_dist_src, rust.dist_src); set(&mut config.verbose_tests, rust.verbose_tests); // in the case "false" is set explicitly, do not overwrite the command line args @@ -660,9 +803,9 @@ impl Config { set(&mut config.lld_enabled, rust.lld); set(&mut config.llvm_tools_enabled, rust.llvm_tools); config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); - config.rustc_default_linker = rust.default_linker.clone(); - config.musl_root = rust.musl_root.clone().map(PathBuf::from); - config.save_toolstates = rust.save_toolstates.clone().map(PathBuf::from); + config.rustc_default_linker = rust.default_linker; + config.musl_root = rust.musl_root.map(PathBuf::from); + config.save_toolstates = rust.save_toolstates.map(PathBuf::from); set(&mut config.deny_warnings, flags.deny_warnings.or(rust.deny_warnings)); set(&mut config.backtrace_on_ice, rust.backtrace_on_ice); set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir); @@ -679,9 +822,9 @@ impl Config { config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); } - if let Some(ref t) = toml.target { + if let Some(t) = toml.target { for (triple, cfg) in t { - let mut target = Target::from_triple(triple); + let mut target = Target::from_triple(&triple); if let Some(ref s) = cfg.llvm_config { target.llvm_config = Some(config.src.join(s)); @@ -695,35 +838,52 @@ impl Config { if let Some(s) = cfg.no_std { target.no_std = s; } - target.cc = cfg.cc.clone().map(PathBuf::from); - target.cxx = cfg.cxx.clone().map(PathBuf::from); - target.ar = cfg.ar.clone().map(PathBuf::from); - target.ranlib = cfg.ranlib.clone().map(PathBuf::from); - target.linker = cfg.linker.clone().map(PathBuf::from); + target.cc = cfg.cc.map(PathBuf::from); + target.cxx = cfg.cxx.map(PathBuf::from); + target.ar = cfg.ar.map(PathBuf::from); + target.ranlib = cfg.ranlib.map(PathBuf::from); + target.linker = cfg.linker.map(PathBuf::from); target.crt_static = cfg.crt_static; - target.musl_root = cfg.musl_root.clone().map(PathBuf::from); - target.musl_libdir = cfg.musl_libdir.clone().map(PathBuf::from); - target.wasi_root = cfg.wasi_root.clone().map(PathBuf::from); - target.qemu_rootfs = cfg.qemu_rootfs.clone().map(PathBuf::from); + target.musl_root = cfg.musl_root.map(PathBuf::from); + target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); + target.wasi_root = cfg.wasi_root.map(PathBuf::from); + target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); - config.target_config.insert(TargetSelection::from_user(triple), target); + config.target_config.insert(TargetSelection::from_user(&triple), target); } } - if let Some(ref t) = toml.dist { - config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from); - config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from); - config.dist_upload_addr = t.upload_addr.clone(); + if config.llvm_from_ci { + let triple = &config.build.triple; + let mut build_target = config + .target_config + .entry(config.build) + .or_insert_with(|| Target::from_triple(&triple)); + + check_ci_llvm!(build_target.llvm_config); + check_ci_llvm!(build_target.llvm_filecheck); + let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin"); + build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); + build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); + } + + if let Some(t) = toml.dist { + config.dist_sign_folder = t.sign_folder.map(PathBuf::from); + config.dist_gpg_password_file = t.gpg_password_file.map(PathBuf::from); + config.dist_upload_addr = t.upload_addr; set(&mut config.rust_dist_src, t.src_tarball); set(&mut config.missing_tools, t.missing_tools); } + // Cargo does not provide a RUSTFMT environment variable, so we + // synthesize it manually. Note that we also later check the config.toml + // and set this to that path if necessary. + let rustfmt = config.initial_rustc.with_file_name(exe("rustfmt", config.build)); + config.initial_rustfmt = if rustfmt.exists() { Some(rustfmt) } else { None }; + // Now that we've reached the end of our configuration, infer the // default values for all options that we haven't otherwise stored yet. - set(&mut config.initial_rustc, build.rustc.map(PathBuf::from)); - set(&mut config.initial_cargo, build.cargo.map(PathBuf::from)); - config.llvm_skip_rebuild = llvm_skip_rebuild.unwrap_or(false); let default = false; @@ -737,6 +897,8 @@ impl Config { config.rust_debug_assertions_std = debug_assertions_std.unwrap_or(config.rust_debug_assertions); + config.rust_debug_logging = debug_logging.unwrap_or(config.rust_debug_assertions); + let with_defaults = |debuginfo_level_specific: Option<u32>| { debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { 1 diff --git a/src/bootstrap/defaults/README.md b/src/bootstrap/defaults/README.md new file mode 100644 index 00000000000..f5b96db1b0f --- /dev/null +++ b/src/bootstrap/defaults/README.md @@ -0,0 +1,12 @@ +# About bootstrap defaults + +These defaults are intended to be a good starting point for working with x.py, +with the understanding that no one set of defaults make sense for everyone. + +They are still experimental, and we'd appreciate your help improving them! +If you use a setting that's not in these defaults that you think +others would benefit from, please [file an issue] or make a PR with the changes. +Similarly, if one of these defaults doesn't match what you use personally, +please open an issue to get it changed. + +[file an issue]: https://github.com/rust-lang/rust/issues/new/choose diff --git a/src/bootstrap/defaults/config.toml.codegen b/src/bootstrap/defaults/config.toml.codegen new file mode 100644 index 00000000000..a9505922ca7 --- /dev/null +++ b/src/bootstrap/defaults/config.toml.codegen @@ -0,0 +1,13 @@ +# These defaults are meant for contributors to the compiler who modify codegen or LLVM +[llvm] +# This enables debug-assertions in LLVM, +# catching logic errors in codegen much earlier in the process. +assertions = true + +[rust] +# This enables `RUSTC_LOG=debug`, avoiding confusing situations +# where adding `debug!()` appears to do nothing. +# However, it makes running the compiler slightly slower. +debug-logging = true +# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower. +incremental = true diff --git a/src/bootstrap/defaults/config.toml.compiler b/src/bootstrap/defaults/config.toml.compiler new file mode 100644 index 00000000000..4772de8a2cb --- /dev/null +++ b/src/bootstrap/defaults/config.toml.compiler @@ -0,0 +1,8 @@ +# These defaults are meant for contributors to the compiler who do not modify codegen or LLVM +[rust] +# This enables `RUSTC_LOG=debug`, avoiding confusing situations +# where adding `debug!()` appears to do nothing. +# However, it makes running the compiler slightly slower. +debug-logging = true +# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower. +incremental = true diff --git a/src/bootstrap/defaults/config.toml.library b/src/bootstrap/defaults/config.toml.library new file mode 100644 index 00000000000..e4316f4d864 --- /dev/null +++ b/src/bootstrap/defaults/config.toml.library @@ -0,0 +1,10 @@ +# These defaults are meant for contributors to the standard library and documentation. +[build] +# When building the standard library, you almost never want to build the compiler itself. +build-stage = 0 +test-stage = 0 +bench-stage = 0 + +[rust] +# This greatly increases the speed of rebuilds, especially when there are only minor changes. However, it makes the initial build slightly slower. +incremental = true diff --git a/src/bootstrap/defaults/config.toml.user b/src/bootstrap/defaults/config.toml.user new file mode 100644 index 00000000000..6647061d88f --- /dev/null +++ b/src/bootstrap/defaults/config.toml.user @@ -0,0 +1,9 @@ +# These defaults are meant for users and distro maintainers building from source, without intending to make multiple changes. +[build] +# When compiling from source, you almost always want a full stage 2 build, +# which has all the latest optimizations from nightly. +build-stage = 2 +test-stage = 2 +doc-stage = 2 +# When compiling from source, you usually want all tools. +extended = true diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index c1022099a02..857e06d846d 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -18,7 +18,6 @@ use build_helper::{output, t}; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::cache::{Interned, INTERNER}; -use crate::channel; use crate::compile; use crate::config::TargetSelection; use crate::tool::{self, Tool}; @@ -47,7 +46,7 @@ pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { } } -fn distdir(builder: &Builder<'_>) -> PathBuf { +pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf { builder.out.join("dist") } @@ -323,8 +322,8 @@ fn make_win_dist( // Warn windows-gnu users that the bundled GCC cannot compile C files builder.create( &target_bin_dir.join("GCC-WARNING.txt"), - "gcc.exe contained in this folder cannot be used for compiling C files - it is only\ - used as a linker. In order to be able to compile projects containing C code use\ + "gcc.exe contained in this folder cannot be used for compiling C files - it is only \ + used as a linker. In order to be able to compile projects containing C code use \ the GCC provided by MinGW or Cygwin.", ); @@ -569,7 +568,7 @@ impl Step for Rustc { &page_dst, &[ ("<INSERT DATE HERE>", &month_year), - ("<INSERT VERSION HERE>", channel::CFG_RELEASE_NUM), + ("<INSERT VERSION HERE>", &builder.version), ], ); } @@ -605,7 +604,9 @@ impl Step for DebuggerScripts { fn make_run(run: RunConfig<'_>) { run.builder.ensure(DebuggerScripts { - sysroot: run.builder.sysroot(run.builder.compiler(run.builder.top_stage, run.host)), + sysroot: run + .builder + .sysroot(run.builder.compiler(run.builder.top_stage, run.build_triple())), host: run.target, }); } @@ -791,6 +792,18 @@ impl Step for RustcDev { let stamp = compile::librustc_stamp(builder, compiler_to_use, target); copy_target_libs(builder, target, &image, &stamp); + // Copy compiler sources. + let dst_src = image.join("lib/rustlib/rustc-src/rust"); + t!(fs::create_dir_all(&dst_src)); + + let src_files = ["Cargo.lock"]; + // This is the reduced set of paths which will become the rustc-dev component + // (essentially the compiler crates and all of their path dependencies). + copy_src_dirs(builder, &builder.src, &["compiler"], &[], &dst_src); + for file in src_files.iter() { + builder.copy(&builder.src.join(file), &dst_src.join(file)); + } + let mut cmd = rust_installer(builder); cmd.arg("generate") .arg("--product-name=Rust") @@ -1020,7 +1033,7 @@ impl Step for Src { copy_src_dirs( builder, &builder.src, - &["library"], + &["library", "src/llvm-project/libunwind"], &[ // not needed and contains symlinks which rustup currently // chokes on when unpacking. @@ -2287,9 +2300,9 @@ impl Step for Extended { } fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) { - let mut parts = channel::CFG_RELEASE_NUM.split('.'); + let mut parts = builder.version.split('.'); cmd.env("CFG_RELEASE_INFO", builder.rust_version()) - .env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM) + .env("CFG_RELEASE_NUM", &builder.version) .env("CFG_RELEASE", builder.rust_release()) .env("CFG_VER_MAJOR", parts.next().unwrap()) .env("CFG_VER_MINOR", parts.next().unwrap()) @@ -2355,15 +2368,10 @@ impl Step for HashSign { cmd.arg(sign); cmd.arg(distdir(builder)); cmd.arg(today.trim()); - cmd.arg(builder.rust_package_vers()); cmd.arg(addr); - cmd.arg(builder.package_vers(&builder.release_num("cargo"))); - cmd.arg(builder.package_vers(&builder.release_num("rls"))); - cmd.arg(builder.package_vers(&builder.release_num("rust-analyzer/crates/rust-analyzer"))); - cmd.arg(builder.package_vers(&builder.release_num("clippy"))); - cmd.arg(builder.package_vers(&builder.release_num("miri"))); - cmd.arg(builder.package_vers(&builder.release_num("rustfmt"))); - cmd.arg(builder.llvm_tools_package_vers()); + cmd.arg(&builder.config.channel); + cmd.arg(&builder.src); + cmd.env("BUILD_MANIFEST_LEGACY", "1"); builder.create_dir(&distdir(builder)); @@ -2380,26 +2388,29 @@ impl Step for HashSign { /// Note: This function does not yet support Windows, but we also don't support /// linking LLVM tools dynamically on Windows yet. fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) { - let src_libdir = builder.llvm_out(target).join("lib"); + if !builder.config.llvm_link_shared { + // We do not need to copy LLVM files into the sysroot if it is not + // dynamically linked; it is already included into librustc_llvm + // statically. + return; + } + // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib + // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely + // clear why this is the case, though. llvm-config will emit the versioned + // paths and we don't want those in the sysroot (as we're expecting + // unversioned paths). if target.contains("apple-darwin") { + let src_libdir = builder.llvm_out(target).join("lib"); let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); if llvm_dylib_path.exists() { builder.install(&llvm_dylib_path, dst_libdir, 0o644); } - return; - } - - // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so. - // Since tools link to the latter rather than the former, we have to - // follow the symlink to find out what to distribute. - let llvm_dylib_path = src_libdir.join("libLLVM.so"); - if llvm_dylib_path.exists() { - let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| { - panic!("dist: Error calling canonicalize path `{}`: {}", llvm_dylib_path.display(), e); - }); - - builder.install(&llvm_dylib_path, dst_libdir, 0o644); + } else if let Ok(llvm_config) = crate::native::prebuilt_llvm_config(builder, target) { + let files = output(Command::new(llvm_config).arg("--libfiles")); + for file in files.lines() { + builder.install(Path::new(file), dst_libdir, 0o644); + } } } @@ -2499,3 +2510,94 @@ impl Step for LlvmTools { Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) } } + +// Tarball intended for internal consumption to ease rustc/std development. +// +// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct RustDev { + pub target: TargetSelection, +} + +impl Step for RustDev { + type Output = Option<PathBuf>; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("rust-dev") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDev { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { + let target = self.target; + + /* run only if llvm-config isn't used */ + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref _s) = config.llvm_config { + builder.info(&format!("Skipping RustDev ({}): external LLVM", target)); + return None; + } + } + + builder.info(&format!("Dist RustDev ({})", target)); + let _time = timeit(builder); + let src = builder.src.join("src/llvm-project/llvm"); + let name = pkgname(builder, "rust-dev"); + + let tmp = tmpdir(builder); + let image = tmp.join("rust-dev-image"); + drop(fs::remove_dir_all(&image)); + + // Prepare the image directory + let dst_bindir = image.join("bin"); + t!(fs::create_dir_all(&dst_bindir)); + + let exe = builder.llvm_out(target).join("bin").join(exe("llvm-config", target)); + builder.install(&exe, &dst_bindir, 0o755); + builder.install(&builder.llvm_filecheck(target), &dst_bindir, 0o755); + + // Copy the include directory as well; needed mostly to build + // librustc_llvm properly (e.g., llvm-config.h is in here). But also + // just broadly useful to be able to link against the bundled LLVM. + builder.cp_r(&builder.llvm_out(target).join("include"), &image.join("include")); + + // Copy libLLVM.so to the target lib dir as well, so the RPATH like + // `$ORIGIN/../lib` can find it. It may also be used as a dependency + // of `rustc-dev` to support the inherited `-lLLVM` when using the + // compiler libraries. + maybe_install_llvm(builder, target, &image.join("lib")); + + // Prepare the overlay + let overlay = tmp.join("rust-dev-overlay"); + drop(fs::remove_dir_all(&overlay)); + builder.create_dir(&overlay); + builder.install(&src.join("README.txt"), &overlay, 0o644); + builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644); + builder.create(&overlay.join("version"), &builder.rust_version()); + + // Generate the installer tarball + let mut cmd = rust_installer(builder); + cmd.arg("generate") + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=rust-dev-installed.") + .arg("--image-dir") + .arg(&image) + .arg("--work-dir") + .arg(&tmpdir(builder)) + .arg("--output-dir") + .arg(&distdir(builder)) + .arg("--non-installed-overlay") + .arg(&overlay) + .arg(format!("--package-name={}-{}", name, target.triple)) + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg("--component-name=rust-dev"); + + builder.run(&mut cmd); + Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) + } +} diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index d7f3a888edd..aa670bd9a2e 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -66,7 +66,6 @@ book!( Nomicon, "src/doc/nomicon", "nomicon"; Reference, "src/doc/reference", "reference"; RustByExample, "src/doc/rust-by-example", "rust-by-example"; - RustcBook, "src/doc/rustc", "rustc"; RustdocBook, "src/doc/rustdoc", "rustdoc"; ); @@ -434,7 +433,7 @@ impl Step for Std { .arg("-Z") .arg("unstable-options") .arg("--resource-suffix") - .arg(crate::channel::CFG_RELEASE_NUM) + .arg(&builder.version) .arg("--index-page") .arg(&builder.src.join("src/doc/index.md")); @@ -660,7 +659,7 @@ impl Step for ErrorIndex { let mut index = tool::ErrorIndex::command(builder, self.compiler); index.arg("html"); index.arg(out.join("error-index.html")); - index.arg(crate::channel::CFG_RELEASE_NUM); + index.arg(&builder.version); builder.run(&mut index); } @@ -718,3 +717,72 @@ fn symlink_dir_force(config: &Config, src: &Path, dst: &Path) -> io::Result<()> symlink_dir(config, src, dst) } + +#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcBook { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustcBook { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/rustc").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcBook { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Builds the rustc book. + /// + /// The lints are auto-generated by a tool, and then merged into the book + /// in the "md-doc" directory in the build output directory. Then + /// "rustbook" is used to convert it to HTML. + fn run(self, builder: &Builder<'_>) { + let out_base = builder.md_doc_out(self.target).join("rustc"); + t!(fs::create_dir_all(&out_base)); + let out_listing = out_base.join("src/lints"); + builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base); + builder.info(&format!("Generating lint docs ({})", self.target)); + + let rustc = builder.rustc(self.compiler); + // The tool runs `rustc` for extracting output examples, so it needs a + // functional sysroot. + builder.ensure(compile::Std { compiler: self.compiler, target: self.target }); + let mut cmd = builder.tool_cmd(Tool::LintDocs); + cmd.arg("--src"); + cmd.arg(builder.src.join("compiler")); + cmd.arg("--out"); + cmd.arg(&out_listing); + cmd.arg("--rustc"); + cmd.arg(&rustc); + cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg()); + if builder.config.verbose() { + cmd.arg("--verbose"); + } + // If the lib directories are in an unusual location (changed in + // config.toml), then this needs to explicitly update the dylib search + // path. + builder.add_rustc_lib_path(self.compiler, &mut cmd); + builder.run(&mut cmd); + // Run rustbook/mdbook to generate the HTML pages. + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str("rustc"), + src: INTERNER.intern_path(out_base), + }); + if is_explicit_request(builder, "src/doc/rustc") { + let out = builder.doc_out(self.target); + let index = out.join("rustc").join("index.html"); + open(builder, &index); + } + } +} diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 2db4bb07a9f..c1a9d4fcd23 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -7,6 +7,7 @@ use std::env; use std::path::PathBuf; use std::process; +use build_helper::t; use getopts::Options; use crate::builder::Builder; @@ -19,9 +20,10 @@ pub struct Flags { pub on_fail: Option<String>, pub stage: Option<u32>, pub keep_stage: Vec<u32>, + pub keep_stage_std: Vec<u32>, - pub host: Vec<TargetSelection>, - pub target: Vec<TargetSelection>, + pub host: Option<Vec<TargetSelection>>, + pub target: Option<Vec<TargetSelection>>, pub config: Option<PathBuf>, pub jobs: Option<u32>, pub cmd: Subcommand, @@ -45,6 +47,9 @@ pub enum Subcommand { paths: Vec<PathBuf>, }, Check { + // Whether to run checking over all targets (e.g., unit / integration + // tests). + all_targets: bool, paths: Vec<PathBuf>, }, Clippy { @@ -88,6 +93,9 @@ pub enum Subcommand { Run { paths: Vec<PathBuf>, }, + Setup { + path: String, + }, } impl Default for Subcommand { @@ -98,7 +106,6 @@ impl Default for Subcommand { impl Flags { pub fn parse(args: &[String]) -> Flags { - let mut extra_help = String::new(); let mut subcommand_help = String::from( "\ Usage: x.py <subcommand> [options] [<paths>...] @@ -145,6 +152,13 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", (pass multiple times to keep e.g., both stages 0 and 1)", "N", ); + opts.optmulti( + "", + "keep-stage-std", + "stage(s) of the standard library to keep without recompiling \ + (pass multiple times to keep e.g., both stages 0 and 1)", + "N", + ); opts.optopt("", "src", "path to the root of the rust checkout", "DIR"); let j_msg = format!( "number of jobs to run in parallel; \ @@ -170,16 +184,6 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", "VALUE", ); - // fn usage() - let usage = - |exit_code: i32, opts: &Options, subcommand_help: &str, extra_help: &str| -> ! { - println!("{}", opts.usage(subcommand_help)); - if !extra_help.is_empty() { - println!("{}", extra_help); - } - process::exit(exit_code); - }; - // We can't use getopt to parse the options until we have completed specifying which // options are valid, but under the current implementation, some options are conditional on // the subcommand. Therefore we must manually identify the subcommand first, so that we can @@ -202,6 +206,7 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", || (s == "install") || (s == "run") || (s == "r") + || (s == "setup") }); let subcommand = match subcommand { Some(s) => s, @@ -248,6 +253,9 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", `/<build_base>/rustfix_missing_coverage.txt`", ); } + "check" => { + opts.optflag("", "all-targets", "Check all targets"); + } "bench" => { opts.optmulti("", "test-args", "extra arguments", "ARGS"); } @@ -263,12 +271,38 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`", _ => {} }; + // fn usage() + let usage = |exit_code: i32, opts: &Options, verbose: bool, subcommand_help: &str| -> ! { + let mut extra_help = String::new(); + + // All subcommands except `clean` can have an optional "Available paths" section + if verbose { + let config = Config::parse(&["build".to_string()]); + let build = Build::new(config); + + let maybe_rules_help = Builder::get_help(&build, subcommand.as_str()); + extra_help.push_str(maybe_rules_help.unwrap_or_default().as_str()); + } else if !(subcommand.as_str() == "clean" || subcommand.as_str() == "fmt") { + extra_help.push_str( + format!("Run `./x.py {} -h -v` to see a list of available paths.", subcommand) + .as_str(), + ); + } + + println!("{}", opts.usage(subcommand_help)); + if !extra_help.is_empty() { + println!("{}", extra_help); + } + process::exit(exit_code); + }; + // Done specifying what options are possible, so do the getopts parsing let matches = opts.parse(&args[..]).unwrap_or_else(|e| { // Invalid argument/option format println!("\n{}\n", e); - usage(1, &opts, &subcommand_help, &extra_help); + usage(1, &opts, false, &subcommand_help); }); + // Extra sanity check to make sure we didn't hit this crazy corner case: // // ./x.py --frobulate clean build @@ -430,35 +464,35 @@ Arguments: At least a tool needs to be called.", ); } + "setup" => { + subcommand_help.push_str( + "\n +Arguments: + This subcommand accepts a 'profile' to use for builds. For example: + + ./x.py setup library + + The profile is optional and you will be prompted interactively if it is not given.", + ); + } _ => {} }; // Get any optional paths which occur after the subcommand - let paths = matches.free[1..].iter().map(|p| p.into()).collect::<Vec<PathBuf>>(); + let mut paths = matches.free[1..].iter().map(|p| p.into()).collect::<Vec<PathBuf>>(); let cfg_file = env::var_os("BOOTSTRAP_CONFIG").map(PathBuf::from); - - // All subcommands except `clean` can have an optional "Available paths" section - if matches.opt_present("verbose") { - let config = Config::parse(&["build".to_string()]); - let build = Build::new(config); - - let maybe_rules_help = Builder::get_help(&build, subcommand.as_str()); - extra_help.push_str(maybe_rules_help.unwrap_or_default().as_str()); - } else if !(subcommand.as_str() == "clean" || subcommand.as_str() == "fmt") { - extra_help.push_str( - format!("Run `./x.py {} -h -v` to see a list of available paths.", subcommand) - .as_str(), - ); - } + let verbose = matches.opt_present("verbose"); // User passed in -h/--help? if matches.opt_present("help") { - usage(0, &opts, &subcommand_help, &extra_help); + usage(0, &opts, verbose, &subcommand_help); } let cmd = match subcommand.as_str() { "build" | "b" => Subcommand::Build { paths }, - "check" | "c" => Subcommand::Check { paths }, + "check" | "c" => { + Subcommand::Check { paths, all_targets: matches.opt_present("all-targets") } + } "clippy" => Subcommand::Clippy { paths }, "fix" => Subcommand::Fix { paths }, "test" | "t" => Subcommand::Test { @@ -483,7 +517,7 @@ Arguments: "clean" => { if !paths.is_empty() { println!("\nclean does not take a path argument\n"); - usage(1, &opts, &subcommand_help, &extra_help); + usage(1, &opts, verbose, &subcommand_help); } Subcommand::Clean { all: matches.opt_present("all") } @@ -494,12 +528,26 @@ Arguments: "run" | "r" => { if paths.is_empty() { println!("\nrun requires at least a path!\n"); - usage(1, &opts, &subcommand_help, &extra_help); + usage(1, &opts, verbose, &subcommand_help); } Subcommand::Run { paths } } + "setup" => { + let path = if paths.len() > 1 { + println!("\nat most one profile can be passed to setup\n"); + usage(1, &opts, verbose, &subcommand_help) + } else if let Some(path) = paths.pop() { + t!(path.into_os_string().into_string().map_err(|path| format!( + "{} is not a valid UTF8 string", + path.to_string_lossy() + ))) + } else { + t!(crate::setup::interactive_path()) + }; + Subcommand::Setup { path } + } _ => { - usage(1, &opts, &subcommand_help, &extra_help); + usage(1, &opts, verbose, &subcommand_help); } }; @@ -508,7 +556,9 @@ Arguments: println!("--stage not supported for x.py check, always treated as stage 0"); process::exit(1); } - if matches.opt_str("keep-stage").is_some() { + if matches.opt_str("keep-stage").is_some() + || matches.opt_str("keep-stage-std").is_some() + { println!("--keep-stage not supported for x.py check, only one stage available"); process::exit(1); } @@ -526,14 +576,31 @@ Arguments: .into_iter() .map(|j| j.parse().expect("`keep-stage` should be a number")) .collect(), - host: split(&matches.opt_strs("host")) - .into_iter() - .map(|x| TargetSelection::from_user(&x)) - .collect::<Vec<_>>(), - target: split(&matches.opt_strs("target")) + keep_stage_std: matches + .opt_strs("keep-stage-std") .into_iter() - .map(|x| TargetSelection::from_user(&x)) - .collect::<Vec<_>>(), + .map(|j| j.parse().expect("`keep-stage-std` should be a number")) + .collect(), + host: if matches.opt_present("host") { + Some( + split(&matches.opt_strs("host")) + .into_iter() + .map(|x| TargetSelection::from_user(&x)) + .collect::<Vec<_>>(), + ) + } else { + None + }, + target: if matches.opt_present("target") { + Some( + split(&matches.opt_strs("target")) + .into_iter() + .map(|x| TargetSelection::from_user(&x)) + .collect::<Vec<_>>(), + ) + } else { + None + }, config: cfg_file, jobs: matches.opt_str("jobs").map(|j| j.parse().expect("`jobs` should be a number")), cmd, @@ -620,7 +687,7 @@ impl Subcommand { } fn split(s: &[String]) -> Vec<String> { - s.iter().flat_map(|s| s.split(',')).map(|s| s.to_string()).collect() + s.iter().flat_map(|s| s.split(',')).filter(|s| !s.is_empty()).map(|s| s.to_string()).collect() } fn parse_deny_warnings(matches: &getopts::Matches) -> Option<bool> { diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs index 6f93082e675..0ae9f9712d5 100644 --- a/src/bootstrap/format.rs +++ b/src/bootstrap/format.rs @@ -105,15 +105,13 @@ pub fn format(build: &Build, check: bool) { eprintln!("./x.py fmt is not supported on this channel"); std::process::exit(1); }); - let src = build.src.clone(); - let walker = WalkBuilder::new(&build.src).types(matcher).overrides(ignore_fmt).build_parallel(); + let src = &build.src; + let walker = WalkBuilder::new(src).types(matcher).overrides(ignore_fmt).build_parallel(); walker.run(|| { - let src = src.clone(); - let rustfmt_path = rustfmt_path.clone(); Box::new(move |entry| { let entry = t!(entry); if entry.file_type().map_or(false, |t| t.is_file()) { - rustfmt(&src, &rustfmt_path, &entry.path(), check); + rustfmt(src, &rustfmt_path, &entry.path(), check); } ignore::WalkState::Continue }) diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs index d9ee3bc90fb..074f5cd73f3 100644 --- a/src/bootstrap/install.rs +++ b/src/bootstrap/install.rs @@ -192,7 +192,7 @@ install!((self, builder, _config), builder.ensure(dist::Docs { host: self.target }); install_docs(builder, self.compiler.stage, self.target); }; - Std, "library/std", true, only_hosts: true, { + Std, "library/std", true, only_hosts: false, { for target in &builder.targets { builder.ensure(dist::Std { compiler: self.compiler, diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 54651214363..4cc72f5f39c 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -103,8 +103,6 @@ //! More documentation can be found in each respective module below, and you can //! also check out the `src/bootstrap/README.md` file for more information. -#![feature(drain_filter)] - use std::cell::{Cell, RefCell}; use std::collections::{HashMap, HashSet}; use std::env; @@ -143,6 +141,7 @@ mod metadata; mod native; mod run; mod sanity; +mod setup; mod test; mod tool; mod toolstate; @@ -167,7 +166,7 @@ mod job { use crate::cache::{Interned, INTERNER}; pub use crate::config::Config; -use crate::flags::Subcommand; +pub use crate::flags::Subcommand; const LLVM_TOOLS: &[&str] = &[ "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility @@ -220,6 +219,9 @@ pub struct Build { /// User-specified configuration from `config.toml`. config: Config, + // Version information + version: String, + // Properties derived from the above configuration src: PathBuf, out: PathBuf, @@ -300,9 +302,6 @@ pub enum Mode { /// Build librustc, and compiler libraries, placing output in the "stageN-rustc" directory. Rustc, - /// Build codegen libraries, placing output in the "stageN-codegen" directory - Codegen, - /// Build a tool, placing output in the "stage0-bootstrap-tools" /// directory. This is for miscellaneous sets of tools that are built /// using the bootstrap stage0 compiler in its entirety (target libraries @@ -385,6 +384,10 @@ impl Build { .unwrap() .to_path_buf(); + let version = std::fs::read_to_string(src.join("src").join("version")) + .expect("failed to read src/version"); + let version = version.trim(); + let mut build = Build { initial_rustc: config.initial_rustc.clone(), initial_cargo: config.initial_cargo.clone(), @@ -400,6 +403,7 @@ impl Build { targets: config.targets.clone(), config, + version: version.to_string(), src, out, @@ -438,8 +442,7 @@ impl Build { .next() .unwrap() .trim(); - let my_version = channel::CFG_RELEASE_NUM; - if local_release.split('.').take(2).eq(my_version.split('.').take(2)) { + if local_release.split('.').take(2).eq(version.split('.').take(2)) { build.verbose(&format!("auto-detected local-rebuild {}", local_release)); build.local_rebuild = true; } @@ -468,6 +471,10 @@ impl Build { return clean::clean(self, all); } + if let Subcommand::Setup { path: include_name } = &self.config.cmd { + return setup::setup(&self.config.src, include_name); + } + { let builder = builder::Builder::new(&self); if let Some(path) = builder.paths.get(0) { @@ -546,6 +553,16 @@ impl Build { if self.config.llvm_enabled() { features.push_str(" llvm"); } + + // If debug logging is on, then we want the default for tracing: + // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26 + // which is everything (including debug/trace/etc.) + // if its unset, if debug_assertions is on, then debug_logging will also be on + // as well as tracing *ignoring* this feature when debug_assertions is on + if !self.config.rust_debug_logging { + features.push_str(" max_level_info"); + } + features } @@ -572,7 +589,6 @@ impl Build { let suffix = match mode { Mode::Std => "-std", Mode::Rustc => "-rustc", - Mode::Codegen => "-codegen", Mode::ToolBootstrap => "-bootstrap-tools", Mode::ToolStd | Mode::ToolRustc => "-tools", }; @@ -617,6 +633,10 @@ impl Build { /// /// If no custom `llvm-config` was specified then Rust's llvm will be used. fn is_rust_llvm(&self, target: TargetSelection) -> bool { + if self.config.llvm_from_ci && target == self.config.build { + return true; + } + match self.config.target_config.get(&target) { Some(ref c) => c.llvm_config.is_none(), None => true, @@ -777,7 +797,7 @@ impl Build { match which { GitRepo::Rustc => { - let sha = self.rust_sha().unwrap_or(channel::CFG_RELEASE_NUM); + let sha = self.rust_sha().unwrap_or(&self.version); Some(format!("/rustc/{}", sha)) } GitRepo::Llvm => Some(String::from("/rustc/llvm")), @@ -850,7 +870,7 @@ impl Build { } /// Returns the path to the linker for the given target if it needs to be overridden. - fn linker(&self, target: TargetSelection, can_use_lld: bool) -> Option<&Path> { + fn linker(&self, target: TargetSelection) -> Option<&Path> { if let Some(linker) = self.config.target_config.get(&target).and_then(|c| c.linker.as_ref()) { Some(linker) @@ -863,13 +883,19 @@ impl Build { && !target.contains("msvc") { Some(self.cc(target)) - } else if can_use_lld && self.config.use_lld && self.build == target { + } else if self.config.use_lld && !self.is_fuse_ld_lld(target) && self.build == target { Some(&self.initial_lld) } else { None } } + // LLD is used through `-fuse-ld=lld` rather than directly. + // Only MSVC targets use LLD directly at the moment. + fn is_fuse_ld_lld(&self, target: TargetSelection) -> bool { + self.config.use_lld && !target.contains("msvc") + } + /// Returns if this target should statically link the C runtime, if specified fn crt_static(&self, target: TargetSelection) -> Option<bool> { if target.contains("pc-windows-msvc") { @@ -1002,7 +1028,7 @@ impl Build { /// Returns the value of `release` above for Rust itself. fn rust_release(&self) -> String { - self.release(channel::CFG_RELEASE_NUM) + self.release(&self.version) } /// Returns the "package version" for a component given the `num` release @@ -1022,7 +1048,7 @@ impl Build { /// Returns the value of `package_vers` above for Rust itself. fn rust_package_vers(&self) -> String { - self.package_vers(channel::CFG_RELEASE_NUM) + self.package_vers(&self.version) } /// Returns the value of `package_vers` above for Cargo @@ -1056,7 +1082,7 @@ impl Build { } fn llvm_tools_package_vers(&self) -> String { - self.package_vers(channel::CFG_RELEASE_NUM) + self.package_vers(&self.version) } fn llvm_tools_vers(&self) -> String { @@ -1073,7 +1099,7 @@ impl Build { /// Note that this is a descriptive string which includes the commit date, /// sha, version, etc. fn rust_version(&self) -> String { - self.rust_info.version(self, channel::CFG_RELEASE_NUM) + self.rust_info.version(self, &self.version) } /// Returns the full commit hash. diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index a0c79e38f9d..6bba00ee85e 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -19,7 +19,6 @@ use std::process::Command; use build_helper::{output, t}; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::channel; use crate::config::TargetSelection; use crate::util::{self, exe}; use crate::GitRepo; @@ -129,6 +128,12 @@ impl Step for Llvm { Err(m) => m, }; + if builder.config.llvm_link_shared + && (target.contains("windows") || target.contains("apple-darwin")) + { + panic!("shared linking to LLVM is not currently supported on {}", target.triple); + } + builder.info(&format!("Building LLVM for {}", target)); t!(stamp.remove()); let _time = util::timeit(&builder); @@ -169,7 +174,6 @@ impl Step for Llvm { .define("LLVM_INCLUDE_TESTS", "OFF") .define("LLVM_INCLUDE_DOCS", "OFF") .define("LLVM_INCLUDE_BENCHMARKS", "OFF") - .define("WITH_POLLY", "OFF") .define("LLVM_ENABLE_TERMINFO", "OFF") .define("LLVM_ENABLE_LIBEDIT", "OFF") .define("LLVM_ENABLE_BINDINGS", "OFF") @@ -206,7 +210,10 @@ impl Step for Llvm { // which saves both memory during parallel links and overall disk space // for the tools. We don't do this on every platform as it doesn't work // equally well everywhere. - if builder.llvm_link_tools_dynamically(target) { + // + // If we're not linking rustc to a dynamic LLVM, though, then don't link + // tools to it. + if builder.llvm_link_tools_dynamically(target) && builder.config.llvm_link_shared { cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); } @@ -293,7 +300,7 @@ impl Step for Llvm { // release number on the dev channel. cfg.define("LLVM_VERSION_SUFFIX", "-rust-dev"); } else { - let suffix = format!("-rust-{}-{}", channel::CFG_RELEASE_NUM, builder.config.channel); + let suffix = format!("-rust-{}-{}", builder.version, builder.config.channel); cfg.define("LLVM_VERSION_SUFFIX", suffix); } @@ -305,10 +312,6 @@ impl Step for Llvm { cfg.define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "YES"); } - if let Some(ref python) = builder.config.python { - cfg.define("PYTHON_EXECUTABLE", python); - } - configure_cmake(builder, target, &mut cfg, true); // FIXME: we don't actually need to build all LLVM tools and all LLVM @@ -627,7 +630,14 @@ impl Step for TestHelpers { if builder.config.dry_run { return; } - let target = self.target; + // The x86_64-fortanix-unknown-sgx target doesn't have a working C + // toolchain. However, some x86_64 ELF objects can be linked + // without issues. Use this hack to compile the test helpers. + let target = if self.target == "x86_64-fortanix-unknown-sgx" { + TargetSelection::from_user("x86_64-unknown-linux-gnu") + } else { + self.target + }; let dst = builder.test_helpers_out(target); let src = builder.src.join("src/test/auxiliary/rust_test_helpers.c"); if up_to_date(&src, &dst.join("librust_test_helpers.a")) { @@ -651,7 +661,6 @@ impl Step for TestHelpers { } cfg.compiler(builder.cc(target)); } - cfg.cargo_metadata(false) .out_dir(&dst) .target(&target.triple) diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs index 34170898a54..da34d8b9d70 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/run.rs @@ -1,5 +1,7 @@ use crate::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::dist::distdir; use crate::tool::Tool; +use build_helper::output; use std::process::Command; #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] @@ -10,7 +12,7 @@ impl Step for ExpandYamlAnchors { /// Runs the `expand-yaml_anchors` tool. /// - /// This tool in `src/tools` read the CI configuration files written in YAML and expands the + /// This tool in `src/tools` reads the CI configuration files written in YAML and expands the /// anchors in them, since GitHub Actions doesn't support them. fn run(self, builder: &Builder<'_>) { builder.info("Expanding YAML anchors in the GitHub Actions configuration"); @@ -65,3 +67,42 @@ impl Step for InstallGitHook { run.builder.ensure(InstallGitHook); } } +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct BuildManifest; + +impl Step for BuildManifest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/build-manifest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BuildManifest); + } + + fn run(self, builder: &Builder<'_>) { + // This gets called by `promote-release` + // (https://github.com/rust-lang/promote-release). + let mut cmd = builder.tool_cmd(Tool::BuildManifest); + let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") + }); + let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") + }); + + let today = output(Command::new("date").arg("+%Y-%m-%d")); + + cmd.arg(sign); + cmd.arg(distdir(builder)); + cmd.arg(today.trim()); + cmd.arg(addr); + cmd.arg(&builder.config.channel); + cmd.arg(&builder.src); + + builder.create_dir(&distdir(builder)); + builder.run(&mut cmd); + } +} diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 4d6612a376a..6826d177a4a 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -30,8 +30,8 @@ impl Finder { Self { cache: HashMap::new(), path: env::var_os("PATH").unwrap_or_default() } } - pub fn maybe_have<S: AsRef<OsStr>>(&mut self, cmd: S) -> Option<PathBuf> { - let cmd: OsString = cmd.as_ref().into(); + pub fn maybe_have<S: Into<OsString>>(&mut self, cmd: S) -> Option<PathBuf> { + let cmd: OsString = cmd.into(); let path = &self.path; self.cache .entry(cmd.clone()) diff --git a/src/bootstrap/setup.rs b/src/bootstrap/setup.rs new file mode 100644 index 00000000000..9d3a889aa00 --- /dev/null +++ b/src/bootstrap/setup.rs @@ -0,0 +1,88 @@ +use crate::t; +use std::path::{Path, PathBuf}; +use std::{ + env, fs, + io::{self, Write}, +}; + +pub fn setup(src_path: &Path, include_name: &str) { + let cfg_file = env::var_os("BOOTSTRAP_CONFIG").map(PathBuf::from); + + if cfg_file.as_ref().map_or(false, |f| f.exists()) { + let file = cfg_file.unwrap(); + println!( + "error: you asked `x.py` to setup a new config file, but one already exists at `{}`", + file.display() + ); + println!( + "help: try adding `profile = \"{}\"` at the top of {}", + include_name, + file.display() + ); + println!( + "note: this will use the configuration in {}/src/bootstrap/defaults/config.toml.{}", + src_path.display(), + include_name + ); + std::process::exit(1); + } + + let path = cfg_file.unwrap_or_else(|| src_path.join("config.toml")); + let settings = format!( + "# Includes one of the default files in src/bootstrap/defaults\n\ + profile = \"{}\"\n", + include_name + ); + t!(fs::write(path, settings)); + + let include_path = + format!("{}/src/bootstrap/defaults/config.toml.{}", src_path.display(), include_name); + println!("`x.py` will now use the configuration at {}", include_path); + + let suggestions = match include_name { + "codegen" | "compiler" => &["check", "build", "test"][..], + "library" => &["check", "build", "test library/std", "doc"], + "user" => &["dist", "build"], + _ => return, + }; + + println!("To get started, try one of the following commands:"); + for cmd in suggestions { + println!("- `x.py {}`", cmd); + } + + if include_name != "user" { + println!( + "For more suggestions, see https://rustc-dev-guide.rust-lang.org/building/suggested.html" + ); + } +} + +// Used to get the path for `Subcommand::Setup` +pub fn interactive_path() -> io::Result<String> { + let mut input = String::new(); + println!( + "Welcome to the Rust project! What do you want to do with x.py? +a) Contribute to the standard library +b) Contribute to the compiler +c) Contribute to the compiler, and also modify LLVM or codegen +d) Install Rust from source" + ); + let template = loop { + print!("Please choose one (a/b/c/d): "); + io::stdout().flush()?; + io::stdin().read_line(&mut input)?; + break match input.trim().to_lowercase().as_str() { + "a" | "lib" | "library" => "library", + "b" | "compiler" => "compiler", + "c" | "llvm" => "llvm", + "d" | "user" | "maintainer" => "maintainer", + _ => { + println!("error: unrecognized option '{}'", input.trim()); + println!("note: press Ctrl+C to exit"); + continue; + } + }; + }; + Ok(template.to_owned()) +} diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index a7c9b99f45f..00522ee6b67 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -270,7 +270,7 @@ impl Step for Rls { &[], ); - builder.add_rustc_lib_path(compiler, &mut cargo); + cargo.add_rustc_lib_path(builder, compiler); cargo.arg("--").args(builder.config.cmd.test_args()); if try_run(builder, &mut cargo.into()) { @@ -328,7 +328,7 @@ impl Step for Rustfmt { t!(fs::create_dir_all(&dir)); cargo.env("RUSTFMT_TEST_DIR", dir); - builder.add_rustc_lib_path(compiler, &mut cargo); + cargo.add_rustc_lib_path(builder, compiler); if try_run(builder, &mut cargo.into()) { builder.save_toolstate("rustfmt", ToolState::TestPass); @@ -449,7 +449,7 @@ impl Step for Miri { cargo.arg("--").args(builder.config.cmd.test_args()); - builder.add_rustc_lib_path(compiler, &mut cargo); + cargo.add_rustc_lib_path(builder, compiler); if !try_run(builder, &mut cargo.into()) { return; @@ -554,7 +554,7 @@ impl Step for Clippy { cargo.arg("--").args(builder.config.cmd.test_args()); - builder.add_rustc_lib_path(compiler, &mut cargo); + cargo.add_rustc_lib_path(builder, compiler); builder.run(&mut cargo.into()); } @@ -584,7 +584,7 @@ impl Step for RustdocTheme { } fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.host); + let compiler = run.builder.compiler(run.builder.top_stage, run.target); run.builder.ensure(RustdocTheme { compiler }); } @@ -600,8 +600,11 @@ impl Step for RustdocTheme { .env("CFG_RELEASE_CHANNEL", &builder.config.channel) .env("RUSTDOC_REAL", builder.rustdoc(self.compiler)) .env("RUSTC_BOOTSTRAP", "1"); - if let Some(linker) = builder.linker(self.compiler.host, true) { - cmd.env("RUSTC_TARGET_LINKER", linker); + if let Some(linker) = builder.linker(self.compiler.host) { + cmd.env("RUSTDOC_LINKER", linker); + } + if builder.is_fuse_ld_lld(self.compiler.host) { + cmd.env("RUSTDOC_FUSE_LD_LLD", "1"); } try_run(builder, &mut cmd); } @@ -633,7 +636,7 @@ impl Step for RustdocJSStd { .arg("--crate-name") .arg("std") .arg("--resource-suffix") - .arg(crate::channel::CFG_RELEASE_NUM) + .arg(&builder.version) .arg("--doc-folder") .arg(builder.doc_out(self.target)) .arg("--test-folder") @@ -648,7 +651,6 @@ impl Step for RustdocJSStd { #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct RustdocJSNotStd { - pub host: TargetSelection, pub target: TargetSelection, pub compiler: Compiler, } @@ -663,8 +665,8 @@ impl Step for RustdocJSNotStd { } fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.host); - run.builder.ensure(RustdocJSNotStd { host: run.host, target: run.target, compiler }); + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocJSNotStd { target: run.target, compiler }); } fn run(self, builder: &Builder<'_>) { @@ -685,7 +687,6 @@ impl Step for RustdocJSNotStd { #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct RustdocUi { - pub host: TargetSelection, pub target: TargetSelection, pub compiler: Compiler, } @@ -700,8 +701,8 @@ impl Step for RustdocUi { } fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.host); - run.builder.ensure(RustdocUi { host: run.host, target: run.target, compiler }); + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocUi { target: run.target, compiler }); } fn run(self, builder: &Builder<'_>) { @@ -870,7 +871,7 @@ macro_rules! test_definitions { } fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.host); + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); run.builder.ensure($name { compiler, target: run.target }); } @@ -1061,17 +1062,22 @@ impl Step for Compiletest { flags.push("-Zunstable-options".to_string()); flags.push(builder.config.cmd.rustc_args().join(" ")); - // Don't use LLD here since we want to test that rustc finds and uses a linker by itself. - if let Some(linker) = builder.linker(target, false) { + if let Some(linker) = builder.linker(target) { cmd.arg("--linker").arg(linker); } let mut hostflags = flags.clone(); hostflags.push(format!("-Lnative={}", builder.test_helpers_out(compiler.host).display())); + if builder.is_fuse_ld_lld(compiler.host) { + hostflags.push("-Clink-args=-fuse-ld=lld".to_string()); + } cmd.arg("--host-rustcflags").arg(hostflags.join(" ")); let mut targetflags = flags; targetflags.push(format!("-Lnative={}", builder.test_helpers_out(target).display())); + if builder.is_fuse_ld_lld(target) { + targetflags.push("-Clink-args=-fuse-ld=lld".to_string()); + } cmd.arg("--target-rustcflags").arg(targetflags.join(" ")); cmd.arg("--docck-python").arg(builder.python()); @@ -1275,6 +1281,8 @@ impl Step for Compiletest { cmd.arg("--rustfix-coverage"); } + cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo); + builder.ci_env.force_coloring_in_ci(&mut cmd); builder.info(&format!( @@ -1414,7 +1422,7 @@ macro_rules! test_book { fn make_run(run: RunConfig<'_>) { run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.host), + compiler: run.builder.compiler(run.builder.top_stage, run.target), }); } @@ -1461,7 +1469,7 @@ impl Step for ErrorIndex { // error_index_generator depends on librustdoc. Use the compiler that // is normally used to build rustdoc for other tests (like compiletest // tests in src/test/rustdoc) so that it shares the same artifacts. - let compiler = run.builder.compiler_for(run.builder.top_stage, run.host, run.host); + let compiler = run.builder.compiler_for(run.builder.top_stage, run.target, run.target); run.builder.ensure(ErrorIndex { compiler }); } @@ -1565,7 +1573,7 @@ impl Step for CrateLibrustc { fn make_run(run: RunConfig<'_>) { let builder = run.builder; - let compiler = builder.compiler(builder.top_stage, run.host); + let compiler = builder.compiler(builder.top_stage, run.build_triple()); for krate in builder.in_tree_crates("rustc-main") { if krate.path.ends_with(&run.path) { @@ -1612,7 +1620,7 @@ impl Step for CrateNotDefault { fn make_run(run: RunConfig<'_>) { let builder = run.builder; - let compiler = builder.compiler(builder.top_stage, run.host); + let compiler = builder.compiler(builder.top_stage, run.build_triple()); let test_kind = builder.kind.into(); @@ -1660,7 +1668,7 @@ impl Step for Crate { fn make_run(run: RunConfig<'_>) { let builder = run.builder; - let compiler = builder.compiler(builder.top_stage, run.host); + let compiler = builder.compiler(builder.top_stage, run.build_triple()); let make = |mode: Mode, krate: &CargoCrate| { let test_kind = builder.kind.into(); @@ -1800,7 +1808,7 @@ impl Step for CrateRustdoc { let test_kind = builder.kind.into(); - builder.ensure(CrateRustdoc { host: run.host, test_kind }); + builder.ensure(CrateRustdoc { host: run.target, test_kind }); } fn run(self, builder: &Builder<'_>) { @@ -2016,6 +2024,8 @@ impl Step for Bootstrap { .current_dir(builder.src.join("src/bootstrap")) .env("RUSTFLAGS", "-Cdebuginfo=2") .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) + .env("BOOTSTRAP_OUTPUT_DIRECTORY", &builder.config.out) + .env("BOOTSTRAP_INITIAL_CARGO", &builder.config.initial_cargo) .env("RUSTC_BOOTSTRAP", "1") .env("RUSTC", &builder.initial_rustc); if let Some(flags) = option_env!("RUSTFLAGS") { @@ -2046,7 +2056,6 @@ impl Step for Bootstrap { #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct TierCheck { pub compiler: Compiler, - target: TargetSelection, } impl Step for TierCheck { @@ -2059,18 +2068,19 @@ impl Step for TierCheck { } fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler_for(run.builder.top_stage, run.host, run.host); - run.builder.ensure(TierCheck { compiler, target: run.host }); + let compiler = + run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); + run.builder.ensure(TierCheck { compiler }); } /// Tests the Platform Support page in the rustc book. fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std { compiler: self.compiler, target: self.target }); + builder.ensure(compile::Std { compiler: self.compiler, target: self.compiler.host }); let mut cargo = tool::prepare_tool_cargo( builder, self.compiler, - Mode::ToolRustc, - self.target, + Mode::ToolStd, + self.compiler.host, "run", "src/tools/tier-check", SourceType::InTree, diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 5d9bdc9d843..790e324a55f 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -7,7 +7,6 @@ use std::process::{exit, Command}; use build_helper::t; use crate::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; -use crate::channel; use crate::channel::GitInfo; use crate::compile; use crate::config::TargetSelection; @@ -162,13 +161,15 @@ impl Step for ToolBuild { "the following dependencies are duplicated although they \ have the same features enabled:" ); - for (id, cur, prev) in duplicates.drain_filter(|(_, cur, prev)| cur.2 == prev.2) { + let (same, different): (Vec<_>, Vec<_>) = + duplicates.into_iter().partition(|(_, cur, prev)| cur.2 == prev.2); + for (id, cur, prev) in same { println!(" {}", id); // same features println!(" `{}` ({:?})\n `{}` ({:?})", cur.0, cur.1, prev.0, prev.1); } println!("the following dependencies have different features:"); - for (id, cur, prev) in duplicates { + for (id, cur, prev) in different { println!(" {}", id); let cur_features: HashSet<_> = cur.2.into_iter().collect(); let prev_features: HashSet<_> = prev.2.into_iter().collect(); @@ -253,7 +254,7 @@ pub fn prepare_tool_cargo( cargo.env("CFG_RELEASE", builder.rust_release()); cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel); cargo.env("CFG_VERSION", builder.rust_version()); - cargo.env("CFG_RELEASE_NUM", channel::CFG_RELEASE_NUM); + cargo.env("CFG_RELEASE_NUM", &builder.version); let info = GitInfo::new(builder.config.ignore_git, &dir); if let Some(sha) = info.sha() { @@ -366,6 +367,7 @@ bootstrap_tool!( RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; InstallGitHook, "src/tools/install-git-hook", "install-git-hook"; + LintDocs, "src/tools/lint-docs", "lint-docs"; ); #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] @@ -468,8 +470,13 @@ impl Step for Rustdoc { } fn make_run(run: RunConfig<'_>) { - run.builder - .ensure(Rustdoc { compiler: run.builder.compiler(run.builder.top_stage, run.host) }); + run.builder.ensure(Rustdoc { + // Note: this is somewhat unique in that we actually want a *target* + // compiler here, because rustdoc *is* a compiler. We won't be using + // this as the compiler to build with, but rather this is "what + // compiler are we producing"? + compiler: run.builder.compiler(run.builder.top_stage, run.target), + }); } fn run(self, builder: &Builder<'_>) -> PathBuf { @@ -698,6 +705,10 @@ impl<'a> Builder<'a> { } add_dylib_path(lib_paths, &mut cmd); + + // Provide a RUSTC for this command to use. + cmd.env("RUSTC", &self.initial_rustc); + cmd } } |