From 0e272de69f4a9c889e5f1a024a88b3e1f60cb6c5 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 16 Nov 2016 12:31:19 -0800 Subject: mk: Switch rustbuild to the default build system This commit switches the default build system for Rust from the makefiles to rustbuild. The rustbuild build system has been in development for almost a year now and has become quite mature over time. This commit is an implementation of the proposal on [internals] which slates deletion of the makefiles on 2016-01-02. [internals]: https://internals.rust-lang.org/t/proposal-for-promoting-rustbuild-to-official-status/4368 This commit also updates various documentation in `README.md`, `CONTRIBUTING.md`, `src/bootstrap/README.md`, and throughout the source code of rustbuild itself. Closes #37858 --- src/bootstrap/README.md | 29 ++--- src/bootstrap/bootstrap.py | 93 ++++++++++---- src/bootstrap/cc.rs | 23 +++- src/bootstrap/check.rs | 54 +++++--- src/bootstrap/clean.rs | 3 + src/bootstrap/flags.rs | 1 + src/bootstrap/lib.rs | 96 +++++++++++++- src/bootstrap/mk/Makefile.in | 20 +-- src/bootstrap/native.rs | 14 +- src/bootstrap/sanity.rs | 12 +- src/bootstrap/step.rs | 296 ++++++++++++++++++++++++++++++++++--------- src/bootstrap/util.rs | 17 +++ 12 files changed, 516 insertions(+), 142 deletions(-) (limited to 'src/bootstrap') diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 24d716c1195..d0b501e4d89 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -66,17 +66,6 @@ The script accepts commands, flags, and filters to determine what to do: * `doc` - a command for building documentation. Like above can take arguments for what to document. -If you're more used to `./configure` and `make`, however, then you can also -configure the build system to use rustbuild instead of the old makefiles: - -``` -./configure --enable-rustbuild -make -``` - -Afterwards the `Makefile` which is generated will have a few commands like -`make check`, `make tidy`, etc. - ## Configuring rustbuild There are currently two primary methods for configuring the rustbuild build @@ -90,6 +79,13 @@ be found at `src/bootstrap/config.toml.example`, and the configuration file can also be passed as `--config path/to/config.toml` if the build system is being invoked manually (via the python script). +Finally, rustbuild makes use of the [gcc-rs crate] which has [its own +method][env-vars] of configuring C compilers and C flags via environment +variables. + +[gcc-rs crate]: https://github.com/alexcrichton/gcc-rs +[env-vars]: https://github.com/alexcrichton/gcc-rs#external-configuration-via-environment-variables + ## Build stages The rustbuild build system goes through a few phases to actually build the @@ -273,16 +269,17 @@ After that, each module in rustbuild should have enough documentation to keep you up and running. Some general areas that you may be interested in modifying are: -* Adding a new build tool? Take a look at `build/step.rs` for examples of other - tools, as well as `build/mod.rs`. +* Adding a new build tool? Take a look at `bootstrap/step.rs` for examples of + other tools. * Adding a new compiler crate? Look no further! Adding crates can be done by adding a new directory with `Cargo.toml` followed by configuring all `Cargo.toml` files accordingly. * Adding a new dependency from crates.io? We're still working on that, so hold off on that for now. -* Adding a new configuration option? Take a look at `build/config.rs` or perhaps - `build/flags.rs` and then modify the build elsewhere to read that option. -* Adding a sanity check? Take a look at `build/sanity.rs`. +* Adding a new configuration option? Take a look at `bootstrap/config.rs` or + perhaps `bootstrap/flags.rs` and then modify the build elsewhere to read that + option. +* Adding a sanity check? Take a look at `bootstrap/sanity.rs`. If you have any questions feel free to reach out on `#rust-internals` on IRC or open an issue in the bug tracker! diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index a3fabbb3e80..0dda7f12007 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -30,32 +30,37 @@ def get(url, path, verbose=False): sha_path = sha_file.name try: - download(sha_path, sha_url, verbose) + download(sha_path, sha_url, False, verbose) if os.path.exists(path): if verify(path, sha_path, False): - print("using already-download file " + path) + if verbose: + print("using already-download file " + path) return else: - print("ignoring already-download file " + path + " due to failed verification") + if verbose: + print("ignoring already-download file " + path + " due to failed verification") os.unlink(path) - download(temp_path, url, verbose) - if not verify(temp_path, sha_path, True): + download(temp_path, url, True, verbose) + if not verify(temp_path, sha_path, verbose): raise RuntimeError("failed verification") - print("moving {} to {}".format(temp_path, path)) + if verbose: + print("moving {} to {}".format(temp_path, path)) shutil.move(temp_path, path) finally: - delete_if_present(sha_path) - delete_if_present(temp_path) + delete_if_present(sha_path, verbose) + delete_if_present(temp_path, verbose) -def delete_if_present(path): +def delete_if_present(path, verbose): if os.path.isfile(path): - print("removing " + path) + if verbose: + print("removing " + path) os.unlink(path) -def download(path, url, verbose): - print("downloading {} to {}".format(url, path)) +def download(path, url, probably_big, verbose): + if probably_big or verbose: + print("downloading {}".format(url)) # see http://serverfault.com/questions/301128/how-to-download if sys.platform == 'win32': run(["PowerShell.exe", "/nologo", "-Command", @@ -63,17 +68,22 @@ def download(path, url, verbose): ".DownloadFile('{}', '{}')".format(url, path)], verbose=verbose) else: - run(["curl", "-o", path, url], verbose=verbose) + if probably_big or verbose: + option = "-#" + else: + option = "-s" + run(["curl", option, "-Sf", "-o", path, url], verbose=verbose) def verify(path, sha_path, verbose): - print("verifying " + path) + if verbose: + print("verifying " + path) with open(path, "rb") as f: found = hashlib.sha256(f.read()).hexdigest() with open(sha_path, "r") as f: expected, _ = f.readline().split() verified = found == expected - if not verified and verbose: + if not verified: print("invalid checksum:\n" " found: {}\n" " expected: {}".format(found, expected)) @@ -144,6 +154,7 @@ class RustBuild(object): if self.rustc().startswith(self.bin_root()) and \ (not os.path.exists(self.rustc()) or self.rustc_out_of_date()): + self.print_what_it_means_to_bootstrap() if os.path.exists(self.bin_root()): shutil.rmtree(self.bin_root()) channel = self.stage0_rustc_channel() @@ -167,6 +178,7 @@ class RustBuild(object): if self.cargo().startswith(self.bin_root()) and \ (not os.path.exists(self.cargo()) or self.cargo_out_of_date()): + self.print_what_it_means_to_bootstrap() channel = self.stage0_cargo_channel() filename = "cargo-{}-{}.tar.gz".format(channel, self.build) url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date() @@ -251,7 +263,27 @@ class RustBuild(object): else: return '' + def print_what_it_means_to_bootstrap(self): + if hasattr(self, 'printed'): + return + self.printed = True + if os.path.exists(self.bootstrap_binary()): + return + if not '--help' in sys.argv or len(sys.argv) == 1: + return + + print('info: the build system for Rust is written in Rust, so this') + print(' script is now going to download a stage0 rust compiler') + print(' and then compile the build system itself') + print('') + print('info: in the meantime you can read more about rustbuild at') + print(' src/bootstrap/README.md before the download finishes') + + def bootstrap_binary(self): + return os.path.join(self.build_dir, "bootstrap/debug/bootstrap") + def build_bootstrap(self): + self.print_what_it_means_to_bootstrap() build_dir = os.path.join(self.build_dir, "bootstrap") if self.clean and os.path.exists(build_dir): shutil.rmtree(build_dir) @@ -408,22 +440,31 @@ def main(): rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \ 'CFG_ENABLE_VENDOR' in rb.config_mk + if 'SUDO_USER' in os.environ: + if os.environ['USER'] != os.environ['SUDO_USER']: + rb.use_vendored_sources = True + print('info: looks like you are running this command under `sudo`') + print(' and so in order to preserve your $HOME this will now') + print(' use vendored sources by default. Note that if this') + print(' does not work you should run a normal build first') + print(' before running a command like `sudo make intall`') + if rb.use_vendored_sources: if not os.path.exists('.cargo'): os.makedirs('.cargo') - f = open('.cargo/config','w') - f.write(""" - [source.crates-io] - replace-with = 'vendored-sources' - registry = 'https://example.com' - - [source.vendored-sources] - directory = '{}/src/vendor' - """.format(rb.rust_root)) - f.close() + with open('.cargo/config','w') as f: + f.write(""" + [source.crates-io] + replace-with = 'vendored-sources' + registry = 'https://example.com' + + [source.vendored-sources] + directory = '{}/src/vendor' + """.format(rb.rust_root)) else: if os.path.exists('.cargo'): shutil.rmtree('.cargo') + data = stage0_data(rb.rust_root) rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1) rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1) @@ -438,7 +479,7 @@ def main(): sys.stdout.flush() # Run the bootstrap - args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")] + args = [rb.bootstrap_binary()] args.extend(sys.argv[1:]) env = os.environ.copy() env["BUILD"] = rb.build diff --git a/src/bootstrap/cc.rs b/src/bootstrap/cc.rs index e2bde4a6586..aa70e24d952 100644 --- a/src/bootstrap/cc.rs +++ b/src/bootstrap/cc.rs @@ -51,7 +51,7 @@ pub fn find(build: &mut Build) { if let Some(cc) = config.and_then(|c| c.cc.as_ref()) { cfg.compiler(cc); } else { - set_compiler(&mut cfg, "gcc", target, config); + set_compiler(&mut cfg, "gcc", target, config, build); } let compiler = cfg.get_compiler(); @@ -72,7 +72,7 @@ pub fn find(build: &mut Build) { if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) { cfg.compiler(cxx); } else { - set_compiler(&mut cfg, "g++", host, config); + set_compiler(&mut cfg, "g++", host, config, build); } let compiler = cfg.get_compiler(); build.verbose(&format!("CXX_{} = {:?}", host, compiler.path())); @@ -83,7 +83,8 @@ pub fn find(build: &mut Build) { fn set_compiler(cfg: &mut gcc::Config, gnu_compiler: &str, target: &str, - config: Option<&Target>) { + config: Option<&Target>, + build: &Build) { match target { // When compiling for android we may have the NDK configured in the // config.toml in which case we look there. Otherwise the default @@ -119,6 +120,22 @@ fn set_compiler(cfg: &mut gcc::Config, } } + "mips-unknown-linux-musl" => { + cfg.compiler("mips-linux-musl-gcc"); + } + "mipsel-unknown-linux-musl" => { + cfg.compiler("mipsel-linux-musl-gcc"); + } + + t if t.contains("musl") => { + if let Some(root) = build.musl_root(target) { + let guess = root.join("bin/musl-gcc"); + if guess.exists() { + cfg.compiler(guess); + } + } + } + _ => {} } } diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index b67eab38f5d..c5675fd46cb 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -//! Implementation of the various `check-*` targets of the build system. +//! Implementation of the test-related targets of the build system. //! //! This file implements the various regression test suites that we execute on //! our CI. @@ -62,6 +62,8 @@ impl fmt::Display for TestKind { pub fn linkcheck(build: &Build, stage: u32, host: &str) { println!("Linkcheck stage{} ({})", stage, host); let compiler = Compiler::new(stage, host); + + let _time = util::timeit(); build.run(build.tool_cmd(&compiler, "linkchecker") .arg(build.out.join(host).join("doc"))); } @@ -87,6 +89,7 @@ pub fn cargotest(build: &Build, stage: u32, host: &str) { let out_dir = build.out.join("ct"); t!(fs::create_dir_all(&out_dir)); + let _time = util::timeit(); build.run(build.tool_cmd(compiler, "cargotest") .env("PATH", newpath) .arg(&build.cargo) @@ -119,7 +122,8 @@ pub fn compiletest(build: &Build, target: &str, mode: &str, suite: &str) { - println!("Check compiletest {} ({} -> {})", suite, compiler.host, target); + println!("Check compiletest suite={} mode={} ({} -> {})", + suite, mode, compiler.host, target); let mut cmd = build.tool_cmd(compiler, "compiletest"); // compiletest currently has... a lot of arguments, so let's just pass all @@ -213,6 +217,9 @@ pub fn compiletest(build: &Build, // Running a C compiler on MSVC requires a few env vars to be set, to be // sure to set them here. + // + // Note that if we encounter `PATH` we make sure to append to our own `PATH` + // rather than stomp over it. if target.contains("msvc") { for &(ref k, ref v) in build.cc[target].0.env() { if k != "PATH" { @@ -221,6 +228,7 @@ pub fn compiletest(build: &Build, } } cmd.env("RUSTC_BOOTSTRAP", "1"); + build.add_rust_test_threads(&mut cmd); cmd.arg("--adb-path").arg("adb"); cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); @@ -232,6 +240,7 @@ pub fn compiletest(build: &Build, cmd.arg("--android-cross-path").arg(""); } + let _time = util::timeit(); build.run(&mut cmd); } @@ -244,6 +253,7 @@ pub fn docs(build: &Build, compiler: &Compiler) { // Do a breadth-first traversal of the `src/doc` directory and just run // tests for all files that end in `*.md` let mut stack = vec![build.src.join("src/doc")]; + let _time = util::timeit(); while let Some(p) = stack.pop() { if p.is_dir() { @@ -272,6 +282,8 @@ pub fn error_index(build: &Build, compiler: &Compiler) { let dir = testdir(build, compiler.host); t!(fs::create_dir_all(&dir)); let output = dir.join("error-index.md"); + + let _time = util::timeit(); build.run(build.tool_cmd(compiler, "error_index_generator") .arg("markdown") .arg(&output) @@ -283,6 +295,7 @@ pub fn error_index(build: &Build, compiler: &Compiler) { fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) { let mut cmd = Command::new(build.rustdoc(compiler)); build.add_rustc_lib_path(compiler, &mut cmd); + build.add_rust_test_threads(&mut cmd); cmd.arg("--test"); cmd.arg(markdown); @@ -366,16 +379,25 @@ pub fn krate(build: &Build, dylib_path.insert(0, build.sysroot_libdir(compiler, target)); cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + if target.contains("android") { + cargo.arg("--no-run"); + } else if target.contains("emscripten") { + cargo.arg("--no-run"); + } + + cargo.arg("--"); + if build.config.quiet_tests { - cargo.arg("--"); cargo.arg("--quiet"); } + let _time = util::timeit(); + if target.contains("android") { - build.run(cargo.arg("--no-run")); + build.run(&mut cargo); krate_android(build, compiler, target, mode); } else if target.contains("emscripten") { - build.run(cargo.arg("--no-run")); + build.run(&mut cargo); krate_emscripten(build, compiler, target, mode); } else { cargo.args(&build.flags.cmd.test_args()); @@ -402,14 +424,17 @@ fn krate_android(build: &Build, target, compiler.host, test_file_name); + let quiet = if build.config.quiet_tests { "--quiet" } else { "" }; let program = format!("(cd {dir}; \ LD_LIBRARY_PATH=./{target} ./{test} \ --logfile {log} \ + {quiet} \ {args})", dir = ADB_TEST_DIR, target = target, test = test_file_name, log = log, + quiet = quiet, args = build.flags.cmd.test_args().join(" ")); let output = output(Command::new("adb").arg("shell").arg(&program)); @@ -438,18 +463,13 @@ fn krate_emscripten(build: &Build, let test_file_name = test.to_string_lossy().into_owned(); println!("running {}", test_file_name); let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured"); - let status = Command::new(nodejs) - .arg(&test_file_name) - .stderr(::std::process::Stdio::inherit()) - .status(); - match status { - Ok(status) => { - if !status.success() { - panic!("some tests failed"); - } - } - Err(e) => panic!(format!("failed to execute command: {}", e)), - }; + let mut cmd = Command::new(nodejs); + cmd.arg(&test_file_name) + .stderr(::std::process::Stdio::inherit()); + if build.config.quiet_tests { + cmd.arg("--quiet"); + } + build.run(&mut cmd); } } diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs index 75bcbfee6ee..e7655458aed 100644 --- a/src/bootstrap/clean.rs +++ b/src/bootstrap/clean.rs @@ -46,6 +46,9 @@ fn rm_rf(build: &Build, path: &Path) { if !path.exists() { return } + if path.is_file() { + return do_op(path, "remove file", |p| fs::remove_file(p)); + } for file in t!(fs::read_dir(path)) { let file = t!(file).path(); diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index a7d80e4cdc4..7a2d56fc5d3 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -239,6 +239,7 @@ To learn more about a subcommand, run `./x.py -h` install: m.opt_present("install"), } } + "--help" => usage(0, &opts), cmd => { println!("unknown command: {}", cmd); usage(1, &opts); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 590c967d147..912b5864c81 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -13,9 +13,56 @@ //! This module, and its descendants, are the implementation of the Rust build //! system. Most of this build system is backed by Cargo but the outer layer //! here serves as the ability to orchestrate calling Cargo, sequencing Cargo -//! builds, building artifacts like LLVM, etc. +//! builds, building artifacts like LLVM, etc. The goals of rustbuild are: //! -//! More documentation can be found in each respective module below. +//! * To be an easily understandable, easily extensible, and maintainable build +//! system. +//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka +//! crates.io and Cargo. +//! * A standard interface to build across all platforms, including MSVC +//! +//! ## Architecture +//! +//! Although this build system defers most of the complicated logic to Cargo +//! itself, it still needs to maintain a list of targets and dependencies which +//! it can itself perform. Rustbuild is made up of a list of rules with +//! dependencies amongst them (created in the `step` module) and then knows how +//! to execute each in sequence. Each time rustbuild is invoked, it will simply +//! iterate through this list of steps and execute each serially in turn. For +//! each step rustbuild relies on the step internally being incremental and +//! parallel. Note, though, that the `-j` parameter to rustbuild gets forwarded +//! to appropriate test harnesses and such. +//! +//! Most of the "meaty" steps that matter are backed by Cargo, which does indeed +//! have its own parallelism and incremental management. Later steps, like +//! tests, aren't incremental and simply run the entire suite currently. +//! +//! When you execute `x.py build`, the steps which are executed are: +//! +//! * First, the python script is run. This will automatically download the +//! stage0 rustc and cargo according to `src/stage0.txt`, or using the cached +//! versions if they're available. These are then used to compile rustbuild +//! itself (using Cargo). Finally, control is then transferred to rustbuild. +//! +//! * Rustbuild takes over, performs sanity checks, probes the environment, +//! reads configuration, builds up a list of steps, and then starts executing +//! them. +//! +//! * The stage0 libstd is compiled +//! * The stage0 libtest is compiled +//! * The stage0 librustc is compiled +//! * The stage1 compiler is assembled +//! * The stage1 libstd, libtest, librustc are compiled +//! * The stage2 compiler is assembled +//! * The stage2 libstd, libtest, librustc are compiled +//! +//! Each step is driven by a separate Cargo project and rustbuild orchestrates +//! copying files between steps and otherwise preparing for Cargo to run. +//! +//! ## Further information +//! +//! More documentation can be found in each respective module below, and you can +//! also check out the `src/bootstrap/README.md` file for more information. extern crate build_helper; extern crate cmake; @@ -28,6 +75,7 @@ extern crate toml; use std::collections::HashMap; use std::env; +use std::ffi::OsString; use std::fs::{self, File}; use std::path::{Component, PathBuf, Path}; use std::process::Command; @@ -128,6 +176,7 @@ pub struct Build { cc: HashMap)>, cxx: HashMap, crates: HashMap, + is_sudo: bool, } #[derive(Debug)] @@ -187,6 +236,16 @@ impl Build { }; let local_rebuild = config.local_rebuild; + let is_sudo = match env::var_os("SUDO_USER") { + Some(sudo_user) => { + match env::var_os("USER") { + Some(user) => user != sudo_user, + None => false, + } + } + None => false, + }; + Build { flags: flags, config: config, @@ -208,6 +267,7 @@ impl Build { crates: HashMap::new(), lldb_version: None, lldb_python_dir: None, + is_sudo: is_sudo, } } @@ -414,7 +474,7 @@ impl Build { // how the actual compiler itself is called. // // These variables are primarily all read by - // src/bootstrap/{rustc,rustdoc.rs} + // src/bootstrap/bin/{rustc.rs,rustdoc.rs} cargo.env("RUSTC", self.out.join("bootstrap/debug/rustc")) .env("RUSTC_REAL", self.compiler_path(compiler)) .env("RUSTC_STAGE", stage.to_string()) @@ -435,6 +495,7 @@ impl Build { // Enable usage of unstable features cargo.env("RUSTC_BOOTSTRAP", "1"); + self.add_rust_test_threads(&mut cargo); // Specify some various options for build scripts used throughout // the build. @@ -458,7 +519,7 @@ impl Build { if self.config.rust_optimize && cmd != "bench" { cargo.arg("--release"); } - if self.config.vendor { + if self.config.vendor || self.is_sudo { cargo.arg("--frozen"); } return cargo @@ -492,12 +553,30 @@ impl Build { fn tool_cmd(&self, compiler: &Compiler, tool: &str) -> Command { let mut cmd = Command::new(self.tool(&compiler, tool)); let host = compiler.host; - let paths = vec![ + let mut paths = vec![ self.cargo_out(compiler, Mode::Libstd, host).join("deps"), self.cargo_out(compiler, Mode::Libtest, host).join("deps"), self.cargo_out(compiler, Mode::Librustc, host).join("deps"), self.cargo_out(compiler, Mode::Tool, host).join("deps"), ]; + + // On MSVC a tool may invoke a C compiler (e.g. compiletest in run-make + // mode) and that C compiler may need some extra PATH modification. Do + // so here. + if compiler.host.contains("msvc") { + let curpaths = env::var_os("PATH").unwrap_or(OsString::new()); + let curpaths = env::split_paths(&curpaths).collect::>(); + for &(ref k, ref v) in self.cc[compiler.host].0.env() { + if k != "PATH" { + continue + } + for path in env::split_paths(v) { + if !curpaths.contains(&path) { + paths.push(path); + } + } + } + } add_lib_path(paths, &mut cmd); return cmd } @@ -651,6 +730,13 @@ impl Build { add_lib_path(vec![self.rustc_libdir(compiler)], cmd); } + /// Adds the `RUST_TEST_THREADS` env var if necessary + fn add_rust_test_threads(&self, cmd: &mut Command) { + if env::var_os("RUST_TEST_THREADS").is_none() { + cmd.env("RUST_TEST_THREADS", self.jobs().to_string()); + } + } + /// Returns the compiler's libdir where it stores the dynamic libraries that /// it itself links against. /// diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 1e73595ec99..b165048b7b6 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -23,9 +23,14 @@ all: $(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS) $(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS) -# Don’t use $(Q) here, always show how to invoke the bootstrap script directly help: - $(BOOTSTRAP) --help + $(Q)echo 'Welcome to the rustbuild build system!' + $(Q)echo + $(Q)echo This makefile is a thin veneer over the ./x.py script located + $(Q)echo in this directory. To get the full power of the build system + $(Q)echo you can run x.py directly. + $(Q)echo + $(Q)echo To learn more run \`./x.py --help\` clean: $(Q)$(BOOTSTRAP) clean $(BOOTSTRAP_ARGS) @@ -51,15 +56,14 @@ check-cargotest: dist: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) install: -ifeq (root user, $(USER) $(patsubst %,user,$(SUDO_USER))) - $(Q)echo "'sudo make install' is not supported currently." -else $(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS) -endif tidy: $(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS) --stage 0 -check-stage2-android: - $(Q)$(BOOTSTRAP) --step check-target --target arm-linux-androideabi +check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu: + $(Q)$(BOOTSTRAP) test --target arm-linux-androideabi +check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu: + $(Q)$(BOOTSTRAP) test --target x86_64-unknown-linux-gnu + .PHONY: dist diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 96d1b695dd7..ffa3fe1cbf2 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -28,7 +28,7 @@ use cmake; use gcc; use Build; -use util::up_to_date; +use util::{self, up_to_date}; /// Compile LLVM for `target`. pub fn llvm(build: &Build, target: &str) { @@ -58,6 +58,7 @@ pub fn llvm(build: &Build, target: &str) { println!("Building LLVM for {}", target); + let _time = util::timeit(); let _ = fs::remove_dir_all(&dst.join("build")); t!(fs::create_dir_all(&dst.join("build"))); let assertions = if build.config.llvm_assertions {"ON"} else {"OFF"}; @@ -158,6 +159,17 @@ pub fn test_helpers(build: &Build, target: &str) { println!("Building test helpers"); t!(fs::create_dir_all(&dst)); let mut cfg = gcc::Config::new(); + + // We may have found various cross-compilers a little differently due to our + // extra configuration, so inform gcc of these compilers. Note, though, that + // on MSVC we still need gcc's detection of env vars (ugh). + if !target.contains("msvc") { + if let Some(ar) = build.ar(target) { + cfg.archiver(ar); + } + cfg.compiler(build.cc(target)); + } + cfg.cargo_metadata(false) .out_dir(&dst) .target(target) diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 47efa695217..f3fe22698bb 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -41,10 +41,14 @@ pub fn check(build: &mut Build) { } } let have_cmd = |cmd: &OsStr| { - for path in env::split_paths(&path).map(|p| p.join(cmd)) { - if fs::metadata(&path).is_ok() || - fs::metadata(path.with_extension("exe")).is_ok() { - return Some(path); + for path in env::split_paths(&path) { + let target = path.join(cmd); + let mut cmd_alt = cmd.to_os_string(); + cmd_alt.push(".exe"); + if target.exists() || + target.with_extension("exe").exists() || + target.join(cmd_alt).exists() { + return Some(target); } } return None; diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index b8683831af1..ca169bd146c 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -8,6 +8,24 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +//! Definition of steps of the build system. +//! +//! This is where some of the real meat of rustbuild is located, in how we +//! define targets and the dependencies amongst them. This file can sort of be +//! viewed as just defining targets in a makefile which shell out to predefined +//! functions elsewhere about how to execute the target. +//! +//! The primary function here you're likely interested in is the `build_rules` +//! function. This will create a `Rules` structure which basically just lists +//! everything that rustbuild can do. Each rule has a human-readable name, a +//! path associated with it, some dependencies, and then a closure of how to +//! actually perform the rule. +//! +//! All steps below are defined in self-contained units, so adding a new target +//! to the build system should just involve adding the meta information here +//! along with the actual implementation elsewhere. You can find more comments +//! about how to define rules themselves below. + use std::collections::{HashMap, HashSet}; use std::mem; @@ -20,36 +38,6 @@ use install; use native; use {Compiler, Build, Mode}; -#[derive(PartialEq, Eq, Hash, Clone, Debug)] -struct Step<'a> { - name: &'a str, - stage: u32, - host: &'a str, - target: &'a str, -} - -impl<'a> Step<'a> { - fn name(&self, name: &'a str) -> Step<'a> { - Step { name: name, ..*self } - } - - fn stage(&self, stage: u32) -> Step<'a> { - Step { stage: stage, ..*self } - } - - fn host(&self, host: &'a str) -> Step<'a> { - Step { host: host, ..*self } - } - - fn target(&self, target: &'a str) -> Step<'a> { - Step { target: target, ..*self } - } - - fn compiler(&self) -> Compiler<'a> { - Compiler::new(self.stage, self.host) - } -} - pub fn run(build: &Build) { let rules = build_rules(build); let steps = rules.plan(); @@ -57,14 +45,91 @@ pub fn run(build: &Build) { } pub fn build_rules(build: &Build) -> Rules { - let mut rules: Rules = Rules::new(build); + let mut rules = Rules::new(build); + + // This is the first rule that we're going to define for rustbuild, which is + // used to compile LLVM itself. All rules are added through the `rules` + // structure created above and are configured through a builder-style + // interface. + // + // First up we see the `build` method. This represents a rule that's part of + // the top-level `build` subcommand. For example `./x.py build` is what this + // is associating with. Note that this is normally only relevant if you flag + // a rule as `default`, which we'll talk about later. + // + // Next up we'll see two arguments to this method: + // + // * `llvm` - this is the "human readable" name of this target. This name is + // not accessed anywhere outside this file itself (e.g. not in + // the CLI nor elsewhere in rustbuild). The purpose of this is to + // easily define dependencies between rules. That is, other rules + // will depend on this with the name "llvm". + // * `src/llvm` - this is the relevant path to the rule that we're working + // with. This path is the engine behind how commands like + // `./x.py build src/llvm` work. This should typically point + // to the relevant component, but if there's not really a + // path to be assigned here you can pass something like + // `path/to/nowhere` to ignore it. + // + // After we create the rule with the `build` method we can then configure + // various aspects of it. For example this LLVM rule uses `.host(true)` to + // flag that it's a rule only for host targets. In other words, LLVM isn't + // compiled for targets configured through `--target` (e.g. those we're just + // building a standard library for). + // + // Next up the `dep` method will add a dependency to this rule. The closure + // is yielded the step that represents executing the `llvm` rule itself + // (containing information like stage, host, target, ...) and then it must + // return a target that the step depends on. Here LLVM is actually + // interesting where a cross-compiled LLVM depends on the host LLVM, but + // otherwise it has no dependencies. + // + // To handle this we do a bit of dynamic dispatch to see what the dependency + // is. If we're building a LLVM for the build triple, then we don't actually + // have any dependencies! To do that we return a dependency on the "dummy" + // target which does nothing. + // + // If we're build a cross-compiled LLVM, however, we need to assemble the + // libraries from the previous compiler. This step has the same name as + // ours (llvm) but we want it for a different target, so we use the + // builder-style methods on `Step` to configure this target to the build + // triple. + // + // Finally, to finish off this rule, we define how to actually execute it. + // That logic is all defined in the `native` module so we just delegate to + // the relevant function there. The argument to the closure passed to `run` + // is a `Step` (defined below) which encapsulates information like the + // stage, target, host, etc. + rules.build("llvm", "src/llvm") + .host(true) + .dep(move |s| { + if s.target == build.config.build { + dummy(s, build) + } else { + s.target(&build.config.build) + } + }) + .run(move |s| native::llvm(build, s.target)); + + // Ok! After that example rule that's hopefully enough to explain what's + // going on here. You can check out the API docs below and also see a bunch + // more examples of rules directly below as well. + // dummy rule to do nothing, useful when a dep maps to no deps rules.build("dummy", "path/to/nowhere"); - fn dummy<'a>(s: &Step<'a>, build: &'a Build) -> Step<'a> { - s.name("dummy").stage(0) - .target(&build.config.build) - .host(&build.config.build) - } + + // the compiler with no target libraries ready to go + rules.build("rustc", "src/rustc") + .dep(move |s| { + if s.stage == 0 { + dummy(s, build) + } else { + s.name("librustc") + .host(&build.config.build) + .stage(s.stage - 1) + } + }) + .run(move |s| compile::assemble_rustc(build, s.stage, s.target)); // Helper for loading an entire DAG of crates, rooted at `name` let krates = |name: &str| { @@ -85,28 +150,6 @@ pub fn build_rules(build: &Build) -> Rules { return ret }; - rules.build("rustc", "path/to/nowhere") - .dep(move |s| { - if s.stage == 0 { - dummy(s, build) - } else { - s.name("librustc") - .host(&build.config.build) - .stage(s.stage - 1) - } - }) - .run(move |s| compile::assemble_rustc(build, s.stage, s.target)); - rules.build("llvm", "src/llvm") - .host(true) - .dep(move |s| { - if s.target == build.config.build { - dummy(s, build) - } else { - s.target(&build.config.build) - } - }) - .run(move |s| native::llvm(build, s.target)); - // ======================================================================== // Crate compilations // @@ -337,10 +380,10 @@ pub fn build_rules(build: &Build) -> Rules { .host(true) .run(move |s| check::cargotest(build, s.stage, s.target)); rules.test("check-tidy", "src/tools/tidy") - .dep(|s| s.name("tool-tidy")) + .dep(|s| s.name("tool-tidy").stage(0)) .default(true) .host(true) - .run(move |s| check::tidy(build, s.stage, s.target)); + .run(move |s| check::tidy(build, 0, s.target)); rules.test("check-error-index", "src/tools/error_index_generator") .dep(|s| s.name("libstd")) .dep(|s| s.name("tool-error-index").host(s.host)) @@ -457,16 +500,89 @@ pub fn build_rules(build: &Build) -> Rules { .run(move |s| install::install(build, s.stage, s.target)); rules.verify(); - return rules + return rules; + + fn dummy<'a>(s: &Step<'a>, build: &'a Build) -> Step<'a> { + s.name("dummy").stage(0) + .target(&build.config.build) + .host(&build.config.build) + } +} + +#[derive(PartialEq, Eq, Hash, Clone, Debug)] +struct Step<'a> { + /// Human readable name of the rule this step is executing. Possible names + /// are all defined above in `build_rules`. + name: &'a str, + + /// The stage this step is executing in. This is typically 0, 1, or 2. + stage: u32, + + /// This step will likely involve a compiler, and the target that compiler + /// itself is built for is called the host, this variable. Typically this is + /// the target of the build machine itself. + host: &'a str, + + /// The target that this step represents generating. If you're building a + /// standard library for a new suite of targets, for example, this'll be set + /// to those targets. + target: &'a str, +} + +impl<'a> Step<'a> { + /// Creates a new step which is the same as this, except has a new name. + fn name(&self, name: &'a str) -> Step<'a> { + Step { name: name, ..*self } + } + + /// Creates a new step which is the same as this, except has a new stage. + fn stage(&self, stage: u32) -> Step<'a> { + Step { stage: stage, ..*self } + } + + /// Creates a new step which is the same as this, except has a new host. + fn host(&self, host: &'a str) -> Step<'a> { + Step { host: host, ..*self } + } + + /// Creates a new step which is the same as this, except has a new target. + fn target(&self, target: &'a str) -> Step<'a> { + Step { target: target, ..*self } + } + + /// Returns the `Compiler` structure that this step corresponds to. + fn compiler(&self) -> Compiler<'a> { + Compiler::new(self.stage, self.host) + } } struct Rule<'a> { + /// The human readable name of this target, defined in `build_rules`. name: &'a str, + + /// The path associated with this target, used in the `./x.py` driver for + /// easy and ergonomic specification of what to do. path: &'a str, + + /// The "kind" of top-level command that this rule is associated with, only + /// relevant if this is a default rule. kind: Kind, + + /// List of dependencies this rule has. Each dependency is a function from a + /// step that's being executed to another step that should be executed. deps: Vec) -> Step<'a> + 'a>>, + + /// How to actually execute this rule. Takes a step with contextual + /// information and then executes it. run: Box) + 'a>, + + /// Whether or not this is a "default" rule. That basically means that if + /// you run, for example, `./x.py test` whether it's included or not. default: bool, + + /// Whether or not this is a "host" rule, or in other words whether this is + /// only intended for compiler hosts and not for targets that are being + /// generated. host: bool, } @@ -493,6 +609,8 @@ impl<'a> Rule<'a> { } } +/// Builder pattern returned from the various methods on `Rules` which will add +/// the rule to the internal list on `Drop`. struct RuleBuilder<'a: 'b, 'b> { rules: &'b mut Rules<'a>, rule: Rule<'a>, @@ -554,26 +672,35 @@ impl<'a> Rules<'a> { } } + /// Creates a new rule of `Kind::Build` with the specified human readable + /// name and path associated with it. + /// + /// The builder returned should be configured further with information such + /// as how to actually run this rule. fn build<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Build) } + /// Same as `build`, but for `Kind::Test`. fn test<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Test) } + /// Same as `build`, but for `Kind::Bench`. fn bench<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Bench) } + /// Same as `build`, but for `Kind::Doc`. fn doc<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Doc) } + /// Same as `build`, but for `Kind::Dist`. fn dist<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Dist) @@ -634,6 +761,31 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? /// Construct the top-level build steps that we're going to be executing, /// given the subcommand that our build is performing. fn plan(&self) -> Vec> { + // Ok, the logic here is pretty subtle, and involves quite a few + // conditionals. The basic idea here is to: + // + // 1. First, filter all our rules to the relevant ones. This means that + // the command specified corresponds to one of our `Kind` variants, + // and we filter all rules based on that. + // + // 2. Next, we determine which rules we're actually executing. If a + // number of path filters were specified on the command line we look + // for those, otherwise we look for anything tagged `default`. + // + // 3. Finally, we generate some steps with host and target information. + // + // The last step is by far the most complicated and subtle. The basic + // thinking here is that we want to take the cartesian product of + // specified hosts and targets and build rules with that. The list of + // hosts and targets, if not specified, come from the how this build was + // configured. If the rule we're looking at is a host-only rule the we + // ignore the list of targets and instead consider the list of hosts + // also the list of targets. + // + // Once the host and target lists are generated we take the cartesian + // product of the two and then create a step based off them. Note that + // the stage each step is associated was specified with the `--step` + // flag on the command line. let (kind, paths) = match self.build.flags.cmd { Subcommand::Build { ref paths } => (Kind::Build, &paths[..]), Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]), @@ -664,7 +816,18 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? } else { &self.build.config.target }; - let arr = if rule.host {hosts} else {targets}; + // If --target was specified but --host wasn't specified, don't run + // any host-only tests + let arr = if rule.host { + if self.build.flags.target.len() > 0 && + self.build.flags.host.len() == 0 { + &hosts[..0] + } else { + hosts + } + } else { + targets + }; hosts.iter().flat_map(move |host| { arr.iter().map(move |target| { @@ -705,6 +868,15 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? } } + /// Performs topological sort of dependencies rooted at the `step` + /// specified, pushing all results onto the `order` vector provided. + /// + /// In other words, when this method returns, the `order` vector will + /// contain a list of steps which if executed in order will eventually + /// complete the `step` specified as well. + /// + /// The `added` set specified here is the set of steps that are already + /// present in `order` (and hence don't need to be added again). fn fill(&self, step: Step<'a>, order: &mut Vec>, diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index e028c522366..cb5b456a0f2 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -18,6 +18,7 @@ use std::ffi::OsString; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +use std::time::Instant; use filetime::FileTime; @@ -189,3 +190,19 @@ pub fn push_exe_path(mut buf: PathBuf, components: &[&str]) -> PathBuf { buf } + +pub struct TimeIt(Instant); + +/// Returns an RAII structure that prints out how long it took to drop. +pub fn timeit() -> TimeIt { + TimeIt(Instant::now()) +} + +impl Drop for TimeIt { + fn drop(&mut self) { + let time = self.0.elapsed(); + println!("\tfinished in {}.{:03}", + time.as_secs(), + time.subsec_nanos() / 1_000_000); + } +} -- cgit 1.4.1-3-g733a5