diff options
Diffstat (limited to 'src/bootstrap')
| -rw-r--r-- | src/bootstrap/README.md | 29 | ||||
| -rw-r--r-- | src/bootstrap/bootstrap.py | 93 | ||||
| -rw-r--r-- | src/bootstrap/cc.rs | 23 | ||||
| -rw-r--r-- | src/bootstrap/check.rs | 54 | ||||
| -rw-r--r-- | src/bootstrap/clean.rs | 3 | ||||
| -rw-r--r-- | src/bootstrap/flags.rs | 1 | ||||
| -rw-r--r-- | src/bootstrap/lib.rs | 96 | ||||
| -rw-r--r-- | src/bootstrap/mk/Makefile.in | 20 | ||||
| -rw-r--r-- | src/bootstrap/native.rs | 14 | ||||
| -rw-r--r-- | src/bootstrap/sanity.rs | 12 | ||||
| -rw-r--r-- | src/bootstrap/step.rs | 296 | ||||
| -rw-r--r-- | src/bootstrap/util.rs | 17 |
12 files changed, 516 insertions, 142 deletions
diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 24d716c1195..d0b501e4d89 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -66,17 +66,6 @@ The script accepts commands, flags, and filters to determine what to do: * `doc` - a command for building documentation. Like above can take arguments for what to document. -If you're more used to `./configure` and `make`, however, then you can also -configure the build system to use rustbuild instead of the old makefiles: - -``` -./configure --enable-rustbuild -make -``` - -Afterwards the `Makefile` which is generated will have a few commands like -`make check`, `make tidy`, etc. - ## Configuring rustbuild There are currently two primary methods for configuring the rustbuild build @@ -90,6 +79,13 @@ be found at `src/bootstrap/config.toml.example`, and the configuration file can also be passed as `--config path/to/config.toml` if the build system is being invoked manually (via the python script). +Finally, rustbuild makes use of the [gcc-rs crate] which has [its own +method][env-vars] of configuring C compilers and C flags via environment +variables. + +[gcc-rs crate]: https://github.com/alexcrichton/gcc-rs +[env-vars]: https://github.com/alexcrichton/gcc-rs#external-configuration-via-environment-variables + ## Build stages The rustbuild build system goes through a few phases to actually build the @@ -273,16 +269,17 @@ After that, each module in rustbuild should have enough documentation to keep you up and running. Some general areas that you may be interested in modifying are: -* Adding a new build tool? Take a look at `build/step.rs` for examples of other - tools, as well as `build/mod.rs`. +* Adding a new build tool? Take a look at `bootstrap/step.rs` for examples of + other tools. * Adding a new compiler crate? Look no further! Adding crates can be done by adding a new directory with `Cargo.toml` followed by configuring all `Cargo.toml` files accordingly. * Adding a new dependency from crates.io? We're still working on that, so hold off on that for now. -* Adding a new configuration option? Take a look at `build/config.rs` or perhaps - `build/flags.rs` and then modify the build elsewhere to read that option. -* Adding a sanity check? Take a look at `build/sanity.rs`. +* Adding a new configuration option? Take a look at `bootstrap/config.rs` or + perhaps `bootstrap/flags.rs` and then modify the build elsewhere to read that + option. +* Adding a sanity check? Take a look at `bootstrap/sanity.rs`. If you have any questions feel free to reach out on `#rust-internals` on IRC or open an issue in the bug tracker! diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index a3fabbb3e80..0dda7f12007 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -30,32 +30,37 @@ def get(url, path, verbose=False): sha_path = sha_file.name try: - download(sha_path, sha_url, verbose) + download(sha_path, sha_url, False, verbose) if os.path.exists(path): if verify(path, sha_path, False): - print("using already-download file " + path) + if verbose: + print("using already-download file " + path) return else: - print("ignoring already-download file " + path + " due to failed verification") + if verbose: + print("ignoring already-download file " + path + " due to failed verification") os.unlink(path) - download(temp_path, url, verbose) - if not verify(temp_path, sha_path, True): + download(temp_path, url, True, verbose) + if not verify(temp_path, sha_path, verbose): raise RuntimeError("failed verification") - print("moving {} to {}".format(temp_path, path)) + if verbose: + print("moving {} to {}".format(temp_path, path)) shutil.move(temp_path, path) finally: - delete_if_present(sha_path) - delete_if_present(temp_path) + delete_if_present(sha_path, verbose) + delete_if_present(temp_path, verbose) -def delete_if_present(path): +def delete_if_present(path, verbose): if os.path.isfile(path): - print("removing " + path) + if verbose: + print("removing " + path) os.unlink(path) -def download(path, url, verbose): - print("downloading {} to {}".format(url, path)) +def download(path, url, probably_big, verbose): + if probably_big or verbose: + print("downloading {}".format(url)) # see http://serverfault.com/questions/301128/how-to-download if sys.platform == 'win32': run(["PowerShell.exe", "/nologo", "-Command", @@ -63,17 +68,22 @@ def download(path, url, verbose): ".DownloadFile('{}', '{}')".format(url, path)], verbose=verbose) else: - run(["curl", "-o", path, url], verbose=verbose) + if probably_big or verbose: + option = "-#" + else: + option = "-s" + run(["curl", option, "-Sf", "-o", path, url], verbose=verbose) def verify(path, sha_path, verbose): - print("verifying " + path) + if verbose: + print("verifying " + path) with open(path, "rb") as f: found = hashlib.sha256(f.read()).hexdigest() with open(sha_path, "r") as f: expected, _ = f.readline().split() verified = found == expected - if not verified and verbose: + if not verified: print("invalid checksum:\n" " found: {}\n" " expected: {}".format(found, expected)) @@ -144,6 +154,7 @@ class RustBuild(object): if self.rustc().startswith(self.bin_root()) and \ (not os.path.exists(self.rustc()) or self.rustc_out_of_date()): + self.print_what_it_means_to_bootstrap() if os.path.exists(self.bin_root()): shutil.rmtree(self.bin_root()) channel = self.stage0_rustc_channel() @@ -167,6 +178,7 @@ class RustBuild(object): if self.cargo().startswith(self.bin_root()) and \ (not os.path.exists(self.cargo()) or self.cargo_out_of_date()): + self.print_what_it_means_to_bootstrap() channel = self.stage0_cargo_channel() filename = "cargo-{}-{}.tar.gz".format(channel, self.build) url = "https://static.rust-lang.org/cargo-dist/" + self.stage0_cargo_date() @@ -251,7 +263,27 @@ class RustBuild(object): else: return '' + def print_what_it_means_to_bootstrap(self): + if hasattr(self, 'printed'): + return + self.printed = True + if os.path.exists(self.bootstrap_binary()): + return + if not '--help' in sys.argv or len(sys.argv) == 1: + return + + print('info: the build system for Rust is written in Rust, so this') + print(' script is now going to download a stage0 rust compiler') + print(' and then compile the build system itself') + print('') + print('info: in the meantime you can read more about rustbuild at') + print(' src/bootstrap/README.md before the download finishes') + + def bootstrap_binary(self): + return os.path.join(self.build_dir, "bootstrap/debug/bootstrap") + def build_bootstrap(self): + self.print_what_it_means_to_bootstrap() build_dir = os.path.join(self.build_dir, "bootstrap") if self.clean and os.path.exists(build_dir): shutil.rmtree(build_dir) @@ -408,22 +440,31 @@ def main(): rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \ 'CFG_ENABLE_VENDOR' in rb.config_mk + if 'SUDO_USER' in os.environ: + if os.environ['USER'] != os.environ['SUDO_USER']: + rb.use_vendored_sources = True + print('info: looks like you are running this command under `sudo`') + print(' and so in order to preserve your $HOME this will now') + print(' use vendored sources by default. Note that if this') + print(' does not work you should run a normal build first') + print(' before running a command like `sudo make intall`') + if rb.use_vendored_sources: if not os.path.exists('.cargo'): os.makedirs('.cargo') - f = open('.cargo/config','w') - f.write(""" - [source.crates-io] - replace-with = 'vendored-sources' - registry = 'https://example.com' - - [source.vendored-sources] - directory = '{}/src/vendor' - """.format(rb.rust_root)) - f.close() + with open('.cargo/config','w') as f: + f.write(""" + [source.crates-io] + replace-with = 'vendored-sources' + registry = 'https://example.com' + + [source.vendored-sources] + directory = '{}/src/vendor' + """.format(rb.rust_root)) else: if os.path.exists('.cargo'): shutil.rmtree('.cargo') + data = stage0_data(rb.rust_root) rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1) rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1) @@ -438,7 +479,7 @@ def main(): sys.stdout.flush() # Run the bootstrap - args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")] + args = [rb.bootstrap_binary()] args.extend(sys.argv[1:]) env = os.environ.copy() env["BUILD"] = rb.build diff --git a/src/bootstrap/cc.rs b/src/bootstrap/cc.rs index e2bde4a6586..aa70e24d952 100644 --- a/src/bootstrap/cc.rs +++ b/src/bootstrap/cc.rs @@ -51,7 +51,7 @@ pub fn find(build: &mut Build) { if let Some(cc) = config.and_then(|c| c.cc.as_ref()) { cfg.compiler(cc); } else { - set_compiler(&mut cfg, "gcc", target, config); + set_compiler(&mut cfg, "gcc", target, config, build); } let compiler = cfg.get_compiler(); @@ -72,7 +72,7 @@ pub fn find(build: &mut Build) { if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) { cfg.compiler(cxx); } else { - set_compiler(&mut cfg, "g++", host, config); + set_compiler(&mut cfg, "g++", host, config, build); } let compiler = cfg.get_compiler(); build.verbose(&format!("CXX_{} = {:?}", host, compiler.path())); @@ -83,7 +83,8 @@ pub fn find(build: &mut Build) { fn set_compiler(cfg: &mut gcc::Config, gnu_compiler: &str, target: &str, - config: Option<&Target>) { + config: Option<&Target>, + build: &Build) { match target { // When compiling for android we may have the NDK configured in the // config.toml in which case we look there. Otherwise the default @@ -119,6 +120,22 @@ fn set_compiler(cfg: &mut gcc::Config, } } + "mips-unknown-linux-musl" => { + cfg.compiler("mips-linux-musl-gcc"); + } + "mipsel-unknown-linux-musl" => { + cfg.compiler("mipsel-linux-musl-gcc"); + } + + t if t.contains("musl") => { + if let Some(root) = build.musl_root(target) { + let guess = root.join("bin/musl-gcc"); + if guess.exists() { + cfg.compiler(guess); + } + } + } + _ => {} } } diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index b67eab38f5d..c5675fd46cb 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -8,7 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -//! Implementation of the various `check-*` targets of the build system. +//! Implementation of the test-related targets of the build system. //! //! This file implements the various regression test suites that we execute on //! our CI. @@ -62,6 +62,8 @@ impl fmt::Display for TestKind { pub fn linkcheck(build: &Build, stage: u32, host: &str) { println!("Linkcheck stage{} ({})", stage, host); let compiler = Compiler::new(stage, host); + + let _time = util::timeit(); build.run(build.tool_cmd(&compiler, "linkchecker") .arg(build.out.join(host).join("doc"))); } @@ -87,6 +89,7 @@ pub fn cargotest(build: &Build, stage: u32, host: &str) { let out_dir = build.out.join("ct"); t!(fs::create_dir_all(&out_dir)); + let _time = util::timeit(); build.run(build.tool_cmd(compiler, "cargotest") .env("PATH", newpath) .arg(&build.cargo) @@ -119,7 +122,8 @@ pub fn compiletest(build: &Build, target: &str, mode: &str, suite: &str) { - println!("Check compiletest {} ({} -> {})", suite, compiler.host, target); + println!("Check compiletest suite={} mode={} ({} -> {})", + suite, mode, compiler.host, target); let mut cmd = build.tool_cmd(compiler, "compiletest"); // compiletest currently has... a lot of arguments, so let's just pass all @@ -213,6 +217,9 @@ pub fn compiletest(build: &Build, // Running a C compiler on MSVC requires a few env vars to be set, to be // sure to set them here. + // + // Note that if we encounter `PATH` we make sure to append to our own `PATH` + // rather than stomp over it. if target.contains("msvc") { for &(ref k, ref v) in build.cc[target].0.env() { if k != "PATH" { @@ -221,6 +228,7 @@ pub fn compiletest(build: &Build, } } cmd.env("RUSTC_BOOTSTRAP", "1"); + build.add_rust_test_threads(&mut cmd); cmd.arg("--adb-path").arg("adb"); cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); @@ -232,6 +240,7 @@ pub fn compiletest(build: &Build, cmd.arg("--android-cross-path").arg(""); } + let _time = util::timeit(); build.run(&mut cmd); } @@ -244,6 +253,7 @@ pub fn docs(build: &Build, compiler: &Compiler) { // Do a breadth-first traversal of the `src/doc` directory and just run // tests for all files that end in `*.md` let mut stack = vec![build.src.join("src/doc")]; + let _time = util::timeit(); while let Some(p) = stack.pop() { if p.is_dir() { @@ -272,6 +282,8 @@ pub fn error_index(build: &Build, compiler: &Compiler) { let dir = testdir(build, compiler.host); t!(fs::create_dir_all(&dir)); let output = dir.join("error-index.md"); + + let _time = util::timeit(); build.run(build.tool_cmd(compiler, "error_index_generator") .arg("markdown") .arg(&output) @@ -283,6 +295,7 @@ pub fn error_index(build: &Build, compiler: &Compiler) { fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) { let mut cmd = Command::new(build.rustdoc(compiler)); build.add_rustc_lib_path(compiler, &mut cmd); + build.add_rust_test_threads(&mut cmd); cmd.arg("--test"); cmd.arg(markdown); @@ -366,16 +379,25 @@ pub fn krate(build: &Build, dylib_path.insert(0, build.sysroot_libdir(compiler, target)); cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + if target.contains("android") { + cargo.arg("--no-run"); + } else if target.contains("emscripten") { + cargo.arg("--no-run"); + } + + cargo.arg("--"); + if build.config.quiet_tests { - cargo.arg("--"); cargo.arg("--quiet"); } + let _time = util::timeit(); + if target.contains("android") { - build.run(cargo.arg("--no-run")); + build.run(&mut cargo); krate_android(build, compiler, target, mode); } else if target.contains("emscripten") { - build.run(cargo.arg("--no-run")); + build.run(&mut cargo); krate_emscripten(build, compiler, target, mode); } else { cargo.args(&build.flags.cmd.test_args()); @@ -402,14 +424,17 @@ fn krate_android(build: &Build, target, compiler.host, test_file_name); + let quiet = if build.config.quiet_tests { "--quiet" } else { "" }; let program = format!("(cd {dir}; \ LD_LIBRARY_PATH=./{target} ./{test} \ --logfile {log} \ + {quiet} \ {args})", dir = ADB_TEST_DIR, target = target, test = test_file_name, log = log, + quiet = quiet, args = build.flags.cmd.test_args().join(" ")); let output = output(Command::new("adb").arg("shell").arg(&program)); @@ -438,18 +463,13 @@ fn krate_emscripten(build: &Build, let test_file_name = test.to_string_lossy().into_owned(); println!("running {}", test_file_name); let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured"); - let status = Command::new(nodejs) - .arg(&test_file_name) - .stderr(::std::process::Stdio::inherit()) - .status(); - match status { - Ok(status) => { - if !status.success() { - panic!("some tests failed"); - } - } - Err(e) => panic!(format!("failed to execute command: {}", e)), - }; + let mut cmd = Command::new(nodejs); + cmd.arg(&test_file_name) + .stderr(::std::process::Stdio::inherit()); + if build.config.quiet_tests { + cmd.arg("--quiet"); + } + build.run(&mut cmd); } } diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs index 75bcbfee6ee..e7655458aed 100644 --- a/src/bootstrap/clean.rs +++ b/src/bootstrap/clean.rs @@ -46,6 +46,9 @@ fn rm_rf(build: &Build, path: &Path) { if !path.exists() { return } + if path.is_file() { + return do_op(path, "remove file", |p| fs::remove_file(p)); + } for file in t!(fs::read_dir(path)) { let file = t!(file).path(); diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index a7d80e4cdc4..7a2d56fc5d3 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -239,6 +239,7 @@ To learn more about a subcommand, run `./x.py <command> -h` install: m.opt_present("install"), } } + "--help" => usage(0, &opts), cmd => { println!("unknown command: {}", cmd); usage(1, &opts); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 590c967d147..912b5864c81 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -13,9 +13,56 @@ //! This module, and its descendants, are the implementation of the Rust build //! system. Most of this build system is backed by Cargo but the outer layer //! here serves as the ability to orchestrate calling Cargo, sequencing Cargo -//! builds, building artifacts like LLVM, etc. +//! builds, building artifacts like LLVM, etc. The goals of rustbuild are: //! -//! More documentation can be found in each respective module below. +//! * To be an easily understandable, easily extensible, and maintainable build +//! system. +//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka +//! crates.io and Cargo. +//! * A standard interface to build across all platforms, including MSVC +//! +//! ## Architecture +//! +//! Although this build system defers most of the complicated logic to Cargo +//! itself, it still needs to maintain a list of targets and dependencies which +//! it can itself perform. Rustbuild is made up of a list of rules with +//! dependencies amongst them (created in the `step` module) and then knows how +//! to execute each in sequence. Each time rustbuild is invoked, it will simply +//! iterate through this list of steps and execute each serially in turn. For +//! each step rustbuild relies on the step internally being incremental and +//! parallel. Note, though, that the `-j` parameter to rustbuild gets forwarded +//! to appropriate test harnesses and such. +//! +//! Most of the "meaty" steps that matter are backed by Cargo, which does indeed +//! have its own parallelism and incremental management. Later steps, like +//! tests, aren't incremental and simply run the entire suite currently. +//! +//! When you execute `x.py build`, the steps which are executed are: +//! +//! * First, the python script is run. This will automatically download the +//! stage0 rustc and cargo according to `src/stage0.txt`, or using the cached +//! versions if they're available. These are then used to compile rustbuild +//! itself (using Cargo). Finally, control is then transferred to rustbuild. +//! +//! * Rustbuild takes over, performs sanity checks, probes the environment, +//! reads configuration, builds up a list of steps, and then starts executing +//! them. +//! +//! * The stage0 libstd is compiled +//! * The stage0 libtest is compiled +//! * The stage0 librustc is compiled +//! * The stage1 compiler is assembled +//! * The stage1 libstd, libtest, librustc are compiled +//! * The stage2 compiler is assembled +//! * The stage2 libstd, libtest, librustc are compiled +//! +//! Each step is driven by a separate Cargo project and rustbuild orchestrates +//! copying files between steps and otherwise preparing for Cargo to run. +//! +//! ## Further information +//! +//! More documentation can be found in each respective module below, and you can +//! also check out the `src/bootstrap/README.md` file for more information. extern crate build_helper; extern crate cmake; @@ -28,6 +75,7 @@ extern crate toml; use std::collections::HashMap; use std::env; +use std::ffi::OsString; use std::fs::{self, File}; use std::path::{Component, PathBuf, Path}; use std::process::Command; @@ -128,6 +176,7 @@ pub struct Build { cc: HashMap<String, (gcc::Tool, Option<PathBuf>)>, cxx: HashMap<String, gcc::Tool>, crates: HashMap<String, Crate>, + is_sudo: bool, } #[derive(Debug)] @@ -187,6 +236,16 @@ impl Build { }; let local_rebuild = config.local_rebuild; + let is_sudo = match env::var_os("SUDO_USER") { + Some(sudo_user) => { + match env::var_os("USER") { + Some(user) => user != sudo_user, + None => false, + } + } + None => false, + }; + Build { flags: flags, config: config, @@ -208,6 +267,7 @@ impl Build { crates: HashMap::new(), lldb_version: None, lldb_python_dir: None, + is_sudo: is_sudo, } } @@ -414,7 +474,7 @@ impl Build { // how the actual compiler itself is called. // // These variables are primarily all read by - // src/bootstrap/{rustc,rustdoc.rs} + // src/bootstrap/bin/{rustc.rs,rustdoc.rs} cargo.env("RUSTC", self.out.join("bootstrap/debug/rustc")) .env("RUSTC_REAL", self.compiler_path(compiler)) .env("RUSTC_STAGE", stage.to_string()) @@ -435,6 +495,7 @@ impl Build { // Enable usage of unstable features cargo.env("RUSTC_BOOTSTRAP", "1"); + self.add_rust_test_threads(&mut cargo); // Specify some various options for build scripts used throughout // the build. @@ -458,7 +519,7 @@ impl Build { if self.config.rust_optimize && cmd != "bench" { cargo.arg("--release"); } - if self.config.vendor { + if self.config.vendor || self.is_sudo { cargo.arg("--frozen"); } return cargo @@ -492,12 +553,30 @@ impl Build { fn tool_cmd(&self, compiler: &Compiler, tool: &str) -> Command { let mut cmd = Command::new(self.tool(&compiler, tool)); let host = compiler.host; - let paths = vec![ + let mut paths = vec![ self.cargo_out(compiler, Mode::Libstd, host).join("deps"), self.cargo_out(compiler, Mode::Libtest, host).join("deps"), self.cargo_out(compiler, Mode::Librustc, host).join("deps"), self.cargo_out(compiler, Mode::Tool, host).join("deps"), ]; + + // On MSVC a tool may invoke a C compiler (e.g. compiletest in run-make + // mode) and that C compiler may need some extra PATH modification. Do + // so here. + if compiler.host.contains("msvc") { + let curpaths = env::var_os("PATH").unwrap_or(OsString::new()); + let curpaths = env::split_paths(&curpaths).collect::<Vec<_>>(); + for &(ref k, ref v) in self.cc[compiler.host].0.env() { + if k != "PATH" { + continue + } + for path in env::split_paths(v) { + if !curpaths.contains(&path) { + paths.push(path); + } + } + } + } add_lib_path(paths, &mut cmd); return cmd } @@ -651,6 +730,13 @@ impl Build { add_lib_path(vec![self.rustc_libdir(compiler)], cmd); } + /// Adds the `RUST_TEST_THREADS` env var if necessary + fn add_rust_test_threads(&self, cmd: &mut Command) { + if env::var_os("RUST_TEST_THREADS").is_none() { + cmd.env("RUST_TEST_THREADS", self.jobs().to_string()); + } + } + /// Returns the compiler's libdir where it stores the dynamic libraries that /// it itself links against. /// diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 1e73595ec99..b165048b7b6 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -23,9 +23,14 @@ all: $(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS) $(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS) -# Don’t use $(Q) here, always show how to invoke the bootstrap script directly help: - $(BOOTSTRAP) --help + $(Q)echo 'Welcome to the rustbuild build system!' + $(Q)echo + $(Q)echo This makefile is a thin veneer over the ./x.py script located + $(Q)echo in this directory. To get the full power of the build system + $(Q)echo you can run x.py directly. + $(Q)echo + $(Q)echo To learn more run \`./x.py --help\` clean: $(Q)$(BOOTSTRAP) clean $(BOOTSTRAP_ARGS) @@ -51,15 +56,14 @@ check-cargotest: dist: $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) install: -ifeq (root user, $(USER) $(patsubst %,user,$(SUDO_USER))) - $(Q)echo "'sudo make install' is not supported currently." -else $(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS) -endif tidy: $(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS) --stage 0 -check-stage2-android: - $(Q)$(BOOTSTRAP) --step check-target --target arm-linux-androideabi +check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu: + $(Q)$(BOOTSTRAP) test --target arm-linux-androideabi +check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu: + $(Q)$(BOOTSTRAP) test --target x86_64-unknown-linux-gnu + .PHONY: dist diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 96d1b695dd7..ffa3fe1cbf2 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -28,7 +28,7 @@ use cmake; use gcc; use Build; -use util::up_to_date; +use util::{self, up_to_date}; /// Compile LLVM for `target`. pub fn llvm(build: &Build, target: &str) { @@ -58,6 +58,7 @@ pub fn llvm(build: &Build, target: &str) { println!("Building LLVM for {}", target); + let _time = util::timeit(); let _ = fs::remove_dir_all(&dst.join("build")); t!(fs::create_dir_all(&dst.join("build"))); let assertions = if build.config.llvm_assertions {"ON"} else {"OFF"}; @@ -158,6 +159,17 @@ pub fn test_helpers(build: &Build, target: &str) { println!("Building test helpers"); t!(fs::create_dir_all(&dst)); let mut cfg = gcc::Config::new(); + + // We may have found various cross-compilers a little differently due to our + // extra configuration, so inform gcc of these compilers. Note, though, that + // on MSVC we still need gcc's detection of env vars (ugh). + if !target.contains("msvc") { + if let Some(ar) = build.ar(target) { + cfg.archiver(ar); + } + cfg.compiler(build.cc(target)); + } + cfg.cargo_metadata(false) .out_dir(&dst) .target(target) diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 47efa695217..f3fe22698bb 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -41,10 +41,14 @@ pub fn check(build: &mut Build) { } } let have_cmd = |cmd: &OsStr| { - for path in env::split_paths(&path).map(|p| p.join(cmd)) { - if fs::metadata(&path).is_ok() || - fs::metadata(path.with_extension("exe")).is_ok() { - return Some(path); + for path in env::split_paths(&path) { + let target = path.join(cmd); + let mut cmd_alt = cmd.to_os_string(); + cmd_alt.push(".exe"); + if target.exists() || + target.with_extension("exe").exists() || + target.join(cmd_alt).exists() { + return Some(target); } } return None; diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index b8683831af1..ca169bd146c 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -8,6 +8,24 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +//! Definition of steps of the build system. +//! +//! This is where some of the real meat of rustbuild is located, in how we +//! define targets and the dependencies amongst them. This file can sort of be +//! viewed as just defining targets in a makefile which shell out to predefined +//! functions elsewhere about how to execute the target. +//! +//! The primary function here you're likely interested in is the `build_rules` +//! function. This will create a `Rules` structure which basically just lists +//! everything that rustbuild can do. Each rule has a human-readable name, a +//! path associated with it, some dependencies, and then a closure of how to +//! actually perform the rule. +//! +//! All steps below are defined in self-contained units, so adding a new target +//! to the build system should just involve adding the meta information here +//! along with the actual implementation elsewhere. You can find more comments +//! about how to define rules themselves below. + use std::collections::{HashMap, HashSet}; use std::mem; @@ -20,36 +38,6 @@ use install; use native; use {Compiler, Build, Mode}; -#[derive(PartialEq, Eq, Hash, Clone, Debug)] -struct Step<'a> { - name: &'a str, - stage: u32, - host: &'a str, - target: &'a str, -} - -impl<'a> Step<'a> { - fn name(&self, name: &'a str) -> Step<'a> { - Step { name: name, ..*self } - } - - fn stage(&self, stage: u32) -> Step<'a> { - Step { stage: stage, ..*self } - } - - fn host(&self, host: &'a str) -> Step<'a> { - Step { host: host, ..*self } - } - - fn target(&self, target: &'a str) -> Step<'a> { - Step { target: target, ..*self } - } - - fn compiler(&self) -> Compiler<'a> { - Compiler::new(self.stage, self.host) - } -} - pub fn run(build: &Build) { let rules = build_rules(build); let steps = rules.plan(); @@ -57,14 +45,91 @@ pub fn run(build: &Build) { } pub fn build_rules(build: &Build) -> Rules { - let mut rules: Rules = Rules::new(build); + let mut rules = Rules::new(build); + + // This is the first rule that we're going to define for rustbuild, which is + // used to compile LLVM itself. All rules are added through the `rules` + // structure created above and are configured through a builder-style + // interface. + // + // First up we see the `build` method. This represents a rule that's part of + // the top-level `build` subcommand. For example `./x.py build` is what this + // is associating with. Note that this is normally only relevant if you flag + // a rule as `default`, which we'll talk about later. + // + // Next up we'll see two arguments to this method: + // + // * `llvm` - this is the "human readable" name of this target. This name is + // not accessed anywhere outside this file itself (e.g. not in + // the CLI nor elsewhere in rustbuild). The purpose of this is to + // easily define dependencies between rules. That is, other rules + // will depend on this with the name "llvm". + // * `src/llvm` - this is the relevant path to the rule that we're working + // with. This path is the engine behind how commands like + // `./x.py build src/llvm` work. This should typically point + // to the relevant component, but if there's not really a + // path to be assigned here you can pass something like + // `path/to/nowhere` to ignore it. + // + // After we create the rule with the `build` method we can then configure + // various aspects of it. For example this LLVM rule uses `.host(true)` to + // flag that it's a rule only for host targets. In other words, LLVM isn't + // compiled for targets configured through `--target` (e.g. those we're just + // building a standard library for). + // + // Next up the `dep` method will add a dependency to this rule. The closure + // is yielded the step that represents executing the `llvm` rule itself + // (containing information like stage, host, target, ...) and then it must + // return a target that the step depends on. Here LLVM is actually + // interesting where a cross-compiled LLVM depends on the host LLVM, but + // otherwise it has no dependencies. + // + // To handle this we do a bit of dynamic dispatch to see what the dependency + // is. If we're building a LLVM for the build triple, then we don't actually + // have any dependencies! To do that we return a dependency on the "dummy" + // target which does nothing. + // + // If we're build a cross-compiled LLVM, however, we need to assemble the + // libraries from the previous compiler. This step has the same name as + // ours (llvm) but we want it for a different target, so we use the + // builder-style methods on `Step` to configure this target to the build + // triple. + // + // Finally, to finish off this rule, we define how to actually execute it. + // That logic is all defined in the `native` module so we just delegate to + // the relevant function there. The argument to the closure passed to `run` + // is a `Step` (defined below) which encapsulates information like the + // stage, target, host, etc. + rules.build("llvm", "src/llvm") + .host(true) + .dep(move |s| { + if s.target == build.config.build { + dummy(s, build) + } else { + s.target(&build.config.build) + } + }) + .run(move |s| native::llvm(build, s.target)); + + // Ok! After that example rule that's hopefully enough to explain what's + // going on here. You can check out the API docs below and also see a bunch + // more examples of rules directly below as well. + // dummy rule to do nothing, useful when a dep maps to no deps rules.build("dummy", "path/to/nowhere"); - fn dummy<'a>(s: &Step<'a>, build: &'a Build) -> Step<'a> { - s.name("dummy").stage(0) - .target(&build.config.build) - .host(&build.config.build) - } + + // the compiler with no target libraries ready to go + rules.build("rustc", "src/rustc") + .dep(move |s| { + if s.stage == 0 { + dummy(s, build) + } else { + s.name("librustc") + .host(&build.config.build) + .stage(s.stage - 1) + } + }) + .run(move |s| compile::assemble_rustc(build, s.stage, s.target)); // Helper for loading an entire DAG of crates, rooted at `name` let krates = |name: &str| { @@ -85,28 +150,6 @@ pub fn build_rules(build: &Build) -> Rules { return ret }; - rules.build("rustc", "path/to/nowhere") - .dep(move |s| { - if s.stage == 0 { - dummy(s, build) - } else { - s.name("librustc") - .host(&build.config.build) - .stage(s.stage - 1) - } - }) - .run(move |s| compile::assemble_rustc(build, s.stage, s.target)); - rules.build("llvm", "src/llvm") - .host(true) - .dep(move |s| { - if s.target == build.config.build { - dummy(s, build) - } else { - s.target(&build.config.build) - } - }) - .run(move |s| native::llvm(build, s.target)); - // ======================================================================== // Crate compilations // @@ -337,10 +380,10 @@ pub fn build_rules(build: &Build) -> Rules { .host(true) .run(move |s| check::cargotest(build, s.stage, s.target)); rules.test("check-tidy", "src/tools/tidy") - .dep(|s| s.name("tool-tidy")) + .dep(|s| s.name("tool-tidy").stage(0)) .default(true) .host(true) - .run(move |s| check::tidy(build, s.stage, s.target)); + .run(move |s| check::tidy(build, 0, s.target)); rules.test("check-error-index", "src/tools/error_index_generator") .dep(|s| s.name("libstd")) .dep(|s| s.name("tool-error-index").host(s.host)) @@ -457,16 +500,89 @@ pub fn build_rules(build: &Build) -> Rules { .run(move |s| install::install(build, s.stage, s.target)); rules.verify(); - return rules + return rules; + + fn dummy<'a>(s: &Step<'a>, build: &'a Build) -> Step<'a> { + s.name("dummy").stage(0) + .target(&build.config.build) + .host(&build.config.build) + } +} + +#[derive(PartialEq, Eq, Hash, Clone, Debug)] +struct Step<'a> { + /// Human readable name of the rule this step is executing. Possible names + /// are all defined above in `build_rules`. + name: &'a str, + + /// The stage this step is executing in. This is typically 0, 1, or 2. + stage: u32, + + /// This step will likely involve a compiler, and the target that compiler + /// itself is built for is called the host, this variable. Typically this is + /// the target of the build machine itself. + host: &'a str, + + /// The target that this step represents generating. If you're building a + /// standard library for a new suite of targets, for example, this'll be set + /// to those targets. + target: &'a str, +} + +impl<'a> Step<'a> { + /// Creates a new step which is the same as this, except has a new name. + fn name(&self, name: &'a str) -> Step<'a> { + Step { name: name, ..*self } + } + + /// Creates a new step which is the same as this, except has a new stage. + fn stage(&self, stage: u32) -> Step<'a> { + Step { stage: stage, ..*self } + } + + /// Creates a new step which is the same as this, except has a new host. + fn host(&self, host: &'a str) -> Step<'a> { + Step { host: host, ..*self } + } + + /// Creates a new step which is the same as this, except has a new target. + fn target(&self, target: &'a str) -> Step<'a> { + Step { target: target, ..*self } + } + + /// Returns the `Compiler` structure that this step corresponds to. + fn compiler(&self) -> Compiler<'a> { + Compiler::new(self.stage, self.host) + } } struct Rule<'a> { + /// The human readable name of this target, defined in `build_rules`. name: &'a str, + + /// The path associated with this target, used in the `./x.py` driver for + /// easy and ergonomic specification of what to do. path: &'a str, + + /// The "kind" of top-level command that this rule is associated with, only + /// relevant if this is a default rule. kind: Kind, + + /// List of dependencies this rule has. Each dependency is a function from a + /// step that's being executed to another step that should be executed. deps: Vec<Box<Fn(&Step<'a>) -> Step<'a> + 'a>>, + + /// How to actually execute this rule. Takes a step with contextual + /// information and then executes it. run: Box<Fn(&Step<'a>) + 'a>, + + /// Whether or not this is a "default" rule. That basically means that if + /// you run, for example, `./x.py test` whether it's included or not. default: bool, + + /// Whether or not this is a "host" rule, or in other words whether this is + /// only intended for compiler hosts and not for targets that are being + /// generated. host: bool, } @@ -493,6 +609,8 @@ impl<'a> Rule<'a> { } } +/// Builder pattern returned from the various methods on `Rules` which will add +/// the rule to the internal list on `Drop`. struct RuleBuilder<'a: 'b, 'b> { rules: &'b mut Rules<'a>, rule: Rule<'a>, @@ -554,26 +672,35 @@ impl<'a> Rules<'a> { } } + /// Creates a new rule of `Kind::Build` with the specified human readable + /// name and path associated with it. + /// + /// The builder returned should be configured further with information such + /// as how to actually run this rule. fn build<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Build) } + /// Same as `build`, but for `Kind::Test`. fn test<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Test) } + /// Same as `build`, but for `Kind::Bench`. fn bench<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Bench) } + /// Same as `build`, but for `Kind::Doc`. fn doc<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Doc) } + /// Same as `build`, but for `Kind::Dist`. fn dist<'b>(&'b mut self, name: &'a str, path: &'a str) -> RuleBuilder<'a, 'b> { self.rule(name, path, Kind::Dist) @@ -634,6 +761,31 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? /// Construct the top-level build steps that we're going to be executing, /// given the subcommand that our build is performing. fn plan(&self) -> Vec<Step<'a>> { + // Ok, the logic here is pretty subtle, and involves quite a few + // conditionals. The basic idea here is to: + // + // 1. First, filter all our rules to the relevant ones. This means that + // the command specified corresponds to one of our `Kind` variants, + // and we filter all rules based on that. + // + // 2. Next, we determine which rules we're actually executing. If a + // number of path filters were specified on the command line we look + // for those, otherwise we look for anything tagged `default`. + // + // 3. Finally, we generate some steps with host and target information. + // + // The last step is by far the most complicated and subtle. The basic + // thinking here is that we want to take the cartesian product of + // specified hosts and targets and build rules with that. The list of + // hosts and targets, if not specified, come from the how this build was + // configured. If the rule we're looking at is a host-only rule the we + // ignore the list of targets and instead consider the list of hosts + // also the list of targets. + // + // Once the host and target lists are generated we take the cartesian + // product of the two and then create a step based off them. Note that + // the stage each step is associated was specified with the `--step` + // flag on the command line. let (kind, paths) = match self.build.flags.cmd { Subcommand::Build { ref paths } => (Kind::Build, &paths[..]), Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]), @@ -664,7 +816,18 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? } else { &self.build.config.target }; - let arr = if rule.host {hosts} else {targets}; + // If --target was specified but --host wasn't specified, don't run + // any host-only tests + let arr = if rule.host { + if self.build.flags.target.len() > 0 && + self.build.flags.host.len() == 0 { + &hosts[..0] + } else { + hosts + } + } else { + targets + }; hosts.iter().flat_map(move |host| { arr.iter().map(move |target| { @@ -705,6 +868,15 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd? } } + /// Performs topological sort of dependencies rooted at the `step` + /// specified, pushing all results onto the `order` vector provided. + /// + /// In other words, when this method returns, the `order` vector will + /// contain a list of steps which if executed in order will eventually + /// complete the `step` specified as well. + /// + /// The `added` set specified here is the set of steps that are already + /// present in `order` (and hence don't need to be added again). fn fill(&self, step: Step<'a>, order: &mut Vec<Step<'a>>, diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index e028c522366..cb5b456a0f2 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -18,6 +18,7 @@ use std::ffi::OsString; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +use std::time::Instant; use filetime::FileTime; @@ -189,3 +190,19 @@ pub fn push_exe_path(mut buf: PathBuf, components: &[&str]) -> PathBuf { buf } + +pub struct TimeIt(Instant); + +/// Returns an RAII structure that prints out how long it took to drop. +pub fn timeit() -> TimeIt { + TimeIt(Instant::now()) +} + +impl Drop for TimeIt { + fn drop(&mut self) { + let time = self.0.elapsed(); + println!("\tfinished in {}.{:03}", + time.as_secs(), + time.subsec_nanos() / 1_000_000); + } +} |