diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | src/Cargo.lock | 20 | ||||
| -rw-r--r-- | src/bootstrap/Cargo.toml | 2 | ||||
| -rw-r--r-- | src/bootstrap/README.md | 70 | ||||
| -rw-r--r-- | src/bootstrap/bootstrap.py | 6 | ||||
| -rw-r--r-- | src/bootstrap/check.rs | 91 | ||||
| -rw-r--r-- | src/bootstrap/clean.rs | 22 | ||||
| -rw-r--r-- | src/bootstrap/compile.rs | 28 | ||||
| -rw-r--r-- | src/bootstrap/doc.rs | 39 | ||||
| -rw-r--r-- | src/bootstrap/flags.rs | 239 | ||||
| -rw-r--r-- | src/bootstrap/lib.rs | 271 | ||||
| -rw-r--r-- | src/bootstrap/metadata.rs | 95 | ||||
| -rw-r--r-- | src/bootstrap/mk/Makefile.in | 31 | ||||
| -rw-r--r-- | src/bootstrap/step.rs | 1179 | ||||
| -rw-r--r-- | src/bootstrap/util.rs | 3 | ||||
| -rw-r--r-- | src/librustc_errors/Cargo.toml | 2 | ||||
| -rwxr-xr-x | x.py | 19 |
17 files changed, 1136 insertions, 985 deletions
diff --git a/README.md b/README.md index b79c9703f44..7360651095b 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ ones from MSYS if you have it installed). You'll also need Visual Studio 2013 or newer with the C++ tools. Then all you need to do is to kick off rustbuild. ``` -python .\src\bootstrap\bootstrap.py +python x.py build ``` Currently rustbuild only works with some known versions of Visual Studio. If you @@ -137,7 +137,7 @@ by manually calling the appropriate vcvars file before running the bootstrap. ``` CALL "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\vcvars64.bat" -python .\src\bootstrap\bootstrap.py +python x.py build ``` ## Building Documentation diff --git a/src/Cargo.lock b/src/Cargo.lock index 5826995cc3c..c3a8923c6de 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -40,9 +40,9 @@ name = "bootstrap" version = "0.0.0" dependencies = [ "build_helper 0.1.0", - "cmake 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "cmake 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "gcc 0.3.38 (git+https://github.com/alexcrichton/gcc-rs)", + "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)", "getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", @@ -63,7 +63,7 @@ version = "0.1.0" [[package]] name = "cmake" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)", @@ -134,11 +134,6 @@ version = "0.0.0" [[package]] name = "gcc" version = "0.3.38" -source = "git+https://github.com/alexcrichton/gcc-rs#be620ac6d3ddb498cd0c700d5312c6a4c3c19597" - -[[package]] -name = "gcc" -version = "0.3.38" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -189,7 +184,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "linkchecker" version = "0.1.0" dependencies = [ - "url 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -725,7 +720,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "url" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -743,10 +738,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] -"checksum cmake 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "dfcf5bcece56ef953b8ea042509e9dcbdfe97820b7e20d86beb53df30ed94978" +"checksum cmake 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0e5bcf27e097a184c1df4437654ed98df3d7a516e8508a6ba45d8b092bbdf283" "checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f" "checksum filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "5363ab8e4139b8568a6237db5248646e5a8a2f89bd5ccb02092182b11fd3e922" -"checksum gcc 0.3.38 (git+https://github.com/alexcrichton/gcc-rs)" = "<none>" "checksum gcc 0.3.38 (registry+https://github.com/rust-lang/crates.io-index)" = "553f11439bdefe755bf366b264820f1da70f3aaf3924e594b886beb9c831bcf5" "checksum getopts 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9047cfbd08a437050b363d35ef160452c5fe8ea5187ae0a624708c91581d685" "checksum idna 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1053236e00ce4f668aeca4a769a09b3bf5a682d802abd6f3cb39374f6b162c11" @@ -760,6 +754,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "0590d72182e50e879c4da3b11c6488dae18fccb1ae0c7a3eda18e16795844796" "checksum unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c1f7ceb96afdfeedee42bade65a0d585a6a0106f681b6749c8ff4daa8df30b3f" "checksum unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "26643a2f83bac55f1976fb716c10234485f9202dcd65cfbdf9da49867b271172" -"checksum url 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ba5a45db1d2e0effb7a1c00cc73ffc63a973da8c7d1fcd5b46f24285ade6c54" +"checksum url 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "48ccf7bd87a81b769cf84ad556e034541fb90e1cd6d4bc375c822ed9500cd9d7" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index c9669075438..9d44ca033e4 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -27,7 +27,7 @@ num_cpus = "0.2" toml = "0.1" getopts = "0.2" rustc-serialize = "0.3" -gcc = { git = "https://github.com/alexcrichton/gcc-rs" } +gcc = "0.3.36" libc = "0.2" md5 = "0.1" diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 57d644d635c..f73f41ffae2 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -10,24 +10,72 @@ system. ## Using rustbuild -When configuring Rust via `./configure`, pass the following to enable building -via this build system: +The rustbuild build system has a primary entry point, a top level `x.py` script: ``` -./configure --enable-rustbuild -make +python ./x.py build ``` -Afterwards the `Makefile` which is generated will have a few commands like -`make check`, `make tidy`, etc. For finer-grained control, the -`bootstrap.py` entry point can be used: +Note that if you're on Unix you should be able to execute the script directly: ``` -python src/bootstrap/bootstrap.py +./x.py build ``` -This accepts a number of options like `--stage` and `--step` which can configure -what's actually being done. +The script accepts commands, flags, and filters to determine what to do: + +* `build` - a general purpose command for compiling code. Alone `build` will + bootstrap the entire compiler, and otherwise arguments passed indicate what to + build. For example: + + ``` + # build the whole compiler + ./x.py build + + # build the stage1 compier + ./x.py build --stage 1 + + # build stage0 libstd + ./x.py build --stage 0 src/libstd + + # build a particular crate in stage0 + ./x.py build --stage 0 src/libtest + ``` + +* `test` - a command for executing unit tests. Like the `build` command this + will execute the entire test suite by default, and otherwise it can be used to + select which test suite is run: + + ``` + # run all unit tests + ./x.py test + + # execute the run-pass test suite + ./x.py test src/test/run-pass + + # execute only some tests in the run-pass test suite + ./x.py test src/test/run-pass --filter my-filter + + # execute tests in the standard library in stage0 + ./x.py test --stage 0 src/libstd + + # execute all doc tests + ./x.py test src/doc + ``` + +* `doc` - a command for building documentation. Like above can take arguments + for what to document. + +If you're more used to `./configure` and `make`, however, then you can also +configure the build system to use rustbuild instead of the old makefiles: + +``` +./configure --enable-rustbuild +make +``` + +Afterwards the `Makefile` which is generated will have a few commands like +`make check`, `make tidy`, etc. ## Configuring rustbuild @@ -47,7 +95,7 @@ being invoked manually (via the python script). The rustbuild build system goes through a few phases to actually build the compiler. What actually happens when you invoke rustbuild is: -1. The entry point script, `src/bootstrap/bootstrap.py` is run. This script is +1. The entry point script, `x.py` is run. This script is responsible for downloading the stage0 compiler/Cargo binaries, and it then compiles the build system itself (this folder). Finally, it then invokes the actual `bootstrap` binary build system. diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 2c2260a8e60..76bbb9d22e0 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -399,12 +399,10 @@ def main(): # Run the bootstrap args = [os.path.join(rb.build_dir, "bootstrap/debug/bootstrap")] - args.append('--src') - args.append(rb.rust_root) - args.append('--build') - args.append(rb.build) args.extend(sys.argv[1:]) env = os.environ.copy() + env["BUILD"] = rb.build + env["SRC"] = rb.rust_root env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) rb.run(args, env) diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 0a281b89c57..e5b666ab3b6 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -13,44 +13,19 @@ //! This file implements the various regression test suites that we execute on //! our CI. -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::env; use std::fs; use std::path::{PathBuf, Path}; use std::process::Command; use build_helper::output; -use rustc_serialize::json; use {Build, Compiler, Mode}; use util::{self, dylib_path, dylib_path_var}; const ADB_TEST_DIR: &'static str = "/data/tmp"; -#[derive(RustcDecodable)] -struct Output { - packages: Vec<Package>, - resolve: Resolve, -} - -#[derive(RustcDecodable)] -struct Package { - id: String, - name: String, - source: Option<String>, -} - -#[derive(RustcDecodable)] -struct Resolve { - nodes: Vec<ResolveNode>, -} - -#[derive(RustcDecodable)] -struct ResolveNode { - id: String, - dependencies: Vec<String>, -} - /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. /// /// This tool in `src/tools` will verify the validity of all our links in the @@ -181,7 +156,7 @@ pub fn compiletest(build: &Build, let llvm_version = output(Command::new(&llvm_config).arg("--version")); cmd.arg("--llvm-version").arg(llvm_version); - cmd.args(&build.flags.args); + cmd.args(&build.flags.cmd.test_args()); if build.config.verbose || build.flags.verbose { cmd.arg("--verbose"); @@ -282,7 +257,7 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) { cmd.arg("--test"); cmd.arg(markdown); - let mut test_args = build.flags.args.join(" "); + let mut test_args = build.flags.cmd.test_args().join(" "); if build.config.quiet_tests { test_args.push_str(" --quiet"); } @@ -302,7 +277,8 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) { pub fn krate(build: &Build, compiler: &Compiler, target: &str, - mode: Mode) { + mode: Mode, + krate: Option<&str>) { let (name, path, features, root) = match mode { Mode::Libstd => { ("libstd", "src/rustc/std_shim", build.std_features(), "std_shim") @@ -318,24 +294,6 @@ pub fn krate(build: &Build, println!("Testing {} stage{} ({} -> {})", name, compiler.stage, compiler.host, target); - // Run `cargo metadata` to figure out what crates we're testing. - // - // Down below we're going to call `cargo test`, but to test the right set - // of packages we're going to have to know what `-p` arguments to pass it - // to know what crates to test. Here we run `cargo metadata` to learn about - // the dependency graph and what `-p` arguments there are. - let mut cargo = Command::new(&build.cargo); - cargo.arg("metadata") - .arg("--manifest-path").arg(build.src.join(path).join("Cargo.toml")); - let output = output(&mut cargo); - let output: Output = json::decode(&output).unwrap(); - let id2pkg = output.packages.iter() - .map(|pkg| (&pkg.id, pkg)) - .collect::<HashMap<_, _>>(); - let id2deps = output.resolve.nodes.iter() - .map(|node| (&node.id, &node.dependencies)) - .collect::<HashMap<_, _>>(); - // Build up the base `cargo test` command. // // Pass in some standard flags then iterate over the graph we've discovered @@ -346,24 +304,25 @@ pub fn krate(build: &Build, .arg(build.src.join(path).join("Cargo.toml")) .arg("--features").arg(features); - let mut visited = HashSet::new(); - let root_pkg = output.packages.iter().find(|p| p.name == root).unwrap(); - let mut next = vec![&root_pkg.id]; - while let Some(id) = next.pop() { - // Skip any packages with sources listed, as these come from crates.io - // and we shouldn't be testing them. - if id2pkg[id].source.is_some() { - continue - } - // Right now jemalloc is our only target-specific crate in the sense - // that it's not present on all platforms. Custom skip it here for now, - // but if we add more this probably wants to get more generalized. - if !id.contains("jemalloc") { - cargo.arg("-p").arg(&id2pkg[id].name); + match krate { + Some(krate) => { + cargo.arg("-p").arg(krate); } - for dep in id2deps[id] { - if visited.insert(dep) { - next.push(dep); + None => { + let mut visited = HashSet::new(); + let mut next = vec![root]; + while let Some(name) = next.pop() { + // Right now jemalloc is our only target-specific crate in the sense + // that it's not present on all platforms. Custom skip it here for now, + // but if we add more this probably wants to get more generalized. + if !name.contains("jemalloc") { + cargo.arg("-p").arg(name); + } + for dep in build.crates[name].deps.iter() { + if visited.insert(dep) { + next.push(dep); + } + } } } } @@ -389,7 +348,7 @@ pub fn krate(build: &Build, build.run(cargo.arg("--no-run")); krate_emscripten(build, compiler, target, mode); } else { - cargo.args(&build.flags.args); + cargo.args(&build.flags.cmd.test_args()); build.run(&mut cargo); } } @@ -421,7 +380,7 @@ fn krate_android(build: &Build, target = target, test = test_file_name, log = log, - args = build.flags.args.join(" ")); + args = build.flags.cmd.test_args().join(" ")); let output = output(Command::new("adb").arg("shell").arg(&program)); println!("{}", output); diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs index a1e286e162f..3d338cf4f94 100644 --- a/src/bootstrap/clean.rs +++ b/src/bootstrap/clean.rs @@ -25,17 +25,17 @@ pub fn clean(build: &Build) { rm_rf(build, &build.out.join("tmp")); for host in build.config.host.iter() { - - let out = build.out.join(host); - - rm_rf(build, &out.join("doc")); - - for stage in 0..4 { - rm_rf(build, &out.join(format!("stage{}", stage))); - rm_rf(build, &out.join(format!("stage{}-std", stage))); - rm_rf(build, &out.join(format!("stage{}-rustc", stage))); - rm_rf(build, &out.join(format!("stage{}-tools", stage))); - rm_rf(build, &out.join(format!("stage{}-test", stage))); + let entries = match build.out.join(host).read_dir() { + Ok(iter) => iter, + Err(_) => continue, + }; + + for entry in entries { + let entry = t!(entry); + if entry.file_name().to_str() == Some("llvm") { + continue + } + t!(fs::remove_dir_all(&entry.path())); } } } diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index ff8e4757bd1..5fc4f006729 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -64,8 +64,8 @@ pub fn std<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) { } build.run(&mut cargo); - update_mtime(&libstd_stamp(build, compiler, target)); - std_link(build, target, compiler, compiler.host); + update_mtime(&libstd_stamp(build, &compiler, target)); + std_link(build, target, compiler.stage, compiler.host); } /// Link all libstd rlibs/dylibs into the sysroot location. @@ -74,11 +74,12 @@ pub fn std<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) { /// by `compiler` into `host`'s sysroot. pub fn std_link(build: &Build, target: &str, - compiler: &Compiler, + stage: u32, host: &str) { + let compiler = Compiler::new(stage, &build.config.build); let target_compiler = Compiler::new(compiler.stage, host); let libdir = build.sysroot_libdir(&target_compiler, target); - let out_dir = build.cargo_out(compiler, Mode::Libstd, target); + let out_dir = build.cargo_out(&compiler, Mode::Libstd, target); // If we're linking one compiler host's output into another, then we weren't // called from the `std` method above. In that case we clean out what's @@ -146,7 +147,7 @@ pub fn test<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) { .arg(build.src.join("src/rustc/test_shim/Cargo.toml")); build.run(&mut cargo); update_mtime(&libtest_stamp(build, compiler, target)); - test_link(build, target, compiler, compiler.host); + test_link(build, target, compiler.stage, compiler.host); } /// Link all libtest rlibs/dylibs into the sysroot location. @@ -155,11 +156,12 @@ pub fn test<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) { /// by `compiler` into `host`'s sysroot. pub fn test_link(build: &Build, target: &str, - compiler: &Compiler, + stage: u32, host: &str) { + let compiler = Compiler::new(stage, &build.config.build); let target_compiler = Compiler::new(compiler.stage, host); let libdir = build.sysroot_libdir(&target_compiler, target); - let out_dir = build.cargo_out(compiler, Mode::Libtest, target); + let out_dir = build.cargo_out(&compiler, Mode::Libtest, target); add_to_sysroot(&out_dir, &libdir); } @@ -218,7 +220,7 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) { } build.run(&mut cargo); - rustc_link(build, target, compiler, compiler.host); + rustc_link(build, target, compiler.stage, compiler.host); } /// Link all librustc rlibs/dylibs into the sysroot location. @@ -227,11 +229,12 @@ pub fn rustc<'a>(build: &'a Build, target: &str, compiler: &Compiler<'a>) { /// by `compiler` into `host`'s sysroot. pub fn rustc_link(build: &Build, target: &str, - compiler: &Compiler, + stage: u32, host: &str) { + let compiler = Compiler::new(stage, &build.config.build); let target_compiler = Compiler::new(compiler.stage, host); let libdir = build.sysroot_libdir(&target_compiler, target); - let out_dir = build.cargo_out(compiler, Mode::Librustc, target); + let out_dir = build.cargo_out(&compiler, Mode::Librustc, target); add_to_sysroot(&out_dir, &libdir); } @@ -259,7 +262,10 @@ fn compiler_file(compiler: &Path, file: &str) -> PathBuf { /// must have been previously produced by the `stage - 1` build.config.build /// compiler. pub fn assemble_rustc(build: &Build, stage: u32, host: &str) { - assert!(stage > 0, "the stage0 compiler isn't assembled, it's downloaded"); + // nothing to do in stage0 + if stage == 0 { + return + } // The compiler that we're assembling let target_compiler = Compiler::new(stage, host); diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index c2636384dbb..30c7fefad87 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -19,7 +19,6 @@ use std::fs::{self, File}; use std::io::prelude::*; -use std::path::Path; use std::process::Command; use {Build, Compiler, Mode}; @@ -30,8 +29,9 @@ use util::{up_to_date, cp_r}; /// /// This will not actually generate any documentation if the documentation has /// already been generated. -pub fn rustbook(build: &Build, stage: u32, target: &str, name: &str, out: &Path) { - t!(fs::create_dir_all(out)); +pub fn rustbook(build: &Build, stage: u32, target: &str, name: &str) { + let out = build.doc_out(target); + t!(fs::create_dir_all(&out)); let out = out.join(name); let compiler = Compiler::new(stage, &build.config.build); @@ -57,9 +57,10 @@ pub fn rustbook(build: &Build, stage: u32, target: &str, name: &str, out: &Path) /// `STAMP` alongw ith providing the various header/footer HTML we've cutomized. /// /// In the end, this is just a glorified wrapper around rustdoc! -pub fn standalone(build: &Build, stage: u32, target: &str, out: &Path) { +pub fn standalone(build: &Build, stage: u32, target: &str) { println!("Documenting stage{} standalone ({})", stage, target); - t!(fs::create_dir_all(out)); + let out = build.doc_out(target); + t!(fs::create_dir_all(&out)); let compiler = Compiler::new(stage, &build.config.build); @@ -109,7 +110,7 @@ pub fn standalone(build: &Build, stage: u32, target: &str, out: &Path) { .arg("--html-in-header").arg(&favicon) .arg("--markdown-playground-url") .arg("https://play.rust-lang.org/") - .arg("-o").arg(out) + .arg("-o").arg(&out) .arg(&path); if filename == "reference.md" { @@ -131,9 +132,10 @@ pub fn standalone(build: &Build, stage: u32, target: &str, out: &Path) { /// /// This will generate all documentation for the standard library and its /// dependencies. This is largely just a wrapper around `cargo doc`. -pub fn std(build: &Build, stage: u32, target: &str, out: &Path) { +pub fn std(build: &Build, stage: u32, target: &str) { println!("Documenting stage{} std ({})", stage, target); - t!(fs::create_dir_all(out)); + let out = build.doc_out(target); + t!(fs::create_dir_all(&out)); let compiler = Compiler::new(stage, &build.config.build); let out_dir = build.stage_out(&compiler, Mode::Libstd) .join(target).join("doc"); @@ -146,16 +148,17 @@ pub fn std(build: &Build, stage: u32, target: &str, out: &Path) { .arg(build.src.join("src/rustc/std_shim/Cargo.toml")) .arg("--features").arg(build.std_features()); build.run(&mut cargo); - cp_r(&out_dir, out) + cp_r(&out_dir, &out) } /// Compile all libtest documentation. /// /// This will generate all documentation for libtest and its dependencies. This /// is largely just a wrapper around `cargo doc`. -pub fn test(build: &Build, stage: u32, target: &str, out: &Path) { +pub fn test(build: &Build, stage: u32, target: &str) { println!("Documenting stage{} test ({})", stage, target); - t!(fs::create_dir_all(out)); + let out = build.doc_out(target); + t!(fs::create_dir_all(&out)); let compiler = Compiler::new(stage, &build.config.build); let out_dir = build.stage_out(&compiler, Mode::Libtest) .join(target).join("doc"); @@ -167,16 +170,17 @@ pub fn test(build: &Build, stage: u32, target: &str, out: &Path) { cargo.arg("--manifest-path") .arg(build.src.join("src/rustc/test_shim/Cargo.toml")); build.run(&mut cargo); - cp_r(&out_dir, out) + cp_r(&out_dir, &out) } /// Generate all compiler documentation. /// /// This will generate all documentation for the compiler libraries and their /// dependencies. This is largely just a wrapper around `cargo doc`. -pub fn rustc(build: &Build, stage: u32, target: &str, out: &Path) { +pub fn rustc(build: &Build, stage: u32, target: &str) { println!("Documenting stage{} compiler ({})", stage, target); - t!(fs::create_dir_all(out)); + let out = build.doc_out(target); + t!(fs::create_dir_all(&out)); let compiler = Compiler::new(stage, &build.config.build); let out_dir = build.stage_out(&compiler, Mode::Librustc) .join(target).join("doc"); @@ -189,14 +193,15 @@ pub fn rustc(build: &Build, stage: u32, target: &str, out: &Path) { .arg(build.src.join("src/rustc/Cargo.toml")) .arg("--features").arg(build.rustc_features()); build.run(&mut cargo); - cp_r(&out_dir, out) + cp_r(&out_dir, &out) } /// Generates the HTML rendered error-index by running the /// `error_index_generator` tool. -pub fn error_index(build: &Build, stage: u32, target: &str, out: &Path) { +pub fn error_index(build: &Build, stage: u32, target: &str) { println!("Documenting stage{} error index ({})", stage, target); - t!(fs::create_dir_all(out)); + let out = build.doc_out(target); + t!(fs::create_dir_all(&out)); let compiler = Compiler::new(stage, &build.config.build); let mut index = build.tool_cmd(&compiler, "error_index_generator"); index.arg("html"); diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index d925997f36c..d7516954f12 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -13,30 +13,46 @@ //! This module implements the command-line parsing of the build system which //! has various flags to configure how it's run. +use std::env; use std::fs; use std::path::PathBuf; use std::process; -use std::slice; -use getopts::Options; +use getopts::{Matches, Options}; + +use Build; +use config::Config; +use metadata; +use step; /// Deserialized version of all flags for this compile. pub struct Flags { pub verbose: bool, pub stage: Option<u32>, pub build: String, - pub host: Filter, - pub target: Filter, - pub step: Vec<String>, + pub host: Vec<String>, + pub target: Vec<String>, pub config: Option<PathBuf>, pub src: Option<PathBuf>, pub jobs: Option<u32>, - pub args: Vec<String>, - pub clean: bool, + pub cmd: Subcommand, } -pub struct Filter { - values: Vec<String>, +pub enum Subcommand { + Build { + paths: Vec<PathBuf>, + }, + Doc { + paths: Vec<PathBuf>, + }, + Test { + paths: Vec<PathBuf>, + test_args: Vec<String>, + }, + Clean, + Dist { + install: bool, + }, } impl Flags { @@ -44,29 +60,177 @@ impl Flags { let mut opts = Options::new(); opts.optflag("v", "verbose", "use verbose output"); opts.optopt("", "config", "TOML configuration file for build", "FILE"); + opts.optopt("", "build", "build target of the stage0 compiler", "BUILD"); opts.optmulti("", "host", "host targets to build", "HOST"); - opts.reqopt("", "build", "build target of the stage0 compiler", "BUILD"); - opts.optmulti("", "target", "targets to build", "TARGET"); - opts.optmulti("s", "step", "build step to execute", "STEP"); + opts.optmulti("", "target", "target targets to build", "TARGET"); opts.optopt("", "stage", "stage to build", "N"); - opts.optopt("", "src", "path to repo root", "DIR"); + opts.optopt("", "src", "path to the root of the rust checkout", "DIR"); opts.optopt("j", "jobs", "number of jobs to run in parallel", "JOBS"); - opts.optflag("", "clean", "clean output directory"); opts.optflag("h", "help", "print this help message"); - let usage = |n| -> ! { - let brief = format!("Usage: rust.py [options]"); - print!("{}", opts.usage(&brief)); + let usage = |n, opts: &Options| -> ! { + let command = args.get(0).map(|s| &**s); + let brief = format!("Usage: x.py {} [options] [<args>...]", + command.unwrap_or("<command>")); + + println!("{}", opts.usage(&brief)); + match command { + Some("build") => { + println!("\ +Arguments: + This subcommand accepts a number of positional arguments of directories to + the crates and/or artifacts to compile. For example: + + ./x.py build src/libcore + ./x.py build src/libproc_macro + ./x.py build src/libstd --stage 1 + + If no arguments are passed then the complete artifacts for that stage are + also compiled. + + ./x.py build + ./x.py build --stage 1 + + For a quick build with a usable compile, you can pass: + + ./x.py build --stage 1 src/libtest +"); + } + + Some("test") => { + println!("\ +Arguments: + This subcommand accepts a number of positional arguments of directories to + tests that should be compiled and run. For example: + + ./x.py test src/test/run-pass + ./x.py test src/test/run-pass/assert-* + ./x.py test src/libstd --test-args hash_map + ./x.py test src/libstd --stage 0 + + If no arguments are passed then the complete artifacts for that stage are + compiled and tested. + + ./x.py test + ./x.py test --stage 1 +"); + } + + Some("doc") => { + println!("\ +Arguments: + This subcommand accepts a number of positional arguments of directories of + documentation to build. For example: + + ./x.py doc src/doc/book + ./x.py doc src/doc/nomicon + ./x.py doc src/libstd + + If no arguments are passed then everything is documented: + + ./x.py doc + ./x.py doc --stage 1 +"); + } + + _ => {} + } + + if let Some(command) = command { + if command == "build" || + command == "dist" || + command == "doc" || + command == "test" || + command == "clean" { + println!("Available invocations:"); + if args.iter().any(|a| a == "-v") { + let flags = Flags::parse(&["build".to_string()]); + let mut config = Config::default(); + config.build = flags.build.clone(); + let mut build = Build::new(flags, config); + metadata::build(&mut build); + step::build_rules(&build).print_help(command); + } else { + println!(" ... elided, run `./x.py {} -h -v` to see", + command); + } + + println!(""); + } + } + +println!("\ +Subcommands: + build Compile either the compiler or libraries + test Build and run some test suites + doc Build documentation + clean Clean out build directories + dist Build and/or install distribution artifacts + +To learn more about a subcommand, run `./x.py <command> -h` +"); + process::exit(n); }; - - let m = opts.parse(args).unwrap_or_else(|e| { - println!("failed to parse options: {}", e); - usage(1); - }); - if m.opt_present("h") { - usage(0); + if args.len() == 0 { + println!("a command must be passed"); + usage(1, &opts); } + let parse = |opts: &Options| { + let m = opts.parse(&args[1..]).unwrap_or_else(|e| { + println!("failed to parse options: {}", e); + usage(1, opts); + }); + if m.opt_present("h") { + usage(0, opts); + } + return m + }; + + let cwd = t!(env::current_dir()); + let remaining_as_path = |m: &Matches| { + m.free.iter().map(|p| cwd.join(p)).collect::<Vec<_>>() + }; + + let m: Matches; + let cmd = match &args[0][..] { + "build" => { + m = parse(&opts); + Subcommand::Build { paths: remaining_as_path(&m) } + } + "doc" => { + m = parse(&opts); + Subcommand::Doc { paths: remaining_as_path(&m) } + } + "test" => { + opts.optmulti("", "test-args", "extra arguments", "ARGS"); + m = parse(&opts); + Subcommand::Test { + paths: remaining_as_path(&m), + test_args: m.opt_strs("test-args"), + } + } + "clean" => { + m = parse(&opts); + if m.free.len() > 0 { + println!("clean takes no arguments"); + usage(1, &opts); + } + Subcommand::Clean + } + "dist" => { + opts.optflag("", "install", "run installer as well"); + m = parse(&opts); + Subcommand::Dist { + install: m.opt_present("install"), + } + } + cmd => { + println!("unknown command: {}", cmd); + usage(1, &opts); + } + }; + let cfg_file = m.opt_str("config").map(PathBuf::from).or_else(|| { if fs::metadata("config.toml").is_ok() { @@ -78,26 +242,27 @@ impl Flags { Flags { verbose: m.opt_present("v"), - clean: m.opt_present("clean"), stage: m.opt_str("stage").map(|j| j.parse().unwrap()), - build: m.opt_str("build").unwrap(), - host: Filter { values: m.opt_strs("host") }, - target: Filter { values: m.opt_strs("target") }, - step: m.opt_strs("step"), + build: m.opt_str("build").unwrap_or_else(|| { + env::var("BUILD").unwrap() + }), + host: m.opt_strs("host"), + target: m.opt_strs("target"), config: cfg_file, src: m.opt_str("src").map(PathBuf::from), jobs: m.opt_str("jobs").map(|j| j.parse().unwrap()), - args: m.free.clone(), + cmd: cmd, } } } -impl Filter { - pub fn contains(&self, name: &str) -> bool { - self.values.len() == 0 || self.values.iter().any(|s| s == name) - } - - pub fn iter(&self) -> slice::Iter<String> { - self.values.iter() +impl Subcommand { + pub fn test_args(&self) -> Vec<&str> { + match *self { + Subcommand::Test { ref test_args, .. } => { + test_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => Vec::new(), + } } } diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 7c5a0c7373f..e4af4621206 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -57,6 +57,7 @@ mod channel; mod check; mod clean; mod compile; +mod metadata; mod config; mod dist; mod doc; @@ -76,7 +77,7 @@ mod job { } pub use config::Config; -pub use flags::Flags; +pub use flags::{Flags, Subcommand}; /// A structure representing a Rust compiler. /// @@ -130,6 +131,17 @@ pub struct Build { // Runtime state filled in later on cc: HashMap<String, (gcc::Tool, Option<PathBuf>)>, cxx: HashMap<String, gcc::Tool>, + crates: HashMap<String, Crate>, +} + +#[derive(Debug)] +struct Crate { + name: String, + deps: Vec<String>, + path: PathBuf, + doc_step: String, + build_step: String, + test_step: String, } /// The various "modes" of invoking Cargo. @@ -162,7 +174,9 @@ impl Build { /// By default all build output will be placed in the current directory. pub fn new(flags: Flags, config: Config) -> Build { let cwd = t!(env::current_dir()); - let src = flags.src.clone().unwrap_or(cwd.clone()); + let src = flags.src.clone().or_else(|| { + env::var_os("SRC").map(|x| x.into()) + }).unwrap_or(cwd.clone()); let out = cwd.join("build"); let stage0_root = out.join(&config.build).join("stage0/bin"); @@ -196,6 +210,7 @@ impl Build { package_vers: String::new(), cc: HashMap::new(), cxx: HashMap::new(), + crates: HashMap::new(), gdb_version: None, lldb_version: None, lldb_python_dir: None, @@ -204,13 +219,11 @@ impl Build { /// Executes the entire build, as configured by the flags and configuration. pub fn build(&mut self) { - use step::Source::*; - unsafe { job::setup(); } - if self.flags.clean { + if let Subcommand::Clean = self.flags.cmd { return clean::clean(self); } @@ -232,247 +245,10 @@ impl Build { } self.verbose("updating submodules"); self.update_submodules(); + self.verbose("learning about cargo"); + metadata::build(self); - // The main loop of the build system. - // - // The `step::all` function returns a topographically sorted list of all - // steps that need to be executed as part of this build. Each step has a - // corresponding entry in `step.rs` and indicates some unit of work that - // needs to be done as part of the build. - // - // Almost all of these are simple one-liners that shell out to the - // corresponding functionality in the extra modules, where more - // documentation can be found. - let steps = step::all(self); - - self.verbose("bootstrap build plan:"); - for step in &steps { - self.verbose(&format!("{:?}", step)); - } - - for target in steps { - let doc_out = self.out.join(&target.target).join("doc"); - match target.src { - Llvm { _dummy } => { - native::llvm(self, target.target); - } - TestHelpers { _dummy } => { - native::test_helpers(self, target.target); - } - Libstd { compiler } => { - compile::std(self, target.target, &compiler); - } - Libtest { compiler } => { - compile::test(self, target.target, &compiler); - } - Librustc { compiler } => { - compile::rustc(self, target.target, &compiler); - } - LibstdLink { compiler, host } => { - compile::std_link(self, target.target, &compiler, host); - } - LibtestLink { compiler, host } => { - compile::test_link(self, target.target, &compiler, host); - } - LibrustcLink { compiler, host } => { - compile::rustc_link(self, target.target, &compiler, host); - } - Rustc { stage: 0 } => { - // nothing to do... - } - Rustc { stage } => { - compile::assemble_rustc(self, stage, target.target); - } - ToolLinkchecker { stage } => { - compile::tool(self, stage, target.target, "linkchecker"); - } - ToolRustbook { stage } => { - compile::tool(self, stage, target.target, "rustbook"); - } - ToolErrorIndex { stage } => { - compile::tool(self, stage, target.target, - "error_index_generator"); - } - ToolCargoTest { stage } => { - compile::tool(self, stage, target.target, "cargotest"); - } - ToolTidy { stage } => { - compile::tool(self, stage, target.target, "tidy"); - } - ToolCompiletest { stage } => { - compile::tool(self, stage, target.target, "compiletest"); - } - DocBook { stage } => { - doc::rustbook(self, stage, target.target, "book", &doc_out); - } - DocNomicon { stage } => { - doc::rustbook(self, stage, target.target, "nomicon", - &doc_out); - } - DocStandalone { stage } => { - doc::standalone(self, stage, target.target, &doc_out); - } - DocStd { stage } => { - doc::std(self, stage, target.target, &doc_out); - } - DocTest { stage } => { - doc::test(self, stage, target.target, &doc_out); - } - DocRustc { stage } => { - doc::rustc(self, stage, target.target, &doc_out); - } - DocErrorIndex { stage } => { - doc::error_index(self, stage, target.target, &doc_out); - } - - CheckLinkcheck { stage } => { - check::linkcheck(self, stage, target.target); - } - CheckCargoTest { stage } => { - check::cargotest(self, stage, target.target); - } - CheckTidy { stage } => { - check::tidy(self, stage, target.target); - } - CheckRPass { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-pass", "run-pass"); - } - CheckRPassFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-pass", "run-pass-fulldeps"); - } - CheckCFail { compiler } => { - check::compiletest(self, &compiler, target.target, - "compile-fail", "compile-fail"); - } - CheckCFailFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "compile-fail", "compile-fail-fulldeps") - } - CheckPFail { compiler } => { - check::compiletest(self, &compiler, target.target, - "parse-fail", "parse-fail"); - } - CheckRFail { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-fail", "run-fail"); - } - CheckRFailFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-fail", "run-fail-fulldeps"); - } - CheckPretty { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "pretty"); - } - CheckPrettyRPass { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-pass"); - } - CheckPrettyRPassFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-pass-fulldeps"); - } - CheckPrettyRFail { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-fail"); - } - CheckPrettyRFailFull { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-fail-fulldeps"); - } - CheckPrettyRPassValgrind { compiler } => { - check::compiletest(self, &compiler, target.target, - "pretty", "run-pass-valgrind"); - } - CheckMirOpt { compiler } => { - check::compiletest(self, &compiler, target.target, - "mir-opt", "mir-opt"); - } - CheckCodegen { compiler } => { - if self.config.codegen_tests { - check::compiletest(self, &compiler, target.target, - "codegen", "codegen"); - } - } - CheckCodegenUnits { compiler } => { - check::compiletest(self, &compiler, target.target, - "codegen-units", "codegen-units"); - } - CheckIncremental { compiler } => { - check::compiletest(self, &compiler, target.target, - "incremental", "incremental"); - } - CheckUi { compiler } => { - check::compiletest(self, &compiler, target.target, - "ui", "ui"); - } - CheckDebuginfo { compiler } => { - if target.target.contains("msvc") { - // nothing to do - } else if target.target.contains("apple") { - check::compiletest(self, &compiler, target.target, - "debuginfo-lldb", "debuginfo"); - } else { - check::compiletest(self, &compiler, target.target, - "debuginfo-gdb", "debuginfo"); - } - } - CheckRustdoc { compiler } => { - check::compiletest(self, &compiler, target.target, - "rustdoc", "rustdoc"); - } - CheckRPassValgrind { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-pass-valgrind", "run-pass-valgrind"); - } - CheckDocs { compiler } => { - check::docs(self, &compiler); - } - CheckErrorIndex { compiler } => { - check::error_index(self, &compiler); - } - CheckRMake { compiler } => { - check::compiletest(self, &compiler, target.target, - "run-make", "run-make") - } - CheckCrateStd { compiler } => { - check::krate(self, &compiler, target.target, Mode::Libstd) - } - CheckCrateTest { compiler } => { - check::krate(self, &compiler, target.target, Mode::Libtest) - } - CheckCrateRustc { compiler } => { - check::krate(self, &compiler, target.target, Mode::Librustc) - } - - DistDocs { stage } => dist::docs(self, stage, target.target), - DistMingw { _dummy } => dist::mingw(self, target.target), - DistRustc { stage } => dist::rustc(self, stage, target.target), - DistStd { compiler } => dist::std(self, &compiler, target.target), - DistSrc { _dummy } => dist::rust_src(self), - - Install { stage } => install::install(self, stage, target.target), - - DebuggerScripts { stage } => { - let compiler = Compiler::new(stage, target.target); - dist::debugger_scripts(self, - &self.sysroot(&compiler), - target.target); - } - - AndroidCopyLibs { compiler } => { - check::android_copy_libs(self, &compiler, target.target); - } - - // pseudo-steps - Dist { .. } | - Doc { .. } | - CheckTarget { .. } | - Check { .. } => {} - } - } + step::run(self); } /// Updates all git submodules that we have. @@ -812,6 +588,11 @@ impl Build { self.out.join(target).join("llvm") } + /// Output directory for all documentation for a target + fn doc_out(&self, target: &str) -> PathBuf { + self.out.join(target).join("doc") + } + /// Returns true if no custom `llvm-config` is set for the specified target. /// /// If no custom `llvm-config` was specified then Rust's llvm will be used. diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/metadata.rs new file mode 100644 index 00000000000..bf5cc6a4ad8 --- /dev/null +++ b/src/bootstrap/metadata.rs @@ -0,0 +1,95 @@ +// Copyright 2016 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::collections::HashMap; +use std::process::Command; +use std::path::PathBuf; + +use build_helper::output; +use rustc_serialize::json; + +use {Build, Crate}; + +#[derive(RustcDecodable)] +struct Output { + packages: Vec<Package>, + resolve: Resolve, +} + +#[derive(RustcDecodable)] +struct Package { + id: String, + name: String, + source: Option<String>, + manifest_path: String, +} + +#[derive(RustcDecodable)] +struct Resolve { + nodes: Vec<ResolveNode>, +} + +#[derive(RustcDecodable)] +struct ResolveNode { + id: String, + dependencies: Vec<String>, +} + +pub fn build(build: &mut Build) { + build_krate(build, "src/rustc/std_shim"); + build_krate(build, "src/rustc/test_shim"); + build_krate(build, "src/rustc"); +} + +fn build_krate(build: &mut Build, krate: &str) { + // Run `cargo metadata` to figure out what crates we're testing. + // + // Down below we're going to call `cargo test`, but to test the right set + // of packages we're going to have to know what `-p` arguments to pass it + // to know what crates to test. Here we run `cargo metadata` to learn about + // the dependency graph and what `-p` arguments there are. + let mut cargo = Command::new(&build.cargo); + cargo.arg("metadata") + .arg("--manifest-path").arg(build.src.join(krate).join("Cargo.toml")); + let output = output(&mut cargo); + let output: Output = json::decode(&output).unwrap(); + let mut id2name = HashMap::new(); + for package in output.packages { + if package.source.is_none() { + id2name.insert(package.id, package.name.clone()); + let mut path = PathBuf::from(package.manifest_path); + path.pop(); + build.crates.insert(package.name.clone(), Crate { + build_step: format!("build-crate-{}", package.name), + doc_step: format!("doc-crate-{}", package.name), + test_step: format!("test-crate-{}", package.name), + name: package.name, + deps: Vec::new(), + path: path, + }); + } + } + + for node in output.resolve.nodes { + let name = match id2name.get(&node.id) { + Some(name) => name, + None => continue, + }; + + let krate = build.crates.get_mut(name).unwrap(); + for dep in node.dependencies.iter() { + let dep = match id2name.get(dep) { + Some(dep) => dep, + None => continue, + }; + krate.deps.push(dep.clone()); + } + } +} diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 0762ed98472..d4031077639 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -17,47 +17,46 @@ else BOOTSTRAP_ARGS := endif -BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py $(BOOTSTRAP_ARGS) +BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py all: - $(Q)$(BOOTSTRAP) + $(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS) + $(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS) # Don’t use $(Q) here, always show how to invoke the bootstrap script directly help: $(BOOTSTRAP) --help clean: - $(Q)$(BOOTSTRAP) --clean + $(Q)$(BOOTSTRAP) clean $(BOOTSTRAP_ARGS) rustc-stage1: - $(Q)$(BOOTSTRAP) --step libtest --stage 1 + $(Q)$(BOOTSTRAP) build --stage 1 src/libtest $(BOOTSTRAP_ARGS) rustc-stage2: - $(Q)$(BOOTSTRAP) --step libtest --stage 2 + $(Q)$(BOOTSTRAP) build --stage 2 src/libtest $(BOOTSTRAP_ARGS) docs: doc doc: - $(Q)$(BOOTSTRAP) --step doc -style: - $(Q)$(BOOTSTRAP) --step doc-style + $(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS) nomicon: - $(Q)$(BOOTSTRAP) --step doc-nomicon + $(Q)$(BOOTSTRAP) doc src/doc/nomicon $(BOOTSTRAP_ARGS) book: - $(Q)$(BOOTSTRAP) --step doc-book + $(Q)$(BOOTSTRAP) doc src/doc/book $(BOOTSTRAP_ARGS) standalone-docs: - $(Q)$(BOOTSTRAP) --step doc-standalone + $(Q)$(BOOTSTRAP) doc src/doc $(BOOTSTRAP_ARGS) check: - $(Q)$(BOOTSTRAP) --step check + $(Q)$(BOOTSTRAP) test $(BOOTSTRAP_ARGS) check-cargotest: - $(Q)$(BOOTSTRAP) --step check-cargotest + $(Q)$(BOOTSTRAP) test src/tools/cargotest $(BOOTSTRAP_ARGS) dist: - $(Q)$(BOOTSTRAP) --step dist + $(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS) install: ifeq (root user, $(USER) $(patsubst %,user,$(SUDO_USER))) $(Q)echo "'sudo make install' is not supported currently." else - $(Q)$(BOOTSTRAP) --step install + $(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS) endif tidy: - $(Q)$(BOOTSTRAP) --step check-tidy --stage 0 + $(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS) .PHONY: dist diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index 3bf0f211921..4a7cfa1cc6a 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -8,600 +8,683 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -//! Major workhorse of rustbuild, definition and dependencies between stages of -//! the copmile. -//! -//! The primary purpose of this module is to define the various `Step`s of -//! execution of the build. Each `Step` has a corresponding `Source` indicating -//! what it's actually doing along with a number of dependencies which must be -//! executed first. -//! -//! This module will take the CLI as input and calculate the steps required for -//! the build requested, ensuring that all intermediate pieces are in place. -//! Essentially this module is a `make`-replacement, but not as good. - -use std::collections::HashSet; - -use {Build, Compiler}; - -#[derive(Hash, Eq, PartialEq, Clone, Debug)] -pub struct Step<'a> { - pub src: Source<'a>, - pub target: &'a str, +use std::collections::{HashMap, HashSet}; +use std::mem; + +use check; +use compile; +use dist; +use doc; +use flags::Subcommand; +use install; +use native; +use {Compiler, Build, Mode}; + +#[derive(PartialEq, Eq, Hash, Clone, Debug)] +struct Step<'a> { + name: &'a str, + stage: u32, + host: &'a str, + target: &'a str, } -/// Macro used to iterate over all targets that are recognized by the build -/// system. -/// -/// Whenever a new step is added it will involve adding an entry here, updating -/// the dependencies section below, and then adding an implementation of the -/// step in `build/mod.rs`. -/// -/// This macro takes another macro as an argument and then calls that macro with -/// all steps that the build system knows about. -macro_rules! targets { - ($m:ident) => { - $m! { - // Step representing building the stageN compiler. This is just the - // compiler executable itself, not any of the support libraries - (rustc, Rustc { stage: u32 }), - - // Steps for the two main cargo builds. These are parameterized over - // the compiler which is producing the artifact. - (libstd, Libstd { compiler: Compiler<'a> }), - (libtest, Libtest { compiler: Compiler<'a> }), - (librustc, Librustc { compiler: Compiler<'a> }), - - // Links the target produced by the compiler provided into the - // host's directory also provided. - (libstd_link, LibstdLink { - compiler: Compiler<'a>, - host: &'a str - }), - (libtest_link, LibtestLink { - compiler: Compiler<'a>, - host: &'a str - }), - (librustc_link, LibrustcLink { - compiler: Compiler<'a>, - host: &'a str - }), - - // Various tools that we can build as part of the build. - (tool_linkchecker, ToolLinkchecker { stage: u32 }), - (tool_rustbook, ToolRustbook { stage: u32 }), - (tool_error_index, ToolErrorIndex { stage: u32 }), - (tool_cargotest, ToolCargoTest { stage: u32 }), - (tool_tidy, ToolTidy { stage: u32 }), - (tool_compiletest, ToolCompiletest { stage: u32 }), - - // Steps for long-running native builds. Ideally these wouldn't - // actually exist and would be part of build scripts, but for now - // these are here. - // - // There aren't really any parameters to this, but empty structs - // with braces are unstable so we just pick something that works. - (llvm, Llvm { _dummy: () }), - (test_helpers, TestHelpers { _dummy: () }), - (debugger_scripts, DebuggerScripts { stage: u32 }), - - // Steps for various pieces of documentation that we can generate, - // the 'doc' step is just a pseudo target to depend on a bunch of - // others. - (doc, Doc { stage: u32 }), - (doc_book, DocBook { stage: u32 }), - (doc_nomicon, DocNomicon { stage: u32 }), - (doc_standalone, DocStandalone { stage: u32 }), - (doc_std, DocStd { stage: u32 }), - (doc_test, DocTest { stage: u32 }), - (doc_rustc, DocRustc { stage: u32 }), - (doc_error_index, DocErrorIndex { stage: u32 }), - - // Steps for running tests. The 'check' target is just a pseudo - // target to depend on a bunch of others. - (check, Check { stage: u32, compiler: Compiler<'a> }), - (check_target, CheckTarget { stage: u32, compiler: Compiler<'a> }), - (check_linkcheck, CheckLinkcheck { stage: u32 }), - (check_cargotest, CheckCargoTest { stage: u32 }), - (check_tidy, CheckTidy { stage: u32 }), - (check_rpass, CheckRPass { compiler: Compiler<'a> }), - (check_rpass_full, CheckRPassFull { compiler: Compiler<'a> }), - (check_rpass_valgrind, CheckRPassValgrind { compiler: Compiler<'a> }), - (check_rfail, CheckRFail { compiler: Compiler<'a> }), - (check_rfail_full, CheckRFailFull { compiler: Compiler<'a> }), - (check_cfail, CheckCFail { compiler: Compiler<'a> }), - (check_cfail_full, CheckCFailFull { compiler: Compiler<'a> }), - (check_pfail, CheckPFail { compiler: Compiler<'a> }), - (check_pretty, CheckPretty { compiler: Compiler<'a> }), - (check_pretty_rpass, CheckPrettyRPass { compiler: Compiler<'a> }), - (check_pretty_rpass_full, CheckPrettyRPassFull { compiler: Compiler<'a> }), - (check_pretty_rfail, CheckPrettyRFail { compiler: Compiler<'a> }), - (check_pretty_rfail_full, CheckPrettyRFailFull { compiler: Compiler<'a> }), - (check_pretty_rpass_valgrind, CheckPrettyRPassValgrind { compiler: Compiler<'a> }), - (check_codegen, CheckCodegen { compiler: Compiler<'a> }), - (check_codegen_units, CheckCodegenUnits { compiler: Compiler<'a> }), - (check_incremental, CheckIncremental { compiler: Compiler<'a> }), - (check_ui, CheckUi { compiler: Compiler<'a> }), - (check_mir_opt, CheckMirOpt { compiler: Compiler<'a> }), - (check_debuginfo, CheckDebuginfo { compiler: Compiler<'a> }), - (check_rustdoc, CheckRustdoc { compiler: Compiler<'a> }), - (check_docs, CheckDocs { compiler: Compiler<'a> }), - (check_error_index, CheckErrorIndex { compiler: Compiler<'a> }), - (check_rmake, CheckRMake { compiler: Compiler<'a> }), - (check_crate_std, CheckCrateStd { compiler: Compiler<'a> }), - (check_crate_test, CheckCrateTest { compiler: Compiler<'a> }), - (check_crate_rustc, CheckCrateRustc { compiler: Compiler<'a> }), - - // Distribution targets, creating tarballs - (dist, Dist { stage: u32 }), - (dist_docs, DistDocs { stage: u32 }), - (dist_mingw, DistMingw { _dummy: () }), - (dist_rustc, DistRustc { stage: u32 }), - (dist_std, DistStd { compiler: Compiler<'a> }), - (dist_src, DistSrc { _dummy: () }), - - // install target - (install, Install { stage: u32 }), - - // Misc targets - (android_copy_libs, AndroidCopyLibs { compiler: Compiler<'a> }), - } +impl<'a> Step<'a> { + fn name(&self, name: &'a str) -> Step<'a> { + Step { name: name, ..*self } } -} -// Define the `Source` enum by iterating over all the steps and peeling out just -// the types that we want to define. + fn stage(&self, stage: u32) -> Step<'a> { + Step { stage: stage, ..*self } + } -macro_rules! item { ($a:item) => ($a) } + fn host(&self, host: &'a str) -> Step<'a> { + Step { host: host, ..*self } + } -macro_rules! define_source { - ($(($short:ident, $name:ident { $($args:tt)* }),)*) => { - item! { - #[derive(Hash, Eq, PartialEq, Clone, Debug)] - pub enum Source<'a> { - $($name { $($args)* }),* - } - } + fn target(&self, target: &'a str) -> Step<'a> { + Step { target: target, ..*self } } -} -targets!(define_source); - -/// Calculate a list of all steps described by `build`. -/// -/// This will inspect the flags passed in on the command line and use that to -/// build up a list of steps to execute. These steps will then be transformed -/// into a topologically sorted list which when executed left-to-right will -/// correctly sequence the entire build. -pub fn all(build: &Build) -> Vec<Step> { - build.verbose("inferred build steps:"); - - let mut ret = Vec::new(); - let mut all = HashSet::new(); - for target in top_level(build) { - fill(build, &target, &mut ret, &mut all); - } - return ret; - - fn fill<'a>(build: &'a Build, - target: &Step<'a>, - ret: &mut Vec<Step<'a>>, - set: &mut HashSet<Step<'a>>) { - if set.insert(target.clone()) { - for dep in target.deps(build) { - build.verbose(&format!("{:?}\n -> {:?}", target, dep)); - fill(build, &dep, ret, set); - } - ret.push(target.clone()); - } + fn compiler(&self) -> Compiler<'a> { + Compiler::new(self.stage, self.host) } } -/// Determines what top-level targets are requested as part of this build, -/// returning them as a list. -fn top_level(build: &Build) -> Vec<Step> { - let mut targets = Vec::new(); - let stage = build.flags.stage.unwrap_or(2); +pub fn run(build: &Build) { + let rules = build_rules(build); + let steps = rules.plan(); + rules.run(&steps); +} - let host = Step { - src: Source::Llvm { _dummy: () }, - target: build.flags.host.iter().next() - .unwrap_or(&build.config.build), - }; - let target = Step { - src: Source::Llvm { _dummy: () }, - target: build.flags.target.iter().next().map(|x| &x[..]) - .unwrap_or(host.target) +pub fn build_rules(build: &Build) -> Rules { + let mut rules: Rules = Rules::new(build); + // dummy rule to do nothing, useful when a dep maps to no deps + rules.build("dummy", "path/to/nowhere"); + fn dummy<'a>(s: &Step<'a>, build: &'a Build) -> Step<'a> { + s.name("dummy").stage(0) + .target(&build.config.build) + .host(&build.config.build) + } + + // Helper for loading an entire DAG of crates, rooted at `name` + let krates = |name: &str| { + let mut ret = Vec::new(); + let mut list = vec![name]; + let mut visited = HashSet::new(); + while let Some(krate) = list.pop() { + let default = krate == name; + let krate = &build.crates[krate]; + let path = krate.path.strip_prefix(&build.src).unwrap(); + ret.push((krate, path.to_str().unwrap(), default)); + for dep in krate.deps.iter() { + if visited.insert(dep) && dep != "build_helper" { + list.push(dep); + } + } + } + return ret }; - // First, try to find steps on the command line. - add_steps(build, stage, &host, &target, &mut targets); + rules.build("rustc", "path/to/nowhere") + .dep(move |s| { + if s.stage == 0 { + dummy(s, build) + } else { + s.name("librustc") + .host(&build.config.build) + .stage(s.stage - 1) + } + }) + .run(move |s| compile::assemble_rustc(build, s.stage, s.target)); + rules.build("llvm", "src/llvm") + .host(true) + .run(move |s| native::llvm(build, s.target)); + + // ======================================================================== + // Crate compilations + // + // Tools used during the build system but not shipped + rules.build("libstd", "src/libstd") + .dep(|s| s.name("build-crate-std_shim")); + rules.build("libtest", "src/libtest") + .dep(|s| s.name("build-crate-test_shim")); + rules.build("librustc", "src/librustc") + .dep(|s| s.name("build-crate-rustc-main")); + for (krate, path, _default) in krates("std_shim") { + rules.build(&krate.build_step, path) + .dep(|s| s.name("rustc").target(s.host)) + .dep(move |s| { + if s.host == build.config.build { + dummy(s, build) + } else { + s.host(&build.config.build) + } + }) + .run(move |s| { + if s.host == build.config.build { + compile::std(build, s.target, &s.compiler()) + } else { + compile::std_link(build, s.target, s.stage, s.host) + } + }); + } + for (krate, path, default) in krates("test_shim") { + rules.build(&krate.build_step, path) + .dep(|s| s.name("libstd")) + .dep(move |s| { + if s.host == build.config.build { + dummy(s, build) + } else { + s.host(&build.config.build) + } + }) + .default(default) + .run(move |s| { + if s.host == build.config.build { + compile::test(build, s.target, &s.compiler()) + } else { + compile::test_link(build, s.target, s.stage, s.host) + } + }); + } + for (krate, path, default) in krates("rustc-main") { + rules.build(&krate.build_step, path) + .dep(|s| s.name("libtest")) + .dep(move |s| s.name("llvm").host(&build.config.build).stage(0)) + .dep(move |s| { + if s.host == build.config.build { + dummy(s, build) + } else { + s.host(&build.config.build) + } + }) + .host(true) + .default(default) + .run(move |s| { + if s.host == build.config.build { + compile::rustc(build, s.target, &s.compiler()) + } else { + compile::rustc_link(build, s.target, s.stage, s.host) + } + }); + } - // If none are specified, then build everything. - if targets.len() == 0 { - let t = Step { - src: Source::Llvm { _dummy: () }, - target: &build.config.build, + // ======================================================================== + // Test targets + // + // Various unit tests and tests suites we can run + { + let mut suite = |name, path, dir, mode| { + rules.test(name, path) + .dep(|s| s.name("libtest")) + .dep(|s| s.name("tool-compiletest").target(s.host)) + .dep(|s| s.name("test-helpers")) + .dep(move |s| { + if s.target.contains("android") { + s.name("android-copy-libs") + } else { + dummy(s, build) + } + }) + .default(true) + .run(move |s| { + check::compiletest(build, &s.compiler(), s.target, dir, mode) + }); }; - if build.config.docs { - targets.push(t.doc(stage)); - } - for host in build.config.host.iter() { - if !build.flags.host.contains(host) { - continue - } - let host = t.target(host); - if host.target == build.config.build { - targets.push(host.librustc(host.compiler(stage))); - } else { - targets.push(host.librustc_link(t.compiler(stage), host.target)); - } - for target in build.config.target.iter() { - if !build.flags.target.contains(target) { - continue - } - if host.target == build.config.build { - targets.push(host.target(target) - .libtest(host.compiler(stage))); - } else { - targets.push(host.target(target) - .libtest_link(t.compiler(stage), host.target)); - } - } + suite("check-rpass", "src/test/run-pass", "run-pass", "run-pass"); + suite("check-cfail", "src/test/compile-fail", "compile-fail", "compile-fail"); + suite("check-pfail", "src/test/parse-fail", "parse-fail", "parse-fail"); + suite("check-rfail", "src/test/run-fail", "run-fail", "run-fail"); + suite("check-rpass-valgrind", "src/test/run-pass-valgrind", + "run-pass-valgrind", "run-pass-valgrind"); + suite("check-mir-opt", "src/test/mir-opt", "mir-opt", "mir-opt"); + if build.config.codegen_tests { + suite("check-codegen", "src/test/codegen", "codegen", "codegen"); } + suite("check-codegen-units", "src/test/codegen-units", "codegen-units", + "codegen-units"); + suite("check-incremental", "src/test/incremental", "incremental", + "incremental"); + suite("check-ui", "src/test/ui", "ui", "ui"); + suite("check-pretty", "src/test/pretty", "pretty", "pretty"); + suite("check-pretty-rpass", "src/test/run-pass/pretty", "pretty", + "run-pass"); + suite("check-pretty-rfail", "src/test/run-pass/pretty", "pretty", + "run-fail"); + suite("check-pretty-valgrind", "src/test/run-pass-valgrind", "pretty", + "run-pass-valgrind"); } - targets -} + if build.config.build.contains("msvc") { + // nothing to do for debuginfo tests + } else if build.config.build.contains("apple") { + rules.test("check-debuginfo", "src/test/debuginfo") + .dep(|s| s.name("libtest")) + .dep(|s| s.name("tool-compiletest").host(s.host)) + .dep(|s| s.name("test-helpers")) + .dep(|s| s.name("debugger-scripts")) + .run(move |s| check::compiletest(build, &s.compiler(), s.target, + "debuginfo-lldb", "debuginfo")); + } else { + rules.test("check-debuginfo", "src/test/debuginfo") + .dep(|s| s.name("libtest")) + .dep(|s| s.name("tool-compiletest").host(s.host)) + .dep(|s| s.name("test-helpers")) + .dep(|s| s.name("debugger-scripts")) + .run(move |s| check::compiletest(build, &s.compiler(), s.target, + "debuginfo-gdb", "debuginfo")); + } -fn add_steps<'a>(build: &'a Build, - stage: u32, - host: &Step<'a>, - target: &Step<'a>, - targets: &mut Vec<Step<'a>>) { - struct Context<'a> { - stage: u32, - compiler: Compiler<'a>, - _dummy: (), - host: &'a str, - } - for step in build.flags.step.iter() { - - // The macro below insists on hygienic access to all local variables, so - // we shove them all in a struct and subvert hygiene by accessing struct - // fields instead, - let cx = Context { - stage: stage, - compiler: host.target(&build.config.build).compiler(stage), - _dummy: (), - host: host.target, + rules.test("debugger-scripts", "src/etc/lldb_batchmode.py") + .run(move |s| dist::debugger_scripts(build, &build.sysroot(&s.compiler()), + s.target)); + + { + let mut suite = |name, path, dir, mode| { + rules.test(name, path) + .dep(|s| s.name("librustc")) + .dep(|s| s.name("tool-compiletest").target(s.host)) + .default(true) + .host(true) + .run(move |s| { + check::compiletest(build, &s.compiler(), s.target, dir, mode) + }); }; - macro_rules! add_step { - ($(($short:ident, $name:ident { $($arg:ident: $t:ty),* }),)*) => ({$( - let name = stringify!($short).replace("_", "-"); - if &step[..] == &name[..] { - targets.push(target.$short($(cx.$arg),*)); - continue - } - drop(name); - )*}) + + suite("check-rpass-full", "src/test/run-pass-fulldeps", + "run-pass", "run-pass-fulldeps"); + suite("check-cfail-full", "src/test/compile-fail-fulldeps", + "compile-fail", "compile-fail-fulldeps"); + suite("check-rmake", "src/test/run-make", "run-make", "run-make"); + suite("check-rustdoc", "src/test/rustdoc", "rustdoc", "rustdoc"); + suite("check-pretty-rpass-full", "src/test/run-pass-fulldeps", + "pretty", "run-pass-fulldeps"); + suite("check-pretty-rfail-full", "src/test/run-fail-fulldeps", + "pretty", "run-fail-fulldeps"); + } + + for (krate, path, _default) in krates("std_shim") { + rules.test(&krate.test_step, path) + .dep(|s| s.name("libtest")) + .run(move |s| check::krate(build, &s.compiler(), s.target, + Mode::Libstd, Some(&krate.name))); + } + rules.test("check-std-all", "path/to/nowhere") + .dep(|s| s.name("libtest")) + .default(true) + .run(move |s| check::krate(build, &s.compiler(), s.target, Mode::Libstd, + None)); + for (krate, path, _default) in krates("test_shim") { + rules.test(&krate.test_step, path) + .dep(|s| s.name("libtest")) + .run(move |s| check::krate(build, &s.compiler(), s.target, + Mode::Libtest, Some(&krate.name))); + } + rules.test("check-test-all", "path/to/nowhere") + .dep(|s| s.name("libtest")) + .default(true) + .run(move |s| check::krate(build, &s.compiler(), s.target, Mode::Libtest, + None)); + for (krate, path, _default) in krates("rustc-main") { + rules.test(&krate.test_step, path) + .dep(|s| s.name("libtest")) + .host(true) + .run(move |s| check::krate(build, &s.compiler(), s.target, + Mode::Librustc, Some(&krate.name))); + } + rules.test("check-rustc-all", "path/to/nowhere") + .dep(|s| s.name("libtest")) + .default(true) + .host(true) + .run(move |s| check::krate(build, &s.compiler(), s.target, Mode::Librustc, + None)); + + rules.test("check-linkchecker", "src/tools/linkchecker") + .dep(|s| s.name("tool-linkchecker")) + .dep(|s| s.name("default:doc")) + .default(true) + .run(move |s| check::linkcheck(build, s.stage, s.target)); + rules.test("check-cargotest", "src/tools/cargotest") + .dep(|s| s.name("tool-cargotest")) + .dep(|s| s.name("librustc")) + .run(move |s| check::cargotest(build, s.stage, s.target)); + rules.test("check-tidy", "src/tools/tidy") + .dep(|s| s.name("tool-tidy")) + .default(true) + .run(move |s| check::tidy(build, s.stage, s.target)); + rules.test("check-error-index", "src/tools/error_index_generator") + .dep(|s| s.name("libstd")) + .dep(|s| s.name("tool-error-index").host(s.host)) + .default(true) + .run(move |s| check::error_index(build, &s.compiler())); + rules.test("check-docs", "src/doc") + .dep(|s| s.name("libtest")) + .default(true) + .run(move |s| check::docs(build, &s.compiler())); + + rules.build("test-helpers", "src/rt/rust_test_helpers.c") + .run(move |s| native::test_helpers(build, s.target)); + rules.test("android-copy-libs", "path/to/nowhere") + .dep(|s| s.name("libtest")) + .run(move |s| check::android_copy_libs(build, &s.compiler(), s.target)); + + // ======================================================================== + // Build tools + // + // Tools used during the build system but not shipped + rules.build("tool-rustbook", "src/tools/rustbook") + .dep(|s| s.name("librustc")) + .run(move |s| compile::tool(build, s.stage, s.target, "rustbook")); + rules.build("tool-error-index", "src/tools/error_index_generator") + .dep(|s| s.name("librustc")) + .run(move |s| compile::tool(build, s.stage, s.target, "error_index_generator")); + rules.build("tool-tidy", "src/tools/tidy") + .dep(|s| s.name("libstd")) + .run(move |s| compile::tool(build, s.stage, s.target, "tidy")); + rules.build("tool-linkchecker", "src/tools/linkchecker") + .dep(|s| s.name("libstd")) + .run(move |s| compile::tool(build, s.stage, s.target, "linkchecker")); + rules.build("tool-cargotest", "src/tools/cargotest") + .dep(|s| s.name("libstd")) + .run(move |s| compile::tool(build, s.stage, s.target, "cargotest")); + rules.build("tool-compiletest", "src/tools/compiletest") + .dep(|s| s.name("libtest")) + .run(move |s| compile::tool(build, s.stage, s.target, "compiletest")); + + // ======================================================================== + // Documentation targets + rules.doc("doc-book", "src/doc/book") + .dep(move |s| s.name("tool-rustbook").target(&build.config.build)) + .default(build.config.docs) + .run(move |s| doc::rustbook(build, s.stage, s.target, "book")); + rules.doc("doc-nomicon", "src/doc/nomicon") + .dep(move |s| s.name("tool-rustbook").target(&build.config.build)) + .default(build.config.docs) + .run(move |s| doc::rustbook(build, s.stage, s.target, "nomicon")); + rules.doc("doc-standalone", "src/doc") + .dep(move |s| s.name("rustc").target(&build.config.build)) + .default(build.config.docs) + .run(move |s| doc::standalone(build, s.stage, s.target)); + rules.doc("doc-error-index", "src/tools/error_index_generator") + .dep(move |s| s.name("tool-error-index").target(&build.config.build)) + .default(build.config.docs) + .run(move |s| doc::error_index(build, s.stage, s.target)); + for (krate, path, default) in krates("std_shim") { + rules.doc(&krate.doc_step, path) + .dep(|s| s.name("libstd")) + .default(default && build.config.docs) + .run(move |s| doc::std(build, s.stage, s.target)); + } + for (krate, path, default) in krates("test_shim") { + rules.doc(&krate.doc_step, path) + .dep(|s| s.name("libtest")) + .default(default && build.config.docs) + .run(move |s| doc::test(build, s.stage, s.target)); + } + for (krate, path, default) in krates("rustc-main") { + rules.doc(&krate.doc_step, path) + .dep(|s| s.name("librustc")) + .host(true) + .default(default && build.config.compiler_docs) + .run(move |s| doc::rustc(build, s.stage, s.target)); + } + + // ======================================================================== + // Distribution targets + rules.dist("dist-rustc", "src/librustc") + .dep(|s| s.name("rustc")) + .host(true) + .default(true) + .run(move |s| dist::rustc(build, s.stage, s.target)); + rules.dist("dist-std", "src/libstd") + .dep(move |s| { + // We want to package up as many target libraries as possible + // for the `rust-std` package, so if this is a host target we + // depend on librustc and otherwise we just depend on libtest. + if build.config.host.iter().any(|t| t == s.target) { + s.name("librustc") + } else { + s.name("libtest") + } + }) + .default(true) + .run(move |s| dist::std(build, &s.compiler(), s.target)); + rules.dist("dist-mingw", "path/to/nowhere") + .run(move |s| dist::mingw(build, s.target)); + rules.dist("dist-src", "src") + .default(true) + .host(true) + .run(move |_| dist::rust_src(build)); + rules.dist("dist-docs", "src/doc") + .dep(|s| s.name("default:doc")) + .run(move |s| dist::docs(build, s.stage, s.target)); + rules.dist("install", "src") + .dep(|s| s.name("default:dist")) + .run(move |s| install::install(build, s.stage, s.target)); + + rules.verify(); + return rules +} + +struct Rule<'a> { + name: &'a str, + path: &'a str, + kind: Kind, + deps: Vec<Box<Fn(&Step<'a>) -> Step<'a> + 'a>>, + run: Box<Fn(&Step<'a>) + 'a>, + default: bool, + host: bool, +} + +#[derive(PartialEq)] +enum Kind { + Build, + Test, + Dist, + Doc, +} + +impl<'a> Rule<'a> { + fn new(name: &'a str, path: &'a str, kind: Kind) -> Rule<'a> { + Rule { + name: name, + deps: Vec::new(), + run: Box::new(|_| ()), + path: path, + kind: kind, + default: false, + host: false, } + } +} + +struct RuleBuilder<'a: 'b, 'b> { + rules: &'b mut Rules<'a>, + rule: Rule<'a>, +} - targets!(add_step); +impl<'a, 'b> RuleBuilder<'a, 'b> { + fn dep<F>(&mut self, f: F) -> &mut Self + where F: Fn(&Step<'a>) -> Step<'a> + 'a, + { + self.rule.deps.push(Box::new(f)); + self + } + + fn run<F>(&mut self, f: F) -> &mut Self + where F: Fn(&Step<'a>) + 'a, + { + self.rule.run = Box::new(f); + self + } + + fn default(&mut self, default: bool) -> &mut Self { + self.rule.default = default; + self + } - panic!("unknown step: {}", step); + fn host(&mut self, host: bool) -> &mut Self { + self.rule.host = host; + self } } -macro_rules! constructors { - ($(($short:ident, $name:ident { $($arg:ident: $t:ty),* }),)*) => {$( - fn $short(&self, $($arg: $t),*) -> Step<'a> { - Step { - src: Source::$name { $($arg: $arg),* }, - target: self.target, - } +impl<'a, 'b> Drop for RuleBuilder<'a, 'b> { + fn drop(&mut self) { + let rule = mem::replace(&mut self.rule, Rule::new("", "", Kind::Build)); + let prev = self.rules.rules.insert(rule.name, rule); + if let Some(prev) = prev { + panic!("duplicate rule named: {}", prev.name); } - )*} + } } -impl<'a> Step<'a> { - fn compiler(&self, stage: u32) -> Compiler<'a> { - Compiler::new(stage, self.target) +pub struct Rules<'a> { + build: &'a Build, + sbuild: Step<'a>, + rules: HashMap<&'a str, Rule<'a>>, +} + +impl<'a> Rules<'a> { + fn new(build: &'a Build) -> Rules<'a> { + Rules { + build: build, + sbuild: Step { + stage: build.flags.stage.unwrap_or(2), + target: &build.config.build, + host: &build.config.build, + name: "", + }, + rules: HashMap::new(), + } } - fn target(&self, target: &'a str) -> Step<'a> { - Step { target: target, src: self.src.clone() } + fn build<'b>(&'b mut self, name: &'a str, path: &'a str) + -> RuleBuilder<'a, 'b> { + self.rule(name, path, Kind::Build) } - // Define ergonomic constructors for each step defined above so they can be - // easily constructed. - targets!(constructors); + fn test<'b>(&'b mut self, name: &'a str, path: &'a str) + -> RuleBuilder<'a, 'b> { + self.rule(name, path, Kind::Test) + } - /// Mapping of all dependencies for rustbuild. - /// - /// This function receives a step, the build that we're building for, and - /// then returns a list of all the dependencies of that step. - pub fn deps(&self, build: &'a Build) -> Vec<Step<'a>> { - match self.src { - Source::Rustc { stage: 0 } => { - Vec::new() - } - Source::Rustc { stage } => { - let compiler = Compiler::new(stage - 1, &build.config.build); - vec![self.librustc(compiler)] - } - Source::Librustc { compiler } => { - vec![self.libtest(compiler), self.llvm(())] - } - Source::Libtest { compiler } => { - vec![self.libstd(compiler)] - } - Source::Libstd { compiler } => { - vec![self.rustc(compiler.stage).target(compiler.host)] - } - Source::LibrustcLink { compiler, host } => { - vec![self.librustc(compiler), - self.libtest_link(compiler, host)] - } - Source::LibtestLink { compiler, host } => { - vec![self.libtest(compiler), self.libstd_link(compiler, host)] - } - Source::LibstdLink { compiler, host } => { - vec![self.libstd(compiler), - self.target(host).rustc(compiler.stage)] - } - Source::Llvm { _dummy } => Vec::new(), - Source::TestHelpers { _dummy } => Vec::new(), - Source::DebuggerScripts { stage: _ } => Vec::new(), - - // Note that all doc targets depend on artifacts from the build - // architecture, not the target (which is where we're generating - // docs into). - Source::DocStd { stage } => { - let compiler = self.target(&build.config.build).compiler(stage); - vec![self.libstd(compiler)] - } - Source::DocTest { stage } => { - let compiler = self.target(&build.config.build).compiler(stage); - vec![self.libtest(compiler)] - } - Source::DocBook { stage } | - Source::DocNomicon { stage } => { - vec![self.target(&build.config.build).tool_rustbook(stage)] - } - Source::DocErrorIndex { stage } => { - vec![self.target(&build.config.build).tool_error_index(stage)] - } - Source::DocStandalone { stage } => { - vec![self.target(&build.config.build).rustc(stage)] - } - Source::DocRustc { stage } => { - vec![self.doc_test(stage)] - } - Source::Doc { stage } => { - let mut deps = vec![ - self.doc_book(stage), self.doc_nomicon(stage), - self.doc_standalone(stage), self.doc_std(stage), - self.doc_error_index(stage), - ]; - - if build.config.compiler_docs { - deps.push(self.doc_rustc(stage)); - } + fn doc<'b>(&'b mut self, name: &'a str, path: &'a str) + -> RuleBuilder<'a, 'b> { + self.rule(name, path, Kind::Doc) + } - deps - } - Source::Check { stage, compiler } => { - // Check is just a pseudo step which means check all targets, - // so just depend on checking all targets. - build.config.target.iter().map(|t| { - self.target(t).check_target(stage, compiler) - }).collect() - } - Source::CheckTarget { stage, compiler } => { - // CheckTarget here means run all possible test suites for this - // target. Most of the time, however, we can't actually run - // anything if we're not the build triple as we could be cross - // compiling. - // - // As a result, the base set of targets here is quite stripped - // down from the standard set of targets. These suites have - // their own internal logic to run in cross-compiled situations - // if they'll run at all. For example compiletest knows that - // when testing Android targets we ship artifacts to the - // emulator. - // - // When in doubt the rule of thumb for adding to this list is - // "should this test suite run on the android bot?" - let mut base = vec![ - self.check_rpass(compiler), - self.check_rfail(compiler), - self.check_crate_std(compiler), - self.check_crate_test(compiler), - self.check_debuginfo(compiler), - ]; - - // If we're testing the build triple, then we know we can - // actually run binaries and such, so we run all possible tests - // that we know about. - if self.target == build.config.build { - base.extend(vec![ - // docs-related - self.check_docs(compiler), - self.check_error_index(compiler), - self.check_rustdoc(compiler), - - // UI-related - self.check_cfail(compiler), - self.check_pfail(compiler), - self.check_ui(compiler), - - // codegen-related - self.check_incremental(compiler), - self.check_codegen(compiler), - self.check_codegen_units(compiler), - - // misc compiletest-test suites - self.check_rpass_full(compiler), - self.check_rfail_full(compiler), - self.check_cfail_full(compiler), - self.check_pretty_rpass_full(compiler), - self.check_pretty_rfail_full(compiler), - self.check_rpass_valgrind(compiler), - self.check_rmake(compiler), - self.check_mir_opt(compiler), - - // crates - self.check_crate_rustc(compiler), - - // pretty - self.check_pretty(compiler), - self.check_pretty_rpass(compiler), - self.check_pretty_rfail(compiler), - self.check_pretty_rpass_valgrind(compiler), - - // misc - self.check_linkcheck(stage), - self.check_tidy(stage), - - // can we make the distributables? - self.dist(stage), - ]); - } - base - } - Source::CheckLinkcheck { stage } => { - vec![self.tool_linkchecker(stage), self.doc(stage)] - } - Source::CheckCargoTest { stage } => { - vec![self.tool_cargotest(stage), - self.librustc(self.compiler(stage))] - } - Source::CheckTidy { stage } => { - vec![self.tool_tidy(stage)] - } - Source::CheckMirOpt { compiler} | - Source::CheckPrettyRPass { compiler } | - Source::CheckPrettyRFail { compiler } | - Source::CheckRFail { compiler } | - Source::CheckPFail { compiler } | - Source::CheckCodegen { compiler } | - Source::CheckCodegenUnits { compiler } | - Source::CheckIncremental { compiler } | - Source::CheckUi { compiler } | - Source::CheckPretty { compiler } | - Source::CheckCFail { compiler } | - Source::CheckRPassValgrind { compiler } | - Source::CheckRPass { compiler } => { - let mut base = vec![ - self.libtest(compiler), - self.target(compiler.host).tool_compiletest(compiler.stage), - self.test_helpers(()), - ]; - if self.target.contains("android") { - base.push(self.android_copy_libs(compiler)); - } - base - } - Source::CheckDebuginfo { compiler } => { - vec![ - self.libtest(compiler), - self.target(compiler.host).tool_compiletest(compiler.stage), - self.test_helpers(()), - self.debugger_scripts(compiler.stage), - ] - } - Source::CheckRustdoc { compiler } | - Source::CheckRPassFull { compiler } | - Source::CheckRFailFull { compiler } | - Source::CheckCFailFull { compiler } | - Source::CheckPrettyRPassFull { compiler } | - Source::CheckPrettyRFailFull { compiler } | - Source::CheckPrettyRPassValgrind { compiler } | - Source::CheckRMake { compiler } => { - vec![self.librustc(compiler), - self.target(compiler.host).tool_compiletest(compiler.stage)] - } - Source::CheckDocs { compiler } => { - vec![self.libtest(compiler)] - } - Source::CheckErrorIndex { compiler } => { - vec![self.libstd(compiler), - self.target(compiler.host).tool_error_index(compiler.stage)] - } - Source::CheckCrateStd { compiler } => { - vec![self.libtest(compiler)] - } - Source::CheckCrateTest { compiler } => { - vec![self.libtest(compiler)] - } - Source::CheckCrateRustc { compiler } => { - vec![self.libtest(compiler)] - } + fn dist<'b>(&'b mut self, name: &'a str, path: &'a str) + -> RuleBuilder<'a, 'b> { + self.rule(name, path, Kind::Dist) + } - Source::ToolLinkchecker { stage } | - Source::ToolTidy { stage } => { - vec![self.libstd(self.compiler(stage))] - } - Source::ToolErrorIndex { stage } | - Source::ToolRustbook { stage } => { - vec![self.librustc(self.compiler(stage))] - } - Source::ToolCargoTest { stage } => { - vec![self.libstd(self.compiler(stage))] - } - Source::ToolCompiletest { stage } => { - vec![self.libtest(self.compiler(stage))] - } + fn rule<'b>(&'b mut self, + name: &'a str, + path: &'a str, + kind: Kind) -> RuleBuilder<'a, 'b> { + RuleBuilder { + rules: self, + rule: Rule::new(name, path, kind), + } + } + + /// Verify the dependency graph defined by all our rules are correct, e.g. + /// everything points to a valid something else. + fn verify(&self) { + for rule in self.rules.values() { + for dep in rule.deps.iter() { + let dep = dep(&self.sbuild.name(rule.name)); + if self.rules.contains_key(&dep.name) || dep.name.starts_with("default:") { + continue } + panic!("\ + +invalid rule dependency graph detected, was a rule added and maybe typo'd? + + `{}` depends on `{}` which does not exist - Source::DistDocs { stage } => vec![self.doc(stage)], - Source::DistMingw { _dummy: _ } => Vec::new(), - Source::DistRustc { stage } => { - vec![self.rustc(stage)] +", rule.name, dep.name); } - Source::DistStd { compiler } => { - // We want to package up as many target libraries as possible - // for the `rust-std` package, so if this is a host target we - // depend on librustc and otherwise we just depend on libtest. - if build.config.host.iter().any(|t| t == self.target) { - vec![self.librustc(compiler)] + } + } + + pub fn print_help(&self, command: &str) { + let kind = match command { + "build" => Kind::Build, + "doc" => Kind::Doc, + "test" => Kind::Test, + "dist" => Kind::Dist, + _ => return, + }; + let rules = self.rules.values().filter(|r| r.kind == kind); + let rules = rules.filter(|r| !r.path.contains("nowhere")); + let mut rules = rules.collect::<Vec<_>>(); + rules.sort_by_key(|r| r.path); + + println!("Available paths:\n"); + for rule in rules { + print!(" ./x.py {} {}", command, rule.path); + + println!(""); + } + } + + /// Construct the top-level build steps that we're going to be executing, + /// given the subcommand that our build is performing. + fn plan(&self) -> Vec<Step<'a>> { + let (kind, paths) = match self.build.flags.cmd { + Subcommand::Build { ref paths } => (Kind::Build, &paths[..]), + Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]), + Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]), + Subcommand::Dist { install } => { + if install { + return vec![self.sbuild.name("install")] } else { - vec![self.libtest(compiler)] - } - } - Source::DistSrc { _dummy: _ } => Vec::new(), - - Source::Dist { stage } => { - let mut base = Vec::new(); - - for host in build.config.host.iter() { - let host = self.target(host); - base.push(host.dist_src(())); - base.push(host.dist_rustc(stage)); - if host.target.contains("windows-gnu") { - base.push(host.dist_mingw(())); - } - - let compiler = self.compiler(stage); - for target in build.config.target.iter() { - let target = self.target(target); - if build.config.docs { - base.push(target.dist_docs(stage)); - } - base.push(target.dist_std(compiler)); - } + (Kind::Dist, &[][..]) } - base } + Subcommand::Clean => panic!(), + }; - Source::Install { stage } => { - vec![self.dist(stage)] - } + self.rules.values().filter(|rule| rule.kind == kind).filter(|rule| { + (paths.len() == 0 && rule.default) || paths.iter().any(|path| { + path.ends_with(rule.path) + }) + }).flat_map(|rule| { + let hosts = if self.build.flags.host.len() > 0 { + &self.build.flags.host + } else { + &self.build.config.host + }; + let targets = if self.build.flags.target.len() > 0 { + &self.build.flags.target + } else { + &self.build.config.target + }; + let arr = if rule.host {hosts} else {targets}; + + hosts.iter().flat_map(move |host| { + arr.iter().map(move |target| { + self.sbuild.name(rule.name).target(target).host(host) + }) + }) + }).collect() + } + + /// Execute all top-level targets indicated by `steps`. + /// + /// This will take the list returned by `plan` and then execute each step + /// along with all required dependencies as it goes up the chain. + fn run(&self, steps: &[Step<'a>]) { + self.build.verbose("bootstrap top targets:"); + for step in steps.iter() { + self.build.verbose(&format!("\t{:?}", step)); + } + + // Using `steps` as the top-level targets, make a topological ordering + // of what we need to do. + let mut order = Vec::new(); + let mut added = HashSet::new(); + for step in steps.iter().cloned() { + self.fill(step, &mut order, &mut added); + } - Source::AndroidCopyLibs { compiler } => { - vec![self.libtest(compiler)] + // Print out what we're doing for debugging + self.build.verbose("bootstrap build plan:"); + for step in order.iter() { + self.build.verbose(&format!("\t{:?}", step)); + } + + // And finally, iterate over everything and execute it. + for step in order.iter() { + (self.rules[step.name].run)(step); + } + } + + fn fill(&self, + step: Step<'a>, + order: &mut Vec<Step<'a>>, + added: &mut HashSet<Step<'a>>) { + if !added.insert(step.clone()) { + return + } + for dep in self.rules[step.name].deps.iter() { + let dep = dep(&step); + if dep.name.starts_with("default:") { + let kind = match &dep.name[8..] { + "doc" => Kind::Doc, + "dist" => Kind::Dist, + kind => panic!("unknown kind: `{}`", kind), + }; + let rules = self.rules.values().filter(|r| r.default); + for rule in rules.filter(|r| r.kind == kind) { + self.fill(dep.name(rule.name), order, added); + } + } else { + self.fill(dep, order, added); } } + order.push(step); } } diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index 6c0a32a54d9..d552f5928a9 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -57,8 +57,7 @@ pub fn cp_r(src: &Path, dst: &Path) { let name = path.file_name().unwrap(); let dst = dst.join(name); if t!(f.file_type()).is_dir() { - let _ = fs::remove_dir_all(&dst); - t!(fs::create_dir(&dst)); + t!(fs::create_dir_all(&dst)); cp_r(&path, &dst); } else { let _ = fs::remove_file(&dst); diff --git a/src/librustc_errors/Cargo.toml b/src/librustc_errors/Cargo.toml index 128c270eb35..c92e4d8f5ab 100644 --- a/src/librustc_errors/Cargo.toml +++ b/src/librustc_errors/Cargo.toml @@ -11,4 +11,4 @@ crate-type = ["dylib"] [dependencies] log = { path = "../liblog" } serialize = { path = "../libserialize" } -syntax_pos = { path = "../libsyntax_pos" } \ No newline at end of file +syntax_pos = { path = "../libsyntax_pos" } diff --git a/x.py b/x.py new file mode 100755 index 00000000000..54148b0d2b2 --- /dev/null +++ b/x.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# Copyright 2016 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import sys +import os +dir = os.path.dirname(__file__) +sys.path.append(os.path.abspath(os.path.join(dir, "src", "bootstrap"))) + +import bootstrap + +bootstrap.main() |