Add support for test suites emulated in QEMU

This commit adds support to the build system to execute test suites that cannot
run natively but can instead run inside of a QEMU emulator. A proof-of-concept
builder was added for the `arm-unknown-linux-gnueabihf` target to show off how
this might work.

In general the architecture is to have a server running inside of the emulator
which a local client connects to. The protocol between the server/client
supports compiling tests on the host and running them on the target inside the
emulator.

Closes #33114

5 files changed, 160 insertions, 42 deletions

diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index 4aca843558f..6df925f924d 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -26,7 +26,7 @@ use build_helper::output;
 
 use {Build, Compiler, Mode};
 use dist;
-use util::{self, dylib_path, dylib_path_var};
+use util::{self, dylib_path, dylib_path_var, exe};
 
 const ADB_TEST_DIR: &'static str = "/data/tmp";
 
@@ -221,6 +221,12 @@ pub fn compiletest(build: &Build,
            .arg("--llvm-cxxflags").arg("");
     }
 
+    if build.qemu_rootfs(target).is_some() {
+        cmd.arg("--qemu-test-client")
+           .arg(build.tool(&Compiler::new(0, &build.config.build),
+                           "qemu-test-client"));
+    }
+
     // Running a C compiler on MSVC requires a few env vars to be set, to be
     // sure to set them here.
     //
@@ -403,9 +409,9 @@ pub fn krate(build: &Build,
     dylib_path.insert(0, build.sysroot_libdir(&compiler, target));
     cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
-    if target.contains("android") {
-        cargo.arg("--no-run");
-    } else if target.contains("emscripten") {
+    if target.contains("android") ||
+       target.contains("emscripten") ||
+       build.qemu_rootfs(target).is_some() {
         cargo.arg("--no-run");
     }
 
@@ -423,6 +429,9 @@ pub fn krate(build: &Build,
     } else if target.contains("emscripten") {
         build.run(&mut cargo);
         krate_emscripten(build, &compiler, target, mode);
+    } else if build.qemu_rootfs(target).is_some() {
+        build.run(&mut cargo);
+        krate_qemu(build, &compiler, target, mode);
     } else {
         cargo.args(&build.flags.cmd.test_args());
         build.run(&mut cargo);
@@ -480,23 +489,46 @@ fn krate_emscripten(build: &Build,
                     compiler: &Compiler,
                     target: &str,
                     mode: Mode) {
-     let mut tests = Vec::new();
-     let out_dir = build.cargo_out(compiler, mode, target);
-     find_tests(&out_dir, target, &mut tests);
-     find_tests(&out_dir.join("deps"), target, &mut tests);
-
-     for test in tests {
-         let test_file_name = test.to_string_lossy().into_owned();
-         println!("running {}", test_file_name);
-         let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured");
-         let mut cmd = Command::new(nodejs);
-         cmd.arg(&test_file_name);
-         if build.config.quiet_tests {
-             cmd.arg("--quiet");
-         }
-         build.run(&mut cmd);
-     }
- }
+    let mut tests = Vec::new();
+    let out_dir = build.cargo_out(compiler, mode, target);
+    find_tests(&out_dir, target, &mut tests);
+    find_tests(&out_dir.join("deps"), target, &mut tests);
+
+    for test in tests {
+        let test_file_name = test.to_string_lossy().into_owned();
+        println!("running {}", test_file_name);
+        let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured");
+        let mut cmd = Command::new(nodejs);
+        cmd.arg(&test_file_name);
+        if build.config.quiet_tests {
+            cmd.arg("--quiet");
+        }
+        build.run(&mut cmd);
+    }
+}
+
+fn krate_qemu(build: &Build,
+              compiler: &Compiler,
+              target: &str,
+              mode: Mode) {
+    let mut tests = Vec::new();
+    let out_dir = build.cargo_out(compiler, mode, target);
+    find_tests(&out_dir, target, &mut tests);
+    find_tests(&out_dir.join("deps"), target, &mut tests);
+
+    let tool = build.tool(&Compiler::new(0, &build.config.build),
+                          "qemu-test-client");
+    for test in tests {
+        let mut cmd = Command::new(&tool);
+        cmd.arg("run")
+           .arg(&test);
+        if build.config.quiet_tests {
+            cmd.arg("--quiet");
+        }
+        cmd.args(&build.flags.cmd.test_args());
+        build.run(&mut cmd);
+    }
+}
 
 
 fn find_tests(dir: &Path,
@@ -516,13 +548,15 @@ fn find_tests(dir: &Path,
     }
 }
 
-pub fn android_copy_libs(build: &Build,
-                         compiler: &Compiler,
-                         target: &str) {
-    if !target.contains("android") {
-        return
+pub fn emulator_copy_libs(build: &Build, compiler: &Compiler, target: &str) {
+    if target.contains("android") {
+        android_copy_libs(build, compiler, target)
+    } else if let Some(s) = build.qemu_rootfs(target) {
+        qemu_copy_libs(build, compiler, target, s)
     }
+}
 
+fn android_copy_libs(build: &Build, compiler: &Compiler, target: &str) {
     println!("Android copy libs to emulator ({})", target);
     build.run(Command::new("adb").arg("wait-for-device"));
     build.run(Command::new("adb").arg("remount"));
@@ -548,6 +582,39 @@ pub fn android_copy_libs(build: &Build,
     }
 }
 
+fn qemu_copy_libs(build: &Build,
+                  compiler: &Compiler,
+                  target: &str,
+                  rootfs: &Path) {
+    println!("QEMU copy libs to emulator ({})", target);
+    assert!(target.starts_with("arm"), "only works with arm for now");
+    t!(fs::create_dir_all(build.out.join("tmp")));
+
+    // Copy our freshly compiled test server over to the rootfs
+    let server = build.cargo_out(compiler, Mode::Tool, target)
+                      .join(exe("qemu-test-server", target));
+    t!(fs::copy(&server, rootfs.join("testd")));
+
+    // Spawn the emulator and wait for it to come online
+    let tool = build.tool(&Compiler::new(0, &build.config.build),
+                          "qemu-test-client");
+    build.run(Command::new(&tool)
+                      .arg("spawn-emulator")
+                      .arg(rootfs)
+                      .arg(build.out.join("tmp")));
+
+    // Push all our dylibs to the emulator
+    for f in t!(build.sysroot_libdir(compiler, target).read_dir()) {
+        let f = t!(f);
+        let name = f.file_name().into_string().unwrap();
+        if util::is_dylib(&name) {
+            build.run(Command::new(&tool)
+                              .arg("push")
+                              .arg(f.path()));
+        }
+    }
+}
+
 /// Run "distcheck", a 'make check' from a tarball
 pub fn distcheck(build: &Build) {
     if build.config.build != "x86_64-unknown-linux-gnu" {
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index 7c35151a6d2..398ca1b2f22 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -382,10 +382,10 @@ fn add_to_sysroot(out_dir: &Path, sysroot_dst: &Path) {
 ///
 /// This will build the specified tool with the specified `host` compiler in
 /// `stage` into the normal cargo output directory.
-pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
-    println!("Building stage{} tool {} ({})", stage, tool, host);
+pub fn tool(build: &Build, stage: u32, target: &str, tool: &str) {
+    println!("Building stage{} tool {} ({})", stage, tool, target);
 
-    let compiler = Compiler::new(stage, host);
+    let compiler = Compiler::new(stage, &build.config.build);
 
     // FIXME: need to clear out previous tool and ideally deps, may require
     //        isolating output directories or require a pseudo shim step to
@@ -396,7 +396,7 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
     // let out_dir = build.cargo_out(stage, &host, Mode::Librustc, target);
     // build.clear_if_dirty(&out_dir, &libstd_stamp(build, stage, &host, target));
 
-    let mut cargo = build.cargo(&compiler, Mode::Tool, host, "build");
+    let mut cargo = build.cargo(&compiler, Mode::Tool, target, "build");
     cargo.arg("--manifest-path")
          .arg(build.src.join(format!("src/tools/{}/Cargo.toml", tool)));
 
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index e035f8157ff..351ac446b49 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -114,6 +114,7 @@ pub struct Target {
     pub cxx: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
     pub musl_root: Option<PathBuf>,
+    pub qemu_rootfs: Option<PathBuf>,
 }
 
 /// Structure of the `config.toml` file that configuration is read from.
@@ -222,6 +223,7 @@ struct TomlTarget {
     cxx: Option<String>,
     android_ndk: Option<String>,
     musl_root: Option<String>,
+    qemu_rootfs: Option<String>,
 }
 
 impl Config {
@@ -360,6 +362,7 @@ impl Config {
                 target.cxx = cfg.cxx.clone().map(PathBuf::from);
                 target.cc = cfg.cc.clone().map(PathBuf::from);
                 target.musl_root = cfg.musl_root.clone().map(PathBuf::from);
+                target.qemu_rootfs = cfg.qemu_rootfs.clone().map(PathBuf::from);
 
                 config.target_config.insert(triple.clone(), target);
             }
@@ -562,6 +565,12 @@ impl Config {
                                                .map(|s| s.to_string())
                                                .collect();
                 }
+                "CFG_QEMU_ARMHF_ROOTFS" if value.len() > 0 => {
+                    let target = "arm-unknown-linux-gnueabihf".to_string();
+                    let target = self.target_config.entry(target)
+                                     .or_insert(Target::default());
+                    target.qemu_rootfs = Some(parse_configure_path(value));
+                }
                 _ => {}
             }
         }
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index db2fe2db813..6347e08ca58 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -886,6 +886,17 @@ impl Build {
             .map(|p| &**p)
     }
 
+    /// Returns the root of the "rootfs" image that this target will be using,
+    /// if one was configured.
+    ///
+    /// If `Some` is returned then that means that tests for this target are
+    /// emulated with QEMU and binaries will need to be shipped to the emulator.
+    fn qemu_rootfs(&self, target: &str) -> Option<&Path> {
+        self.config.target_config.get(target)
+            .and_then(|t| t.qemu_rootfs.as_ref())
+            .map(|p| &**p)
+    }
+
     /// Path to the python interpreter to use
     fn python(&self) -> &Path {
         self.config.python.as_ref().unwrap()
diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs
index 3932a7cf8c5..ee5b61062fe 100644
--- a/src/bootstrap/step.rs
+++ b/src/bootstrap/step.rs
@@ -295,7 +295,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
                  .dep(|s| s.name("libtest"))
                  .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
                  .dep(|s| s.name("test-helpers"))
-                 .dep(|s| s.name("android-copy-libs"))
+                 .dep(|s| s.name("emulator-copy-libs"))
                  .default(mode != "pretty") // pretty tests don't run everywhere
                  .run(move |s| {
                      check::compiletest(build, &s.compiler(), s.target, mode, dir)
@@ -333,7 +333,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
              .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
              .dep(|s| s.name("test-helpers"))
              .dep(|s| s.name("debugger-scripts"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .run(move |s| check::compiletest(build, &s.compiler(), s.target,
                                          "debuginfo-gdb", "debuginfo"));
         let mut rule = rules.test("check-debuginfo", "src/test/debuginfo");
@@ -387,14 +387,14 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     for (krate, path, _default) in krates("std_shim") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Libstd, TestKind::Test,
                                         Some(&krate.name)));
     }
     rules.test("check-std-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .dep(|s| s.name("android-copy-libs"))
+         .dep(|s| s.name("emulator-copy-libs"))
          .default(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
                                     Mode::Libstd, TestKind::Test, None));
@@ -403,14 +403,14 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     for (krate, path, _default) in krates("std_shim") {
         rules.bench(&krate.bench_step, path)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Libstd, TestKind::Bench,
                                         Some(&krate.name)));
     }
     rules.bench("bench-std-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .dep(|s| s.name("android-copy-libs"))
+         .dep(|s| s.name("emulator-copy-libs"))
          .default(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
                                     Mode::Libstd, TestKind::Bench, None));
@@ -418,21 +418,21 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     for (krate, path, _default) in krates("test_shim") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Libtest, TestKind::Test,
                                         Some(&krate.name)));
     }
     rules.test("check-test-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .dep(|s| s.name("android-copy-libs"))
+         .dep(|s| s.name("emulator-copy-libs"))
          .default(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
                                     Mode::Libtest, TestKind::Test, None));
     for (krate, path, _default) in krates("rustc-main") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("librustc"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .host(true)
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Librustc, TestKind::Test,
@@ -440,7 +440,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     }
     rules.test("check-rustc-all", "path/to/nowhere")
          .dep(|s| s.name("librustc"))
-         .dep(|s| s.name("android-copy-libs"))
+         .dep(|s| s.name("emulator-copy-libs"))
          .default(true)
          .host(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
@@ -481,9 +481,34 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
 
     rules.build("test-helpers", "src/rt/rust_test_helpers.c")
          .run(move |s| native::test_helpers(build, s.target));
-    rules.test("android-copy-libs", "path/to/nowhere")
+
+    // Some test suites are run inside emulators, and most of our test binaries
+    // are linked dynamically which means we need to ship the standard library
+    // and such to the emulator ahead of time. This step represents this and is
+    // a dependency of all test suites.
+    //
+    // Most of the time this step is a noop (the `check::emulator_copy_libs`
+    // only does work if necessary). For some steps such as shipping data to
+    // QEMU we have to build our own tools so we've got conditional dependencies
+    // on those programs as well. Note that the QEMU client is built for the
+    // build target (us) and the server is built for the target.
+    rules.test("emulator-copy-libs", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .run(move |s| check::android_copy_libs(build, &s.compiler(), s.target));
+         .dep(move |s| {
+             if build.qemu_rootfs(s.target).is_some() {
+                s.name("tool-qemu-test-client").target(s.host).stage(0)
+             } else {
+                 Step::noop()
+             }
+         })
+         .dep(move |s| {
+             if build.qemu_rootfs(s.target).is_some() {
+                s.name("tool-qemu-test-server")
+             } else {
+                 Step::noop()
+             }
+         })
+         .run(move |s| check::emulator_copy_libs(build, &s.compiler(), s.target));
 
     rules.test("check-bootstrap", "src/bootstrap")
          .default(true)
@@ -516,6 +541,12 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     rules.build("tool-build-manifest", "src/tools/build-manifest")
          .dep(|s| s.name("libstd"))
          .run(move |s| compile::tool(build, s.stage, s.target, "build-manifest"));
+    rules.build("tool-qemu-test-server", "src/tools/qemu-test-server")
+         .dep(|s| s.name("libstd"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "qemu-test-server"));
+    rules.build("tool-qemu-test-client", "src/tools/qemu-test-client")
+         .dep(|s| s.name("libstd"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "qemu-test-client"));
 
     // ========================================================================
     // Documentation targets