Rollup merge of #39400 - alexcrichton:arm-cross-test, r=brson

Add support for test suites emulated in QEMU

This commit adds support to the build system to execute test suites that cannot
run natively but can instead run inside of a QEMU emulator. A proof-of-concept
builder was added for the `arm-unknown-linux-gnueabihf` target to show off how
this might work.

In general the architecture is to have a server running inside of the emulator
which a local client connects to. The protocol between the server/client
supports compiling tests on the host and running them on the target inside the
emulator.

Closes #33114

5 files changed, 160 insertions, 42 deletions

diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index 19aac0f36bb..0ebd1055132 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -26,7 +26,7 @@ use build_helper::output;
 
 use {Build, Compiler, Mode};
 use dist;
-use util::{self, dylib_path, dylib_path_var};
+use util::{self, dylib_path, dylib_path_var, exe};
 
 const ADB_TEST_DIR: &'static str = "/data/tmp";
 
@@ -221,6 +221,12 @@ pub fn compiletest(build: &Build,
            .arg("--llvm-cxxflags").arg("");
     }
 
+    if build.qemu_rootfs(target).is_some() {
+        cmd.arg("--qemu-test-client")
+           .arg(build.tool(&Compiler::new(0, &build.config.build),
+                           "qemu-test-client"));
+    }
+
     // Running a C compiler on MSVC requires a few env vars to be set, to be
     // sure to set them here.
     //
@@ -403,9 +409,9 @@ pub fn krate(build: &Build,
     dylib_path.insert(0, build.sysroot_libdir(&compiler, target));
     cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
-    if target.contains("android") {
-        cargo.arg("--no-run");
-    } else if target.contains("emscripten") {
+    if target.contains("android") ||
+       target.contains("emscripten") ||
+       build.qemu_rootfs(target).is_some() {
         cargo.arg("--no-run");
     }
 
@@ -423,6 +429,9 @@ pub fn krate(build: &Build,
     } else if target.contains("emscripten") {
         build.run(&mut cargo);
         krate_emscripten(build, &compiler, target, mode);
+    } else if build.qemu_rootfs(target).is_some() {
+        build.run(&mut cargo);
+        krate_qemu(build, &compiler, target, mode);
     } else {
         cargo.args(&build.flags.cmd.test_args());
         build.run(&mut cargo);
@@ -480,23 +489,46 @@ fn krate_emscripten(build: &Build,
                     compiler: &Compiler,
                     target: &str,
                     mode: Mode) {
-     let mut tests = Vec::new();
-     let out_dir = build.cargo_out(compiler, mode, target);
-     find_tests(&out_dir, target, &mut tests);
-     find_tests(&out_dir.join("deps"), target, &mut tests);
-
-     for test in tests {
-         let test_file_name = test.to_string_lossy().into_owned();
-         println!("running {}", test_file_name);
-         let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured");
-         let mut cmd = Command::new(nodejs);
-         cmd.arg(&test_file_name);
-         if build.config.quiet_tests {
-             cmd.arg("--quiet");
-         }
-         build.run(&mut cmd);
-     }
- }
+    let mut tests = Vec::new();
+    let out_dir = build.cargo_out(compiler, mode, target);
+    find_tests(&out_dir, target, &mut tests);
+    find_tests(&out_dir.join("deps"), target, &mut tests);
+
+    for test in tests {
+        let test_file_name = test.to_string_lossy().into_owned();
+        println!("running {}", test_file_name);
+        let nodejs = build.config.nodejs.as_ref().expect("nodejs not configured");
+        let mut cmd = Command::new(nodejs);
+        cmd.arg(&test_file_name);
+        if build.config.quiet_tests {
+            cmd.arg("--quiet");
+        }
+        build.run(&mut cmd);
+    }
+}
+
+fn krate_qemu(build: &Build,
+              compiler: &Compiler,
+              target: &str,
+              mode: Mode) {
+    let mut tests = Vec::new();
+    let out_dir = build.cargo_out(compiler, mode, target);
+    find_tests(&out_dir, target, &mut tests);
+    find_tests(&out_dir.join("deps"), target, &mut tests);
+
+    let tool = build.tool(&Compiler::new(0, &build.config.build),
+                          "qemu-test-client");
+    for test in tests {
+        let mut cmd = Command::new(&tool);
+        cmd.arg("run")
+           .arg(&test);
+        if build.config.quiet_tests {
+            cmd.arg("--quiet");
+        }
+        cmd.args(&build.flags.cmd.test_args());
+        build.run(&mut cmd);
+    }
+}
 
 
 fn find_tests(dir: &Path,
@@ -516,13 +548,15 @@ fn find_tests(dir: &Path,
     }
 }
 
-pub fn android_copy_libs(build: &Build,
-                         compiler: &Compiler,
-                         target: &str) {
-    if !target.contains("android") {
-        return
+pub fn emulator_copy_libs(build: &Build, compiler: &Compiler, target: &str) {
+    if target.contains("android") {
+        android_copy_libs(build, compiler, target)
+    } else if let Some(s) = build.qemu_rootfs(target) {
+        qemu_copy_libs(build, compiler, target, s)
     }
+}
 
+fn android_copy_libs(build: &Build, compiler: &Compiler, target: &str) {
     println!("Android copy libs to emulator ({})", target);
     build.run(Command::new("adb").arg("wait-for-device"));
     build.run(Command::new("adb").arg("remount"));
@@ -548,6 +582,39 @@ pub fn android_copy_libs(build: &Build,
     }
 }
 
+fn qemu_copy_libs(build: &Build,
+                  compiler: &Compiler,
+                  target: &str,
+                  rootfs: &Path) {
+    println!("QEMU copy libs to emulator ({})", target);
+    assert!(target.starts_with("arm"), "only works with arm for now");
+    t!(fs::create_dir_all(build.out.join("tmp")));
+
+    // Copy our freshly compiled test server over to the rootfs
+    let server = build.cargo_out(compiler, Mode::Tool, target)
+                      .join(exe("qemu-test-server", target));
+    t!(fs::copy(&server, rootfs.join("testd")));
+
+    // Spawn the emulator and wait for it to come online
+    let tool = build.tool(&Compiler::new(0, &build.config.build),
+                          "qemu-test-client");
+    build.run(Command::new(&tool)
+                      .arg("spawn-emulator")
+                      .arg(rootfs)
+                      .arg(build.out.join("tmp")));
+
+    // Push all our dylibs to the emulator
+    for f in t!(build.sysroot_libdir(compiler, target).read_dir()) {
+        let f = t!(f);
+        let name = f.file_name().into_string().unwrap();
+        if util::is_dylib(&name) {
+            build.run(Command::new(&tool)
+                              .arg("push")
+                              .arg(f.path()));
+        }
+    }
+}
+
 /// Run "distcheck", a 'make check' from a tarball
 pub fn distcheck(build: &Build) {
     if build.config.build != "x86_64-unknown-linux-gnu" {
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index 776b91028a1..81dd42a1e90 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -382,10 +382,10 @@ fn add_to_sysroot(out_dir: &Path, sysroot_dst: &Path) {
 ///
 /// This will build the specified tool with the specified `host` compiler in
 /// `stage` into the normal cargo output directory.
-pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
-    println!("Building stage{} tool {} ({})", stage, tool, host);
+pub fn tool(build: &Build, stage: u32, target: &str, tool: &str) {
+    println!("Building stage{} tool {} ({})", stage, tool, target);
 
-    let compiler = Compiler::new(stage, host);
+    let compiler = Compiler::new(stage, &build.config.build);
 
     // FIXME: need to clear out previous tool and ideally deps, may require
     //        isolating output directories or require a pseudo shim step to
@@ -396,7 +396,7 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
     // let out_dir = build.cargo_out(stage, &host, Mode::Librustc, target);
     // build.clear_if_dirty(&out_dir, &libstd_stamp(build, stage, &host, target));
 
-    let mut cargo = build.cargo(&compiler, Mode::Tool, host, "build");
+    let mut cargo = build.cargo(&compiler, Mode::Tool, target, "build");
     cargo.arg("--manifest-path")
          .arg(build.src.join(format!("src/tools/{}/Cargo.toml", tool)));
 
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 6e077691b3a..4e67a14345b 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -114,6 +114,7 @@ pub struct Target {
     pub cxx: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
     pub musl_root: Option<PathBuf>,
+    pub qemu_rootfs: Option<PathBuf>,
 }
 
 /// Structure of the `config.toml` file that configuration is read from.
@@ -222,6 +223,7 @@ struct TomlTarget {
     cxx: Option<String>,
     android_ndk: Option<String>,
     musl_root: Option<String>,
+    qemu_rootfs: Option<String>,
 }
 
 impl Config {
@@ -360,6 +362,7 @@ impl Config {
                 target.cxx = cfg.cxx.clone().map(PathBuf::from);
                 target.cc = cfg.cc.clone().map(PathBuf::from);
                 target.musl_root = cfg.musl_root.clone().map(PathBuf::from);
+                target.qemu_rootfs = cfg.qemu_rootfs.clone().map(PathBuf::from);
 
                 config.target_config.insert(triple.clone(), target);
             }
@@ -562,6 +565,12 @@ impl Config {
                                                .map(|s| s.to_string())
                                                .collect();
                 }
+                "CFG_QEMU_ARMHF_ROOTFS" if value.len() > 0 => {
+                    let target = "arm-unknown-linux-gnueabihf".to_string();
+                    let target = self.target_config.entry(target)
+                                     .or_insert(Target::default());
+                    target.qemu_rootfs = Some(parse_configure_path(value));
+                }
                 _ => {}
             }
         }
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index df1218752d1..e58dcc9fce9 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -878,6 +878,17 @@ impl Build {
             .map(|p| &**p)
     }
 
+    /// Returns the root of the "rootfs" image that this target will be using,
+    /// if one was configured.
+    ///
+    /// If `Some` is returned then that means that tests for this target are
+    /// emulated with QEMU and binaries will need to be shipped to the emulator.
+    fn qemu_rootfs(&self, target: &str) -> Option<&Path> {
+        self.config.target_config.get(target)
+            .and_then(|t| t.qemu_rootfs.as_ref())
+            .map(|p| &**p)
+    }
+
     /// Path to the python interpreter to use
     fn python(&self) -> &Path {
         self.config.python.as_ref().unwrap()
diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs
index 3932a7cf8c5..ee5b61062fe 100644
--- a/src/bootstrap/step.rs
+++ b/src/bootstrap/step.rs
@@ -295,7 +295,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
                  .dep(|s| s.name("libtest"))
                  .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
                  .dep(|s| s.name("test-helpers"))
-                 .dep(|s| s.name("android-copy-libs"))
+                 .dep(|s| s.name("emulator-copy-libs"))
                  .default(mode != "pretty") // pretty tests don't run everywhere
                  .run(move |s| {
                      check::compiletest(build, &s.compiler(), s.target, mode, dir)
@@ -333,7 +333,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
              .dep(|s| s.name("tool-compiletest").target(s.host).stage(0))
              .dep(|s| s.name("test-helpers"))
              .dep(|s| s.name("debugger-scripts"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .run(move |s| check::compiletest(build, &s.compiler(), s.target,
                                          "debuginfo-gdb", "debuginfo"));
         let mut rule = rules.test("check-debuginfo", "src/test/debuginfo");
@@ -387,14 +387,14 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     for (krate, path, _default) in krates("std_shim") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Libstd, TestKind::Test,
                                         Some(&krate.name)));
     }
     rules.test("check-std-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .dep(|s| s.name("android-copy-libs"))
+         .dep(|s| s.name("emulator-copy-libs"))
          .default(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
                                     Mode::Libstd, TestKind::Test, None));
@@ -403,14 +403,14 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     for (krate, path, _default) in krates("std_shim") {
         rules.bench(&krate.bench_step, path)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Libstd, TestKind::Bench,
                                         Some(&krate.name)));
     }
     rules.bench("bench-std-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .dep(|s| s.name("android-copy-libs"))
+         .dep(|s| s.name("emulator-copy-libs"))
          .default(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
                                     Mode::Libstd, TestKind::Bench, None));
@@ -418,21 +418,21 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     for (krate, path, _default) in krates("test_shim") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("libtest"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Libtest, TestKind::Test,
                                         Some(&krate.name)));
     }
     rules.test("check-test-all", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .dep(|s| s.name("android-copy-libs"))
+         .dep(|s| s.name("emulator-copy-libs"))
          .default(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
                                     Mode::Libtest, TestKind::Test, None));
     for (krate, path, _default) in krates("rustc-main") {
         rules.test(&krate.test_step, path)
              .dep(|s| s.name("librustc"))
-             .dep(|s| s.name("android-copy-libs"))
+             .dep(|s| s.name("emulator-copy-libs"))
              .host(true)
              .run(move |s| check::krate(build, &s.compiler(), s.target,
                                         Mode::Librustc, TestKind::Test,
@@ -440,7 +440,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     }
     rules.test("check-rustc-all", "path/to/nowhere")
          .dep(|s| s.name("librustc"))
-         .dep(|s| s.name("android-copy-libs"))
+         .dep(|s| s.name("emulator-copy-libs"))
          .default(true)
          .host(true)
          .run(move |s| check::krate(build, &s.compiler(), s.target,
@@ -481,9 +481,34 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
 
     rules.build("test-helpers", "src/rt/rust_test_helpers.c")
          .run(move |s| native::test_helpers(build, s.target));
-    rules.test("android-copy-libs", "path/to/nowhere")
+
+    // Some test suites are run inside emulators, and most of our test binaries
+    // are linked dynamically which means we need to ship the standard library
+    // and such to the emulator ahead of time. This step represents this and is
+    // a dependency of all test suites.
+    //
+    // Most of the time this step is a noop (the `check::emulator_copy_libs`
+    // only does work if necessary). For some steps such as shipping data to
+    // QEMU we have to build our own tools so we've got conditional dependencies
+    // on those programs as well. Note that the QEMU client is built for the
+    // build target (us) and the server is built for the target.
+    rules.test("emulator-copy-libs", "path/to/nowhere")
          .dep(|s| s.name("libtest"))
-         .run(move |s| check::android_copy_libs(build, &s.compiler(), s.target));
+         .dep(move |s| {
+             if build.qemu_rootfs(s.target).is_some() {
+                s.name("tool-qemu-test-client").target(s.host).stage(0)
+             } else {
+                 Step::noop()
+             }
+         })
+         .dep(move |s| {
+             if build.qemu_rootfs(s.target).is_some() {
+                s.name("tool-qemu-test-server")
+             } else {
+                 Step::noop()
+             }
+         })
+         .run(move |s| check::emulator_copy_libs(build, &s.compiler(), s.target));
 
     rules.test("check-bootstrap", "src/bootstrap")
          .default(true)
@@ -516,6 +541,12 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
     rules.build("tool-build-manifest", "src/tools/build-manifest")
          .dep(|s| s.name("libstd"))
          .run(move |s| compile::tool(build, s.stage, s.target, "build-manifest"));
+    rules.build("tool-qemu-test-server", "src/tools/qemu-test-server")
+         .dep(|s| s.name("libstd"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "qemu-test-server"));
+    rules.build("tool-qemu-test-client", "src/tools/qemu-test-client")
+         .dep(|s| s.name("libstd"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "qemu-test-client"));
 
     // ========================================================================
     // Documentation targets