From 7bc2cbf5db3e20e6f97494b70b2a1d65ac2e61fc Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 26 Apr 2017 08:52:19 -0700 Subject: travis: Parallelize tests on Android Currently our slowest test suite on android, run-pass, takes over 5 times longer than the x86_64 component (~400 -> ~2200s). Typically QEMU emulation does indeed add overhead, but not 5x for this kind of workload. One of the slowest parts of the Android process is that *compilation* happens serially. Tests themselves need to run single-threaded on the emulator (due to how the test harness works) and this forces the compiles themselves to be single threaded. Now Travis gives us more than one core per machine, so it'd be much better if we could take advantage of them! The emulator itself is still fundamentally single-threaded, but we should see a nice speedup by sending binaries for it to run much more quickly. It turns out that we've already got all the tools to do this in-tree. The qemu-test-{server,client} that are in use for the ARM Linux testing are a perfect match for the Android emulator. This commit migrates the custom adb management code in compiletest/rustbuild to the same qemu-test-{server,client} implementation that ARM Linux uses. This allows us to lift the parallelism restriction on the compiletest test suites, namely run-pass. Consequently although we'll still basically run the tests themselves in single threaded mode we'll be able to compile all of them in parallel, keeping the pipeline much more full and using more cores for the work at hand. Additionally the architecture here should be a bit speedier as it should have less overhead than adb which is a whole new process on both the host and the emulator! Locally on an 8 core machine I've seen the run-pass test suite speed up from taking nearly an hour to only taking 6 minutes. I don't think we'll see quite a drastic speedup on Travis but I'm hoping this change can place the Android tests well below 2 hours instead of just above 2 hours. Because the client/server here are now repurposed for more than just QEMU, they've been renamed to `remote-test-{server,client}`. Note that this PR does not currently modify how debuginfo tests are executed on Android. While parallelizable it wouldn't be quite as easy, so that's left to another day. Thankfully that test suite is much smaller than the run-pass test suite. As a final fix I discovered that the ARM and Android test suites were actually running all library unit tests (e.g. stdtest, coretest, etc) twice. I've corrected that to only run tests once which should also give a nice boost in overall cycle time here. --- src/Cargo.lock | 16 +- src/Cargo.toml | 4 +- src/bootstrap/check.rs | 140 +++----------- src/bootstrap/lib.rs | 6 + src/bootstrap/step.rs | 16 +- src/etc/adb_run_wrapper.sh | 35 ---- src/test/run-pass/vector-sort-panic-safe.rs | 13 +- src/tools/compiletest/src/common.rs | 4 +- src/tools/compiletest/src/main.rs | 40 ++-- src/tools/compiletest/src/runtest.rs | 215 ++------------------- src/tools/qemu-test-client/Cargo.toml | 6 - src/tools/qemu-test-client/src/main.rs | 221 ---------------------- src/tools/qemu-test-server/Cargo.toml | 6 - src/tools/qemu-test-server/src/main.rs | 232 ----------------------- src/tools/remote-test-client/Cargo.toml | 6 + src/tools/remote-test-client/src/main.rs | 281 ++++++++++++++++++++++++++++ src/tools/remote-test-server/Cargo.toml | 6 + src/tools/remote-test-server/src/main.rs | 257 +++++++++++++++++++++++++ 18 files changed, 640 insertions(+), 864 deletions(-) delete mode 100755 src/etc/adb_run_wrapper.sh delete mode 100644 src/tools/qemu-test-client/Cargo.toml delete mode 100644 src/tools/qemu-test-client/src/main.rs delete mode 100644 src/tools/qemu-test-server/Cargo.toml delete mode 100644 src/tools/qemu-test-server/src/main.rs create mode 100644 src/tools/remote-test-client/Cargo.toml create mode 100644 src/tools/remote-test-client/src/main.rs create mode 100644 src/tools/remote-test-server/Cargo.toml create mode 100644 src/tools/remote-test-server/src/main.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index 659feef80a4..5d97ccaabbf 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -366,14 +366,6 @@ dependencies = [ "bitflags 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "qemu-test-client" -version = "0.1.0" - -[[package]] -name = "qemu-test-server" -version = "0.1.0" - [[package]] name = "quick-error" version = "1.1.0" @@ -403,6 +395,14 @@ name = "regex-syntax" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "remote-test-client" +version = "0.1.0" + +[[package]] +name = "remote-test-server" +version = "0.1.0" + [[package]] name = "rls-data" version = "0.1.0" diff --git a/src/Cargo.toml b/src/Cargo.toml index 0dafbb8428e..8f6150c6438 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -11,8 +11,8 @@ members = [ "tools/rustbook", "tools/tidy", "tools/build-manifest", - "tools/qemu-test-client", - "tools/qemu-test-server", + "tools/remote-test-client", + "tools/remote-test-server", ] # Curiously, compiletest will segfault if compiled with opt-level=3 on 64-bit diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 8ab07e9e5b5..1bcec2cdede 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -28,7 +28,7 @@ use {Build, Compiler, Mode}; use dist; use util::{self, dylib_path, dylib_path_var, exe}; -const ADB_TEST_DIR: &'static str = "/data/tmp"; +const ADB_TEST_DIR: &'static str = "/data/tmp/work"; /// The two modes of the test runner; tests or benchmarks. #[derive(Copy, Clone)] @@ -243,10 +243,10 @@ pub fn compiletest(build: &Build, .arg("--llvm-cxxflags").arg(""); } - if build.qemu_rootfs(target).is_some() { - cmd.arg("--qemu-test-client") + if build.remote_tested(target) { + cmd.arg("--remote-test-client") .arg(build.tool(&Compiler::new(0, &build.config.build), - "qemu-test-client")); + "remote-test-client")); } // Running a C compiler on MSVC requires a few env vars to be set, to be @@ -445,9 +445,7 @@ pub fn krate(build: &Build, dylib_path.insert(0, build.sysroot_libdir(&compiler, target)); cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - if target.contains("android") || - target.contains("emscripten") || - build.qemu_rootfs(target).is_some() { + if target.contains("emscripten") || build.remote_tested(target) { cargo.arg("--no-run"); } @@ -459,75 +457,24 @@ pub fn krate(build: &Build, let _time = util::timeit(); - if target.contains("android") { - build.run(&mut cargo); - krate_android(build, &compiler, target, mode); - } else if target.contains("emscripten") { + if target.contains("emscripten") { build.run(&mut cargo); krate_emscripten(build, &compiler, target, mode); - } else if build.qemu_rootfs(target).is_some() { + } else if build.remote_tested(target) { build.run(&mut cargo); - krate_qemu(build, &compiler, target, mode); + krate_remote(build, &compiler, target, mode); } else { cargo.args(&build.flags.cmd.test_args()); build.run(&mut cargo); } } -fn krate_android(build: &Build, - compiler: &Compiler, - target: &str, - mode: Mode) { - let mut tests = Vec::new(); - let out_dir = build.cargo_out(compiler, mode, target); - find_tests(&out_dir, target, &mut tests); - find_tests(&out_dir.join("deps"), target, &mut tests); - - for test in tests { - build.run(Command::new("adb").arg("push").arg(&test).arg(ADB_TEST_DIR)); - - let test_file_name = test.file_name().unwrap().to_string_lossy(); - let log = format!("{}/check-stage{}-T-{}-H-{}-{}.log", - ADB_TEST_DIR, - compiler.stage, - target, - compiler.host, - test_file_name); - let quiet = if build.config.quiet_tests { "--quiet" } else { "" }; - let program = format!("(cd {dir}; \ - LD_LIBRARY_PATH=./{target} ./{test} \ - --logfile {log} \ - {quiet} \ - {args})", - dir = ADB_TEST_DIR, - target = target, - test = test_file_name, - log = log, - quiet = quiet, - args = build.flags.cmd.test_args().join(" ")); - - let output = output(Command::new("adb").arg("shell").arg(&program)); - println!("{}", output); - - t!(fs::create_dir_all(build.out.join("tmp"))); - build.run(Command::new("adb") - .arg("pull") - .arg(&log) - .arg(build.out.join("tmp"))); - build.run(Command::new("adb").arg("shell").arg("rm").arg(&log)); - if !output.contains("result: ok") { - panic!("some tests failed"); - } - } -} - fn krate_emscripten(build: &Build, compiler: &Compiler, target: &str, mode: Mode) { let mut tests = Vec::new(); let out_dir = build.cargo_out(compiler, mode, target); - find_tests(&out_dir, target, &mut tests); find_tests(&out_dir.join("deps"), target, &mut tests); for test in tests { @@ -543,17 +490,16 @@ fn krate_emscripten(build: &Build, } } -fn krate_qemu(build: &Build, - compiler: &Compiler, - target: &str, - mode: Mode) { +fn krate_remote(build: &Build, + compiler: &Compiler, + target: &str, + mode: Mode) { let mut tests = Vec::new(); let out_dir = build.cargo_out(compiler, mode, target); - find_tests(&out_dir, target, &mut tests); find_tests(&out_dir.join("deps"), target, &mut tests); let tool = build.tool(&Compiler::new(0, &build.config.build), - "qemu-test-client"); + "remote-test-client"); for test in tests { let mut cmd = Command::new(&tool); cmd.arg("run") @@ -566,7 +512,6 @@ fn krate_qemu(build: &Build, } } - fn find_tests(dir: &Path, target: &str, dst: &mut Vec) { @@ -585,59 +530,28 @@ fn find_tests(dir: &Path, } pub fn emulator_copy_libs(build: &Build, compiler: &Compiler, target: &str) { - if target.contains("android") { - android_copy_libs(build, compiler, target) - } else if let Some(s) = build.qemu_rootfs(target) { - qemu_copy_libs(build, compiler, target, s) - } -} - -fn android_copy_libs(build: &Build, compiler: &Compiler, target: &str) { - println!("Android copy libs to emulator ({})", target); - build.run(Command::new("adb").arg("wait-for-device")); - build.run(Command::new("adb").arg("remount")); - build.run(Command::new("adb").args(&["shell", "rm", "-r", ADB_TEST_DIR])); - build.run(Command::new("adb").args(&["shell", "mkdir", ADB_TEST_DIR])); - build.run(Command::new("adb") - .arg("push") - .arg(build.src.join("src/etc/adb_run_wrapper.sh")) - .arg(ADB_TEST_DIR)); - - let target_dir = format!("{}/{}", ADB_TEST_DIR, target); - build.run(Command::new("adb").args(&["shell", "mkdir", &target_dir])); - - for f in t!(build.sysroot_libdir(compiler, target).read_dir()) { - let f = t!(f); - let name = f.file_name().into_string().unwrap(); - if util::is_dylib(&name) { - build.run(Command::new("adb") - .arg("push") - .arg(f.path()) - .arg(&target_dir)); - } + if !build.remote_tested(target) { + return } -} -fn qemu_copy_libs(build: &Build, - compiler: &Compiler, - target: &str, - rootfs: &Path) { - println!("QEMU copy libs to emulator ({})", target); - assert!(target.starts_with("arm"), "only works with arm for now"); + println!("REMOTE copy libs to emulator ({})", target); t!(fs::create_dir_all(build.out.join("tmp"))); - // Copy our freshly compiled test server over to the rootfs let server = build.cargo_out(compiler, Mode::Tool, target) - .join(exe("qemu-test-server", target)); - t!(fs::copy(&server, rootfs.join("testd"))); + .join(exe("remote-test-server", target)); // Spawn the emulator and wait for it to come online let tool = build.tool(&Compiler::new(0, &build.config.build), - "qemu-test-client"); - build.run(Command::new(&tool) - .arg("spawn-emulator") - .arg(rootfs) - .arg(build.out.join("tmp"))); + "remote-test-client"); + let mut cmd = Command::new(&tool); + cmd.arg("spawn-emulator") + .arg(target) + .arg(&server) + .arg(build.out.join("tmp")); + if let Some(rootfs) = build.qemu_rootfs(target) { + cmd.arg(rootfs); + } + build.run(&mut cmd); // Push all our dylibs to the emulator for f in t!(build.sysroot_libdir(compiler, target).read_dir()) { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index bbfab388950..74c58844741 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -945,6 +945,12 @@ impl Build { .map(|p| &**p) } + /// Returns whether the target will be tested using the `remote-test-client` + /// and `remote-test-server` binaries. + fn remote_tested(&self, target: &str) -> bool { + self.qemu_rootfs(target).is_some() || target.contains("android") + } + /// Returns the root of the "rootfs" image that this target will be using, /// if one was configured. /// diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index d811e1122c4..a4d6f91fbef 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -513,15 +513,15 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.test("emulator-copy-libs", "path/to/nowhere") .dep(|s| s.name("libtest")) .dep(move |s| { - if build.qemu_rootfs(s.target).is_some() { - s.name("tool-qemu-test-client").target(s.host).stage(0) + if build.remote_tested(s.target) { + s.name("tool-remote-test-client").target(s.host).stage(0) } else { Step::noop() } }) .dep(move |s| { - if build.qemu_rootfs(s.target).is_some() { - s.name("tool-qemu-test-server") + if build.remote_tested(s.target) { + s.name("tool-remote-test-server") } else { Step::noop() } @@ -566,14 +566,14 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .dep(|s| s.name("maybe-clean-tools")) .dep(|s| s.name("libstd-tool")) .run(move |s| compile::tool(build, s.stage, s.target, "build-manifest")); - rules.build("tool-qemu-test-server", "src/tools/qemu-test-server") + rules.build("tool-remote-test-server", "src/tools/remote-test-server") .dep(|s| s.name("maybe-clean-tools")) .dep(|s| s.name("libstd-tool")) - .run(move |s| compile::tool(build, s.stage, s.target, "qemu-test-server")); - rules.build("tool-qemu-test-client", "src/tools/qemu-test-client") + .run(move |s| compile::tool(build, s.stage, s.target, "remote-test-server")); + rules.build("tool-remote-test-client", "src/tools/remote-test-client") .dep(|s| s.name("maybe-clean-tools")) .dep(|s| s.name("libstd-tool")) - .run(move |s| compile::tool(build, s.stage, s.target, "qemu-test-client")); + .run(move |s| compile::tool(build, s.stage, s.target, "remote-test-client")); rules.build("tool-cargo", "cargo") .dep(|s| s.name("maybe-clean-tools")) .dep(|s| s.name("libstd-tool")) diff --git a/src/etc/adb_run_wrapper.sh b/src/etc/adb_run_wrapper.sh deleted file mode 100755 index bd6c483156f..00000000000 --- a/src/etc/adb_run_wrapper.sh +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2014 The Rust Project Developers. See the COPYRIGHT -# file at the top-level directory of this distribution and at -# http://rust-lang.org/COPYRIGHT. -# -# Licensed under the Apache License, Version 2.0 or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. -# -# ignore-tidy-linelength -# -# usage : adb_run_wrapper [test dir - where test executables exist] [test executable] -# - -TEST_PATH=$1 -BIN_PATH=/system/bin -if [ -d "$TEST_PATH" ] -then - shift - RUN=$1 - - if [ ! -z "$RUN" ] - then - shift - - # The length of binary path (i.e. ./$RUN) should be shorter than 128 characters. - cd $TEST_PATH - TEST_EXEC_ENV=22 LD_LIBRARY_PATH=$TEST_PATH PATH=$BIN_PATH:$TEST_PATH ./$RUN $@ 1>$TEST_PATH/$RUN.stdout 2>$TEST_PATH/$RUN.stderr - L_RET=$? - - echo $L_RET > $TEST_PATH/$RUN.exitcode - - fi -fi diff --git a/src/test/run-pass/vector-sort-panic-safe.rs b/src/test/run-pass/vector-sort-panic-safe.rs index 87f1968918c..8ad6ca0abb0 100644 --- a/src/test/run-pass/vector-sort-panic-safe.rs +++ b/src/test/run-pass/vector-sort-panic-safe.rs @@ -13,9 +13,11 @@ #![feature(rand)] #![feature(const_fn)] -use std::sync::atomic::{AtomicUsize, Ordering}; use std::__rand::{thread_rng, Rng}; +use std::panic; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::thread; +use std::cell::Cell; const MAX_LEN: usize = 80; @@ -76,6 +78,7 @@ fn test(input: &[DropCounter]) { let mut panic_countdown = panic_countdown; v.sort_by(|a, b| { if panic_countdown == 0 { + SILENCE_PANIC.with(|s| s.set(true)); panic!(); } panic_countdown -= 1; @@ -94,7 +97,15 @@ fn test(input: &[DropCounter]) { } } +thread_local!(static SILENCE_PANIC: Cell = Cell::new(false)); + fn main() { + let prev = panic::take_hook(); + panic::set_hook(Box::new(move |info| { + if !SILENCE_PANIC.with(|s| s.get()) { + prev(info); + } + })); for len in (1..20).chain(70..MAX_LEN) { // Test on a random array. let mut rng = thread_rng(); diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index eb8cdcee6e6..df41e786be5 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -185,8 +185,8 @@ pub struct Config { // Print one character per test instead of one line pub quiet: bool, - // where to find the qemu test client process, if we're using it - pub qemu_test_client: Option, + // where to find the remote test client process, if we're using it + pub remote_test_client: Option, // Configuration for various run-make tests frobbing things like C compilers // or querying about various LLVM component information. diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs index 09d21221a83..0e4901ef1ab 100644 --- a/src/tools/compiletest/src/main.rs +++ b/src/tools/compiletest/src/main.rs @@ -106,7 +106,7 @@ pub fn parse_config(args: Vec ) -> Config { reqopt("", "llvm-components", "list of LLVM components built in", "LIST"), reqopt("", "llvm-cxxflags", "C++ flags for LLVM", "FLAGS"), optopt("", "nodejs", "the name of nodejs", "PATH"), - optopt("", "qemu-test-client", "path to the qemu test client", "PATH"), + optopt("", "remote-test-client", "path to the remote test client", "PATH"), optflag("h", "help", "show this message")]; let (argv0, args_) = args.split_first().unwrap(); @@ -177,9 +177,7 @@ pub fn parse_config(args: Vec ) -> Config { llvm_version: matches.opt_str("llvm-version"), android_cross_path: opt_path(matches, "android-cross-path"), adb_path: opt_str2(matches.opt_str("adb-path")), - adb_test_dir: format!("{}/{}", - opt_str2(matches.opt_str("adb-test-dir")), - opt_str2(matches.opt_str("target"))), + adb_test_dir: opt_str2(matches.opt_str("adb-test-dir")), adb_device_status: opt_str2(matches.opt_str("target")).contains("android") && "(none)" != opt_str2(matches.opt_str("adb-test-dir")) && @@ -187,7 +185,7 @@ pub fn parse_config(args: Vec ) -> Config { lldb_python_dir: matches.opt_str("lldb-python-dir"), verbose: matches.opt_present("verbose"), quiet: matches.opt_present("quiet"), - qemu_test_client: matches.opt_str("qemu-test-client").map(PathBuf::from), + remote_test_client: matches.opt_str("remote-test-client").map(PathBuf::from), cc: matches.opt_str("cc").unwrap(), cxx: matches.opt_str("cxx").unwrap(), @@ -252,27 +250,14 @@ pub fn run_tests(config: &Config) { if let DebugInfoGdb = config.mode { println!("{} debug-info test uses tcp 5039 port.\ please reserve it", config.target); - } - - // android debug-info test uses remote debugger - // so, we test 1 thread at once. - // also trying to isolate problems with adb_run_wrapper.sh ilooping - match config.mode { - // These tests don't actually run code or don't run for android, so - // we don't need to limit ourselves there - Mode::Ui | - Mode::CompileFail | - Mode::ParseFail | - Mode::RunMake | - Mode::Codegen | - Mode::CodegenUnits | - Mode::Pretty | - Mode::Rustdoc => {} - - _ => { - env::set_var("RUST_TEST_THREADS", "1"); - } + // android debug-info test uses remote debugger so, we test 1 thread + // at once as they're all sharing the same TCP port to communicate + // over. + // + // we should figure out how to lift this restriction! (run them all + // on different ports allocated dynamically). + env::set_var("RUST_TEST_THREADS", "1"); } } @@ -296,9 +281,10 @@ pub fn run_tests(config: &Config) { } DebugInfoGdb => { - if config.qemu_test_client.is_some() { + if config.remote_test_client.is_some() && + !config.target.contains("android"){ println!("WARNING: debuginfo tests are not available when \ - testing with QEMU"); + testing with remote"); return } } diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 7fb296c19f6..3d40cc419fa 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -24,7 +24,6 @@ use util::logv; use std::collections::HashSet; use std::env; -use std::fmt; use std::fs::{self, File, create_dir_all}; use std::io::prelude::*; use std::io::{self, BufReader}; @@ -469,7 +468,9 @@ actual:\n\ let debugger_run_result; match &*self.config.target { - "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => { + "arm-linux-androideabi" | + "armv7-linux-androideabi" | + "aarch64-linux-android" => { cmds = cmds.replace("run", "continue"); @@ -534,6 +535,7 @@ actual:\n\ exe_file.file_name().unwrap().to_str() .unwrap()); + debug!("adb arg: {}", adb_arg); let mut process = procsrv::run_background("", &self.config.adb_path , @@ -590,7 +592,7 @@ actual:\n\ }; debugger_run_result = ProcRes { - status: Status::Normal(status), + status: status, stdout: out, stderr: err, cmdline: cmdline @@ -841,7 +843,7 @@ actual:\n\ self.dump_output(&out, &err); ProcRes { - status: Status::Normal(status), + status: status, stdout: out, stderr: err, cmdline: format!("{:?}", cmd) @@ -1190,25 +1192,20 @@ actual:\n\ let env = self.props.exec_env.clone(); match &*self.config.target { - - "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => { - self._arm_exec_compiled_test(env) - } - // This is pretty similar to below, we're transforming: // // program arg1 arg2 // // into // - // qemu-test-client run program:support-lib.so arg1 arg2 + // remote-test-client run program:support-lib.so arg1 arg2 // // The test-client program will upload `program` to the emulator // along with all other support libraries listed (in this case // `support-lib.so`. It will then execute the program on the // emulator with the arguments specified (in the environment we give // the process) and then report back the same result. - _ if self.config.qemu_test_client.is_some() => { + _ if self.config.remote_test_client.is_some() => { let aux_dir = self.aux_output_dir_name(); let mut args = self.make_run_args(); let mut program = args.prog.clone(); @@ -1224,7 +1221,7 @@ actual:\n\ } args.args.insert(0, program); args.args.insert(0, "run".to_string()); - args.prog = self.config.qemu_test_client.clone().unwrap() + args.prog = self.config.remote_test_client.clone().unwrap() .into_os_string().into_string().unwrap(); self.compose_and_run(args, env, @@ -1324,13 +1321,6 @@ actual:\n\ aux_testpaths.file.display()), &auxres); } - - match &*self.config.target { - "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => { - self._arm_push_aux_shared_library(); - } - _ => {} - } } self.compose_and_run(args, @@ -1564,7 +1554,7 @@ actual:\n\ input).expect(&format!("failed to exec `{}`", prog)); self.dump_output(&out, &err); return ProcRes { - status: Status::Normal(status), + status: status, stdout: out, stderr: err, cmdline: cmdline, @@ -1698,157 +1688,6 @@ actual:\n\ println!("---------------------------------------------------"); } - fn _arm_exec_compiled_test(&self, env: Vec<(String, String)>) -> ProcRes { - let args = self.make_run_args(); - let cmdline = self.make_cmdline("", &args.prog, &args.args); - - // get bare program string - let mut tvec: Vec = args.prog - .split('/') - .map(str::to_owned) - .collect(); - let prog_short = tvec.pop().unwrap(); - - // copy to target - let copy_result = procsrv::run("", - &self.config.adb_path, - None, - &[ - "push".to_owned(), - args.prog.clone(), - self.config.adb_test_dir.clone() - ], - vec![("".to_owned(), "".to_owned())], - Some("".to_owned())) - .expect(&format!("failed to exec `{}`", self.config.adb_path)); - - if self.config.verbose { - println!("push ({}) {} {} {}", - self.config.target, - args.prog, - copy_result.out, - copy_result.err); - } - - logv(self.config, format!("executing ({}) {}", self.config.target, cmdline)); - - let mut runargs = Vec::new(); - - // run test via adb_run_wrapper - runargs.push("shell".to_owned()); - for (key, val) in env { - runargs.push(format!("{}={}", key, val)); - } - runargs.push(format!("{}/../adb_run_wrapper.sh", self.config.adb_test_dir)); - runargs.push(format!("{}", self.config.adb_test_dir)); - runargs.push(format!("{}", prog_short)); - - for tv in &args.args { - runargs.push(tv.to_owned()); - } - procsrv::run("", - &self.config.adb_path, - None, - &runargs, - vec![("".to_owned(), "".to_owned())], Some("".to_owned())) - .expect(&format!("failed to exec `{}`", self.config.adb_path)); - - // get exitcode of result - runargs = Vec::new(); - runargs.push("shell".to_owned()); - runargs.push("cat".to_owned()); - runargs.push(format!("{}/{}.exitcode", self.config.adb_test_dir, prog_short)); - - let procsrv::Result{ out: exitcode_out, err: _, status: _ } = - procsrv::run("", - &self.config.adb_path, - None, - &runargs, - vec![("".to_owned(), "".to_owned())], - Some("".to_owned())) - .expect(&format!("failed to exec `{}`", self.config.adb_path)); - - let mut exitcode: i32 = 0; - for c in exitcode_out.chars() { - if !c.is_numeric() { break; } - exitcode = exitcode * 10 + match c { - '0' ... '9' => c as i32 - ('0' as i32), - _ => 101, - } - } - - // get stdout of result - runargs = Vec::new(); - runargs.push("shell".to_owned()); - runargs.push("cat".to_owned()); - runargs.push(format!("{}/{}.stdout", self.config.adb_test_dir, prog_short)); - - let procsrv::Result{ out: stdout_out, err: _, status: _ } = - procsrv::run("", - &self.config.adb_path, - None, - &runargs, - vec![("".to_owned(), "".to_owned())], - Some("".to_owned())) - .expect(&format!("failed to exec `{}`", self.config.adb_path)); - - // get stderr of result - runargs = Vec::new(); - runargs.push("shell".to_owned()); - runargs.push("cat".to_owned()); - runargs.push(format!("{}/{}.stderr", self.config.adb_test_dir, prog_short)); - - let procsrv::Result{ out: stderr_out, err: _, status: _ } = - procsrv::run("", - &self.config.adb_path, - None, - &runargs, - vec![("".to_owned(), "".to_owned())], - Some("".to_owned())) - .expect(&format!("failed to exec `{}`", self.config.adb_path)); - - self.dump_output(&stdout_out, &stderr_out); - - ProcRes { - status: Status::Parsed(exitcode), - stdout: stdout_out, - stderr: stderr_out, - cmdline: cmdline - } - } - - fn _arm_push_aux_shared_library(&self) { - let tdir = self.aux_output_dir_name(); - - let dirs = fs::read_dir(&tdir).unwrap(); - for file in dirs { - let file = file.unwrap().path(); - if file.extension().and_then(|s| s.to_str()) == Some("so") { - // FIXME (#9639): This needs to handle non-utf8 paths - let copy_result = procsrv::run("", - &self.config.adb_path, - None, - &[ - "push".to_owned(), - file.to_str() - .unwrap() - .to_owned(), - self.config.adb_test_dir.to_owned(), - ], - vec![("".to_owned(), - "".to_owned())], - Some("".to_owned())) - .expect(&format!("failed to exec `{}`", self.config.adb_path)); - - if self.config.verbose { - println!("push ({}) {:?} {} {}", - self.config.target, file.display(), - copy_result.out, copy_result.err); - } - } - } - } - // codegen tests (using FileCheck) fn compile_test_and_save_ir(&self) -> ProcRes { @@ -2347,7 +2186,7 @@ actual:\n\ let output = cmd.output().expect("failed to spawn `make`"); if !output.status.success() { let res = ProcRes { - status: Status::Normal(output.status), + status: output.status, stdout: String::from_utf8_lossy(&output.stdout).into_owned(), stderr: String::from_utf8_lossy(&output.stderr).into_owned(), cmdline: format!("{:?}", cmd), @@ -2594,17 +2433,12 @@ struct ProcArgs { } pub struct ProcRes { - status: Status, + status: ExitStatus, stdout: String, stderr: String, cmdline: String, } -enum Status { - Parsed(i32), - Normal(ExitStatus), -} - impl ProcRes { pub fn fatal(&self, err: Option<&str>) -> ! { if let Some(e) = err { @@ -2628,31 +2462,6 @@ impl ProcRes { } } -impl Status { - fn code(&self) -> Option { - match *self { - Status::Parsed(i) => Some(i), - Status::Normal(ref e) => e.code(), - } - } - - fn success(&self) -> bool { - match *self { - Status::Parsed(i) => i == 0, - Status::Normal(ref e) => e.success(), - } - } -} - -impl fmt::Display for Status { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - Status::Parsed(i) => write!(f, "exit code: {}", i), - Status::Normal(ref e) => e.fmt(f), - } - } -} - enum TargetLocation { ThisFile(PathBuf), ThisDirectory(PathBuf), diff --git a/src/tools/qemu-test-client/Cargo.toml b/src/tools/qemu-test-client/Cargo.toml deleted file mode 100644 index eb326c01de4..00000000000 --- a/src/tools/qemu-test-client/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "qemu-test-client" -version = "0.1.0" -authors = ["The Rust Project Developers"] - -[dependencies] diff --git a/src/tools/qemu-test-client/src/main.rs b/src/tools/qemu-test-client/src/main.rs deleted file mode 100644 index b7ff4116eb5..00000000000 --- a/src/tools/qemu-test-client/src/main.rs +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright 2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -/// This is a small client program intended to pair with `qemu-test-server` in -/// this repository. This client connects to the server over TCP and is used to -/// push artifacts and run tests on the server instead of locally. -/// -/// Here is also where we bake in the support to spawn the QEMU emulator as -/// well. - -use std::env; -use std::fs::File; -use std::io::prelude::*; -use std::io::{self, BufWriter}; -use std::net::TcpStream; -use std::path::Path; -use std::process::{Command, Stdio}; -use std::thread; -use std::time::Duration; - -macro_rules! t { - ($e:expr) => (match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {}", stringify!($e), e), - }) -} - -fn main() { - let mut args = env::args().skip(1); - - match &args.next().unwrap()[..] { - "spawn-emulator" => { - spawn_emulator(Path::new(&args.next().unwrap()), - Path::new(&args.next().unwrap())) - } - "push" => { - push(Path::new(&args.next().unwrap())) - } - "run" => { - run(args.next().unwrap(), args.collect()) - } - cmd => panic!("unknown command: {}", cmd), - } -} - -fn spawn_emulator(rootfs: &Path, tmpdir: &Path) { - // Generate a new rootfs image now that we've updated the test server - // executable. This is the equivalent of: - // - // find $rootfs -print 0 | cpio --null -o --format=newc > rootfs.img - let rootfs_img = tmpdir.join("rootfs.img"); - let mut cmd = Command::new("cpio"); - cmd.arg("--null") - .arg("-o") - .arg("--format=newc") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .current_dir(rootfs); - let mut child = t!(cmd.spawn()); - let mut stdin = child.stdin.take().unwrap(); - let rootfs = rootfs.to_path_buf(); - thread::spawn(move || add_files(&mut stdin, &rootfs, &rootfs)); - t!(io::copy(&mut child.stdout.take().unwrap(), - &mut t!(File::create(&rootfs_img)))); - assert!(t!(child.wait()).success()); - - // Start up the emulator, in the background - let mut cmd = Command::new("qemu-system-arm"); - cmd.arg("-M").arg("vexpress-a15") - .arg("-m").arg("1024") - .arg("-kernel").arg("/tmp/zImage") - .arg("-initrd").arg(&rootfs_img) - .arg("-dtb").arg("/tmp/vexpress-v2p-ca15-tc1.dtb") - .arg("-append").arg("console=ttyAMA0 root=/dev/ram rdinit=/sbin/init init=/sbin/init") - .arg("-nographic") - .arg("-redir").arg("tcp:12345::12345"); - t!(cmd.spawn()); - - // Wait for the emulator to come online - loop { - let dur = Duration::from_millis(100); - if let Ok(mut client) = TcpStream::connect("127.0.0.1:12345") { - t!(client.set_read_timeout(Some(dur))); - t!(client.set_write_timeout(Some(dur))); - if client.write_all(b"ping").is_ok() { - let mut b = [0; 4]; - if client.read_exact(&mut b).is_ok() { - break - } - } - } - thread::sleep(dur); - } - - fn add_files(w: &mut Write, root: &Path, cur: &Path) { - for entry in t!(cur.read_dir()) { - let entry = t!(entry); - let path = entry.path(); - let to_print = path.strip_prefix(root).unwrap(); - t!(write!(w, "{}\u{0}", to_print.to_str().unwrap())); - if t!(entry.file_type()).is_dir() { - add_files(w, root, &path); - } - } - } -} - -fn push(path: &Path) { - let client = t!(TcpStream::connect("127.0.0.1:12345")); - let mut client = BufWriter::new(client); - t!(client.write_all(b"push")); - t!(client.write_all(path.file_name().unwrap().to_str().unwrap().as_bytes())); - t!(client.write_all(&[0])); - let mut file = t!(File::open(path)); - t!(io::copy(&mut file, &mut client)); - t!(client.flush()); - println!("done pushing {:?}", path); -} - -fn run(files: String, args: Vec) { - let client = t!(TcpStream::connect("127.0.0.1:12345")); - let mut client = BufWriter::new(client); - t!(client.write_all(b"run ")); - - // Send over the args - for arg in args { - t!(client.write_all(arg.as_bytes())); - t!(client.write_all(&[0])); - } - t!(client.write_all(&[0])); - - // Send over env vars - for (k, v) in env::vars() { - if k != "PATH" && k != "LD_LIBRARY_PATH" { - t!(client.write_all(k.as_bytes())); - t!(client.write_all(&[0])); - t!(client.write_all(v.as_bytes())); - t!(client.write_all(&[0])); - } - } - t!(client.write_all(&[0])); - - // Send over support libraries - let mut files = files.split(':'); - let exe = files.next().unwrap(); - for file in files.map(Path::new) { - t!(client.write_all(file.file_name().unwrap().to_str().unwrap().as_bytes())); - t!(client.write_all(&[0])); - send(&file, &mut client); - } - t!(client.write_all(&[0])); - - // Send over the client executable as the last piece - send(exe.as_ref(), &mut client); - - println!("uploaded {:?}, waiting for result", exe); - - // Ok now it's time to read all the output. We're receiving "frames" - // representing stdout/stderr, so we decode all that here. - let mut header = [0; 5]; - let mut stderr_done = false; - let mut stdout_done = false; - let mut client = t!(client.into_inner()); - let mut stdout = io::stdout(); - let mut stderr = io::stderr(); - while !stdout_done || !stderr_done { - t!(client.read_exact(&mut header)); - let amt = ((header[1] as u64) << 24) | - ((header[2] as u64) << 16) | - ((header[3] as u64) << 8) | - ((header[4] as u64) << 0); - if header[0] == 0 { - if amt == 0 { - stdout_done = true; - } else { - t!(io::copy(&mut (&mut client).take(amt), &mut stdout)); - t!(stdout.flush()); - } - } else { - if amt == 0 { - stderr_done = true; - } else { - t!(io::copy(&mut (&mut client).take(amt), &mut stderr)); - t!(stderr.flush()); - } - } - } - - // Finally, read out the exit status - let mut status = [0; 5]; - t!(client.read_exact(&mut status)); - let code = ((status[1] as i32) << 24) | - ((status[2] as i32) << 16) | - ((status[3] as i32) << 8) | - ((status[4] as i32) << 0); - if status[0] == 0 { - std::process::exit(code); - } else { - println!("died due to signal {}", code); - std::process::exit(3); - } -} - -fn send(path: &Path, dst: &mut Write) { - let mut file = t!(File::open(&path)); - let amt = t!(file.metadata()).len(); - t!(dst.write_all(&[ - (amt >> 24) as u8, - (amt >> 16) as u8, - (amt >> 8) as u8, - (amt >> 0) as u8, - ])); - t!(io::copy(&mut file, dst)); -} diff --git a/src/tools/qemu-test-server/Cargo.toml b/src/tools/qemu-test-server/Cargo.toml deleted file mode 100644 index af445a25935..00000000000 --- a/src/tools/qemu-test-server/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "qemu-test-server" -version = "0.1.0" -authors = ["The Rust Project Developers"] - -[dependencies] diff --git a/src/tools/qemu-test-server/src/main.rs b/src/tools/qemu-test-server/src/main.rs deleted file mode 100644 index 1c5d7b915ba..00000000000 --- a/src/tools/qemu-test-server/src/main.rs +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -/// This is a small server which is intended to run inside of an emulator. This -/// server pairs with the `qemu-test-client` program in this repository. The -/// `qemu-test-client` connects to this server over a TCP socket and performs -/// work such as: -/// -/// 1. Pushing shared libraries to the server -/// 2. Running tests through the server -/// -/// The server supports running tests concurrently and also supports tests -/// themselves having support libraries. All data over the TCP sockets is in a -/// basically custom format suiting our needs. - -use std::fs::{self, File, Permissions}; -use std::io::prelude::*; -use std::io::{self, BufReader}; -use std::net::{TcpListener, TcpStream}; -use std::os::unix::prelude::*; -use std::sync::{Arc, Mutex}; -use std::path::Path; -use std::str; -use std::sync::atomic::{AtomicUsize, ATOMIC_USIZE_INIT, Ordering}; -use std::thread; -use std::process::{Command, Stdio}; - -macro_rules! t { - ($e:expr) => (match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {}", stringify!($e), e), - }) -} - -static TEST: AtomicUsize = ATOMIC_USIZE_INIT; - -fn main() { - println!("starting test server"); - let listener = t!(TcpListener::bind("10.0.2.15:12345")); - println!("listening!"); - - let work = Path::new("/tmp/work"); - t!(fs::create_dir_all(work)); - - let lock = Arc::new(Mutex::new(())); - - for socket in listener.incoming() { - let mut socket = t!(socket); - let mut buf = [0; 4]; - t!(socket.read_exact(&mut buf)); - if &buf[..] == b"ping" { - t!(socket.write_all(b"pong")); - } else if &buf[..] == b"push" { - handle_push(socket, work); - } else if &buf[..] == b"run " { - let lock = lock.clone(); - thread::spawn(move || handle_run(socket, work, &lock)); - } else { - panic!("unknown command {:?}", buf); - } - } -} - -fn handle_push(socket: TcpStream, work: &Path) { - let mut reader = BufReader::new(socket); - let mut filename = Vec::new(); - t!(reader.read_until(0, &mut filename)); - filename.pop(); // chop off the 0 - let filename = t!(str::from_utf8(&filename)); - - let path = work.join(filename); - t!(io::copy(&mut reader, &mut t!(File::create(&path)))); - t!(fs::set_permissions(&path, Permissions::from_mode(0o755))); -} - -struct RemoveOnDrop<'a> { - inner: &'a Path, -} - -impl<'a> Drop for RemoveOnDrop<'a> { - fn drop(&mut self) { - t!(fs::remove_dir_all(self.inner)); - } -} - -fn handle_run(socket: TcpStream, work: &Path, lock: &Mutex<()>) { - let mut arg = Vec::new(); - let mut reader = BufReader::new(socket); - - // Allocate ourselves a directory that we'll delete when we're done to save - // space. - let n = TEST.fetch_add(1, Ordering::SeqCst); - let path = work.join(format!("test{}", n)); - let exe = path.join("exe"); - t!(fs::create_dir(&path)); - let _a = RemoveOnDrop { inner: &path }; - - // First up we'll get a list of arguments delimited with 0 bytes. An empty - // argument means that we're done. - let mut cmd = Command::new(&exe); - while t!(reader.read_until(0, &mut arg)) > 1 { - cmd.arg(t!(str::from_utf8(&arg[..arg.len() - 1]))); - arg.truncate(0); - } - - // Next we'll get a bunch of env vars in pairs delimited by 0s as well - arg.truncate(0); - while t!(reader.read_until(0, &mut arg)) > 1 { - let key_len = arg.len() - 1; - let val_len = t!(reader.read_until(0, &mut arg)) - 1; - { - let key = &arg[..key_len]; - let val = &arg[key_len + 1..][..val_len]; - let key = t!(str::from_utf8(key)); - let val = t!(str::from_utf8(val)); - cmd.env(key, val); - } - arg.truncate(0); - } - - // The section of code from here down to where we drop the lock is going to - // be a critical section for us. On Linux you can't execute a file which is - // open somewhere for writing, as you'll receive the error "text file busy". - // Now here we never have the text file open for writing when we spawn it, - // so why do we still need a critical section? - // - // Process spawning first involves a `fork` on Unix, which clones all file - // descriptors into the child process. This means that it's possible for us - // to open the file for writing (as we're downloading it), then some other - // thread forks, then we close the file and try to exec. At that point the - // other thread created a child process with the file open for writing, and - // we attempt to execute it, so we get an error. - // - // This race is resolve by ensuring that only one thread can writ ethe file - // and spawn a child process at once. Kinda an unfortunate solution, but we - // don't have many other choices with this sort of setup! - // - // In any case the lock is acquired here, before we start writing any files. - // It's then dropped just after we spawn the child. That way we don't lock - // the execution of the child, just the creation of its files. - let lock = lock.lock(); - - // Next there's a list of dynamic libraries preceded by their filenames. - arg.truncate(0); - while t!(reader.read_until(0, &mut arg)) > 1 { - let dst = path.join(t!(str::from_utf8(&arg[..arg.len() - 1]))); - let amt = read_u32(&mut reader) as u64; - t!(io::copy(&mut reader.by_ref().take(amt), - &mut t!(File::create(&dst)))); - t!(fs::set_permissions(&dst, Permissions::from_mode(0o755))); - arg.truncate(0); - } - - // Finally we'll get the binary. The other end will tell us how big the - // binary is and then we'll download it all to the exe path we calculated - // earlier. - let amt = read_u32(&mut reader) as u64; - t!(io::copy(&mut reader.by_ref().take(amt), - &mut t!(File::create(&exe)))); - t!(fs::set_permissions(&exe, Permissions::from_mode(0o755))); - - // Support libraries were uploaded to `work` earlier, so make sure that's - // in `LD_LIBRARY_PATH`. Also include our own current dir which may have - // had some libs uploaded. - cmd.env("LD_LIBRARY_PATH", - format!("{}:{}", work.display(), path.display())); - - // Spawn the child and ferry over stdout/stderr to the socket in a framed - // fashion (poor man's style) - let mut child = t!(cmd.stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()); - drop(lock); - let mut stdout = child.stdout.take().unwrap(); - let mut stderr = child.stderr.take().unwrap(); - let socket = Arc::new(Mutex::new(reader.into_inner())); - let socket2 = socket.clone(); - let thread = thread::spawn(move || my_copy(&mut stdout, 0, &*socket2)); - my_copy(&mut stderr, 1, &*socket); - thread.join().unwrap(); - - // Finally send over the exit status. - let status = t!(child.wait()); - let (which, code) = match status.code() { - Some(n) => (0, n), - None => (1, status.signal().unwrap()), - }; - t!(socket.lock().unwrap().write_all(&[ - which, - (code >> 24) as u8, - (code >> 16) as u8, - (code >> 8) as u8, - (code >> 0) as u8, - ])); -} - -fn my_copy(src: &mut Read, which: u8, dst: &Mutex) { - let mut b = [0; 1024]; - loop { - let n = t!(src.read(&mut b)); - let mut dst = dst.lock().unwrap(); - t!(dst.write_all(&[ - which, - (n >> 24) as u8, - (n >> 16) as u8, - (n >> 8) as u8, - (n >> 0) as u8, - ])); - if n > 0 { - t!(dst.write_all(&b[..n])); - } else { - break - } - } -} - -fn read_u32(r: &mut Read) -> u32 { - let mut len = [0; 4]; - t!(r.read_exact(&mut len)); - ((len[0] as u32) << 24) | - ((len[1] as u32) << 16) | - ((len[2] as u32) << 8) | - ((len[3] as u32) << 0) -} diff --git a/src/tools/remote-test-client/Cargo.toml b/src/tools/remote-test-client/Cargo.toml new file mode 100644 index 00000000000..54739101f1e --- /dev/null +++ b/src/tools/remote-test-client/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "remote-test-client" +version = "0.1.0" +authors = ["The Rust Project Developers"] + +[dependencies] diff --git a/src/tools/remote-test-client/src/main.rs b/src/tools/remote-test-client/src/main.rs new file mode 100644 index 00000000000..265354ff800 --- /dev/null +++ b/src/tools/remote-test-client/src/main.rs @@ -0,0 +1,281 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +/// This is a small client program intended to pair with `remote-test-server` in +/// this repository. This client connects to the server over TCP and is used to +/// push artifacts and run tests on the server instead of locally. +/// +/// Here is also where we bake in the support to spawn the QEMU emulator as +/// well. + +use std::env; +use std::fs::{self, File}; +use std::io::prelude::*; +use std::io::{self, BufWriter}; +use std::net::TcpStream; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::Duration; + +macro_rules! t { + ($e:expr) => (match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + }) +} + +fn main() { + let mut args = env::args().skip(1); + + match &args.next().unwrap()[..] { + "spawn-emulator" => { + spawn_emulator(&args.next().unwrap(), + Path::new(&args.next().unwrap()), + Path::new(&args.next().unwrap()), + args.next().map(|s| s.into())) + } + "push" => { + push(Path::new(&args.next().unwrap())) + } + "run" => { + run(args.next().unwrap(), args.collect()) + } + cmd => panic!("unknown command: {}", cmd), + } +} + +fn spawn_emulator(target: &str, + server: &Path, + tmpdir: &Path, + rootfs: Option) { + if target.contains("android") { + start_android_emulator(server); + } else { + let rootfs = rootfs.as_ref().expect("need rootfs on non-android"); + start_qemu_emulator(rootfs, server, tmpdir); + } + + // Wait for the emulator to come online + loop { + let dur = Duration::from_millis(100); + if let Ok(mut client) = TcpStream::connect("127.0.0.1:12345") { + t!(client.set_read_timeout(Some(dur))); + t!(client.set_write_timeout(Some(dur))); + if client.write_all(b"ping").is_ok() { + let mut b = [0; 4]; + if client.read_exact(&mut b).is_ok() { + break + } + } + } + thread::sleep(dur); + } +} + +fn start_android_emulator(server: &Path) { + println!("waiting for device to come online"); + let status = Command::new("adb") + .arg("wait-for-device") + .status() + .unwrap(); + assert!(status.success()); + + println!("pushing server"); + let status = Command::new("adb") + .arg("push") + .arg(server) + .arg("/data/tmp/testd") + .status() + .unwrap(); + assert!(status.success()); + + println!("forwarding tcp"); + let status = Command::new("adb") + .arg("forward") + .arg("tcp:12345") + .arg("tcp:12345") + .status() + .unwrap(); + assert!(status.success()); + + println!("executing server"); + Command::new("adb") + .arg("shell") + .arg("/data/tmp/testd") + .spawn() + .unwrap(); +} + +fn start_qemu_emulator(rootfs: &Path, server: &Path, tmpdir: &Path) { + // Generate a new rootfs image now that we've updated the test server + // executable. This is the equivalent of: + // + // find $rootfs -print 0 | cpio --null -o --format=newc > rootfs.img + t!(fs::copy(server, rootfs.join("testd"))); + let rootfs_img = tmpdir.join("rootfs.img"); + let mut cmd = Command::new("cpio"); + cmd.arg("--null") + .arg("-o") + .arg("--format=newc") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .current_dir(rootfs); + let mut child = t!(cmd.spawn()); + let mut stdin = child.stdin.take().unwrap(); + let rootfs = rootfs.to_path_buf(); + thread::spawn(move || add_files(&mut stdin, &rootfs, &rootfs)); + t!(io::copy(&mut child.stdout.take().unwrap(), + &mut t!(File::create(&rootfs_img)))); + assert!(t!(child.wait()).success()); + + // Start up the emulator, in the background + let mut cmd = Command::new("qemu-system-arm"); + cmd.arg("-M").arg("vexpress-a15") + .arg("-m").arg("1024") + .arg("-kernel").arg("/tmp/zImage") + .arg("-initrd").arg(&rootfs_img) + .arg("-dtb").arg("/tmp/vexpress-v2p-ca15-tc1.dtb") + .arg("-append").arg("console=ttyAMA0 root=/dev/ram rdinit=/sbin/init init=/sbin/init") + .arg("-nographic") + .arg("-redir").arg("tcp:12345::12345"); + t!(cmd.spawn()); + + fn add_files(w: &mut Write, root: &Path, cur: &Path) { + for entry in t!(cur.read_dir()) { + let entry = t!(entry); + let path = entry.path(); + let to_print = path.strip_prefix(root).unwrap(); + t!(write!(w, "{}\u{0}", to_print.to_str().unwrap())); + if t!(entry.file_type()).is_dir() { + add_files(w, root, &path); + } + } + } +} + +fn push(path: &Path) { + let client = t!(TcpStream::connect("127.0.0.1:12345")); + let mut client = BufWriter::new(client); + t!(client.write_all(b"push")); + send(path, &mut client); + t!(client.flush()); + + // Wait for an acknowledgement that all the data was received. No idea + // why this is necessary, seems like it shouldn't be! + let mut client = client.into_inner().unwrap(); + let mut buf = [0; 4]; + t!(client.read_exact(&mut buf)); + assert_eq!(&buf, b"ack "); + println!("done pushing {:?}", path); +} + +fn run(files: String, args: Vec) { + let client = t!(TcpStream::connect("127.0.0.1:12345")); + let mut client = BufWriter::new(client); + t!(client.write_all(b"run ")); + + // Send over the args + for arg in args { + t!(client.write_all(arg.as_bytes())); + t!(client.write_all(&[0])); + } + t!(client.write_all(&[0])); + + // Send over env vars + // + // Don't send over *everything* though as some env vars are set by and used + // by the client. + for (k, v) in env::vars() { + match &k[..] { + "PATH" | + "LD_LIBRARY_PATH" | + "PWD" => continue, + _ => {} + } + t!(client.write_all(k.as_bytes())); + t!(client.write_all(&[0])); + t!(client.write_all(v.as_bytes())); + t!(client.write_all(&[0])); + } + t!(client.write_all(&[0])); + + // Send over support libraries + let mut files = files.split(':'); + let exe = files.next().unwrap(); + for file in files.map(Path::new) { + send(&file, &mut client); + } + t!(client.write_all(&[0])); + + // Send over the client executable as the last piece + send(exe.as_ref(), &mut client); + + println!("uploaded {:?}, waiting for result", exe); + + // Ok now it's time to read all the output. We're receiving "frames" + // representing stdout/stderr, so we decode all that here. + let mut header = [0; 5]; + let mut stderr_done = false; + let mut stdout_done = false; + let mut client = t!(client.into_inner()); + let mut stdout = io::stdout(); + let mut stderr = io::stderr(); + while !stdout_done || !stderr_done { + t!(client.read_exact(&mut header)); + let amt = ((header[1] as u64) << 24) | + ((header[2] as u64) << 16) | + ((header[3] as u64) << 8) | + ((header[4] as u64) << 0); + if header[0] == 0 { + if amt == 0 { + stdout_done = true; + } else { + t!(io::copy(&mut (&mut client).take(amt), &mut stdout)); + t!(stdout.flush()); + } + } else { + if amt == 0 { + stderr_done = true; + } else { + t!(io::copy(&mut (&mut client).take(amt), &mut stderr)); + t!(stderr.flush()); + } + } + } + + // Finally, read out the exit status + let mut status = [0; 5]; + t!(client.read_exact(&mut status)); + let code = ((status[1] as i32) << 24) | + ((status[2] as i32) << 16) | + ((status[3] as i32) << 8) | + ((status[4] as i32) << 0); + if status[0] == 0 { + std::process::exit(code); + } else { + println!("died due to signal {}", code); + std::process::exit(3); + } +} + +fn send(path: &Path, dst: &mut Write) { + t!(dst.write_all(path.file_name().unwrap().to_str().unwrap().as_bytes())); + t!(dst.write_all(&[0])); + let mut file = t!(File::open(&path)); + let amt = t!(file.metadata()).len(); + t!(dst.write_all(&[ + (amt >> 24) as u8, + (amt >> 16) as u8, + (amt >> 8) as u8, + (amt >> 0) as u8, + ])); + t!(io::copy(&mut file, dst)); +} diff --git a/src/tools/remote-test-server/Cargo.toml b/src/tools/remote-test-server/Cargo.toml new file mode 100644 index 00000000000..8704296289e --- /dev/null +++ b/src/tools/remote-test-server/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "remote-test-server" +version = "0.1.0" +authors = ["The Rust Project Developers"] + +[dependencies] diff --git a/src/tools/remote-test-server/src/main.rs b/src/tools/remote-test-server/src/main.rs new file mode 100644 index 00000000000..308ccdbef77 --- /dev/null +++ b/src/tools/remote-test-server/src/main.rs @@ -0,0 +1,257 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +/// This is a small server which is intended to run inside of an emulator. This +/// server pairs with the `remote-test-client` program in this repository. The +/// `remote-test-client` connects to this server over a TCP socket and performs +/// work such as: +/// +/// 1. Pushing shared libraries to the server +/// 2. Running tests through the server +/// +/// The server supports running tests concurrently and also supports tests +/// themselves having support libraries. All data over the TCP sockets is in a +/// basically custom format suiting our needs. + +use std::cmp; +use std::fs::{self, File, Permissions}; +use std::io::prelude::*; +use std::io::{self, BufReader}; +use std::net::{TcpListener, TcpStream}; +use std::os::unix::prelude::*; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::str; +use std::sync::atomic::{AtomicUsize, ATOMIC_USIZE_INIT, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; + +macro_rules! t { + ($e:expr) => (match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + }) +} + +static TEST: AtomicUsize = ATOMIC_USIZE_INIT; + +fn main() { + println!("starting test server"); + let (listener, work) = if cfg!(target_os = "android") { + (t!(TcpListener::bind("0.0.0.0:12345")), "/data/tmp/work") + } else { + (t!(TcpListener::bind("10.0.2.15:12345")), "/tmp/work") + }; + println!("listening!"); + + let work = Path::new(work); + t!(fs::create_dir_all(work)); + + let lock = Arc::new(Mutex::new(())); + + for socket in listener.incoming() { + let mut socket = t!(socket); + let mut buf = [0; 4]; + if socket.read_exact(&mut buf).is_err() { + continue + } + if &buf[..] == b"ping" { + t!(socket.write_all(b"pong")); + } else if &buf[..] == b"push" { + handle_push(socket, work); + } else if &buf[..] == b"run " { + let lock = lock.clone(); + thread::spawn(move || handle_run(socket, work, &lock)); + } else { + panic!("unknown command {:?}", buf); + } + } +} + +fn handle_push(socket: TcpStream, work: &Path) { + let mut reader = BufReader::new(socket); + recv(&work, &mut reader); + + let mut socket = reader.into_inner(); + t!(socket.write_all(b"ack ")); +} + +struct RemoveOnDrop<'a> { + inner: &'a Path, +} + +impl<'a> Drop for RemoveOnDrop<'a> { + fn drop(&mut self) { + t!(fs::remove_dir_all(self.inner)); + } +} + +fn handle_run(socket: TcpStream, work: &Path, lock: &Mutex<()>) { + let mut arg = Vec::new(); + let mut reader = BufReader::new(socket); + + // Allocate ourselves a directory that we'll delete when we're done to save + // space. + let n = TEST.fetch_add(1, Ordering::SeqCst); + let path = work.join(format!("test{}", n)); + t!(fs::create_dir(&path)); + let _a = RemoveOnDrop { inner: &path }; + + // First up we'll get a list of arguments delimited with 0 bytes. An empty + // argument means that we're done. + let mut args = Vec::new(); + while t!(reader.read_until(0, &mut arg)) > 1 { + args.push(t!(str::from_utf8(&arg[..arg.len() - 1])).to_string()); + arg.truncate(0); + } + + // Next we'll get a bunch of env vars in pairs delimited by 0s as well + let mut env = Vec::new(); + arg.truncate(0); + while t!(reader.read_until(0, &mut arg)) > 1 { + let key_len = arg.len() - 1; + let val_len = t!(reader.read_until(0, &mut arg)) - 1; + { + let key = &arg[..key_len]; + let val = &arg[key_len + 1..][..val_len]; + let key = t!(str::from_utf8(key)).to_string(); + let val = t!(str::from_utf8(val)).to_string(); + env.push((key, val)); + } + arg.truncate(0); + } + + // The section of code from here down to where we drop the lock is going to + // be a critical section for us. On Linux you can't execute a file which is + // open somewhere for writing, as you'll receive the error "text file busy". + // Now here we never have the text file open for writing when we spawn it, + // so why do we still need a critical section? + // + // Process spawning first involves a `fork` on Unix, which clones all file + // descriptors into the child process. This means that it's possible for us + // to open the file for writing (as we're downloading it), then some other + // thread forks, then we close the file and try to exec. At that point the + // other thread created a child process with the file open for writing, and + // we attempt to execute it, so we get an error. + // + // This race is resolve by ensuring that only one thread can writ ethe file + // and spawn a child process at once. Kinda an unfortunate solution, but we + // don't have many other choices with this sort of setup! + // + // In any case the lock is acquired here, before we start writing any files. + // It's then dropped just after we spawn the child. That way we don't lock + // the execution of the child, just the creation of its files. + let lock = lock.lock(); + + // Next there's a list of dynamic libraries preceded by their filenames. + while t!(reader.fill_buf())[0] != 0 { + recv(&path, &mut reader); + } + assert_eq!(t!(reader.read(&mut [0])), 1); + + // Finally we'll get the binary. The other end will tell us how big the + // binary is and then we'll download it all to the exe path we calculated + // earlier. + let exe = recv(&path, &mut reader); + + let mut cmd = Command::new(&exe); + for arg in args { + cmd.arg(arg); + } + for (k, v) in env { + cmd.env(k, v); + } + + // Support libraries were uploaded to `work` earlier, so make sure that's + // in `LD_LIBRARY_PATH`. Also include our own current dir which may have + // had some libs uploaded. + cmd.env("LD_LIBRARY_PATH", + format!("{}:{}", work.display(), path.display())); + + // Spawn the child and ferry over stdout/stderr to the socket in a framed + // fashion (poor man's style) + let mut child = t!(cmd.stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()); + drop(lock); + let mut stdout = child.stdout.take().unwrap(); + let mut stderr = child.stderr.take().unwrap(); + let socket = Arc::new(Mutex::new(reader.into_inner())); + let socket2 = socket.clone(); + let thread = thread::spawn(move || my_copy(&mut stdout, 0, &*socket2)); + my_copy(&mut stderr, 1, &*socket); + thread.join().unwrap(); + + // Finally send over the exit status. + let status = t!(child.wait()); + let (which, code) = match status.code() { + Some(n) => (0, n), + None => (1, status.signal().unwrap()), + }; + t!(socket.lock().unwrap().write_all(&[ + which, + (code >> 24) as u8, + (code >> 16) as u8, + (code >> 8) as u8, + (code >> 0) as u8, + ])); +} + +fn recv(dir: &Path, io: &mut B) -> PathBuf { + let mut filename = Vec::new(); + t!(io.read_until(0, &mut filename)); + + // We've got some tests with *really* long names. We try to name the test + // executable the same on the target as it is on the host to aid with + // debugging, but the targets we're emulating are often more restrictive + // than the hosts as well. + // + // To ensure we can run a maximum number of tests without modifications we + // just arbitrarily truncate the filename to 50 bytes. That should + // hopefully allow us to still identify what's running while staying under + // the filesystem limits. + let len = cmp::min(filename.len() - 1, 50); + let dst = dir.join(t!(str::from_utf8(&filename[..len]))); + let amt = read_u32(io) as u64; + t!(io::copy(&mut io.take(amt), + &mut t!(File::create(&dst)))); + t!(fs::set_permissions(&dst, Permissions::from_mode(0o755))); + return dst +} + +fn my_copy(src: &mut Read, which: u8, dst: &Mutex) { + let mut b = [0; 1024]; + loop { + let n = t!(src.read(&mut b)); + let mut dst = dst.lock().unwrap(); + t!(dst.write_all(&[ + which, + (n >> 24) as u8, + (n >> 16) as u8, + (n >> 8) as u8, + (n >> 0) as u8, + ])); + if n > 0 { + t!(dst.write_all(&b[..n])); + } else { + break + } + } +} + +fn read_u32(r: &mut Read) -> u32 { + let mut len = [0; 4]; + t!(r.read_exact(&mut len)); + ((len[0] as u32) << 24) | + ((len[1] as u32) << 16) | + ((len[2] as u32) << 8) | + ((len[3] as u32) << 0) +} -- cgit 1.4.1-3-g733a5