Add support for test suites emulated in QEMU

This commit adds support to the build system to execute test suites that cannot
run natively but can instead run inside of a QEMU emulator. A proof-of-concept
builder was added for the `arm-unknown-linux-gnueabihf` target to show off how
this might work.

In general the architecture is to have a server running inside of the emulator
which a local client connects to. The protocol between the server/client
supports compiling tests on the host and running them on the target inside the
emulator.

Closes #33114

7 files changed, 517 insertions, 1 deletions

diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
index 1aeb76c0a0e..eb8cdcee6e6 100644
--- a/src/tools/compiletest/src/common.rs
+++ b/src/tools/compiletest/src/common.rs
@@ -185,6 +185,9 @@ pub struct Config {
     // Print one character per test instead of one line
     pub quiet: bool,
 
+    // where to find the qemu test client process, if we're using it
+    pub qemu_test_client: Option<PathBuf>,
+
     // Configuration for various run-make tests frobbing things like C compilers
     // or querying about various LLVM component information.
     pub cc: String,
diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs
index 43d02479fb1..40ba66a15d5 100644
--- a/src/tools/compiletest/src/main.rs
+++ b/src/tools/compiletest/src/main.rs
@@ -116,6 +116,7 @@ pub fn parse_config(args: Vec<String> ) -> Config {
           reqopt("", "llvm-components", "list of LLVM components built in", "LIST"),
           reqopt("", "llvm-cxxflags", "C++ flags for LLVM", "FLAGS"),
           optopt("", "nodejs", "the name of nodejs", "PATH"),
+          optopt("", "qemu-test-client", "path to the qemu test client", "PATH"),
           optflag("h", "help", "show this message")];
 
     let (argv0, args_) = args.split_first().unwrap();
@@ -196,6 +197,7 @@ pub fn parse_config(args: Vec<String> ) -> Config {
         lldb_python_dir: matches.opt_str("lldb-python-dir"),
         verbose: matches.opt_present("verbose"),
         quiet: matches.opt_present("quiet"),
+        qemu_test_client: matches.opt_str("qemu-test-client").map(PathBuf::from),
 
         cc: matches.opt_str("cc").unwrap(),
         cxx: matches.opt_str("cxx").unwrap(),
@@ -302,6 +304,14 @@ pub fn run_tests(config: &Config) {
             // time.
             env::set_var("RUST_TEST_THREADS", "1");
         }
+
+        DebugInfoGdb => {
+            if config.qemu_test_client.is_some() {
+                println!("WARNING: debuginfo tests are not available when \
+                          testing with QEMU");
+                return
+            }
+        }
         _ => { /* proceed */ }
     }
 
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index 05d6e21e9aa..04bd792be5c 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -1190,7 +1190,45 @@ actual:\n\
             "arm-linux-androideabi" | "armv7-linux-androideabi" | "aarch64-linux-android" => {
                 self._arm_exec_compiled_test(env)
             }
-            _=> {
+
+            // This is pretty similar to below, we're transforming:
+            //
+            //      program arg1 arg2
+            //
+            // into
+            //
+            //      qemu-test-client run program:support-lib.so arg1 arg2
+            //
+            // The test-client program will upload `program` to the emulator
+            // along with all other support libraries listed (in this case
+            // `support-lib.so`. It will then execute the program on the
+            // emulator with the arguments specified (in the environment we give
+            // the process) and then report back the same result.
+            _ if self.config.qemu_test_client.is_some() => {
+                let aux_dir = self.aux_output_dir_name();
+                let mut args = self.make_run_args();
+                let mut program = args.prog.clone();
+                if let Ok(entries) = aux_dir.read_dir() {
+                    for entry in entries {
+                        let entry = entry.unwrap();
+                        if !entry.path().is_file() {
+                            continue
+                        }
+                        program.push_str(":");
+                        program.push_str(entry.path().to_str().unwrap());
+                    }
+                }
+                args.args.insert(0, program);
+                args.args.insert(0, "run".to_string());
+                args.prog = self.config.qemu_test_client.clone().unwrap()
+                                .into_os_string().into_string().unwrap();
+                self.compose_and_run(args,
+                                     env,
+                                     self.config.run_lib_path.to_str().unwrap(),
+                                     Some(aux_dir.to_str().unwrap()),
+                                     None)
+            }
+            _ => {
                 let aux_dir = self.aux_output_dir_name();
                 self.compose_and_run(self.make_run_args(),
                                      env,
diff --git a/src/tools/qemu-test-client/Cargo.toml b/src/tools/qemu-test-client/Cargo.toml
new file mode 100644
index 00000000000..eb326c01de4
--- /dev/null
+++ b/src/tools/qemu-test-client/Cargo.toml
@@ -0,0 +1,6 @@
+[package]
+name = "qemu-test-client"
+version = "0.1.0"
+authors = ["The Rust Project Developers"]
+
+[dependencies]
diff --git a/src/tools/qemu-test-client/src/main.rs b/src/tools/qemu-test-client/src/main.rs
new file mode 100644
index 00000000000..b7ff4116eb5
--- /dev/null
+++ b/src/tools/qemu-test-client/src/main.rs
@@ -0,0 +1,221 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+/// This is a small client program intended to pair with `qemu-test-server` in
+/// this repository. This client connects to the server over TCP and is used to
+/// push artifacts and run tests on the server instead of locally.
+///
+/// Here is also where we bake in the support to spawn the QEMU emulator as
+/// well.
+
+use std::env;
+use std::fs::File;
+use std::io::prelude::*;
+use std::io::{self, BufWriter};
+use std::net::TcpStream;
+use std::path::Path;
+use std::process::{Command, Stdio};
+use std::thread;
+use std::time::Duration;
+
+macro_rules! t {
+    ($e:expr) => (match $e {
+        Ok(e) => e,
+        Err(e) => panic!("{} failed with {}", stringify!($e), e),
+    })
+}
+
+fn main() {
+    let mut args = env::args().skip(1);
+
+    match &args.next().unwrap()[..] {
+        "spawn-emulator" => {
+            spawn_emulator(Path::new(&args.next().unwrap()),
+                           Path::new(&args.next().unwrap()))
+        }
+        "push" => {
+            push(Path::new(&args.next().unwrap()))
+        }
+        "run" => {
+            run(args.next().unwrap(), args.collect())
+        }
+        cmd => panic!("unknown command: {}", cmd),
+    }
+}
+
+fn spawn_emulator(rootfs: &Path, tmpdir: &Path) {
+    // Generate a new rootfs image now that we've updated the test server
+    // executable. This is the equivalent of:
+    //
+    //      find $rootfs -print 0 | cpio --null -o --format=newc > rootfs.img
+    let rootfs_img = tmpdir.join("rootfs.img");
+    let mut cmd = Command::new("cpio");
+    cmd.arg("--null")
+       .arg("-o")
+       .arg("--format=newc")
+       .stdin(Stdio::piped())
+       .stdout(Stdio::piped())
+       .current_dir(rootfs);
+    let mut child = t!(cmd.spawn());
+    let mut stdin = child.stdin.take().unwrap();
+    let rootfs = rootfs.to_path_buf();
+    thread::spawn(move || add_files(&mut stdin, &rootfs, &rootfs));
+    t!(io::copy(&mut child.stdout.take().unwrap(),
+                &mut t!(File::create(&rootfs_img))));
+    assert!(t!(child.wait()).success());
+
+    // Start up the emulator, in the background
+    let mut cmd = Command::new("qemu-system-arm");
+    cmd.arg("-M").arg("vexpress-a15")
+       .arg("-m").arg("1024")
+       .arg("-kernel").arg("/tmp/zImage")
+       .arg("-initrd").arg(&rootfs_img)
+       .arg("-dtb").arg("/tmp/vexpress-v2p-ca15-tc1.dtb")
+       .arg("-append").arg("console=ttyAMA0 root=/dev/ram rdinit=/sbin/init init=/sbin/init")
+       .arg("-nographic")
+       .arg("-redir").arg("tcp:12345::12345");
+    t!(cmd.spawn());
+
+    // Wait for the emulator to come online
+    loop {
+        let dur = Duration::from_millis(100);
+        if let Ok(mut client) = TcpStream::connect("127.0.0.1:12345") {
+            t!(client.set_read_timeout(Some(dur)));
+            t!(client.set_write_timeout(Some(dur)));
+            if client.write_all(b"ping").is_ok() {
+                let mut b = [0; 4];
+                if client.read_exact(&mut b).is_ok() {
+                    break
+                }
+            }
+        }
+        thread::sleep(dur);
+    }
+
+    fn add_files(w: &mut Write, root: &Path, cur: &Path) {
+        for entry in t!(cur.read_dir()) {
+            let entry = t!(entry);
+            let path = entry.path();
+            let to_print = path.strip_prefix(root).unwrap();
+            t!(write!(w, "{}\u{0}", to_print.to_str().unwrap()));
+            if t!(entry.file_type()).is_dir() {
+                add_files(w, root, &path);
+            }
+        }
+    }
+}
+
+fn push(path: &Path) {
+    let client = t!(TcpStream::connect("127.0.0.1:12345"));
+    let mut client = BufWriter::new(client);
+    t!(client.write_all(b"push"));
+    t!(client.write_all(path.file_name().unwrap().to_str().unwrap().as_bytes()));
+    t!(client.write_all(&[0]));
+    let mut file = t!(File::open(path));
+    t!(io::copy(&mut file, &mut client));
+    t!(client.flush());
+    println!("done pushing {:?}", path);
+}
+
+fn run(files: String, args: Vec<String>) {
+    let client = t!(TcpStream::connect("127.0.0.1:12345"));
+    let mut client = BufWriter::new(client);
+    t!(client.write_all(b"run "));
+
+    // Send over the args
+    for arg in args {
+        t!(client.write_all(arg.as_bytes()));
+        t!(client.write_all(&[0]));
+    }
+    t!(client.write_all(&[0]));
+
+    // Send over env vars
+    for (k, v) in env::vars() {
+        if k != "PATH" && k != "LD_LIBRARY_PATH" {
+            t!(client.write_all(k.as_bytes()));
+            t!(client.write_all(&[0]));
+            t!(client.write_all(v.as_bytes()));
+            t!(client.write_all(&[0]));
+        }
+    }
+    t!(client.write_all(&[0]));
+
+    // Send over support libraries
+    let mut files = files.split(':');
+    let exe = files.next().unwrap();
+    for file in files.map(Path::new) {
+        t!(client.write_all(file.file_name().unwrap().to_str().unwrap().as_bytes()));
+        t!(client.write_all(&[0]));
+        send(&file, &mut client);
+    }
+    t!(client.write_all(&[0]));
+
+    // Send over the client executable as the last piece
+    send(exe.as_ref(), &mut client);
+
+    println!("uploaded {:?}, waiting for result", exe);
+
+    // Ok now it's time to read all the output. We're receiving "frames"
+    // representing stdout/stderr, so we decode all that here.
+    let mut header = [0; 5];
+    let mut stderr_done = false;
+    let mut stdout_done = false;
+    let mut client = t!(client.into_inner());
+    let mut stdout = io::stdout();
+    let mut stderr = io::stderr();
+    while !stdout_done || !stderr_done {
+        t!(client.read_exact(&mut header));
+        let amt = ((header[1] as u64) << 24) |
+                  ((header[2] as u64) << 16) |
+                  ((header[3] as u64) <<  8) |
+                  ((header[4] as u64) <<  0);
+        if header[0] == 0 {
+            if amt == 0 {
+                stdout_done = true;
+            } else {
+                t!(io::copy(&mut (&mut client).take(amt), &mut stdout));
+                t!(stdout.flush());
+            }
+        } else {
+            if amt == 0 {
+                stderr_done = true;
+            } else {
+                t!(io::copy(&mut (&mut client).take(amt), &mut stderr));
+                t!(stderr.flush());
+            }
+        }
+    }
+
+    // Finally, read out the exit status
+    let mut status = [0; 5];
+    t!(client.read_exact(&mut status));
+    let code = ((status[1] as i32) << 24) |
+               ((status[2] as i32) << 16) |
+               ((status[3] as i32) <<  8) |
+               ((status[4] as i32) <<  0);
+    if status[0] == 0 {
+        std::process::exit(code);
+    } else {
+        println!("died due to signal {}", code);
+        std::process::exit(3);
+    }
+}
+
+fn send(path: &Path, dst: &mut Write) {
+    let mut file = t!(File::open(&path));
+    let amt = t!(file.metadata()).len();
+    t!(dst.write_all(&[
+        (amt >> 24) as u8,
+        (amt >> 16) as u8,
+        (amt >>  8) as u8,
+        (amt >>  0) as u8,
+    ]));
+    t!(io::copy(&mut file, dst));
+}
diff --git a/src/tools/qemu-test-server/Cargo.toml b/src/tools/qemu-test-server/Cargo.toml
new file mode 100644
index 00000000000..af445a25935
--- /dev/null
+++ b/src/tools/qemu-test-server/Cargo.toml
@@ -0,0 +1,6 @@
+[package]
+name = "qemu-test-server"
+version = "0.1.0"
+authors = ["The Rust Project Developers"]
+
+[dependencies]
diff --git a/src/tools/qemu-test-server/src/main.rs b/src/tools/qemu-test-server/src/main.rs
new file mode 100644
index 00000000000..1c5d7b915ba
--- /dev/null
+++ b/src/tools/qemu-test-server/src/main.rs
@@ -0,0 +1,232 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+/// This is a small server which is intended to run inside of an emulator. This
+/// server pairs with the `qemu-test-client` program in this repository. The
+/// `qemu-test-client` connects to this server over a TCP socket and performs
+/// work such as:
+///
+/// 1. Pushing shared libraries to the server
+/// 2. Running tests through the server
+///
+/// The server supports running tests concurrently and also supports tests
+/// themselves having support libraries. All data over the TCP sockets is in a
+/// basically custom format suiting our needs.
+
+use std::fs::{self, File, Permissions};
+use std::io::prelude::*;
+use std::io::{self, BufReader};
+use std::net::{TcpListener, TcpStream};
+use std::os::unix::prelude::*;
+use std::sync::{Arc, Mutex};
+use std::path::Path;
+use std::str;
+use std::sync::atomic::{AtomicUsize, ATOMIC_USIZE_INIT, Ordering};
+use std::thread;
+use std::process::{Command, Stdio};
+
+macro_rules! t {
+    ($e:expr) => (match $e {
+        Ok(e) => e,
+        Err(e) => panic!("{} failed with {}", stringify!($e), e),
+    })
+}
+
+static TEST: AtomicUsize = ATOMIC_USIZE_INIT;
+
+fn main() {
+    println!("starting test server");
+    let listener = t!(TcpListener::bind("10.0.2.15:12345"));
+    println!("listening!");
+
+    let work = Path::new("/tmp/work");
+    t!(fs::create_dir_all(work));
+
+    let lock = Arc::new(Mutex::new(()));
+
+    for socket in listener.incoming() {
+        let mut socket = t!(socket);
+        let mut buf = [0; 4];
+        t!(socket.read_exact(&mut buf));
+        if &buf[..] == b"ping" {
+            t!(socket.write_all(b"pong"));
+        } else if &buf[..] == b"push" {
+            handle_push(socket, work);
+        } else if &buf[..] == b"run " {
+            let lock = lock.clone();
+            thread::spawn(move || handle_run(socket, work, &lock));
+        } else {
+            panic!("unknown command {:?}", buf);
+        }
+    }
+}
+
+fn handle_push(socket: TcpStream, work: &Path) {
+    let mut reader = BufReader::new(socket);
+    let mut filename = Vec::new();
+    t!(reader.read_until(0, &mut filename));
+    filename.pop(); // chop off the 0
+    let filename = t!(str::from_utf8(&filename));
+
+    let path = work.join(filename);
+    t!(io::copy(&mut reader, &mut t!(File::create(&path))));
+    t!(fs::set_permissions(&path, Permissions::from_mode(0o755)));
+}
+
+struct RemoveOnDrop<'a> {
+    inner: &'a Path,
+}
+
+impl<'a> Drop for RemoveOnDrop<'a> {
+    fn drop(&mut self) {
+        t!(fs::remove_dir_all(self.inner));
+    }
+}
+
+fn handle_run(socket: TcpStream, work: &Path, lock: &Mutex<()>) {
+    let mut arg = Vec::new();
+    let mut reader = BufReader::new(socket);
+
+    // Allocate ourselves a directory that we'll delete when we're done to save
+    // space.
+    let n = TEST.fetch_add(1, Ordering::SeqCst);
+    let path = work.join(format!("test{}", n));
+    let exe = path.join("exe");
+    t!(fs::create_dir(&path));
+    let _a = RemoveOnDrop { inner: &path };
+
+    // First up we'll get a list of arguments delimited with 0 bytes. An empty
+    // argument means that we're done.
+    let mut cmd = Command::new(&exe);
+    while t!(reader.read_until(0, &mut arg)) > 1 {
+        cmd.arg(t!(str::from_utf8(&arg[..arg.len() - 1])));
+        arg.truncate(0);
+    }
+
+    // Next we'll get a bunch of env vars in pairs delimited by 0s as well
+    arg.truncate(0);
+    while t!(reader.read_until(0, &mut arg)) > 1 {
+        let key_len = arg.len() - 1;
+        let val_len = t!(reader.read_until(0, &mut arg)) - 1;
+        {
+            let key = &arg[..key_len];
+            let val = &arg[key_len + 1..][..val_len];
+            let key = t!(str::from_utf8(key));
+            let val = t!(str::from_utf8(val));
+            cmd.env(key, val);
+        }
+        arg.truncate(0);
+    }
+
+    // The section of code from here down to where we drop the lock is going to
+    // be a critical section for us. On Linux you can't execute a file which is
+    // open somewhere for writing, as you'll receive the error "text file busy".
+    // Now here we never have the text file open for writing when we spawn it,
+    // so why do we still need a critical section?
+    //
+    // Process spawning first involves a `fork` on Unix, which clones all file
+    // descriptors into the child process. This means that it's possible for us
+    // to open the file for writing (as we're downloading it), then some other
+    // thread forks, then we close the file and try to exec. At that point the
+    // other thread created a child process with the file open for writing, and
+    // we attempt to execute it, so we get an error.
+    //
+    // This race is resolve by ensuring that only one thread can writ ethe file
+    // and spawn a child process at once. Kinda an unfortunate solution, but we
+    // don't have many other choices with this sort of setup!
+    //
+    // In any case the lock is acquired here, before we start writing any files.
+    // It's then dropped just after we spawn the child. That way we don't lock
+    // the execution of the child, just the creation of its files.
+    let lock = lock.lock();
+
+    // Next there's a list of dynamic libraries preceded by their filenames.
+    arg.truncate(0);
+    while t!(reader.read_until(0, &mut arg)) > 1 {
+        let dst = path.join(t!(str::from_utf8(&arg[..arg.len() - 1])));
+        let amt = read_u32(&mut reader) as u64;
+        t!(io::copy(&mut reader.by_ref().take(amt),
+                    &mut t!(File::create(&dst))));
+        t!(fs::set_permissions(&dst, Permissions::from_mode(0o755)));
+        arg.truncate(0);
+    }
+
+    // Finally we'll get the binary. The other end will tell us how big the
+    // binary is and then we'll download it all to the exe path we calculated
+    // earlier.
+    let amt = read_u32(&mut reader) as u64;
+    t!(io::copy(&mut reader.by_ref().take(amt),
+                &mut t!(File::create(&exe))));
+    t!(fs::set_permissions(&exe, Permissions::from_mode(0o755)));
+
+    // Support libraries were uploaded to `work` earlier, so make sure that's
+    // in `LD_LIBRARY_PATH`. Also include our own current dir which may have
+    // had some libs uploaded.
+    cmd.env("LD_LIBRARY_PATH",
+            format!("{}:{}", work.display(), path.display()));
+
+    // Spawn the child and ferry over stdout/stderr to the socket in a framed
+    // fashion (poor man's style)
+    let mut child = t!(cmd.stdin(Stdio::null())
+                          .stdout(Stdio::piped())
+                          .stderr(Stdio::piped())
+                          .spawn());
+    drop(lock);
+    let mut stdout = child.stdout.take().unwrap();
+    let mut stderr = child.stderr.take().unwrap();
+    let socket = Arc::new(Mutex::new(reader.into_inner()));
+    let socket2 = socket.clone();
+    let thread = thread::spawn(move || my_copy(&mut stdout, 0, &*socket2));
+    my_copy(&mut stderr, 1, &*socket);
+    thread.join().unwrap();
+
+    // Finally send over the exit status.
+    let status = t!(child.wait());
+    let (which, code) = match status.code() {
+        Some(n) => (0, n),
+        None => (1, status.signal().unwrap()),
+    };
+    t!(socket.lock().unwrap().write_all(&[
+        which,
+        (code >> 24) as u8,
+        (code >> 16) as u8,
+        (code >>  8) as u8,
+        (code >>  0) as u8,
+    ]));
+}
+
+fn my_copy(src: &mut Read, which: u8, dst: &Mutex<Write>) {
+    let mut b = [0; 1024];
+    loop {
+        let n = t!(src.read(&mut b));
+        let mut dst = dst.lock().unwrap();
+        t!(dst.write_all(&[
+            which,
+            (n >> 24) as u8,
+            (n >> 16) as u8,
+            (n >>  8) as u8,
+            (n >>  0) as u8,
+        ]));
+        if n > 0 {
+            t!(dst.write_all(&b[..n]));
+        } else {
+            break
+        }
+    }
+}
+
+fn read_u32(r: &mut Read) -> u32 {
+    let mut len = [0; 4];
+    t!(r.read_exact(&mut len));
+    ((len[0] as u32) << 24) |
+    ((len[1] as u32) << 16) |
+    ((len[2] as u32) <<  8) |
+    ((len[3] as u32) <<  0)
+}