Move test.sh to y.rs test

11 files changed, 605 insertions, 238 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 61da6a2491c..732edd66196 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -22,4 +22,4 @@ task:
     - # Reduce amount of benchmark runs as they are slow
     - export COMPILE_RUNS=2
     - export RUN_RUNS=2
-    - ./test.sh
+    - ./y.rs test
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index aa556a21bf8..a36c570ddc8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -103,7 +103,7 @@ jobs:
         # Enable extra checks
         export CG_CLIF_ENABLE_VERIFIER=1
 
-        ./test.sh
+        ./y.rs test
 
     - name: Package prebuilt cg_clif
       run: tar cvfJ cg_clif.tar.xz build
diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs
index 48faec8bc4b..04f8be6f820 100644
--- a/build_system/build_backend.rs
+++ b/build_system/build_backend.rs
@@ -1,12 +1,11 @@
 use std::env;
-use std::path::{Path, PathBuf};
 use std::process::Command;
 
 pub(crate) fn build_backend(
     channel: &str,
     host_triple: &str,
     use_unstable_features: bool,
-) -> PathBuf {
+)  {
     let mut cmd = Command::new("cargo");
     cmd.arg("build").arg("--target").arg(host_triple);
 
@@ -38,6 +37,4 @@ pub(crate) fn build_backend(
 
     eprintln!("[BUILD] rustc_codegen_cranelift");
     super::utils::spawn_and_wait(cmd);
-
-    Path::new("target").join(host_triple).join(channel)
 }
diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs
index 16cce83dd9c..dddd5a673c5 100644
--- a/build_system/build_sysroot.rs
+++ b/build_system/build_sysroot.rs
@@ -10,10 +10,12 @@ pub(crate) fn build_sysroot(
     channel: &str,
     sysroot_kind: SysrootKind,
     target_dir: &Path,
-    cg_clif_build_dir: PathBuf,
+    cg_clif_build_dir: &Path,
     host_triple: &str,
     target_triple: &str,
 ) {
+    eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
+
     if target_dir.exists() {
         fs::remove_dir_all(target_dir).unwrap();
     }
@@ -35,11 +37,17 @@ pub(crate) fn build_sysroot(
 
     // Build and copy rustc and cargo wrappers
     for wrapper in ["rustc-clif", "cargo-clif"] {
+        let wrapper_name = if cfg!(windows) {
+            format!("{wrapper}.exe")
+        } else {
+            wrapper.to_string()
+        };
+
         let mut build_cargo_wrapper_cmd = Command::new("rustc");
         build_cargo_wrapper_cmd
             .arg(PathBuf::from("scripts").join(format!("{wrapper}.rs")))
             .arg("-o")
-            .arg(target_dir.join(wrapper))
+            .arg(target_dir.join(wrapper_name))
             .arg("-g");
         spawn_and_wait(build_cargo_wrapper_cmd);
     }
diff --git a/build_system/mod.rs b/build_system/mod.rs
index b897b7fbacf..d1f8ad3e817 100644
--- a/build_system/mod.rs
+++ b/build_system/mod.rs
@@ -1,5 +1,5 @@
 use std::env;
-use std::path::PathBuf;
+use std::path::{PathBuf, Path};
 use std::process;
 
 mod build_backend;
@@ -8,6 +8,7 @@ mod config;
 mod prepare;
 mod rustc_info;
 mod utils;
+mod tests;
 
 fn usage() {
     eprintln!("Usage:");
@@ -15,6 +16,9 @@ fn usage() {
     eprintln!(
         "  ./y.rs build [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--no-unstable-features]"
     );
+    eprintln!(
+        "  ./y.rs test [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--no-unstable-features]"
+    );
 }
 
 macro_rules! arg_error {
@@ -25,11 +29,13 @@ macro_rules! arg_error {
     }};
 }
 
+#[derive(PartialEq, Debug)]
 enum Command {
     Build,
+    Test,
 }
 
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub(crate) enum SysrootKind {
     None,
     Clif,
@@ -52,6 +58,7 @@ pub fn main() {
             process::exit(0);
         }
         Some("build") => Command::Build,
+        Some("test") => Command::Test,
         Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
         Some(command) => arg_error!("Unknown command {}", command),
         None => {
@@ -115,14 +122,27 @@ pub fn main() {
         process::exit(1);
     }
 
-    let cg_clif_build_dir =
+    let cg_clif_build_dir = Path::new("target").join(&host_triple).join(&channel);
+
+    if command == Command::Test {
+        // TODO: Should we also build_backend here?
+        tests::run_tests(
+            channel,
+            sysroot_kind,
+            &target_dir,
+            &cg_clif_build_dir,
+            &host_triple,
+            &target_triple,
+        ).expect("Failed to run tests");
+    } else {
         build_backend::build_backend(channel, &host_triple, use_unstable_features);
-    build_sysroot::build_sysroot(
-        channel,
-        sysroot_kind,
-        &target_dir,
-        cg_clif_build_dir,
-        &host_triple,
-        &target_triple,
-    );
+        build_sysroot::build_sysroot(
+            channel,
+            sysroot_kind,
+            &target_dir,
+            &cg_clif_build_dir,
+            &host_triple,
+            &target_triple,
+        );
+    }
 }
diff --git a/build_system/prepare.rs b/build_system/prepare.rs
index 8bb00352d3f..7e0fd182d98 100644
--- a/build_system/prepare.rs
+++ b/build_system/prepare.rs
@@ -50,8 +50,7 @@ pub(crate) fn prepare() {
     spawn_and_wait(build_cmd);
     fs::copy(
         Path::new("simple-raytracer/target/debug").join(get_file_name("main", "bin")),
-        // FIXME use get_file_name here too once testing is migrated to rust
-        "simple-raytracer/raytracer_cg_llvm",
+        Path::new("simple-raytracer").join(get_file_name("raytracer_cg_llvm", "bin")),
     )
     .unwrap();
 }
diff --git a/build_system/tests.rs b/build_system/tests.rs
new file mode 100644
index 00000000000..74e4b51095f
--- /dev/null
+++ b/build_system/tests.rs
@@ -0,0 +1,504 @@
+use super::build_sysroot;
+use super::config;
+use super::utils::{spawn_and_wait, spawn_and_wait_with_input};
+use build_system::SysrootKind;
+use std::env;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
+
+struct TestCase {
+    config: &'static str,
+    func: &'static dyn Fn(&TestRunner),
+}
+
+impl TestCase {
+    const fn new(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self {
+        Self { config, func }
+    }
+}
+
+const NO_SYSROOT_SUITE: &[TestCase] = &[
+    TestCase::new("build.mini_core", &|runner| {
+        runner.run_rustc(["example/mini_core.rs", "--crate-name", "mini_core", "--crate-type", "lib,dylib", "--target", &runner.target_triple]);
+    }),
+
+    TestCase::new("build.example", &|runner| {
+        runner.run_rustc(["example/example.rs", "--crate-type", "lib", "--target", &runner.target_triple]);
+    }),
+
+    TestCase::new("jit.mini_core_hello_world", &|runner| {
+        let mut jit_cmd = runner.rustc_command(["-Zunstable-options", "-Cllvm-args=mode=jit", "-Cprefer-dynamic", "example/mini_core_hello_world.rs", "--cfg", "jit", "--target", &runner.host_triple]);
+        jit_cmd.env("CG_CLIF_JIT_ARGS", "abc bcd");
+        spawn_and_wait(jit_cmd);
+
+        eprintln!("[JIT-lazy] mini_core_hello_world");
+        let mut jit_cmd = runner.rustc_command(["-Zunstable-options", "-Cllvm-args=mode=jit-lazy", "-Cprefer-dynamic", "example/mini_core_hello_world.rs", "--cfg", "jit", "--target", &runner.host_triple]);
+        jit_cmd.env("CG_CLIF_JIT_ARGS", "abc bcd");
+        spawn_and_wait(jit_cmd);
+    }),
+
+    TestCase::new("aot.mini_core_hello_world", &|runner| {
+        runner.run_rustc(["example/mini_core_hello_world.rs", "--crate-name", "mini_core_hello_world", "--crate-type", "bin", "-g", "--target", &runner.target_triple]);
+        runner.run_out_command("mini_core_hello_world", ["abc", "bcd"]);
+    }),
+];
+
+
+const BASE_SYSROOT_SUITE: &[TestCase] = &[
+    TestCase::new("aot.arbitrary_self_types_pointers_and_wrappers", &|runner| {
+        runner.run_rustc(["example/arbitrary_self_types_pointers_and_wrappers.rs", "--crate-name", "arbitrary_self_types_pointers_and_wrappers", "--crate-type", "bin", "--target", &runner.target_triple]);
+        runner.run_out_command("arbitrary_self_types_pointers_and_wrappers", []);
+    }),
+
+    TestCase::new("aot.issue_91827_extern_types", &|runner| {
+        runner.run_rustc(["example/issue-91827-extern-types.rs", "--crate-name", "issue_91827_extern_types", "--crate-type", "bin", "--target", &runner.target_triple]);
+        runner.run_out_command("issue_91827_extern_types", []);
+    }),
+
+    TestCase::new("build.alloc_system", &|runner| {
+        runner.run_rustc(["example/alloc_system.rs", "--crate-type", "lib", "--target", &runner.target_triple]);
+    }),
+
+    TestCase::new("aot.alloc_example", &|runner| {
+        runner.run_rustc(["example/alloc_example.rs", "--crate-type", "bin", "--target", &runner.target_triple]);
+        runner.run_out_command("alloc_example", []);
+    }),
+
+    TestCase::new("jit.std_example", &|runner| {
+        runner.run_rustc(["-Zunstable-options", "-Cllvm-args=mode=jit", "-Cprefer-dynamic", "example/std_example.rs", "--target", &runner.host_triple]);
+
+        eprintln!("[JIT-lazy] std_example");
+        runner.run_rustc(["-Zunstable-options", "-Cllvm-args=mode=jit-lazy", "-Cprefer-dynamic", "example/std_example.rs", "--target", &runner.host_triple]);
+    }),
+
+    TestCase::new("aot.std_example", &|runner| {
+        runner.run_rustc(["example/std_example.rs", "--crate-type", "bin", "--target", &runner.target_triple]);
+        runner.run_out_command("std_example", ["arg"]);
+    }),
+
+    TestCase::new("aot.dst_field_align", &|runner| {
+        runner.run_rustc(["example/dst-field-align.rs", "--crate-name", "dst_field_align", "--crate-type", "bin", "--target", &runner.target_triple]);
+        runner.run_out_command("dst_field_align", []);
+    }),
+
+    TestCase::new("aot.subslice-patterns-const-eval", &|runner| {
+        runner.run_rustc(["example/subslice-patterns-const-eval.rs", "--crate-type", "bin", "-Cpanic=abort", "--target", &runner.target_triple]);
+        runner.run_out_command("subslice-patterns-const-eval", []);
+    }),
+
+    TestCase::new("aot.track-caller-attribute", &|runner| {
+        runner.run_rustc(["example/track-caller-attribute.rs", "--crate-type", "bin", "-Cpanic=abort", "--target", &runner.target_triple]);
+        runner.run_out_command("track-caller-attribute", []);
+    }),
+
+    TestCase::new("aot.float-minmax-pass", &|runner| {
+        runner.run_rustc(["example/float-minmax-pass.rs", "--crate-type", "bin", "-Cpanic=abort", "--target", &runner.target_triple]);
+        runner.run_out_command("float-minmax-pass", []);
+    }),
+
+    TestCase::new("aot.mod_bench", &|runner| {
+        runner.run_rustc(["example/mod_bench.rs", "--crate-type", "bin", "--target", &runner.target_triple]);
+        runner.run_out_command("mod_bench", []);
+    }),
+];
+
+const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
+    TestCase::new("test.rust-random/rand", &|runner| {
+        runner.in_dir(["rand"], |runner| {
+            runner.run_cargo(["clean"]);
+
+            if runner.host_triple == runner.target_triple {
+                eprintln!("[TEST] rust-random/rand");
+                runner.run_cargo(["test", "--workspace"]);
+            } else {
+                eprintln!("[AOT] rust-random/rand");
+                runner.run_cargo(["build", "--workspace", "--target", &runner.target_triple, "--tests"]);
+            }
+        });
+    }),
+
+    TestCase::new("bench.simple-raytracer", &|runner| {
+        runner.in_dir(["simple-raytracer"], |runner| {
+            let run_runs = env::var("RUN_RUNS").unwrap_or("10".to_string());
+
+            if runner.host_triple == runner.target_triple {
+                eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
+                let mut bench_compile = Command::new("hyperfine");
+                bench_compile.arg("--runs");
+                bench_compile.arg(&run_runs);
+                bench_compile.arg("--warmup");
+                bench_compile.arg("1");
+                bench_compile.arg("--prepare");
+                bench_compile.arg(format!("{:?}", runner.cargo_command(["clean"])));
+
+                if cfg!(windows) {
+                    bench_compile.arg("cmd /C \"set RUSTFLAGS= && cargo build\"");
+                } else {
+                    bench_compile.arg("RUSTFLAGS='' cargo build");
+                }
+
+                bench_compile.arg(format!("{:?}", runner.cargo_command(["build"])));
+                spawn_and_wait(bench_compile);
+
+
+
+                eprintln!("[BENCH RUN] ebobby/simple-raytracer");
+                fs::copy(PathBuf::from("./target/debug/main"), PathBuf::from("raytracer_cg_clif")).unwrap();
+
+                let mut bench_run = Command::new("hyperfine");
+                bench_run.arg("--runs");
+                bench_run.arg(&run_runs);
+                bench_run.arg(PathBuf::from("./raytracer_cg_llvm"));
+                bench_run.arg(PathBuf::from("./raytracer_cg_clif"));
+                spawn_and_wait(bench_run);
+            } else {
+                runner.run_cargo(["clean"]);
+                eprintln!("[BENCH COMPILE] ebobby/simple-raytracer (skipped)");
+                eprintln!("[COMPILE] ebobby/simple-raytracer");
+                runner.run_cargo(["build", "--target", &runner.target_triple]);
+                eprintln!("[BENCH RUN] ebobby/simple-raytracer (skipped)");
+            }
+        });
+    }),
+
+    TestCase::new("test.libcore", &|runner| {
+        runner.in_dir(["build_sysroot", "sysroot_src", "library", "core", "tests"], |runner| {
+            runner.run_cargo(["clean"]);
+
+            if runner.host_triple == runner.target_triple {
+                runner.run_cargo(["test"]);
+            } else {
+                eprintln!("Cross-Compiling: Not running tests");
+                runner.run_cargo(["build", "--target", &runner.target_triple, "--tests"]);
+            }
+        });
+    }),
+
+    TestCase::new("test.regex-shootout-regex-dna", &|runner| {
+        runner.in_dir(["regex"], |runner| {
+            runner.run_cargo(["clean"]);
+
+            // newer aho_corasick versions throw a deprecation warning
+            let mut lint_rust_flags = runner.rust_flags.clone();
+            lint_rust_flags.push("--cap-lints".to_string());
+            lint_rust_flags.push("warn".to_string());
+
+            let mut build_cmd = runner.cargo_command(["build", "--example", "shootout-regex-dna", "--target", &runner.target_triple]);
+            build_cmd.env("RUSTFLAGS", lint_rust_flags.join(" "));
+
+            spawn_and_wait(build_cmd);
+
+            if runner.host_triple == runner.target_triple {
+                let mut run_cmd = runner.cargo_command(["run", "--example", "shootout-regex-dna", "--target", &runner.target_triple]);
+                run_cmd.env("RUSTFLAGS", lint_rust_flags.join(" "));
+
+
+                let input = fs::read_to_string(PathBuf::from("examples/regexdna-input.txt")).unwrap();
+                let expected_path = PathBuf::from("examples/regexdna-output.txt");
+                let expected = fs::read_to_string(&expected_path).unwrap();
+
+                let output = spawn_and_wait_with_input(run_cmd, input);
+                // Make sure `[codegen mono items] start` doesn't poison the diff
+                let output = output.lines()
+                    .filter(|line| !line.contains("codegen mono items"))
+                    .filter(|line| !line.contains("Spawned thread"))
+                    .chain(Some("")) // This just adds the trailing newline
+                    .collect::<Vec<&str>>()
+                    .join("\r\n");
+
+
+                if output != expected {
+                    let res_path = PathBuf::from("res.txt");
+                    fs::write(&res_path, &output).unwrap();
+
+                    if cfg!(windows) {
+                        println!("Output files don't match!");
+                        println!("Expected Output:\n{}", expected);
+                        println!("Actual Output:\n{}", output);
+                    } else {
+                        let mut diff = Command::new("diff");
+                        diff.arg("-u");
+                        diff.arg(res_path);
+                        diff.arg(expected_path);
+                        spawn_and_wait(diff);
+                    }
+
+                    std::process::exit(1);
+                }
+            }
+        });
+    }),
+
+    TestCase::new("test.regex", &|runner| {
+        runner.in_dir(["regex"], |runner| {
+            runner.run_cargo(["clean"]);
+
+            if runner.host_triple == runner.target_triple {
+                runner.run_cargo(["test", "--tests", "--", "--exclude-should-panic", "--test-threads", "1", "-Zunstable-options", "-q"]);
+            } else {
+                eprintln!("Cross-Compiling: Not running tests");
+                runner.run_cargo(["build", "--tests", "--target", &runner.target_triple]);
+            }
+        });
+    }),
+
+    TestCase::new("test.portable-simd", &|runner| {
+        runner.in_dir(["portable-simd"], |runner| {
+            runner.run_cargo(["clean"]);
+            runner.run_cargo(["build", "--all-targets", "--target", &runner.target_triple]);
+
+            if runner.host_triple == runner.target_triple {
+                runner.run_cargo(["test", "-q"]);
+            }
+        });
+    }),
+];
+
+
+
+pub(crate) fn run_tests(
+    channel: &str,
+    sysroot_kind: SysrootKind,
+    target_dir: &Path,
+    cg_clif_build_dir: &Path,
+    host_triple: &str,
+    target_triple: &str,
+) -> Result<()> {
+    let runner = TestRunner::new(host_triple.to_string(), target_triple.to_string());
+
+    if config::get_bool("testsuite.no_sysroot") {
+        build_sysroot::build_sysroot(
+            channel,
+            SysrootKind::None,
+            &target_dir,
+            cg_clif_build_dir,
+            &host_triple,
+            &target_triple,
+        );
+
+        let _ = remove_out_dir();
+        runner.run_testsuite(NO_SYSROOT_SUITE);
+    } else {
+        eprintln!("[SKIP] no_sysroot tests");
+    }
+
+    let run_base_sysroot = config::get_bool("testsuite.base_sysroot");
+    let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot");
+
+    if run_base_sysroot || run_extended_sysroot {
+        build_sysroot::build_sysroot(
+            channel,
+            sysroot_kind,
+            &target_dir,
+            cg_clif_build_dir,
+            &host_triple,
+            &target_triple,
+        );
+    }
+
+    if run_base_sysroot {
+        runner.run_testsuite(BASE_SYSROOT_SUITE);
+    } else {
+        eprintln!("[SKIP] base_sysroot tests");
+    }
+
+    if run_extended_sysroot {
+        runner.run_testsuite(EXTENDED_SYSROOT_SUITE);
+    } else {
+        eprintln!("[SKIP] extended_sysroot tests");
+    }
+
+    Ok(())
+}
+
+
+fn remove_out_dir() -> Result<()> {
+    let out_dir = Path::new("target").join("out");
+    Ok(fs::remove_dir_all(out_dir)?)
+}
+
+struct TestRunner {
+    root_dir: PathBuf,
+    out_dir: PathBuf,
+    jit_supported: bool,
+    rust_flags: Vec<String>,
+    run_wrapper: Vec<String>,
+    host_triple: String,
+    target_triple: String,
+}
+
+impl TestRunner {
+    pub fn new(host_triple: String, target_triple: String) -> Self {
+        let root_dir = env::current_dir().unwrap();
+
+        let mut out_dir = root_dir.clone();
+        out_dir.push("target");
+        out_dir.push("out");
+
+        let is_native = host_triple == target_triple;
+        let jit_supported = target_triple.contains("x86_64") && is_native;
+
+        let env_rust_flags = env::var("RUSTFLAGS").ok();
+        let env_run_wrapper = env::var("RUN_WRAPPER").ok();
+
+        let mut rust_flags: Vec<&str> = env_rust_flags.iter().map(|s| s.as_str()).collect();
+        let mut run_wrapper: Vec<&str> = env_run_wrapper.iter().map(|s| s.as_str()).collect();
+
+        if !is_native {
+            match target_triple.as_str() {
+                "aarch64-unknown-linux-gnu" => {
+                    // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
+                    rust_flags.insert(0, "-Clinker=aarch64-linux-gnu-gcc");
+                    run_wrapper.extend(["qemu-aarch64", "-L", "/usr/aarch64-linux-gnu"]);
+                },
+                "x86_64-pc-windows-gnu" => {
+                    // We are cross-compiling for Windows. Run tests in wine.
+                    run_wrapper.push("wine".into());
+                }
+                _ => {
+                    println!("Unknown non-native platform");
+                }
+            }
+        }
+
+        // FIXME fix `#[linkage = "extern_weak"]` without this
+        if host_triple.contains("darwin") {
+            rust_flags.push("-Clink-arg=-undefined");
+            rust_flags.push("-Clink-arg=dynamic_lookup");
+        }
+
+        Self {
+            root_dir,
+            out_dir,
+            jit_supported,
+            rust_flags: rust_flags.iter().map(|s| s.to_string()).collect(),
+            run_wrapper: run_wrapper.iter().map(|s| s.to_string()).collect(),
+            host_triple,
+            target_triple,
+        }
+    }
+
+    pub fn run_testsuite(&self, tests: &[TestCase]) {
+        for &TestCase { config, func } in tests {
+            let is_jit_test = config.contains("jit");
+            let (tag, testname) = config.split_once('.').unwrap();
+            let tag = tag.to_uppercase();
+
+            if !config::get_bool(config) || (is_jit_test && !self.jit_supported) {
+                eprintln!("[{tag}] {testname} (skipped)");
+                continue;
+            } else {
+                eprintln!("[{tag}] {testname}");
+            }
+
+            func(self);
+        }
+    }
+
+    fn in_dir<'a, I, F>(&self, dir: I, callback: F)
+    where
+        I: IntoIterator<Item = &'a str>,
+        F: FnOnce(&TestRunner),
+    {
+        let current = env::current_dir().unwrap();
+        let mut new = current.clone();
+        for d in dir {
+            new.push(d);
+        }
+
+        env::set_current_dir(new).unwrap();
+        callback(self);
+        env::set_current_dir(current).unwrap();
+    }
+
+    fn rustc_command<I, S>(&self, args: I) -> Command
+    where
+        I: IntoIterator<Item = S>,
+        S: AsRef<OsStr>,
+    {
+        let mut rustc_clif = self.root_dir.clone();
+        rustc_clif.push("build");
+        rustc_clif.push("rustc-clif");
+        if cfg!(windows) {
+            rustc_clif.set_extension("exe");
+        }
+
+        let mut cmd = Command::new(rustc_clif);
+        cmd.args(self.rust_flags.iter());
+        cmd.arg("-L");
+        cmd.arg(format!("crate={}", self.out_dir.display()));
+        cmd.arg("--out-dir");
+        cmd.arg(format!("{}", self.out_dir.display()));
+        cmd.arg("-Cdebuginfo=2");
+        cmd.args(args);
+        cmd.env("RUSTFLAGS", self.rust_flags.join(" "));
+        cmd
+    }
+
+    fn run_rustc<I, S>(&self, args: I)
+    where
+        I: IntoIterator<Item = S>,
+        S: AsRef<OsStr>,
+    {
+        spawn_and_wait(self.rustc_command(args));
+    }
+
+    fn run_out_command<'a, I>(&self, name: &str, args: I)
+    where
+        I: IntoIterator<Item = &'a str>,
+    {
+        let mut full_cmd = vec![];
+
+        // Prepend the RUN_WRAPPER's
+        for rw in self.run_wrapper.iter() {
+            full_cmd.push(rw.to_string());
+        }
+
+        full_cmd.push({
+            let mut out_path = self.out_dir.clone();
+            out_path.push(name);
+            out_path.to_str().unwrap().to_string()
+        });
+
+        for arg in args.into_iter() {
+            full_cmd.push(arg.to_string());
+        }
+
+        let mut cmd_iter = full_cmd.into_iter();
+        let first = cmd_iter.next().unwrap();
+
+        let mut cmd = Command::new(first);
+        cmd.args(cmd_iter);
+
+        spawn_and_wait(cmd);
+    }
+
+    fn cargo_command<I, S>(&self, args: I) -> Command
+    where
+        I: IntoIterator<Item = S>,
+        S: AsRef<OsStr>,
+    {
+        let mut cargo_clif = self.root_dir.clone();
+        cargo_clif.push("build");
+        cargo_clif.push("cargo-clif");
+        if cfg!(windows) {
+            cargo_clif.set_extension("exe");
+        }
+
+        let mut cmd = Command::new(cargo_clif);
+        cmd.args(args);
+        cmd.env("RUSTFLAGS", self.rust_flags.join(" "));
+        cmd
+    }
+
+    fn run_cargo<'a, I>(&self, args: I)
+    where
+        I: IntoIterator<Item = &'a str>,
+    {
+        spawn_and_wait(self.cargo_command(args));
+    }
+}
diff --git a/build_system/utils.rs b/build_system/utils.rs
index 12b5d70fad8..2d2778d2fc0 100644
--- a/build_system/utils.rs
+++ b/build_system/utils.rs
@@ -1,6 +1,7 @@
 use std::fs;
 use std::path::Path;
-use std::process::{self, Command};
+use std::process::{self, Command, Stdio};
+use std::io::Write;
 
 #[track_caller]
 pub(crate) fn try_hard_link(src: impl AsRef<Path>, dst: impl AsRef<Path>) {
@@ -18,6 +19,27 @@ pub(crate) fn spawn_and_wait(mut cmd: Command) {
     }
 }
 
+#[track_caller]
+pub(crate) fn spawn_and_wait_with_input(mut cmd: Command, input: String) -> String {
+    let mut child = cmd
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .spawn()
+        .expect("Failed to spawn child process");
+
+    let mut stdin = child.stdin.take().expect("Failed to open stdin");
+    std::thread::spawn(move || {
+        stdin.write_all(input.as_bytes()).expect("Failed to write to stdin");
+    });
+
+    let output = child.wait_with_output().expect("Failed to read stdout");
+    if !output.status.success() {
+        process::exit(1);
+    }
+
+    String::from_utf8(output.stdout).unwrap()
+}
+
 pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
     for entry in fs::read_dir(from).unwrap() {
         let entry = entry.unwrap();
diff --git a/config.txt b/config.txt
index b14db27d620..5e4d230776d 100644
--- a/config.txt
+++ b/config.txt
@@ -15,3 +15,36 @@
 # This option can be changed while the build system is already running for as long as sysroot
 # building hasn't started yet.
 #keep_sysroot
+
+
+# Testsuite
+#
+# Each test suite item has a corresponding key here. The default is to run all tests.
+# Comment any of these lines to skip individual tests.
+
+testsuite.no_sysroot
+build.mini_core
+build.example
+jit.mini_core_hello_world
+aot.mini_core_hello_world
+
+testsuite.base_sysroot
+aot.arbitrary_self_types_pointers_and_wrappers
+aot.issue_91827_extern_types
+build.alloc_system
+aot.alloc_example
+jit.std_example
+aot.std_example
+aot.dst_field_align
+aot.subslice-patterns-const-eval
+aot.track-caller-attribute
+aot.float-minmax-pass
+aot.mod_bench
+
+testsuite.extended_sysroot
+test.rust-random/rand
+bench.simple-raytracer
+test.libcore
+test.regex-shootout-regex-dna
+test.regex
+test.portable-simd
diff --git a/scripts/tests.sh b/scripts/tests.sh
deleted file mode 100755
index 9b5ffa40960..00000000000
--- a/scripts/tests.sh
+++ /dev/null
@@ -1,203 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-export CG_CLIF_DISPLAY_CG_TIME=1
-export CG_CLIF_DISABLE_INCR_CACHE=1
-
-export HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
-export TARGET_TRIPLE=${TARGET_TRIPLE:-$HOST_TRIPLE}
-
-export RUN_WRAPPER=''
-
-case "$TARGET_TRIPLE" in
-   x86_64*)
-      export JIT_SUPPORTED=1
-      ;;
-   *)
-      export JIT_SUPPORTED=0
-      ;;
-esac
-
-if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
-   export JIT_SUPPORTED=0
-   if [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then
-      # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
-      export RUSTFLAGS='-Clinker=aarch64-linux-gnu-gcc '$RUSTFLAGS
-      export RUN_WRAPPER='qemu-aarch64 -L /usr/aarch64-linux-gnu'
-   elif [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
-      # We are cross-compiling for Windows. Run tests in wine.
-      export RUN_WRAPPER='wine'
-   else
-      echo "Unknown non-native platform"
-   fi
-fi
-
-# FIXME fix `#[linkage = "extern_weak"]` without this
-if [[ "$(uname)" == 'Darwin' ]]; then
-   export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
-fi
-
-MY_RUSTC="$(pwd)/build/rustc-clif $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2"
-
-function no_sysroot_tests() {
-    echo "[BUILD] mini_core"
-    $MY_RUSTC example/mini_core.rs --crate-name mini_core --crate-type lib,dylib --target "$TARGET_TRIPLE"
-
-    echo "[BUILD] example"
-    $MY_RUSTC example/example.rs --crate-type lib --target "$TARGET_TRIPLE"
-
-    if [[ "$JIT_SUPPORTED" = "1" ]]; then
-        echo "[JIT] mini_core_hello_world"
-        CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Zunstable-options -Cllvm-args=mode=jit -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
-
-        echo "[JIT-lazy] mini_core_hello_world"
-        CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Zunstable-options -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
-    else
-        echo "[JIT] mini_core_hello_world (skipped)"
-    fi
-
-    echo "[AOT] mini_core_hello_world"
-    $MY_RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/mini_core_hello_world abc bcd
-    # (echo "break set -n main"; echo "run"; sleep 1; echo "si -c 10"; sleep 1; echo "frame variable") | lldb -- ./target/out/mini_core_hello_world abc bcd
-}
-
-function base_sysroot_tests() {
-    echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
-    $MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
-
-    echo "[AOT] issue_91827_extern_types"
-    $MY_RUSTC example/issue-91827-extern-types.rs --crate-name issue_91827_extern_types --crate-type bin --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/issue_91827_extern_types
-
-    echo "[BUILD] alloc_system"
-    $MY_RUSTC example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE"
-
-    echo "[AOT] alloc_example"
-    $MY_RUSTC example/alloc_example.rs --crate-type bin --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/alloc_example
-
-    if [[ "$JIT_SUPPORTED" = "1" ]]; then
-        echo "[JIT] std_example"
-        $MY_RUSTC -Zunstable-options -Cllvm-args=mode=jit -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE"
-
-        echo "[JIT-lazy] std_example"
-        $MY_RUSTC -Zunstable-options -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE"
-    else
-        echo "[JIT] std_example (skipped)"
-    fi
-
-    echo "[AOT] std_example"
-    $MY_RUSTC example/std_example.rs --crate-type bin --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/std_example arg
-
-    echo "[AOT] dst_field_align"
-    $MY_RUSTC example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/dst_field_align
-
-    echo "[AOT] subslice-patterns-const-eval"
-    $MY_RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/subslice-patterns-const-eval
-
-    echo "[AOT] track-caller-attribute"
-    $MY_RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/track-caller-attribute
-
-    echo "[AOT] float-minmax-pass"
-    $MY_RUSTC example/float-minmax-pass.rs --crate-type bin -Cpanic=abort --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/float-minmax-pass
-
-    echo "[AOT] mod_bench"
-    $MY_RUSTC example/mod_bench.rs --crate-type bin --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/mod_bench
-}
-
-function extended_sysroot_tests() {
-    pushd rand
-    ../build/cargo-clif clean
-    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-        echo "[TEST] rust-random/rand"
-        ../build/cargo-clif test --workspace
-    else
-        echo "[AOT] rust-random/rand"
-        ../build/cargo-clif build --workspace --target $TARGET_TRIPLE --tests
-    fi
-    popd
-
-    pushd simple-raytracer
-    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-        echo "[BENCH COMPILE] ebobby/simple-raytracer"
-        hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "../build/cargo-clif clean" \
-        "RUSTFLAGS='' cargo build" \
-        "../build/cargo-clif build"
-
-        echo "[BENCH RUN] ebobby/simple-raytracer"
-        cp ./target/debug/main ./raytracer_cg_clif
-        hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_clif
-    else
-        ../build/cargo-clif clean
-        echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)"
-        echo "[COMPILE] ebobby/simple-raytracer"
-        ../build/cargo-clif build --target $TARGET_TRIPLE
-        echo "[BENCH RUN] ebobby/simple-raytracer (skipped)"
-    fi
-    popd
-
-    pushd build_sysroot/sysroot_src/library/core/tests
-    echo "[TEST] libcore"
-    ../../../../../build/cargo-clif clean
-    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-        ../../../../../build/cargo-clif test
-    else
-        ../../../../../build/cargo-clif build --target $TARGET_TRIPLE --tests
-    fi
-    popd
-
-    pushd regex
-    echo "[TEST] rust-lang/regex example shootout-regex-dna"
-    ../build/cargo-clif clean
-    export RUSTFLAGS="$RUSTFLAGS --cap-lints warn" # newer aho_corasick versions throw a deprecation warning
-    # Make sure `[codegen mono items] start` doesn't poison the diff
-    ../build/cargo-clif build --example shootout-regex-dna --target $TARGET_TRIPLE
-    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-        cat examples/regexdna-input.txt \
-            | ../build/cargo-clif run --example shootout-regex-dna --target $TARGET_TRIPLE \
-            | grep -v "Spawned thread" > res.txt
-        diff -u res.txt examples/regexdna-output.txt
-    fi
-
-    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-        echo "[TEST] rust-lang/regex tests"
-        ../build/cargo-clif test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
-    else
-        echo "[AOT] rust-lang/regex tests"
-        ../build/cargo-clif build --tests --target $TARGET_TRIPLE
-    fi
-    popd
-
-    pushd portable-simd
-    echo "[TEST] rust-lang/portable-simd"
-    ../build/cargo-clif clean
-    ../build/cargo-clif build --all-targets --target $TARGET_TRIPLE
-    if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
-        ../build/cargo-clif test -q
-    fi
-    popd
-}
-
-case "$1" in
-    "no_sysroot")
-        no_sysroot_tests
-        ;;
-    "base_sysroot")
-        base_sysroot_tests
-        ;;
-    "extended_sysroot")
-        extended_sysroot_tests
-        ;;
-    *)
-        echo "unknown test suite"
-        ;;
-esac
diff --git a/test.sh b/test.sh
deleted file mode 100755
index a10924628bb..00000000000
--- a/test.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-./y.rs build --sysroot none "$@"
-
-rm -r target/out || true
-
-scripts/tests.sh no_sysroot
-
-./y.rs build "$@"
-
-scripts/tests.sh base_sysroot
-scripts/tests.sh extended_sysroot