Merge pull request #1291 from bjorn3/build_system_rework2

Move downloaded test project to downloads/

5 files changed, 136 insertions, 86 deletions

diff --git a/.gitignore b/.gitignore
index 9a4c54f08ca..fae09592c6a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,9 +15,4 @@ perf.data.old
 /build_sysroot/compiler-builtins
 /build_sysroot/rustc_version
 /rust
-/rand
-/regex
-/simple-raytracer
-/portable-simd
-/abi-cafe
-/abi-checker
+/download
diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs
index c2944ce6262..fae5b271636 100644
--- a/build_system/abi_cafe.rs
+++ b/build_system/abi_cafe.rs
@@ -3,6 +3,7 @@ use std::path::Path;
 
 use super::build_sysroot;
 use super::config;
+use super::prepare;
 use super::utils::{cargo_command, spawn_and_wait};
 use super::SysrootKind;
 
@@ -35,9 +36,8 @@ pub(crate) fn run(
     );
 
     eprintln!("Running abi-cafe");
-    let mut abi_cafe_path = env::current_dir().unwrap();
-    abi_cafe_path.push("abi-cafe");
-    env::set_current_dir(&abi_cafe_path.clone()).unwrap();
+    let abi_cafe_path = prepare::ABI_CAFE.source_dir();
+    env::set_current_dir(abi_cafe_path.clone()).unwrap();
 
     let pairs = ["rustc_calls_cgclif", "cgclif_calls_rustc", "cgclif_calls_cc", "cc_calls_cgclif"];
 
diff --git a/build_system/prepare.rs b/build_system/prepare.rs
index 3ad9f87e318..f9ab8ae7041 100644
--- a/build_system/prepare.rs
+++ b/build_system/prepare.rs
@@ -7,56 +7,61 @@ use std::process::Command;
 use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
 use super::utils::{cargo_command, copy_dir_recursively, spawn_and_wait};
 
+pub(crate) const ABI_CAFE: GitRepo = GitRepo::github(
+    "Gankra",
+    "abi-cafe",
+    "4c6dc8c9c687e2b3a760ff2176ce236872b37212",
+    "abi-cafe",
+);
+
+pub(crate) const RAND: GitRepo =
+    GitRepo::github("rust-random", "rand", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", "rand");
+
+pub(crate) const REGEX: GitRepo =
+    GitRepo::github("rust-lang", "regex", "341f207c1071f7290e3f228c710817c280c8dca1", "regex");
+
+pub(crate) const PORTABLE_SIMD: GitRepo = GitRepo::github(
+    "rust-lang",
+    "portable-simd",
+    "d5cd4a8112d958bd3a252327e0d069a6363249bd",
+    "portable-simd",
+);
+
+pub(crate) const SIMPLE_RAYTRACER: GitRepo = GitRepo::github(
+    "ebobby",
+    "simple-raytracer",
+    "804a7a21b9e673a482797aa289a18ed480e4d813",
+    "<none>",
+);
+
 pub(crate) fn prepare() {
+    if Path::new("download").exists() {
+        std::fs::remove_dir_all(Path::new("download")).unwrap();
+    }
+    std::fs::create_dir_all(Path::new("download")).unwrap();
+
     prepare_sysroot();
 
+    // FIXME maybe install this only locally?
     eprintln!("[INSTALL] hyperfine");
     Command::new("cargo").arg("install").arg("hyperfine").spawn().unwrap().wait().unwrap();
 
-    clone_repo_shallow_github(
-        "abi-cafe",
-        "Gankra",
-        "abi-cafe",
-        "4c6dc8c9c687e2b3a760ff2176ce236872b37212",
-    );
-    apply_patches("abi-cafe", Path::new("abi-cafe"));
-
-    clone_repo_shallow_github(
-        "rand",
-        "rust-random",
-        "rand",
-        "0f933f9c7176e53b2a3c7952ded484e1783f0bf1",
-    );
-    apply_patches("rand", Path::new("rand"));
-
-    clone_repo_shallow_github(
-        "regex",
-        "rust-lang",
-        "regex",
-        "341f207c1071f7290e3f228c710817c280c8dca1",
-    );
-
-    clone_repo_shallow_github(
-        "portable-simd",
-        "rust-lang",
-        "portable-simd",
-        "d5cd4a8112d958bd3a252327e0d069a6363249bd",
-    );
-    apply_patches("portable-simd", Path::new("portable-simd"));
-
-    clone_repo_shallow_github(
-        "simple-raytracer",
-        "ebobby",
-        "simple-raytracer",
-        "804a7a21b9e673a482797aa289a18ed480e4d813",
-    );
+    ABI_CAFE.fetch();
+    RAND.fetch();
+    REGEX.fetch();
+    PORTABLE_SIMD.fetch();
+    SIMPLE_RAYTRACER.fetch();
 
     eprintln!("[LLVM BUILD] simple-raytracer");
-    let build_cmd = cargo_command("cargo", "build", None, Path::new("simple-raytracer"));
+    let build_cmd = cargo_command("cargo", "build", None, &SIMPLE_RAYTRACER.source_dir());
     spawn_and_wait(build_cmd);
     fs::copy(
-        Path::new("simple-raytracer/target/debug").join(get_file_name("main", "bin")),
-        Path::new("simple-raytracer").join(get_file_name("raytracer_cg_llvm", "bin")),
+        SIMPLE_RAYTRACER
+            .source_dir()
+            .join("target")
+            .join("debug")
+            .join(get_file_name("main", "bin")),
+        SIMPLE_RAYTRACER.source_dir().join(get_file_name("raytracer_cg_llvm", "bin")),
     )
     .unwrap();
 }
@@ -88,38 +93,78 @@ fn prepare_sysroot() {
     apply_patches("sysroot", &sysroot_src);
 }
 
+pub(crate) struct GitRepo {
+    url: GitRepoUrl,
+    rev: &'static str,
+    patch_name: &'static str,
+}
+
+enum GitRepoUrl {
+    Github { user: &'static str, repo: &'static str },
+}
+
+impl GitRepo {
+    const fn github(
+        user: &'static str,
+        repo: &'static str,
+        rev: &'static str,
+        patch_name: &'static str,
+    ) -> GitRepo {
+        GitRepo { url: GitRepoUrl::Github { user, repo }, rev, patch_name }
+    }
+
+    pub(crate) fn source_dir(&self) -> PathBuf {
+        match self.url {
+            GitRepoUrl::Github { user: _, repo } => {
+                std::env::current_dir().unwrap().join("download").join(repo)
+            }
+        }
+    }
+
+    fn fetch(&self) {
+        match self.url {
+            GitRepoUrl::Github { user, repo } => {
+                clone_repo_shallow_github(&self.source_dir(), user, repo, self.rev);
+            }
+        }
+        apply_patches(self.patch_name, &self.source_dir());
+    }
+}
+
 #[allow(dead_code)]
-fn clone_repo(target_dir: &str, repo: &str, rev: &str) {
+fn clone_repo(download_dir: &Path, repo: &str, rev: &str) {
     eprintln!("[CLONE] {}", repo);
     // Ignore exit code as the repo may already have been checked out
-    Command::new("git").arg("clone").arg(repo).arg(target_dir).spawn().unwrap().wait().unwrap();
+    Command::new("git").arg("clone").arg(repo).arg(&download_dir).spawn().unwrap().wait().unwrap();
 
     let mut clean_cmd = Command::new("git");
-    clean_cmd.arg("checkout").arg("--").arg(".").current_dir(target_dir);
+    clean_cmd.arg("checkout").arg("--").arg(".").current_dir(&download_dir);
     spawn_and_wait(clean_cmd);
 
     let mut checkout_cmd = Command::new("git");
-    checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(target_dir);
+    checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(download_dir);
     spawn_and_wait(checkout_cmd);
 }
 
-fn clone_repo_shallow_github(target_dir: &str, username: &str, repo: &str, rev: &str) {
+fn clone_repo_shallow_github(download_dir: &Path, user: &str, repo: &str, rev: &str) {
     if cfg!(windows) {
         // Older windows doesn't have tar or curl by default. Fall back to using git.
-        clone_repo(target_dir, &format!("https://github.com/{}/{}.git", username, repo), rev);
+        clone_repo(download_dir, &format!("https://github.com/{}/{}.git", user, repo), rev);
         return;
     }
 
-    let archive_url = format!("https://github.com/{}/{}/archive/{}.tar.gz", username, repo, rev);
-    let archive_file = format!("{}.tar.gz", rev);
-    let archive_dir = format!("{}-{}", repo, rev);
+    let downloads_dir = std::env::current_dir().unwrap().join("download");
 
-    eprintln!("[DOWNLOAD] {}/{} from {}", username, repo, archive_url);
+    let archive_url = format!("https://github.com/{}/{}/archive/{}.tar.gz", user, repo, rev);
+    let archive_file = downloads_dir.join(format!("{}.tar.gz", rev));
+    let archive_dir = downloads_dir.join(format!("{}-{}", repo, rev));
+
+    eprintln!("[DOWNLOAD] {}/{} from {}", user, repo, archive_url);
 
     // Remove previous results if they exists
     let _ = std::fs::remove_file(&archive_file);
     let _ = std::fs::remove_dir_all(&archive_dir);
-    let _ = std::fs::remove_dir_all(target_dir);
+    let _ = std::fs::remove_dir_all(&download_dir);
 
     // Download zip archive
     let mut download_cmd = Command::new("curl");
@@ -128,13 +173,13 @@ fn clone_repo_shallow_github(target_dir: &str, username: &str, repo: &str, rev:
 
     // Unpack tar archive
     let mut unpack_cmd = Command::new("tar");
-    unpack_cmd.arg("xf").arg(&archive_file);
+    unpack_cmd.arg("xf").arg(&archive_file).current_dir(downloads_dir);
     spawn_and_wait(unpack_cmd);
 
     // Rename unpacked dir to the expected name
-    std::fs::rename(archive_dir, target_dir).unwrap();
+    std::fs::rename(archive_dir, &download_dir).unwrap();
 
-    init_git_repo(Path::new(target_dir));
+    init_git_repo(&download_dir);
 
     // Cleanup
     std::fs::remove_file(archive_file).unwrap();
@@ -175,6 +220,10 @@ fn get_patches(source_dir: &Path, crate_name: &str) -> Vec<PathBuf> {
 }
 
 fn apply_patches(crate_name: &str, target_dir: &Path) {
+    if crate_name == "<none>" {
+        return;
+    }
+
     for patch in get_patches(&std::env::current_dir().unwrap(), crate_name) {
         eprintln!(
             "[PATCH] {:?} <- {:?}",
diff --git a/build_system/tests.rs b/build_system/tests.rs
index d4f393fc7ff..a414b60f4e0 100644
--- a/build_system/tests.rs
+++ b/build_system/tests.rs
@@ -1,5 +1,6 @@
 use super::build_sysroot;
 use super::config;
+use super::prepare;
 use super::rustc_info::get_wrapper_file_name;
 use super::utils::{cargo_command, hyperfine_command, spawn_and_wait, spawn_and_wait_with_input};
 use build_system::SysrootKind;
@@ -217,7 +218,7 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
 
 const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::new("test.rust-random/rand", &|runner| {
-        runner.in_dir(["rand"], |runner| {
+        runner.in_dir(prepare::RAND.source_dir(), |runner| {
             runner.run_cargo("clean", []);
 
             if runner.host_triple == runner.target_triple {
@@ -230,7 +231,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
         });
     }),
     TestCase::new("bench.simple-raytracer", &|runner| {
-        runner.in_dir(["simple-raytracer"], |runner| {
+        runner.in_dir(prepare::SIMPLE_RAYTRACER.source_dir(), |runner| {
             let run_runs = env::var("RUN_RUNS").unwrap_or("10".to_string()).parse().unwrap();
 
             if runner.host_triple == runner.target_triple {
@@ -273,19 +274,28 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
         });
     }),
     TestCase::new("test.libcore", &|runner| {
-        runner.in_dir(["build_sysroot", "sysroot_src", "library", "core", "tests"], |runner| {
-            runner.run_cargo("clean", []);
+        runner.in_dir(
+            std::env::current_dir()
+                .unwrap()
+                .join("build_sysroot")
+                .join("sysroot_src")
+                .join("library")
+                .join("core")
+                .join("tests"),
+            |runner| {
+                runner.run_cargo("clean", []);
 
-            if runner.host_triple == runner.target_triple {
-                runner.run_cargo("test", []);
-            } else {
-                eprintln!("Cross-Compiling: Not running tests");
-                runner.run_cargo("build", ["--tests"]);
-            }
-        });
+                if runner.host_triple == runner.target_triple {
+                    runner.run_cargo("test", []);
+                } else {
+                    eprintln!("Cross-Compiling: Not running tests");
+                    runner.run_cargo("build", ["--tests"]);
+                }
+            },
+        );
     }),
     TestCase::new("test.regex-shootout-regex-dna", &|runner| {
-        runner.in_dir(["regex"], |runner| {
+        runner.in_dir(prepare::REGEX.source_dir(), |runner| {
             runner.run_cargo("clean", []);
 
             // newer aho_corasick versions throw a deprecation warning
@@ -336,7 +346,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
         });
     }),
     TestCase::new("test.regex", &|runner| {
-        runner.in_dir(["regex"], |runner| {
+        runner.in_dir(prepare::REGEX.source_dir(), |runner| {
             runner.run_cargo("clean", []);
 
             // newer aho_corasick versions throw a deprecation warning
@@ -367,7 +377,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
         });
     }),
     TestCase::new("test.portable-simd", &|runner| {
-        runner.in_dir(["portable-simd"], |runner| {
+        runner.in_dir(prepare::PORTABLE_SIMD.source_dir(), |runner| {
             runner.run_cargo("clean", []);
             runner.run_cargo("build", ["--all-targets", "--target", &runner.target_triple]);
 
@@ -506,16 +516,8 @@ impl TestRunner {
         }
     }
 
-    fn in_dir<'a, I, F>(&self, dir: I, callback: F)
-    where
-        I: IntoIterator<Item = &'a str>,
-        F: FnOnce(&TestRunner),
-    {
+    fn in_dir(&self, new: impl AsRef<Path>, callback: impl FnOnce(&TestRunner)) {
         let current = env::current_dir().unwrap();
-        let mut new = current.clone();
-        for d in dir {
-            new.push(d);
-        }
 
         env::set_current_dir(new).unwrap();
         callback(self);
diff --git a/clean_all.sh b/clean_all.sh
index 0fd9b9455a3..fedab2433aa 100755
--- a/clean_all.sh
+++ b/clean_all.sh
@@ -3,4 +3,8 @@ set -e
 
 rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
 rm -rf target/ build/ perf.data{,.old} y.bin
+rm -rf download/
+
+# Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh
+# FIXME remove at some point in the future
 rm -rf rand/ regex/ simple-raytracer/ portable-simd/ abi-checker/ abi-cafe/