Merge pull request #1338 from bjorn3/build_system_rework5

Avoid clobbering build_system/ and ~/.cargo/bin

16 files changed, 130 insertions, 134 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 76b48d70ab7..7c966aa1ab9 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -6,8 +6,6 @@ task:
     - pkg install -y curl git bash
     - curl https://sh.rustup.rs -sSf --output rustup.sh
     - sh rustup.sh --default-toolchain none -y --profile=minimal
-  cargo_bin_cache:
-    folder: ~/.cargo/bin
   target_cache:
     folder: target
   prepare_script:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b79406879ff..cc9ae19afde 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -62,12 +62,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-installed-crates
-
     - name: Cache cargo registry and index
       uses: actions/cache@v3
       with:
diff --git a/.github/workflows/nightly-cranelift.yml b/.github/workflows/nightly-cranelift.yml
index 0565938ee35..968cd43efd1 100644
--- a/.github/workflows/nightly-cranelift.yml
+++ b/.github/workflows/nightly-cranelift.yml
@@ -13,12 +13,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ubuntu-latest-cargo-installed-crates
-
     - name: Prepare dependencies
       run: |
         git config --global user.email "user@example.com"
diff --git a/.github/workflows/rustc.yml b/.github/workflows/rustc.yml
index bab81b9dc2a..2c7de86f9b9 100644
--- a/.github/workflows/rustc.yml
+++ b/.github/workflows/rustc.yml
@@ -10,12 +10,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
     - name: Cache cargo registry and index
       uses: actions/cache@v3
       with:
@@ -44,12 +38,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
     - name: Cache cargo registry and index
       uses: actions/cache@v3
       with:
diff --git a/.gitignore b/.gitignore
index b443fd58a1b..8012e93f6a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-target
+/target
 **/*.rs.bk
 *.rlib
 *.o
@@ -11,9 +11,6 @@ perf.data.old
 /y.exe
 /y.pdb
 /build
-/build_sysroot/sysroot_src
-/build_sysroot/compiler-builtins
-/build_sysroot/rustc_version
 /dist
 /rust
 /download
diff --git a/.vscode/settings.json b/.vscode/settings.json
index bc914e37d2b..d8650d1e387 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -30,7 +30,7 @@
             ]
         },
         {
-            "sysroot_src": "./build_sysroot/sysroot_src/library",
+            "sysroot_src": "./download/sysroot/sysroot_src/library",
             "crates": [
                 {
                     "root_module": "./example/std_example.rs",
diff --git a/build_system/bench.rs b/build_system/bench.rs
new file mode 100644
index 00000000000..0ad8863223d
--- /dev/null
+++ b/build_system/bench.rs
@@ -0,0 +1,79 @@
+use std::env;
+use std::fs;
+use std::path::Path;
+
+use super::path::{Dirs, RelPath};
+use super::prepare::GitRepo;
+use super::rustc_info::{get_file_name, get_wrapper_file_name};
+use super::utils::{hyperfine_command, is_ci, spawn_and_wait, CargoProject};
+
+pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
+    "ebobby",
+    "simple-raytracer",
+    "804a7a21b9e673a482797aa289a18ed480e4d813",
+    "<none>",
+);
+
+pub(crate) static SIMPLE_RAYTRACER: CargoProject =
+    CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
+
+pub(crate) fn benchmark(dirs: &Dirs) {
+    benchmark_simple_raytracer(dirs);
+}
+
+fn benchmark_simple_raytracer(dirs: &Dirs) {
+    if std::process::Command::new("hyperfine").output().is_err() {
+        eprintln!("Hyperfine not installed");
+        eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine");
+        std::process::exit(1);
+    }
+
+    let run_runs = env::var("RUN_RUNS")
+        .unwrap_or(if is_ci() { "2" } else { "10" }.to_string())
+        .parse()
+        .unwrap();
+
+    eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
+    let cargo_clif = RelPath::DIST.to_path(dirs).join(get_wrapper_file_name("cargo-clif", "bin"));
+    let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs);
+    let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
+
+    let clean_cmd = format!(
+        "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
+        manifest_path = manifest_path.display(),
+        target_dir = target_dir.display(),
+    );
+    let llvm_build_cmd = format!(
+        "cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
+        manifest_path = manifest_path.display(),
+        target_dir = target_dir.display(),
+    );
+    let clif_build_cmd = format!(
+        "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
+        cargo_clif = cargo_clif.display(),
+        manifest_path = manifest_path.display(),
+        target_dir = target_dir.display(),
+    );
+
+    let bench_compile =
+        hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
+
+    spawn_and_wait(bench_compile);
+
+    eprintln!("[BENCH RUN] ebobby/simple-raytracer");
+    fs::copy(
+        target_dir.join("debug").join(get_file_name("main", "bin")),
+        RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_clif", "bin")),
+    )
+    .unwrap();
+
+    let mut bench_run = hyperfine_command(
+        0,
+        run_runs,
+        None,
+        Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
+        Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
+    );
+    bench_run.current_dir(RelPath::BUILD.to_path(dirs));
+    spawn_and_wait(bench_run);
+}
diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs
index cbbf09b9b97..711d4ccc55b 100644
--- a/build_system/build_sysroot.rs
+++ b/build_system/build_sysroot.rs
@@ -146,10 +146,11 @@ pub(crate) fn build_sysroot(
     }
 }
 
-// FIXME move to download/ or dist/
-pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = RelPath::BUILD_SYSROOT.join("rustc_version");
-pub(crate) static SYSROOT_SRC: RelPath = RelPath::BUILD_SYSROOT.join("sysroot_src");
-static STANDARD_LIBRARY: CargoProject = CargoProject::new(&RelPath::BUILD_SYSROOT, "build_sysroot");
+pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
+pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot");
+pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version");
+pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src");
+static STANDARD_LIBRARY: CargoProject = CargoProject::new(&BUILD_SYSROOT, "build_sysroot");
 
 fn build_clif_sysroot_for_triple(
     dirs: &Dirs,
diff --git a/build_system/mod.rs b/build_system/mod.rs
index 2f311aed7c8..76d1d013b0d 100644
--- a/build_system/mod.rs
+++ b/build_system/mod.rs
@@ -5,6 +5,7 @@ use std::process;
 use self::utils::is_ci;
 
 mod abi_cafe;
+mod bench;
 mod build_backend;
 mod build_sysroot;
 mod config;
@@ -20,6 +21,7 @@ USAGE:
     ./y.rs prepare [--out-dir DIR]
     ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
     ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
+    ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
 
 OPTIONS:
     --sysroot none|clif|llvm
@@ -54,6 +56,7 @@ enum Command {
     Prepare,
     Build,
     Test,
+    Bench,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -67,7 +70,7 @@ pub fn main() {
     if env::var("RUST_BACKTRACE").is_err() {
         env::set_var("RUST_BACKTRACE", "1");
     }
-    env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1");
+    env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1"); // FIXME disable this by default
     env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1");
 
     if is_ci() {
@@ -83,6 +86,7 @@ pub fn main() {
         Some("prepare") => Command::Prepare,
         Some("build") => Command::Build,
         Some("test") => Command::Test,
+        Some("bench") => Command::Bench,
         Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
         Some(command) => arg_error!("Unknown command {}", command),
         None => {
@@ -198,5 +202,16 @@ pub fn main() {
                 &target_triple,
             );
         }
+        Command::Bench => {
+            build_sysroot::build_sysroot(
+                &dirs,
+                channel,
+                sysroot_kind,
+                &cg_clif_dylib,
+                &host_triple,
+                &target_triple,
+            );
+            bench::benchmark(&dirs);
+        }
     }
 }
diff --git a/build_system/path.rs b/build_system/path.rs
index e93981f1d64..35ab6f111fe 100644
--- a/build_system/path.rs
+++ b/build_system/path.rs
@@ -42,7 +42,6 @@ impl RelPath {
     pub(crate) const DIST: RelPath = RelPath::Base(PathBase::Dist);
 
     pub(crate) const SCRIPTS: RelPath = RelPath::SOURCE.join("scripts");
-    pub(crate) const BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
     pub(crate) const PATCHES: RelPath = RelPath::SOURCE.join("patches");
 
     pub(crate) const fn join(&'static self, suffix: &'static str) -> RelPath {
diff --git a/build_system/prepare.rs b/build_system/prepare.rs
index 8ac67e8f942..9ad4ddc92c5 100644
--- a/build_system/prepare.rs
+++ b/build_system/prepare.rs
@@ -3,9 +3,11 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
-use super::build_sysroot::{SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
+use crate::build_system::rustc_info::get_default_sysroot;
+
+use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
 use super::path::{Dirs, RelPath};
-use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
+use super::rustc_info::{get_file_name, get_rustc_version};
 use super::utils::{copy_dir_recursively, spawn_and_wait, Compiler};
 
 pub(crate) fn prepare(dirs: &Dirs) {
@@ -31,14 +33,14 @@ pub(crate) fn prepare(dirs: &Dirs) {
     super::tests::RAND_REPO.fetch(dirs);
     super::tests::REGEX_REPO.fetch(dirs);
     super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
-    super::tests::SIMPLE_RAYTRACER_REPO.fetch(dirs);
+    super::bench::SIMPLE_RAYTRACER_REPO.fetch(dirs);
 
     eprintln!("[LLVM BUILD] simple-raytracer");
     let host_compiler = Compiler::host();
-    let build_cmd = super::tests::SIMPLE_RAYTRACER.build(&host_compiler, dirs);
+    let build_cmd = super::bench::SIMPLE_RAYTRACER.build(&host_compiler, dirs);
     spawn_and_wait(build_cmd);
     fs::copy(
-        super::tests::SIMPLE_RAYTRACER
+        super::bench::SIMPLE_RAYTRACER
             .target_dir(dirs)
             .join(&host_compiler.triple)
             .join("debug")
@@ -49,27 +51,27 @@ pub(crate) fn prepare(dirs: &Dirs) {
 }
 
 fn prepare_sysroot(dirs: &Dirs) {
-    let rustc_path = get_rustc_path();
-    let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust");
-    let sysroot_src = SYSROOT_SRC;
-
+    let sysroot_src_orig = get_default_sysroot().join("lib/rustlib/src/rust");
     assert!(sysroot_src_orig.exists());
 
-    sysroot_src.ensure_fresh(dirs);
-    fs::create_dir_all(sysroot_src.to_path(dirs).join("library")).unwrap();
     eprintln!("[COPY] sysroot src");
+
+    BUILD_SYSROOT.ensure_fresh(dirs);
+    copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs));
+
+    fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap();
     copy_dir_recursively(
         &sysroot_src_orig.join("library"),
-        &sysroot_src.to_path(dirs).join("library"),
+        &SYSROOT_SRC.to_path(dirs).join("library"),
     );
 
     let rustc_version = get_rustc_version();
     fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap();
 
     eprintln!("[GIT] init");
-    init_git_repo(&sysroot_src.to_path(dirs));
+    init_git_repo(&SYSROOT_SRC.to_path(dirs));
 
-    apply_patches(dirs, "sysroot", &sysroot_src.to_path(dirs));
+    apply_patches(dirs, "sysroot", &SYSROOT_SRC.to_path(dirs));
 }
 
 pub(crate) struct GitRepo {
diff --git a/build_system/tests.rs b/build_system/tests.rs
index 37c35106af6..5b8b6f2df10 100644
--- a/build_system/tests.rs
+++ b/build_system/tests.rs
@@ -1,11 +1,9 @@
-use super::build_sysroot;
+use super::bench::SIMPLE_RAYTRACER;
+use super::build_sysroot::{self, SYSROOT_SRC};
 use super::config;
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
-use super::rustc_info::{get_file_name, get_wrapper_file_name};
-use super::utils::{
-    hyperfine_command, is_ci, spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler,
-};
+use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler};
 use super::SysrootKind;
 use std::env;
 use std::ffi::OsStr;
@@ -251,18 +249,8 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
 static PORTABLE_SIMD: CargoProject =
     CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd");
 
-pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
-    "ebobby",
-    "simple-raytracer",
-    "804a7a21b9e673a482797aa289a18ed480e4d813",
-    "<none>",
-);
-
-pub(crate) static SIMPLE_RAYTRACER: CargoProject =
-    CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
-
 static LIBCORE_TESTS: CargoProject =
-    CargoProject::new(&RelPath::BUILD_SYSROOT.join("sysroot_src/library/core/tests"), "core_tests");
+    CargoProject::new(&SYSROOT_SRC.join("library/core/tests"), "core_tests");
 
 const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::new("test.rust-random/rand", &|runner| {
@@ -280,67 +268,9 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
             spawn_and_wait(build_cmd);
         }
     }),
-    TestCase::new("bench.simple-raytracer", &|runner| {
-        let run_runs = env::var("RUN_RUNS")
-            .unwrap_or(if is_ci() { "2" } else { "10" }.to_string())
-            .parse()
-            .unwrap();
-
-        if runner.is_native {
-            eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
-            let cargo_clif = RelPath::DIST
-                .to_path(&runner.dirs)
-                .join(get_wrapper_file_name("cargo-clif", "bin"));
-            let manifest_path = SIMPLE_RAYTRACER.manifest_path(&runner.dirs);
-            let target_dir = SIMPLE_RAYTRACER.target_dir(&runner.dirs);
-
-            let clean_cmd = format!(
-                "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
-                manifest_path = manifest_path.display(),
-                target_dir = target_dir.display(),
-            );
-            let llvm_build_cmd = format!(
-                "cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
-                manifest_path = manifest_path.display(),
-                target_dir = target_dir.display(),
-            );
-            let clif_build_cmd = format!(
-                "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
-                cargo_clif = cargo_clif.display(),
-                manifest_path = manifest_path.display(),
-                target_dir = target_dir.display(),
-            );
-
-            let bench_compile =
-                hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
-
-            spawn_and_wait(bench_compile);
-
-            eprintln!("[BENCH RUN] ebobby/simple-raytracer");
-            fs::copy(
-                target_dir.join("debug").join(get_file_name("main", "bin")),
-                RelPath::BUILD
-                    .to_path(&runner.dirs)
-                    .join(get_file_name("raytracer_cg_clif", "bin")),
-            )
-            .unwrap();
-
-            let mut bench_run = hyperfine_command(
-                0,
-                run_runs,
-                None,
-                Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
-                Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
-            );
-            bench_run.current_dir(RelPath::BUILD.to_path(&runner.dirs));
-            spawn_and_wait(bench_run);
-        } else {
-            spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.target_compiler.cargo, &runner.dirs));
-            eprintln!("[BENCH COMPILE] ebobby/simple-raytracer (skipped)");
-            eprintln!("[COMPILE] ebobby/simple-raytracer");
-            spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
-            eprintln!("[BENCH RUN] ebobby/simple-raytracer (skipped)");
-        }
+    TestCase::new("test.simple-raytracer", &|runner| {
+        spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.host_compiler.cargo, &runner.dirs));
+        spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
     }),
     TestCase::new("test.libcore", &|runner| {
         spawn_and_wait(LIBCORE_TESTS.clean(&runner.host_compiler.cargo, &runner.dirs));
diff --git a/clean_all.sh b/clean_all.sh
index 1760e5836ec..cdfc2e143e6 100755
--- a/clean_all.sh
+++ b/clean_all.sh
@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
 set -e
 
-rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
-rm -rf target/ build/ dist/ perf.data{,.old} y.bin
-rm -rf download/
+rm -rf target/ download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb
 
 # Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh
 # FIXME remove at some point in the future
 rm -rf rand/ regex/ simple-raytracer/ portable-simd/ abi-checker/ abi-cafe/
+rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
diff --git a/config.txt b/config.txt
index 258b67e9314..d9912a8158f 100644
--- a/config.txt
+++ b/config.txt
@@ -44,7 +44,7 @@ aot.issue-72793
 
 testsuite.extended_sysroot
 test.rust-random/rand
-bench.simple-raytracer
+test.simple-raytracer
 test.libcore
 test.regex-shootout-regex-dna
 test.regex
diff --git a/scripts/rustup.sh b/scripts/rustup.sh
index bc4c06ed7d2..6111c205444 100755
--- a/scripts/rustup.sh
+++ b/scripts/rustup.sh
@@ -17,10 +17,10 @@ case $1 in
         done
 
         ./clean_all.sh
-        ./y.rs prepare
 
         (cd build_sysroot && cargo update)
 
+        ./y.rs prepare
         ;;
     "commit")
         git add rust-toolchain build_sysroot/Cargo.lock
diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh
index 6c64b7de7da..88bc6445503 100644
--- a/scripts/setup_rust_fork.sh
+++ b/scripts/setup_rust_fork.sh
@@ -51,7 +51,7 @@ popd
 # FIXME remove once inline asm is fully supported
 export RUSTFLAGS="$RUSTFLAGS --cfg=rustix_use_libc"
 
-export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd build_sysroot/sysroot_src; pwd)"
+export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd download/sysroot/sysroot_src; pwd)"
 
 # Allow the testsuite to use llvm tools
 host_triple=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")