Merge commit '3a31c6d8272c14388a34622193baf553636fe470' into sync_cg_clif-2021-07-07

6 files changed, 544 insertions, 0 deletions

diff --git a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
new file mode 100644
index 00000000000..1df2bcc4541
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
@@ -0,0 +1,40 @@
+use std::env;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+pub(crate) fn build_backend(channel: &str, host_triple: &str) -> PathBuf {
+    let mut cmd = Command::new("cargo");
+    cmd.arg("build").arg("--target").arg(host_triple);
+
+    match channel {
+        "debug" => {}
+        "release" => {
+            cmd.arg("--release");
+        }
+        _ => unreachable!(),
+    }
+
+    if cfg!(unix) {
+        if cfg!(target_os = "macos") {
+            cmd.env(
+                "RUSTFLAGS",
+                "-Csplit-debuginfo=unpacked \
+                -Clink-arg=-Wl,-rpath,@loader_path/../lib \
+                -Zosx-rpath-install-name"
+                    .to_string()
+                    + env::var("RUSTFLAGS").as_deref().unwrap_or(""),
+            );
+        } else {
+            cmd.env(
+                "RUSTFLAGS",
+                "-Clink-arg=-Wl,-rpath=$ORIGIN/../lib ".to_string()
+                    + env::var("RUSTFLAGS").as_deref().unwrap_or(""),
+            );
+        }
+    }
+
+    eprintln!("[BUILD] rustc_codegen_cranelift");
+    crate::utils::spawn_and_wait(cmd);
+
+    Path::new("target").join(host_triple).join(channel)
+}
diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
new file mode 100644
index 00000000000..9fb88c27961
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
@@ -0,0 +1,216 @@
+use std::env;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::{self, Command};
+
+use crate::rustc_info::{get_file_name, get_rustc_version};
+use crate::utils::{spawn_and_wait, try_hard_link};
+use crate::SysrootKind;
+
+pub(crate) fn build_sysroot(
+    channel: &str,
+    sysroot_kind: SysrootKind,
+    target_dir: &Path,
+    cg_clif_build_dir: PathBuf,
+    host_triple: &str,
+    target_triple: &str,
+) {
+    if target_dir.exists() {
+        fs::remove_dir_all(target_dir).unwrap();
+    }
+    fs::create_dir_all(target_dir.join("bin")).unwrap();
+    fs::create_dir_all(target_dir.join("lib")).unwrap();
+
+    // Copy the backend
+    for file in ["cg_clif", "cg_clif_build_sysroot"] {
+        try_hard_link(
+            cg_clif_build_dir.join(get_file_name(file, "bin")),
+            target_dir.join("bin").join(get_file_name(file, "bin")),
+        );
+    }
+
+    let cg_clif_dylib = get_file_name("rustc_codegen_cranelift", "dylib");
+    try_hard_link(
+        cg_clif_build_dir.join(&cg_clif_dylib),
+        target_dir
+            .join(if cfg!(windows) {
+                // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
+                // binaries.
+                "bin"
+            } else {
+                "lib"
+            })
+            .join(cg_clif_dylib),
+    );
+
+    // Build and copy cargo wrapper
+    let mut build_cargo_wrapper_cmd = Command::new("rustc");
+    build_cargo_wrapper_cmd
+        .arg("scripts/cargo.rs")
+        .arg("-o")
+        .arg(target_dir.join("cargo"))
+        .arg("-g");
+    spawn_and_wait(build_cargo_wrapper_cmd);
+
+    let default_sysroot = crate::rustc_info::get_default_sysroot();
+
+    let rustlib = target_dir.join("lib").join("rustlib");
+    let host_rustlib_lib = rustlib.join(host_triple).join("lib");
+    let target_rustlib_lib = rustlib.join(target_triple).join("lib");
+    fs::create_dir_all(&host_rustlib_lib).unwrap();
+    fs::create_dir_all(&target_rustlib_lib).unwrap();
+
+    if target_triple == "x86_64-pc-windows-gnu" {
+        if !default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib").exists() {
+            eprintln!(
+                "The x86_64-pc-windows-gnu target needs to be installed first before it is possible \
+                to compile a sysroot for it.",
+            );
+            process::exit(1);
+        }
+        for file in fs::read_dir(
+            default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
+        )
+        .unwrap()
+        {
+            let file = file.unwrap().path();
+            if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") {
+                continue; // only copy object files
+            }
+            try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
+        }
+    }
+
+    match sysroot_kind {
+        SysrootKind::None => {} // Nothing to do
+        SysrootKind::Llvm => {
+            for file in fs::read_dir(
+                default_sysroot.join("lib").join("rustlib").join(host_triple).join("lib"),
+            )
+            .unwrap()
+            {
+                let file = file.unwrap().path();
+                let file_name_str = file.file_name().unwrap().to_str().unwrap();
+                if file_name_str.contains("rustc_")
+                    || file_name_str.contains("chalk")
+                    || file_name_str.contains("tracing")
+                    || file_name_str.contains("regex")
+                {
+                    // These are large crates that are part of the rustc-dev component and are not
+                    // necessary to run regular programs.
+                    continue;
+                }
+                try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap()));
+            }
+
+            if target_triple != host_triple {
+                for file in fs::read_dir(
+                    default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
+                )
+                .unwrap()
+                {
+                    let file = file.unwrap().path();
+                    try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
+                }
+            }
+        }
+        SysrootKind::Clif => {
+            build_clif_sysroot_for_triple(channel, target_dir, host_triple, None);
+
+            if host_triple != target_triple {
+                // When cross-compiling it is often necessary to manually pick the right linker
+                let linker = if target_triple == "aarch64-unknown-linux-gnu" {
+                    Some("aarch64-linux-gnu-gcc")
+                } else {
+                    None
+                };
+                build_clif_sysroot_for_triple(channel, target_dir, target_triple, linker);
+            }
+
+            // Copy std for the host to the lib dir. This is necessary for the jit mode to find
+            // libstd.
+            for file in fs::read_dir(host_rustlib_lib).unwrap() {
+                let file = file.unwrap().path();
+                if file.file_name().unwrap().to_str().unwrap().contains("std-") {
+                    try_hard_link(&file, target_dir.join("lib").join(file.file_name().unwrap()));
+                }
+            }
+        }
+    }
+}
+
+fn build_clif_sysroot_for_triple(
+    channel: &str,
+    target_dir: &Path,
+    triple: &str,
+    linker: Option<&str>,
+) {
+    match fs::read_to_string(Path::new("build_sysroot").join("rustc_version")) {
+        Err(e) => {
+            eprintln!("Failed to get rustc version for patched sysroot source: {}", e);
+            eprintln!("Hint: Try `./y.rs prepare` to patch the sysroot source");
+            process::exit(1);
+        }
+        Ok(source_version) => {
+            let rustc_version = get_rustc_version();
+            if source_version != rustc_version {
+                eprintln!("The patched sysroot source is outdated");
+                eprintln!("Source version: {}", source_version.trim());
+                eprintln!("Rustc version:  {}", rustc_version.trim());
+                eprintln!("Hint: Try `./y.rs prepare` to update the patched sysroot source");
+                process::exit(1);
+            }
+        }
+    }
+
+    let build_dir = Path::new("build_sysroot").join("target").join(triple).join(channel);
+
+    if !crate::config::get_bool("keep_sysroot") {
+        // Cleanup the target dir with the exception of build scripts and the incremental cache
+        for dir in ["build", "deps", "examples", "native"] {
+            if build_dir.join(dir).exists() {
+                fs::remove_dir_all(build_dir.join(dir)).unwrap();
+            }
+        }
+    }
+
+    // Build sysroot
+    let mut build_cmd = Command::new("cargo");
+    build_cmd.arg("build").arg("--target").arg(triple).current_dir("build_sysroot");
+    let mut rustflags = "--clif -Zforce-unstable-if-unmarked".to_string();
+    if channel == "release" {
+        build_cmd.arg("--release");
+        rustflags.push_str(" -Zmir-opt-level=3");
+    }
+    if let Some(linker) = linker {
+        use std::fmt::Write;
+        write!(rustflags, " -Clinker={}", linker).unwrap();
+    }
+    build_cmd.env("RUSTFLAGS", rustflags);
+    build_cmd.env(
+        "RUSTC",
+        env::current_dir().unwrap().join(target_dir).join("bin").join("cg_clif_build_sysroot"),
+    );
+    // FIXME Enable incremental again once rust-lang/rust#74946 is fixed
+    build_cmd.env("CARGO_INCREMENTAL", "0").env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
+    spawn_and_wait(build_cmd);
+
+    // Copy all relevant files to the sysroot
+    for entry in
+        fs::read_dir(Path::new("build_sysroot/target").join(triple).join(channel).join("deps"))
+            .unwrap()
+    {
+        let entry = entry.unwrap();
+        if let Some(ext) = entry.path().extension() {
+            if ext == "rmeta" || ext == "d" || ext == "dSYM" {
+                continue;
+            }
+        } else {
+            continue;
+        };
+        try_hard_link(
+            entry.path(),
+            target_dir.join("lib").join("rustlib").join(triple).join("lib").join(entry.file_name()),
+        );
+    }
+}
diff --git a/compiler/rustc_codegen_cranelift/build_system/config.rs b/compiler/rustc_codegen_cranelift/build_system/config.rs
new file mode 100644
index 00000000000..ef540cf1f82
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/build_system/config.rs
@@ -0,0 +1,55 @@
+use std::{fs, process};
+
+fn load_config_file() -> Vec<(String, Option<String>)> {
+    fs::read_to_string("config.txt")
+        .unwrap()
+        .lines()
+        .map(|line| if let Some((line, _comment)) = line.split_once('#') { line } else { line })
+        .map(|line| line.trim())
+        .filter(|line| !line.is_empty())
+        .map(|line| {
+            if let Some((key, val)) = line.split_once('=') {
+                (key.trim().to_owned(), Some(val.trim().to_owned()))
+            } else {
+                (line.to_owned(), None)
+            }
+        })
+        .collect()
+}
+
+pub(crate) fn get_bool(name: &str) -> bool {
+    let values = load_config_file()
+        .into_iter()
+        .filter(|(key, _)| key == name)
+        .map(|(_, val)| val)
+        .collect::<Vec<_>>();
+    if values.is_empty() {
+        false
+    } else {
+        if values.iter().any(|val| val.is_some()) {
+            eprintln!("Boolean config `{}` has a value", name);
+            process::exit(1);
+        }
+        true
+    }
+}
+
+pub(crate) fn get_value(name: &str) -> Option<String> {
+    let values = load_config_file()
+        .into_iter()
+        .filter(|(key, _)| key == name)
+        .map(|(_, val)| val)
+        .collect::<Vec<_>>();
+    if values.is_empty() {
+        None
+    } else if values.len() == 1 {
+        if values[0].is_none() {
+            eprintln!("Config `{}` missing value", name);
+            process::exit(1);
+        }
+        values.into_iter().next().unwrap()
+    } else {
+        eprintln!("Config `{}` given multiple values: {:?}", name, values);
+        process::exit(1);
+    }
+}
diff --git a/compiler/rustc_codegen_cranelift/build_system/prepare.rs b/compiler/rustc_codegen_cranelift/build_system/prepare.rs
new file mode 100644
index 00000000000..401b8271abc
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/build_system/prepare.rs
@@ -0,0 +1,133 @@
+use std::env;
+use std::ffi::OsStr;
+use std::ffi::OsString;
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+
+use crate::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
+use crate::utils::{copy_dir_recursively, spawn_and_wait};
+
+pub(crate) fn prepare() {
+    prepare_sysroot();
+
+    eprintln!("[INSTALL] hyperfine");
+    Command::new("cargo").arg("install").arg("hyperfine").spawn().unwrap().wait().unwrap();
+
+    clone_repo(
+        "rand",
+        "https://github.com/rust-random/rand.git",
+        "0f933f9c7176e53b2a3c7952ded484e1783f0bf1",
+    );
+    apply_patches("rand", Path::new("rand"));
+
+    clone_repo(
+        "regex",
+        "https://github.com/rust-lang/regex.git",
+        "341f207c1071f7290e3f228c710817c280c8dca1",
+    );
+
+    clone_repo(
+        "simple-raytracer",
+        "https://github.com/ebobby/simple-raytracer",
+        "804a7a21b9e673a482797aa289a18ed480e4d813",
+    );
+
+    eprintln!("[LLVM BUILD] simple-raytracer");
+    let mut build_cmd = Command::new("cargo");
+    build_cmd.arg("build").env_remove("CARGO_TARGET_DIR").current_dir("simple-raytracer");
+    spawn_and_wait(build_cmd);
+    fs::copy(
+        Path::new("simple-raytracer/target/debug").join(get_file_name("main", "bin")),
+        // FIXME use get_file_name here too once testing is migrated to rust
+        "simple-raytracer/raytracer_cg_llvm",
+    )
+    .unwrap();
+}
+
+fn prepare_sysroot() {
+    let rustc_path = get_rustc_path();
+    let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust");
+    let sysroot_src = env::current_dir().unwrap().join("build_sysroot").join("sysroot_src");
+
+    assert!(sysroot_src_orig.exists());
+
+    if sysroot_src.exists() {
+        fs::remove_dir_all(&sysroot_src).unwrap();
+    }
+    fs::create_dir_all(sysroot_src.join("library")).unwrap();
+    eprintln!("[COPY] sysroot src");
+    copy_dir_recursively(&sysroot_src_orig.join("library"), &sysroot_src.join("library"));
+
+    let rustc_version = get_rustc_version();
+    fs::write(
+        Path::new("build_sysroot").join("rustc_version"),
+        &rustc_version,
+    )
+    .unwrap();
+
+    eprintln!("[GIT] init");
+    let mut git_init_cmd = Command::new("git");
+    git_init_cmd.arg("init").arg("-q").current_dir(&sysroot_src);
+    spawn_and_wait(git_init_cmd);
+
+    let mut git_add_cmd = Command::new("git");
+    git_add_cmd.arg("add").arg(".").current_dir(&sysroot_src);
+    spawn_and_wait(git_add_cmd);
+
+    let mut git_commit_cmd = Command::new("git");
+    git_commit_cmd
+        .arg("commit")
+        .arg("-m")
+        .arg("Initial commit")
+        .arg("-q")
+        .current_dir(&sysroot_src);
+    spawn_and_wait(git_commit_cmd);
+
+    apply_patches("sysroot", &sysroot_src);
+
+    clone_repo(
+        "build_sysroot/compiler-builtins",
+        "https://github.com/rust-lang/compiler-builtins.git",
+        "0.1.46",
+    );
+    apply_patches("compiler-builtins", Path::new("build_sysroot/compiler-builtins"));
+}
+
+fn clone_repo(target_dir: &str, repo: &str, rev: &str) {
+    eprintln!("[CLONE] {}", repo);
+    // Ignore exit code as the repo may already have been checked out
+    Command::new("git").arg("clone").arg(repo).arg(target_dir).spawn().unwrap().wait().unwrap();
+
+    let mut clean_cmd = Command::new("git");
+    clean_cmd.arg("checkout").arg("--").arg(".").current_dir(target_dir);
+    spawn_and_wait(clean_cmd);
+
+    let mut checkout_cmd = Command::new("git");
+    checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(target_dir);
+    spawn_and_wait(checkout_cmd);
+}
+
+fn get_patches(crate_name: &str) -> Vec<OsString> {
+    let mut patches: Vec<_> = fs::read_dir("patches")
+        .unwrap()
+        .map(|entry| entry.unwrap().path())
+        .filter(|path| path.extension() == Some(OsStr::new("patch")))
+        .map(|path| path.file_name().unwrap().to_owned())
+        .filter(|file_name| {
+            file_name.to_str().unwrap().split_once("-").unwrap().1.starts_with(crate_name)
+        })
+        .collect();
+    patches.sort();
+    patches
+}
+
+fn apply_patches(crate_name: &str, target_dir: &Path) {
+    for patch in get_patches(crate_name) {
+        eprintln!("[PATCH] {:?} <- {:?}", target_dir.file_name().unwrap(), patch);
+        let patch_arg = env::current_dir().unwrap().join("patches").join(patch);
+        let mut apply_patch_cmd = Command::new("git");
+        apply_patch_cmd.arg("am").arg(patch_arg).arg("-q").current_dir(target_dir);
+        spawn_and_wait(apply_patch_cmd);
+    }
+}
diff --git a/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs b/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs
new file mode 100644
index 00000000000..9206bb02bd3
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs
@@ -0,0 +1,65 @@
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
+pub(crate) fn get_rustc_version() -> String {
+    let version_info =
+        Command::new("rustc").stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout;
+    String::from_utf8(version_info).unwrap()
+}
+
+pub(crate) fn get_host_triple() -> String {
+    let version_info =
+        Command::new("rustc").stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
+    String::from_utf8(version_info)
+        .unwrap()
+        .lines()
+        .to_owned()
+        .find(|line| line.starts_with("host"))
+        .unwrap()
+        .split(":")
+        .nth(1)
+        .unwrap()
+        .trim()
+        .to_owned()
+}
+
+pub(crate) fn get_rustc_path() -> PathBuf {
+    let rustc_path = Command::new("rustup")
+        .stderr(Stdio::inherit())
+        .args(&["which", "rustc"])
+        .output()
+        .unwrap()
+        .stdout;
+    Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned()
+}
+
+pub(crate) fn get_default_sysroot() -> PathBuf {
+    let default_sysroot = Command::new("rustc")
+        .stderr(Stdio::inherit())
+        .args(&["--print", "sysroot"])
+        .output()
+        .unwrap()
+        .stdout;
+    Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned()
+}
+
+pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String {
+    let file_name = Command::new("rustc")
+        .stderr(Stdio::inherit())
+        .args(&[
+            "--crate-name",
+            crate_name,
+            "--crate-type",
+            crate_type,
+            "--print",
+            "file-names",
+            "-",
+        ])
+        .output()
+        .unwrap()
+        .stdout;
+    let file_name = String::from_utf8(file_name).unwrap().trim().to_owned();
+    assert!(!file_name.contains('\n'));
+    assert!(file_name.contains(crate_name));
+    file_name
+}
diff --git a/compiler/rustc_codegen_cranelift/build_system/utils.rs b/compiler/rustc_codegen_cranelift/build_system/utils.rs
new file mode 100644
index 00000000000..12b5d70fad8
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/build_system/utils.rs
@@ -0,0 +1,35 @@
+use std::fs;
+use std::path::Path;
+use std::process::{self, Command};
+
+#[track_caller]
+pub(crate) fn try_hard_link(src: impl AsRef<Path>, dst: impl AsRef<Path>) {
+    let src = src.as_ref();
+    let dst = dst.as_ref();
+    if let Err(_) = fs::hard_link(src, dst) {
+        fs::copy(src, dst).unwrap(); // Fallback to copying if hardlinking failed
+    }
+}
+
+#[track_caller]
+pub(crate) fn spawn_and_wait(mut cmd: Command) {
+    if !cmd.spawn().unwrap().wait().unwrap().success() {
+        process::exit(1);
+    }
+}
+
+pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
+    for entry in fs::read_dir(from).unwrap() {
+        let entry = entry.unwrap();
+        let filename = entry.file_name();
+        if filename == "." || filename == ".." {
+            continue;
+        }
+        if entry.metadata().unwrap().is_dir() {
+            fs::create_dir(to.join(&filename)).unwrap();
+            copy_dir_recursively(&from.join(&filename), &to.join(&filename));
+        } else {
+            fs::copy(from.join(&filename), to.join(&filename)).unwrap();
+        }
+    }
+}