diff options
Diffstat (limited to 'compiler/rustc_codegen_gcc/build_system')
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/Cargo.lock | 9 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/Cargo.toml | 3 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/build.rs | 196 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/cargo.rs | 114 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/clean.rs | 82 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/clone_gcc.rs | 79 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/config.rs | 647 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/info.rs | 19 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/main.rs | 32 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/prepare.rs | 75 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/test.rs | 1225 | ||||
| -rw-r--r-- | compiler/rustc_codegen_gcc/build_system/src/utils.rs | 323 |
12 files changed, 2479 insertions, 325 deletions
diff --git a/compiler/rustc_codegen_gcc/build_system/Cargo.lock b/compiler/rustc_codegen_gcc/build_system/Cargo.lock index 86268e19160..e727561a2bf 100644 --- a/compiler/rustc_codegen_gcc/build_system/Cargo.lock +++ b/compiler/rustc_codegen_gcc/build_system/Cargo.lock @@ -3,5 +3,14 @@ version = 3 [[package]] +name = "boml" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fdb93f04c73bff54305fa437ffea5449c41edcaadfe882f35836206b166ac5" + +[[package]] name = "y" version = "0.1.0" +dependencies = [ + "boml", +] diff --git a/compiler/rustc_codegen_gcc/build_system/Cargo.toml b/compiler/rustc_codegen_gcc/build_system/Cargo.toml index f36709ea036..d2600ed5a03 100644 --- a/compiler/rustc_codegen_gcc/build_system/Cargo.toml +++ b/compiler/rustc_codegen_gcc/build_system/Cargo.toml @@ -3,6 +3,9 @@ name = "y" version = "0.1.0" edition = "2021" +[dependencies] +boml = "0.3.1" + [[bin]] name = "y" path = "src/main.rs" diff --git a/compiler/rustc_codegen_gcc/build_system/src/build.rs b/compiler/rustc_codegen_gcc/build_system/src/build.rs index f1c3701a946..0eabd1d8972 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/build.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/build.rs @@ -1,7 +1,5 @@ -use crate::config::{set_config, ConfigInfo}; -use crate::utils::{ - get_gcc_path, run_command, run_command_with_output_and_env, walk_dir, -}; +use crate::config::{Channel, ConfigInfo}; +use crate::utils::{run_command, run_command_with_output_and_env, walk_dir}; use std::collections::HashMap; use std::ffi::OsStr; use std::fs; @@ -9,33 +7,18 @@ use std::path::Path; #[derive(Default)] struct BuildArg { - codegen_release_channel: bool, - sysroot_release_channel: bool, - sysroot_panic_abort: bool, flags: Vec<String>, - gcc_path: String, + config_info: ConfigInfo, } impl BuildArg { fn new() -> Result<Option<Self>, String> { - let gcc_path = get_gcc_path()?; - let mut build_arg = Self { - gcc_path, - ..Default::default() - }; + let mut build_arg = Self::default(); // We skip binary name and the `build` command. let mut args = std::env::args().skip(2); while let Some(arg) = args.next() { match arg.as_str() { - "--release" => build_arg.codegen_release_channel = true, - "--release-sysroot" => build_arg.sysroot_release_channel = true, - "--no-default-features" => { - build_arg.flags.push("--no-default-features".to_string()); - } - "--sysroot-panic-abort" => { - build_arg.sysroot_panic_abort = true; - }, "--features" => { if let Some(arg) = args.next() { build_arg.flags.push("--features".to_string()); @@ -50,25 +33,11 @@ impl BuildArg { Self::usage(); return Ok(None); } - "--target-triple" => { - if args.next().is_some() { - // Handled in config.rs. - } else { - return Err( - "Expected a value after `--target-triple`, found nothing".to_string() - ); - } - } - "--target" => { - if args.next().is_some() { - // Handled in config.rs. - } else { - return Err( - "Expected a value after `--target`, found nothing".to_string() - ); + arg => { + if !build_arg.config_info.parse_argument(arg, &mut args)? { + return Err(format!("Unknown argument `{}`", arg)); } } - arg => return Err(format!("Unknown argument `{}`", arg)), } } Ok(Some(build_arg)) @@ -79,29 +48,19 @@ impl BuildArg { r#" `build` command help: - --release : Build codegen in release mode - --release-sysroot : Build sysroot in release mode - --sysroot-panic-abort : Build the sysroot without unwinding support. - --no-default-features : Add `--no-default-features` flag - --features [arg] : Add a new feature [arg] - --target-triple [arg] : Set the target triple to [arg] - --help : Show this help -"# - ) + --features [arg] : Add a new feature [arg]"# + ); + ConfigInfo::show_usage(); + println!(" --help : Show this help"); } } -fn build_sysroot( - env: &mut HashMap<String, String>, - args: &BuildArg, - config: &ConfigInfo, -) -> Result<(), String> { - std::env::set_current_dir("build_sysroot") - .map_err(|error| format!("Failed to go to `build_sysroot` directory: {:?}", error))?; +pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Result<(), String> { + let start_dir = Path::new("build_sysroot"); // Cleanup for previous run // Clean target dir except for build scripts and incremental cache let _ = walk_dir( - "target", + start_dir.join("target"), |dir: &Path| { for top in &["debug", "release"] { let _ = fs::remove_dir_all(dir.join(top).join("build")); @@ -138,92 +97,114 @@ fn build_sysroot( |_| Ok(()), ); - let _ = fs::remove_file("Cargo.lock"); - let _ = fs::remove_file("test_target/Cargo.lock"); - let _ = fs::remove_dir_all("sysroot"); + let _ = fs::remove_file(start_dir.join("Cargo.lock")); + let _ = fs::remove_file(start_dir.join("test_target/Cargo.lock")); + let _ = fs::remove_dir_all(start_dir.join("sysroot")); // Builds libs - let mut rustflags = env - .get("RUSTFLAGS") - .cloned() - .unwrap_or_default(); - if args.sysroot_panic_abort { + let mut rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); + if config.sysroot_panic_abort { rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests"); } - env.insert( - "RUSTFLAGS".to_string(), - format!("{} -Zmir-opt-level=3", rustflags), - ); - let channel = if args.sysroot_release_channel { - run_command_with_output_and_env( - &[ - &"cargo", - &"build", - &"--target", - &config.target, - &"--release", - ], - None, - Some(&env), - )?; + rustflags.push_str(" -Z force-unstable-if-unmarked"); + let mut env = env.clone(); + + let mut args: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"build", &"--target", &config.target]; + + if config.no_default_features { + rustflags.push_str(" -Csymbol-mangling-version=v0"); + args.push(&"--no-default-features"); + } + + let channel = if config.sysroot_release_channel { + rustflags.push_str(" -Zmir-opt-level=3"); + args.push(&"--release"); "release" } else { - run_command_with_output_and_env( - &[ - &"cargo", - &"build", - &"--target", - &config.target, - ], - None, - Some(env), - )?; "debug" }; + env.insert("RUSTFLAGS".to_string(), rustflags); + run_command_with_output_and_env(&args, Some(start_dir), Some(&env))?; + // Copy files to sysroot - let sysroot_path = format!("sysroot/lib/rustlib/{}/lib/", config.target_triple); - fs::create_dir_all(&sysroot_path) - .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_path, error))?; + let sysroot_path = start_dir.join(format!("sysroot/lib/rustlib/{}/lib/", config.target_triple)); + fs::create_dir_all(&sysroot_path).map_err(|error| { + format!( + "Failed to create directory `{}`: {:?}", + sysroot_path.display(), + error + ) + })?; let copier = |dir_to_copy: &Path| { + // FIXME: should not use shell command! run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ()) }; walk_dir( - &format!("target/{}/{}/deps", config.target_triple, channel), + start_dir.join(&format!("target/{}/{}/deps", config.target_triple, channel)), copier, copier, )?; // Copy the source files to the sysroot (Rust for Linux needs this). - let sysroot_src_path = "sysroot/lib/rustlib/src/rust"; - fs::create_dir_all(&sysroot_src_path) - .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_src_path, error))?; - run_command(&[&"cp", &"-r", &"sysroot_src/library/", &sysroot_src_path], None)?; + let sysroot_src_path = start_dir.join("sysroot/lib/rustlib/src/rust"); + fs::create_dir_all(&sysroot_src_path).map_err(|error| { + format!( + "Failed to create directory `{}`: {:?}", + sysroot_src_path.display(), + error + ) + })?; + run_command( + &[ + &"cp", + &"-r", + &start_dir.join("sysroot_src/library/"), + &sysroot_src_path, + ], + None, + )?; Ok(()) } -fn build_codegen(args: &BuildArg) -> Result<(), String> { +fn build_codegen(args: &mut BuildArg) -> Result<(), String> { let mut env = HashMap::new(); - env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone()); - env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone()); + env.insert( + "LD_LIBRARY_PATH".to_string(), + args.config_info.gcc_path.clone(), + ); + env.insert( + "LIBRARY_PATH".to_string(), + args.config_info.gcc_path.clone(), + ); + + if args.config_info.no_default_features { + env.insert( + "RUSTFLAGS".to_string(), + "-Csymbol-mangling-version=v0".to_string(), + ); + } let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"rustc"]; - if args.codegen_release_channel { + if args.config_info.channel == Channel::Release { command.push(&"--release"); env.insert("CHANNEL".to_string(), "release".to_string()); env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string()); } else { env.insert("CHANNEL".to_string(), "debug".to_string()); } + if args.config_info.no_default_features { + command.push(&"--no-default-features"); + } let flags = args.flags.iter().map(|s| s.as_str()).collect::<Vec<_>>(); for flag in &flags { command.push(flag); } run_command_with_output_and_env(&command, None, Some(&env))?; - let config = set_config(&mut env, &[], Some(&args.gcc_path))?; + args.config_info.setup(&mut env, false)?; // We voluntarily ignore the error. let _ = fs::remove_dir_all("target/out"); @@ -236,19 +217,16 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> { })?; println!("[BUILD] sysroot"); - build_sysroot( - &mut env, - args, - &config, - )?; + build_sysroot(&env, &args.config_info)?; Ok(()) } pub fn run() -> Result<(), String> { - let args = match BuildArg::new()? { + let mut args = match BuildArg::new()? { Some(args) => args, None => return Ok(()), }; - build_codegen(&args)?; + args.config_info.setup_gcc_path()?; + build_codegen(&mut args)?; Ok(()) } diff --git a/compiler/rustc_codegen_gcc/build_system/src/cargo.rs b/compiler/rustc_codegen_gcc/build_system/src/cargo.rs new file mode 100644 index 00000000000..1cfcdba6b1c --- /dev/null +++ b/compiler/rustc_codegen_gcc/build_system/src/cargo.rs @@ -0,0 +1,114 @@ +use crate::config::ConfigInfo; +use crate::utils::{ + get_toolchain, run_command_with_output_and_env_no_err, rustc_toolchain_version_info, + rustc_version_info, +}; + +use std::collections::HashMap; +use std::ffi::OsStr; +use std::path::PathBuf; + +fn args() -> Result<Option<Vec<String>>, String> { + // We skip the binary and the "cargo" option. + if let Some("--help") = std::env::args().skip(2).next().as_deref() { + usage(); + return Ok(None); + } + let args = std::env::args().skip(2).collect::<Vec<_>>(); + if args.is_empty() { + return Err( + "Expected at least one argument for `cargo` subcommand, found none".to_string(), + ); + } + Ok(Some(args)) +} + +fn usage() { + println!( + r#" +`cargo` command help: + + [args] : Arguments to be passed to the cargo command + --help : Show this help +"# + ) +} + +pub fn run() -> Result<(), String> { + let args = match args()? { + Some(a) => a, + None => return Ok(()), + }; + + // We first need to go to the original location to ensure that the config setup will go as + // expected. + let current_dir = std::env::current_dir() + .and_then(|path| path.canonicalize()) + .map_err(|error| format!("Failed to get current directory path: {:?}", error))?; + let current_exe = std::env::current_exe() + .and_then(|path| path.canonicalize()) + .map_err(|error| format!("Failed to get current exe path: {:?}", error))?; + let mut parent_dir = current_exe + .components() + .map(|comp| comp.as_os_str()) + .collect::<Vec<_>>(); + // We run this script from "build_system/target/release/y", so we need to remove these elements. + for to_remove in &["y", "release", "target", "build_system"] { + if parent_dir + .last() + .map(|part| part == to_remove) + .unwrap_or(false) + { + parent_dir.pop(); + } else { + return Err(format!( + "Build script not executed from `build_system/target/release/y` (in path {})", + current_exe.display(), + )); + } + } + let parent_dir = PathBuf::from(parent_dir.join(&OsStr::new("/"))); + std::env::set_current_dir(&parent_dir).map_err(|error| { + format!( + "Failed to go to `{}` folder: {:?}", + parent_dir.display(), + error + ) + })?; + + let mut env: HashMap<String, String> = std::env::vars().collect(); + ConfigInfo::default().setup(&mut env, false)?; + let toolchain = get_toolchain()?; + + let toolchain_version = rustc_toolchain_version_info(&toolchain)?; + let default_version = rustc_version_info(None)?; + if toolchain_version != default_version { + println!( + "rustc_codegen_gcc is built for {} but the default rustc version is {}.", + toolchain_version.short, default_version.short, + ); + println!("Using {}.", toolchain_version.short); + } + + // We go back to the original folder since we now have set up everything we needed. + std::env::set_current_dir(¤t_dir).map_err(|error| { + format!( + "Failed to go back to `{}` folder: {:?}", + current_dir.display(), + error + ) + })?; + + let rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); + env.insert("RUSTDOCFLAGS".to_string(), rustflags); + let toolchain = format!("+{}", toolchain); + let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &toolchain]; + for arg in &args { + command.push(arg); + } + if run_command_with_output_and_env_no_err(&command, None, Some(&env)).is_err() { + std::process::exit(1); + } + + Ok(()) +} diff --git a/compiler/rustc_codegen_gcc/build_system/src/clean.rs b/compiler/rustc_codegen_gcc/build_system/src/clean.rs new file mode 100644 index 00000000000..cd8e691a0ed --- /dev/null +++ b/compiler/rustc_codegen_gcc/build_system/src/clean.rs @@ -0,0 +1,82 @@ +use crate::utils::{remove_file, run_command}; + +use std::fs::remove_dir_all; +use std::path::Path; + +#[derive(Default)] +enum CleanArg { + /// `clean all` + All, + /// `clean ui-tests` + UiTests, + /// `clean --help` + #[default] + Help, +} + +impl CleanArg { + fn new() -> Result<Self, String> { + // We skip the binary and the "clean" option. + for arg in std::env::args().skip(2) { + return match arg.as_str() { + "all" => Ok(Self::All), + "ui-tests" => Ok(Self::UiTests), + "--help" => Ok(Self::Help), + a => Err(format!("Unknown argument `{}`", a)), + }; + } + Ok(Self::default()) + } +} + +fn usage() { + println!( + r#" +`clean` command help: + + all : Clean all data + ui-tests : Clean ui tests + --help : Show this help +"# + ) +} + +fn clean_all() -> Result<(), String> { + let dirs_to_remove = [ + "target", + "build_sysroot/sysroot", + "build_sysroot/sysroot_src", + "build_sysroot/target", + ]; + for dir in dirs_to_remove { + let _ = remove_dir_all(dir); + } + let dirs_to_remove = ["regex", "rand", "simple-raytracer"]; + for dir in dirs_to_remove { + let _ = remove_dir_all(Path::new(crate::BUILD_DIR).join(dir)); + } + + let files_to_remove = ["build_sysroot/Cargo.lock", "perf.data", "perf.data.old"]; + + for file in files_to_remove { + let _ = remove_file(file); + } + + println!("Successfully ran `clean all`"); + Ok(()) +} + +fn clean_ui_tests() -> Result<(), String> { + let path = Path::new(crate::BUILD_DIR).join("rust/build/x86_64-unknown-linux-gnu/test/ui/"); + run_command(&[&"find", &path, &"-name", &"stamp", &"-delete"], None)?; + Ok(()) +} + +pub fn run() -> Result<(), String> { + match CleanArg::new()? { + CleanArg::All => clean_all()?, + CleanArg::UiTests => clean_ui_tests()?, + CleanArg::Help => usage(), + } + Ok(()) +} diff --git a/compiler/rustc_codegen_gcc/build_system/src/clone_gcc.rs b/compiler/rustc_codegen_gcc/build_system/src/clone_gcc.rs new file mode 100644 index 00000000000..aee46afaeb0 --- /dev/null +++ b/compiler/rustc_codegen_gcc/build_system/src/clone_gcc.rs @@ -0,0 +1,79 @@ +use crate::config::ConfigInfo; +use crate::utils::{git_clone, run_command_with_output}; + +use std::path::{Path, PathBuf}; + +fn show_usage() { + println!( + r#" +`clone-gcc` command help: + + --out-path : Location where the GCC repository will be cloned (default: `./gcc`)"# + ); + ConfigInfo::show_usage(); + println!(" --help : Show this help"); +} + +#[derive(Default)] +struct Args { + out_path: PathBuf, + config_info: ConfigInfo, +} + +impl Args { + fn new() -> Result<Option<Self>, String> { + let mut command_args = Self::default(); + + let mut out_path = None; + + // We skip binary name and the `clone-gcc` command. + let mut args = std::env::args().skip(2); + + while let Some(arg) = args.next() { + match arg.as_str() { + "--out-path" => match args.next() { + Some(path) if !path.is_empty() => out_path = Some(path), + _ => { + return Err("Expected an argument after `--out-path`, found nothing".into()) + } + }, + "--help" => { + show_usage(); + return Ok(None); + } + arg => { + if !command_args.config_info.parse_argument(arg, &mut args)? { + return Err(format!("Unknown option {}", arg)); + } + } + } + } + command_args.out_path = match out_path { + Some(p) => p.into(), + None => PathBuf::from("./gcc"), + }; + return Ok(Some(command_args)); + } +} + +pub fn run() -> Result<(), String> { + let Some(args) = Args::new()? else { + return Ok(()); + }; + + let result = git_clone("https://github.com/antoyo/gcc", Some(&args.out_path), false)?; + if result.ran_clone { + let gcc_commit = args.config_info.get_gcc_commit()?; + println!("Checking out GCC commit `{}`...", gcc_commit); + run_command_with_output( + &[&"git", &"checkout", &gcc_commit], + Some(Path::new(&result.repo_dir)), + )?; + } else { + println!( + "There is already a GCC folder in `{}`, leaving things as is...", + args.out_path.display() + ); + } + Ok(()) +} diff --git a/compiler/rustc_codegen_gcc/build_system/src/config.rs b/compiler/rustc_codegen_gcc/build_system/src/config.rs index 64d9bd73e01..c633ee57d4a 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/config.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/config.rs @@ -1,149 +1,560 @@ -use crate::utils::{get_gcc_path, get_os_name, get_rustc_host_triple}; +use crate::utils::{ + create_symlink, get_os_name, run_command_with_output, rustc_version_info, split_args, +}; use std::collections::HashMap; use std::env as std_env; +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use boml::{types::TomlValue, Toml}; + +#[derive(Default, PartialEq, Eq, Clone, Copy, Debug)] +pub enum Channel { + #[default] + Debug, + Release, +} + +impl Channel { + pub fn as_str(self) -> &'static str { + match self { + Self::Debug => "debug", + Self::Release => "release", + } + } +} + +fn failed_config_parsing(config_file: &Path, err: &str) -> Result<ConfigFile, String> { + Err(format!( + "Failed to parse `{}`: {}", + config_file.display(), + err + )) +} + +#[derive(Default)] +pub struct ConfigFile { + gcc_path: Option<String>, + download_gccjit: Option<bool>, +} + +impl ConfigFile { + pub fn new(config_file: &Path) -> Result<Self, String> { + let content = fs::read_to_string(config_file).map_err(|_| { + format!( + "Failed to read `{}`. Take a look at `Readme.md` to see how to set up the project", + config_file.display(), + ) + })?; + let toml = Toml::parse(&content).map_err(|err| { + format!( + "Error occurred around `{}`: {:?}", + &content[err.start..=err.end], + err.kind + ) + })?; + let mut config = Self::default(); + for (key, value) in toml.iter() { + match (key, value) { + ("gcc-path", TomlValue::String(value)) => { + config.gcc_path = Some(value.as_str().to_string()) + } + ("gcc-path", _) => { + return failed_config_parsing(config_file, "Expected a string for `gcc-path`") + } + ("download-gccjit", TomlValue::Boolean(value)) => { + config.download_gccjit = Some(*value) + } + ("download-gccjit", _) => { + return failed_config_parsing( + config_file, + "Expected a boolean for `download-gccjit`", + ) + } + _ => return failed_config_parsing(config_file, &format!("Unknown key `{}`", key)), + } + } + match (config.gcc_path.as_mut(), config.download_gccjit) { + (None, None | Some(false)) => { + return failed_config_parsing( + config_file, + "At least one of `gcc-path` or `download-gccjit` value must be set", + ) + } + (Some(_), Some(true)) => { + println!( + "WARNING: both `gcc-path` and `download-gccjit` arguments are used, \ + ignoring `gcc-path`" + ); + } + (Some(gcc_path), _) => { + let path = Path::new(gcc_path); + *gcc_path = path + .canonicalize() + .map_err(|err| { + format!("Failed to get absolute path of `{}`: {:?}", gcc_path, err) + })? + .display() + .to_string(); + } + _ => {} + } + Ok(config) + } +} + +#[derive(Default, Debug)] pub struct ConfigInfo { pub target: String, pub target_triple: String, + pub host_triple: String, pub rustc_command: Vec<String>, + pub run_in_vm: bool, + pub cargo_target_dir: String, + pub dylib_ext: String, + pub sysroot_release_channel: bool, + pub channel: Channel, + pub sysroot_panic_abort: bool, + pub cg_backend_path: String, + pub sysroot_path: String, + pub gcc_path: String, + config_file: Option<String>, + // This is used in particular in rust compiler bootstrap because it doesn't run at the root + // of the `cg_gcc` folder, making it complicated for us to get access to local files we need + // like `libgccjit.version` or `config.toml`. + cg_gcc_path: Option<PathBuf>, + // Needed for the `info` command which doesn't want to actually download the lib if needed, + // just to set the `gcc_path` field to display it. + pub no_download: bool, + pub no_default_features: bool, } -// Returns the beginning for the command line of rustc. -pub fn set_config( - env: &mut HashMap<String, String>, - test_flags: &[String], - gcc_path: Option<&str>, -) -> Result<ConfigInfo, String> { - env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string()); - - let gcc_path = match gcc_path { - Some(path) => path.to_string(), - None => get_gcc_path()?, - }; - env.insert("GCC_PATH".to_string(), gcc_path.clone()); - - let os_name = get_os_name()?; - let dylib_ext = match os_name.as_str() { - "Linux" => "so", - "Darwin" => "dylib", - os => return Err(format!("unsupported OS `{}`", os)), - }; - let host_triple = get_rustc_host_triple()?; - let mut linker = None; - let mut target_triple = host_triple.clone(); - let mut target = target_triple.clone(); - - // We skip binary name and the command. - let mut args = std::env::args().skip(2); - - let mut set_target_triple = false; - let mut set_target = false; - while let Some(arg) = args.next() { - match arg.as_str() { - "--target-triple" => { +impl ConfigInfo { + /// Returns `true` if the argument was taken into account. + pub fn parse_argument( + &mut self, + arg: &str, + args: &mut impl Iterator<Item = String>, + ) -> Result<bool, String> { + match arg { + "--target" => { if let Some(arg) = args.next() { - target_triple = arg; - set_target_triple = true; + self.target = arg; } else { + return Err("Expected a value after `--target`, found nothing".to_string()); + } + } + "--target-triple" => match args.next() { + Some(arg) if !arg.is_empty() => self.target_triple = arg.to_string(), + _ => { return Err( "Expected a value after `--target-triple`, found nothing".to_string() - ); + ) } }, - "--target" => { - if let Some(arg) = args.next() { - target = arg; - set_target = true; - } else { - return Err( - "Expected a value after `--target`, found nothing".to_string() - ); + "--out-dir" => match args.next() { + Some(arg) if !arg.is_empty() => { + self.cargo_target_dir = arg.to_string(); + } + _ => return Err("Expected a value after `--out-dir`, found nothing".to_string()), + }, + "--config-file" => match args.next() { + Some(arg) if !arg.is_empty() => { + self.config_file = Some(arg.to_string()); + } + _ => { + return Err("Expected a value after `--config-file`, found nothing".to_string()) } }, - _ => (), + "--release-sysroot" => self.sysroot_release_channel = true, + "--release" => self.channel = Channel::Release, + "--sysroot-panic-abort" => self.sysroot_panic_abort = true, + "--cg_gcc-path" => match args.next() { + Some(arg) if !arg.is_empty() => { + self.cg_gcc_path = Some(arg.into()); + } + _ => { + return Err("Expected a value after `--cg_gcc-path`, found nothing".to_string()) + } + }, + "--no-default-features" => self.no_default_features = true, + _ => return Ok(false), } + Ok(true) } - if set_target_triple && !set_target { - target = target_triple.clone(); + pub fn rustc_command_vec(&self) -> Vec<&dyn AsRef<OsStr>> { + let mut command: Vec<&dyn AsRef<OsStr>> = Vec::with_capacity(self.rustc_command.len()); + for arg in self.rustc_command.iter() { + command.push(arg); + } + command } - if host_triple != target_triple { - linker = Some(format!("-Clinker={}-gcc", target_triple)); + pub fn get_gcc_commit(&self) -> Result<String, String> { + let commit_hash_file = self.compute_path("libgccjit.version"); + let content = fs::read_to_string(&commit_hash_file).map_err(|_| { + format!( + "Failed to read `{}`. Take a look at `Readme.md` to see how to set up the project", + commit_hash_file.display(), + ) + })?; + let commit = content.trim(); + // This is a very simple check to ensure this is not a path. For the rest, it'll just fail + // when trying to download the file so we should be fine. + if commit.contains('/') || commit.contains('\\') { + return Err(format!( + "{}: invalid commit hash `{}`", + commit_hash_file.display(), + commit, + )); + } + Ok(commit.to_string()) } - let current_dir = - std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; - let channel = if let Some(channel) = env.get("CHANNEL") { - channel.as_str() - } else { - "debug" - }; - let cg_backend_path = current_dir - .join("target") - .join(channel) - .join(&format!("librustc_codegen_gcc.{}", dylib_ext)); - let sysroot_path = current_dir.join("build_sysroot/sysroot"); - let mut rustflags = Vec::new(); - if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { - rustflags.push(cg_rustflags.clone()); + + fn download_gccjit_if_needed(&mut self) -> Result<(), String> { + let output_dir = Path::new(crate::BUILD_DIR).join("libgccjit"); + let commit = self.get_gcc_commit()?; + + let output_dir = output_dir.join(&commit); + if !output_dir.is_dir() { + std::fs::create_dir_all(&output_dir).map_err(|err| { + format!( + "failed to create folder `{}`: {:?}", + output_dir.display(), + err, + ) + })?; + } + let output_dir = output_dir.canonicalize().map_err(|err| { + format!( + "Failed to get absolute path of `{}`: {:?}", + output_dir.display(), + err + ) + })?; + + let libgccjit_so_name = "libgccjit.so"; + let libgccjit_so = output_dir.join(libgccjit_so_name); + if !libgccjit_so.is_file() && !self.no_download { + // Download time! + let tempfile_name = format!("{}.download", libgccjit_so_name); + let tempfile = output_dir.join(&tempfile_name); + let is_in_ci = std::env::var("GITHUB_ACTIONS").is_ok(); + + let url = format!( + "https://github.com/antoyo/gcc/releases/download/master-{}/libgccjit.so", + commit, + ); + + println!("Downloading `{}`...", url); + download_gccjit(url, &output_dir, tempfile_name, !is_in_ci)?; + + let libgccjit_so = output_dir.join(libgccjit_so_name); + // If we reach this point, it means the file was correctly downloaded, so let's + // rename it! + std::fs::rename(&tempfile, &libgccjit_so).map_err(|err| { + format!( + "Failed to rename `{}` into `{}`: {:?}", + tempfile.display(), + libgccjit_so.display(), + err, + ) + })?; + + println!("Downloaded libgccjit.so version {} successfully!", commit); + // We need to create a link named `libgccjit.so.0` because that's what the linker is + // looking for. + create_symlink( + &libgccjit_so, + output_dir.join(&format!("{}.0", libgccjit_so_name)), + )?; + } + + self.gcc_path = output_dir.display().to_string(); + println!("Using `{}` as path for libgccjit", self.gcc_path); + Ok(()) } - if let Some(linker) = linker { - rustflags.push(linker.to_string()); + + pub fn compute_path<P: AsRef<Path>>(&self, other: P) -> PathBuf { + match self.cg_gcc_path { + Some(ref path) => path.join(other), + None => PathBuf::new().join(other), + } } - rustflags.extend_from_slice(&[ - "-Csymbol-mangling-version=v0".to_string(), - "-Cdebuginfo=2".to_string(), - format!("-Zcodegen-backend={}", cg_backend_path.display()), - "--sysroot".to_string(), - sysroot_path.display().to_string(), - ]); - - // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. - // TODO(antoyo): remove when we can handle ThinLTO. - if !env.contains_key(&"FAT_LTO".to_string()) { - rustflags.push("-Clto=off".to_string()); + + pub fn setup_gcc_path(&mut self) -> Result<(), String> { + let config_file = match self.config_file.as_deref() { + Some(config_file) => config_file.into(), + None => self.compute_path("config.toml"), + }; + let ConfigFile { + gcc_path, + download_gccjit, + } = ConfigFile::new(&config_file)?; + + if let Some(true) = download_gccjit { + self.download_gccjit_if_needed()?; + return Ok(()); + } + self.gcc_path = match gcc_path { + Some(path) => path, + None => { + return Err(format!( + "missing `gcc-path` value from `{}`", + config_file.display(), + )) + } + }; + Ok(()) } - rustflags.extend_from_slice(test_flags); - // FIXME(antoyo): remove once the atomic shim is gone - if os_name == "Darwin" { + + pub fn setup( + &mut self, + env: &mut HashMap<String, String>, + use_system_gcc: bool, + ) -> Result<(), String> { + env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string()); + + if self.gcc_path.is_empty() && !use_system_gcc { + self.setup_gcc_path()?; + } + env.insert("GCC_PATH".to_string(), self.gcc_path.clone()); + + if self.cargo_target_dir.is_empty() { + match env.get("CARGO_TARGET_DIR").filter(|dir| !dir.is_empty()) { + Some(cargo_target_dir) => self.cargo_target_dir = cargo_target_dir.clone(), + None => self.cargo_target_dir = "target/out".to_string(), + } + } + + let os_name = get_os_name()?; + self.dylib_ext = match os_name.as_str() { + "Linux" => "so", + "Darwin" => "dylib", + os => return Err(format!("unsupported OS `{}`", os)), + } + .to_string(); + let rustc = match env.get("RUSTC") { + Some(r) if !r.is_empty() => r.to_string(), + _ => "rustc".to_string(), + }; + self.host_triple = match rustc_version_info(Some(&rustc))?.host { + Some(host) => host, + None => return Err("no host found".to_string()), + }; + + if self.target_triple.is_empty() { + if let Some(overwrite) = env.get("OVERWRITE_TARGET_TRIPLE") { + self.target_triple = overwrite.clone(); + } + } + if self.target_triple.is_empty() { + self.target_triple = self.host_triple.clone(); + } + if self.target.is_empty() && !self.target_triple.is_empty() { + self.target = self.target_triple.clone(); + } + + let mut linker = None; + + if self.host_triple != self.target_triple { + if self.target_triple.is_empty() { + return Err("Unknown non-native platform".to_string()); + } + linker = Some(format!("-Clinker={}-gcc", self.target_triple)); + self.run_in_vm = true; + } + + let current_dir = + std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; + let channel = if self.channel == Channel::Release { + "release" + } else if let Some(channel) = env.get("CHANNEL") { + channel.as_str() + } else { + "debug" + }; + + let has_builtin_backend = env + .get("BUILTIN_BACKEND") + .map(|backend| !backend.is_empty()) + .unwrap_or(false); + + let mut rustflags = Vec::new(); + if has_builtin_backend { + // It means we're building inside the rustc testsuite, so some options need to be handled + // a bit differently. + self.cg_backend_path = "gcc".to_string(); + + match env.get("RUSTC_SYSROOT") { + Some(rustc_sysroot) if !rustc_sysroot.is_empty() => { + rustflags.extend_from_slice(&["--sysroot".to_string(), rustc_sysroot.clone()]); + } + _ => {} + } + // This should not be needed, but is necessary for the CI in the rust repository. + // FIXME: Remove when the rust CI switches to the master version of libgccjit. + rustflags.push("-Cpanic=abort".to_string()); + } else { + self.cg_backend_path = current_dir + .join("target") + .join(channel) + .join(&format!("librustc_codegen_gcc.{}", self.dylib_ext)) + .display() + .to_string(); + self.sysroot_path = current_dir + .join("build_sysroot/sysroot") + .display() + .to_string(); + rustflags.extend_from_slice(&["--sysroot".to_string(), self.sysroot_path.clone()]); + }; + + // This environment variable is useful in case we want to change options of rustc commands. + if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { + rustflags.extend_from_slice(&split_args(&cg_rustflags)?); + } + if let Some(test_flags) = env.get("TEST_FLAGS") { + rustflags.extend_from_slice(&split_args(&test_flags)?); + } + + if let Some(linker) = linker { + rustflags.push(linker.to_string()); + } + + if self.no_default_features { + rustflags.push("-Csymbol-mangling-version=v0".to_string()); + } + rustflags.extend_from_slice(&[ - "-Clink-arg=-undefined".to_string(), - "-Clink-arg=dynamic_lookup".to_string(), + "-Cdebuginfo=2".to_string(), + format!("-Zcodegen-backend={}", self.cg_backend_path), + ]); + + // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. + // TODO(antoyo): remove when we can handle ThinLTO. + if !env.contains_key(&"FAT_LTO".to_string()) { + rustflags.push("-Clto=off".to_string()); + } + // FIXME(antoyo): remove once the atomic shim is gone + if os_name == "Darwin" { + rustflags.extend_from_slice(&[ + "-Clink-arg=-undefined".to_string(), + "-Clink-arg=dynamic_lookup".to_string(), + ]); + } + env.insert("RUSTFLAGS".to_string(), rustflags.join(" ")); + // display metadata load errors + env.insert("RUSTC_LOG".to_string(), "warn".to_string()); + + let sysroot = current_dir.join(&format!( + "build_sysroot/sysroot/lib/rustlib/{}/lib", + self.target_triple, + )); + let ld_library_path = format!( + "{target}:{sysroot}:{gcc_path}", + // FIXME: It's possible to pick another out directory. Would be nice to have a command + // line option to change it. + target = current_dir.join("target/out").display(), + sysroot = sysroot.display(), + gcc_path = self.gcc_path, + ); + env.insert("LIBRARY_PATH".to_string(), ld_library_path.clone()); + env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone()); + env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path); + + // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc. + // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH. + // Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc + let path = std::env::var("PATH").unwrap_or_default(); + env.insert( + "PATH".to_string(), + format!( + "/opt/gcc/bin:/opt/m68k-unknown-linux-gnu/bin{}{}", + if path.is_empty() { "" } else { ":" }, + path + ), + ); + + self.rustc_command = vec![rustc]; + self.rustc_command.extend_from_slice(&rustflags); + self.rustc_command.extend_from_slice(&[ + "-L".to_string(), + "crate=target/out".to_string(), + "--out-dir".to_string(), + self.cargo_target_dir.clone(), ]); + + if !env.contains_key("RUSTC_LOG") { + env.insert("RUSTC_LOG".to_string(), "warn".to_string()); + } + Ok(()) } - env.insert("RUSTFLAGS".to_string(), rustflags.join(" ")); - // display metadata load errors - env.insert("RUSTC_LOG".to_string(), "warn".to_string()); - - let sysroot = current_dir.join(&format!( - "build_sysroot/sysroot/lib/rustlib/{}/lib", - target_triple - )); - let ld_library_path = format!( - "{target}:{sysroot}:{gcc_path}", - target = current_dir.join("target/out").display(), - sysroot = sysroot.display(), + + pub fn show_usage() { + println!( + "\ + --target-triple [arg] : Set the target triple to [arg] + --target [arg] : Set the target to [arg] + --out-dir : Location where the files will be generated + --release : Build in release mode + --release-sysroot : Build sysroot in release mode + --sysroot-panic-abort : Build the sysroot without unwinding support + --config-file : Location of the config file to be used + --cg_gcc-path : Location of the rustc_codegen_gcc root folder (used + when ran from another directory) + --no-default-features : Add `--no-default-features` flag to cargo commands" + ); + } +} + +fn download_gccjit( + url: String, + output_dir: &Path, + tempfile_name: String, + with_progress_bar: bool, +) -> Result<(), String> { + // Try curl. If that fails and we are on windows, fallback to PowerShell. + let mut ret = run_command_with_output( + &[ + &"curl", + &"--speed-time", + &"30", + &"--speed-limit", + &"10", // timeout if speed is < 10 bytes/sec for > 30 seconds + &"--connect-timeout", + &"30", // timeout if cannot connect within 30 seconds + &"-o", + &tempfile_name, + &"--retry", + &"3", + &"-SRfL", + if with_progress_bar { + &"--progress-bar" + } else { + &"-s" + }, + &url.as_str(), + ], + Some(&output_dir), ); - env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone()); - env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path); - - // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc. - // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH. - // Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc - let path = std::env::var("PATH").unwrap_or_default(); - env.insert("PATH".to_string(), format!("/opt/gcc/bin:{}", path)); - - let mut rustc_command = vec!["rustc".to_string()]; - rustc_command.extend_from_slice(&rustflags); - rustc_command.extend_from_slice(&[ - "-L".to_string(), - "crate=target/out".to_string(), - "--out-dir".to_string(), - "target/out".to_string(), - ]); - Ok(ConfigInfo { - target, - target_triple, - rustc_command, - }) + if ret.is_err() && cfg!(windows) { + eprintln!("Fallback to PowerShell"); + ret = run_command_with_output( + &[ + &"PowerShell.exe", + &"/nologo", + &"-Command", + &"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempfile_name, + ) + .as_str(), + ], + Some(&output_dir), + ); + } + ret } diff --git a/compiler/rustc_codegen_gcc/build_system/src/info.rs b/compiler/rustc_codegen_gcc/build_system/src/info.rs new file mode 100644 index 00000000000..ea38791d38c --- /dev/null +++ b/compiler/rustc_codegen_gcc/build_system/src/info.rs @@ -0,0 +1,19 @@ +use crate::config::ConfigInfo; + +pub fn run() -> Result<(), String> { + let mut config = ConfigInfo::default(); + + // We skip binary name and the `info` command. + let mut args = std::env::args().skip(2); + while let Some(arg) = args.next() { + if arg == "--help" { + println!("Display the path where the libgccjit will be located"); + return Ok(()); + } + config.parse_argument(&arg, &mut args)?; + } + config.no_download = true; + config.setup_gcc_path()?; + println!("{}", config.gcc_path); + Ok(()) +} diff --git a/compiler/rustc_codegen_gcc/build_system/src/main.rs b/compiler/rustc_codegen_gcc/build_system/src/main.rs index bff82b6e3e5..48ffbc7a907 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/main.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/main.rs @@ -2,12 +2,18 @@ use std::env; use std::process; mod build; +mod cargo; +mod clean; +mod clone_gcc; mod config; +mod info; mod prepare; mod rustc_info; mod test; mod utils; +const BUILD_DIR: &str = "build"; + macro_rules! arg_error { ($($err:tt)*) => {{ eprintln!($($err)*); @@ -22,17 +28,25 @@ fn usage() { "\ Available commands for build_system: - prepare : Run prepare command - build : Run build command - test : Run test command - --help : Show this message" + cargo : Run cargo command + clean : Run clean command + prepare : Run prepare command + build : Run build command + test : Run test command + info : Run info command + clone-gcc : Run clone-gcc command + --help : Show this message" ); } pub enum Command { + Cargo, + Clean, + CloneGcc, Prepare, Build, Test, + Info, } fn main() { @@ -41,9 +55,13 @@ fn main() { } let command = match env::args().nth(1).as_deref() { + Some("cargo") => Command::Cargo, + Some("clean") => Command::Clean, Some("prepare") => Command::Prepare, Some("build") => Command::Build, Some("test") => Command::Test, + Some("info") => Command::Info, + Some("clone-gcc") => Command::CloneGcc, Some("--help") => { usage(); process::exit(0); @@ -57,11 +75,15 @@ fn main() { }; if let Err(e) = match command { + Command::Cargo => cargo::run(), + Command::Clean => clean::run(), Command::Prepare => prepare::run(), Command::Build => build::run(), Command::Test => test::run(), + Command::Info => info::run(), + Command::CloneGcc => clone_gcc::run(), } { - eprintln!("Command failed to run: {e:?}"); + eprintln!("Command failed to run: {e}"); process::exit(1); } } diff --git a/compiler/rustc_codegen_gcc/build_system/src/prepare.rs b/compiler/rustc_codegen_gcc/build_system/src/prepare.rs index 6c7c8586834..821c793c7e5 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/prepare.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/prepare.rs @@ -1,10 +1,16 @@ use crate::rustc_info::get_rustc_path; -use crate::utils::{cargo_install, git_clone, run_command, run_command_with_output, walk_dir}; +use crate::utils::{ + cargo_install, git_clone_root_dir, remove_file, run_command, run_command_with_output, walk_dir, +}; use std::fs; use std::path::Path; -fn prepare_libcore(sysroot_path: &Path, libgccjit12_patches: bool, cross_compile: bool) -> Result<(), String> { +fn prepare_libcore( + sysroot_path: &Path, + libgccjit12_patches: bool, + cross_compile: bool, +) -> Result<(), String> { let rustc_path = match get_rustc_path() { Some(path) => path, None => return Err("`rustc` path not found".to_string()), @@ -88,10 +94,14 @@ fn prepare_libcore(sysroot_path: &Path, libgccjit12_patches: bool, cross_compile }, )?; if cross_compile { - walk_dir("cross_patches", |_| Ok(()), |file_path: &Path| { - patches.push(file_path.to_path_buf()); - Ok(()) - })?; + walk_dir( + "patches/cross_patches", + |_| Ok(()), + |file_path: &Path| { + patches.push(file_path.to_path_buf()); + Ok(()) + }, + )?; } if libgccjit12_patches { walk_dir( @@ -121,6 +131,30 @@ fn prepare_libcore(sysroot_path: &Path, libgccjit12_patches: bool, cross_compile )?; } println!("Successfully prepared libcore for building"); + + Ok(()) +} + +// TODO: remove when we can ignore warnings in rustdoc tests. +fn prepare_rand() -> Result<(), String> { + // Apply patch for the rand crate. + let file_path = "patches/crates/0001-Remove-deny-warnings.patch"; + let rand_dir = Path::new("build/rand"); + println!("[GIT] apply `{}`", file_path); + let path = Path::new("../..").join(file_path); + run_command_with_output(&[&"git", &"apply", &path], Some(rand_dir))?; + run_command_with_output(&[&"git", &"add", &"-A"], Some(rand_dir))?; + run_command_with_output( + &[ + &"git", + &"commit", + &"--no-gpg-sign", + &"-m", + &format!("Patch {}", path.display()), + ], + Some(rand_dir), + )?; + Ok(()) } @@ -129,8 +163,7 @@ fn build_raytracer(repo_dir: &Path) -> Result<(), String> { run_command(&[&"cargo", &"build"], Some(repo_dir))?; let mv_target = repo_dir.join("raytracer_cg_llvm"); if mv_target.is_file() { - std::fs::remove_file(&mv_target) - .map_err(|e| format!("Failed to remove file `{}`: {e:?}", mv_target.display()))?; + remove_file(&mv_target)?; } run_command( &[&"mv", &"target/debug/main", &"raytracer_cg_llvm"], @@ -143,28 +176,13 @@ fn clone_and_setup<F>(repo_url: &str, checkout_commit: &str, extra: Option<F>) - where F: Fn(&Path) -> Result<(), String>, { - let clone_result = git_clone(repo_url, None)?; + let clone_result = git_clone_root_dir(repo_url, &Path::new(crate::BUILD_DIR), false)?; if !clone_result.ran_clone { println!("`{}` has already been cloned", clone_result.repo_name); } - let repo_path = Path::new(&clone_result.repo_name); + let repo_path = Path::new(crate::BUILD_DIR).join(&clone_result.repo_name); run_command(&[&"git", &"checkout", &"--", &"."], Some(&repo_path))?; run_command(&[&"git", &"checkout", &checkout_commit], Some(&repo_path))?; - let filter = format!("-{}-", clone_result.repo_name); - walk_dir( - "crate_patches", - |_| Ok(()), - |file_path| { - let patch = file_path.as_os_str().to_str().unwrap(); - if patch.contains(&filter) && patch.ends_with(".patch") { - run_command_with_output( - &[&"git", &"am", &file_path.canonicalize().unwrap()], - Some(&repo_path), - )?; - } - Ok(()) - }, - )?; if let Some(extra) = extra { extra(&repo_path)?; } @@ -210,8 +228,7 @@ impl PrepareArg { --only-libcore : Only setup libcore and don't clone other repositories --cross : Apply the patches needed to do cross-compilation --libgccjit12-patches : Apply patches needed for libgccjit12 - --help : Show this help -"# + --help : Show this help"# ) } } @@ -230,7 +247,7 @@ pub fn run() -> Result<(), String> { let to_clone = &[ ( "https://github.com/rust-random/rand.git", - "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", + "1f4507a8e1cf8050e4ceef95eeda8f64645b6719", None, ), ( @@ -248,6 +265,8 @@ pub fn run() -> Result<(), String> { for (repo_url, checkout_commit, cb) in to_clone { clone_and_setup(repo_url, checkout_commit, *cb)?; } + + prepare_rand()?; } println!("Successfully ran `prepare`"); diff --git a/compiler/rustc_codegen_gcc/build_system/src/test.rs b/compiler/rustc_codegen_gcc/build_system/src/test.rs index 4c8c63e59ab..a4db2fdebef 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/test.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/test.rs @@ -1,15 +1,1222 @@ -use crate::utils::run_command_with_output; +use crate::build; +use crate::config::{Channel, ConfigInfo}; +use crate::utils::{ + get_toolchain, git_clone, git_clone_root_dir, remove_file, run_command, run_command_with_env, + run_command_with_output_and_env, rustc_version_info, split_args, walk_dir, +}; -fn get_args<'a>(args: &mut Vec<&'a dyn AsRef<std::ffi::OsStr>>, extra_args: &'a Vec<String>) { - for extra_arg in extra_args { - args.push(extra_arg); +use std::collections::{BTreeSet, HashMap}; +use std::ffi::OsStr; +use std::fs::{create_dir_all, remove_dir_all, File}; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +type Env = HashMap<String, String>; +type Runner = fn(&Env, &TestArg) -> Result<(), String>; +type Runners = HashMap<&'static str, (&'static str, Runner)>; + +fn get_runners() -> Runners { + let mut runners = HashMap::new(); + + runners.insert( + "--test-rustc", + ("Run all rustc tests", test_rustc as Runner), + ); + runners.insert( + "--test-successful-rustc", + ("Run successful rustc tests", test_successful_rustc), + ); + runners.insert( + "--test-failing-rustc", + ("Run failing rustc tests", test_failing_rustc), + ); + runners.insert( + "--projects", + ("Run the tests of popular crates", test_projects), + ); + runners.insert("--test-libcore", ("Run libcore tests", test_libcore)); + runners.insert("--clean", ("Empty cargo target directory", clean)); + runners.insert("--build-sysroot", ("Build sysroot", build_sysroot)); + runners.insert("--std-tests", ("Run std tests", std_tests)); + runners.insert("--asm-tests", ("Run asm tests", asm_tests)); + runners.insert( + "--extended-tests", + ("Run extended sysroot tests", extended_sysroot_tests), + ); + runners.insert( + "--extended-rand-tests", + ("Run extended rand tests", extended_rand_tests), + ); + runners.insert( + "--extended-regex-example-tests", + ( + "Run extended regex example tests", + extended_regex_example_tests, + ), + ); + runners.insert( + "--extended-regex-tests", + ("Run extended regex tests", extended_regex_tests), + ); + runners.insert("--mini-tests", ("Run mini tests", mini_tests)); + + runners +} + +fn get_number_after_arg( + args: &mut impl Iterator<Item = String>, + option: &str, +) -> Result<usize, String> { + match args.next() { + Some(nb) if !nb.is_empty() => match usize::from_str(&nb) { + Ok(nb) => Ok(nb), + Err(_) => Err(format!( + "Expected a number after `{}`, found `{}`", + option, nb + )), + }, + _ => Err(format!( + "Expected a number after `{}`, found nothing", + option + )), + } +} + +fn show_usage() { + println!( + r#" +`test` command help: + + --release : Build codegen in release mode + --sysroot-panic-abort : Build the sysroot without unwinding support. + --features [arg] : Add a new feature [arg] + --use-system-gcc : Use system installed libgccjit + --build-only : Only build rustc_codegen_gcc then exits + --use-backend : Useful only for rustc testsuite + --nb-parts : Used to split rustc_tests (for CI needs) + --current-part : Used with `--nb-parts`, allows you to specify which parts to test"# + ); + ConfigInfo::show_usage(); + for (option, (doc, _)) in get_runners() { + // FIXME: Instead of using the hard-coded `23` value, better to compute it instead. + let needed_spaces = 23_usize.saturating_sub(option.len()); + let spaces: String = std::iter::repeat(' ').take(needed_spaces).collect(); + println!(" {}{}: {}", option, spaces, doc); + } + println!(" --help : Show this help"); +} + +#[derive(Default, Debug)] +struct TestArg { + build_only: bool, + use_system_gcc: bool, + runners: BTreeSet<String>, + flags: Vec<String>, + backend: Option<String>, + nb_parts: Option<usize>, + current_part: Option<usize>, + sysroot_panic_abort: bool, + config_info: ConfigInfo, +} + +impl TestArg { + fn new() -> Result<Option<Self>, String> { + let mut test_arg = Self::default(); + + // We skip binary name and the `test` command. + let mut args = std::env::args().skip(2); + let runners = get_runners(); + + while let Some(arg) = args.next() { + match arg.as_str() { + "--features" => match args.next() { + Some(feature) if !feature.is_empty() => { + test_arg + .flags + .extend_from_slice(&["--features".into(), feature]); + } + _ => { + return Err("Expected an argument after `--features`, found nothing".into()) + } + }, + "--use-system-gcc" => { + println!("Using system GCC"); + test_arg.use_system_gcc = true; + } + "--build-only" => test_arg.build_only = true, + "--use-backend" => match args.next() { + Some(backend) if !backend.is_empty() => test_arg.backend = Some(backend), + _ => { + return Err( + "Expected an argument after `--use-backend`, found nothing".into() + ) + } + }, + "--nb-parts" => { + test_arg.nb_parts = Some(get_number_after_arg(&mut args, "--nb-parts")?); + } + "--current-part" => { + test_arg.current_part = + Some(get_number_after_arg(&mut args, "--current-part")?); + } + "--sysroot-panic-abort" => { + test_arg.sysroot_panic_abort = true; + } + "--help" => { + show_usage(); + return Ok(None); + } + x if runners.contains_key(x) => { + test_arg.runners.insert(x.into()); + } + arg => { + if !test_arg.config_info.parse_argument(arg, &mut args)? { + return Err(format!("Unknown option {}", arg)); + } + } + } + } + match (test_arg.current_part, test_arg.nb_parts) { + (Some(_), Some(_)) | (None, None) => {} + _ => { + return Err( + "If either `--current-part` or `--nb-parts` is specified, the other one \ + needs to be specified as well!" + .to_string(), + ); + } + } + if test_arg.config_info.no_default_features { + test_arg.flags.push("--no-default-features".into()); + } + Ok(Some(test_arg)) + } + + pub fn is_using_gcc_master_branch(&self) -> bool { + !self.config_info.no_default_features + } +} + +fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> { + if args.backend.is_some() { + return Ok(()); + } + let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"rustc"]; + let mut tmp_env; + let env = if args.config_info.channel == Channel::Release { + tmp_env = env.clone(); + tmp_env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string()); + command.push(&"--release"); + &tmp_env + } else { + &env + }; + for flag in args.flags.iter() { + command.push(flag); + } + run_command_with_output_and_env(&command, None, Some(env)) +} + +fn clean(_env: &Env, args: &TestArg) -> Result<(), String> { + let _ = std::fs::remove_dir_all(&args.config_info.cargo_target_dir); + let path = Path::new(&args.config_info.cargo_target_dir).join("gccjit"); + std::fs::create_dir_all(&path) + .map_err(|error| format!("failed to create folder `{}`: {:?}", path.display(), error)) +} + +fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[BUILD] mini_core"); + let crate_types = if args.config_info.host_triple != args.config_info.target_triple { + "lib" + } else { + "lib,dylib" + } + .to_string(); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/mini_core.rs", + &"--crate-name", + &"mini_core", + &"--crate-type", + &crate_types, + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_output_and_env(&command, None, Some(&env))?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[BUILD] example"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/example.rs", + &"--crate-type", + &"lib", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_output_and_env(&command, None, Some(&env))?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] mini_core_hello_world"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/mini_core_hello_world.rs", + &"--crate-name", + &"mini_core_hello_world", + &"--crate-type", + &"bin", + &"-g", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_output_and_env(&command, None, Some(&env))?; + + let command: &[&dyn AsRef<OsStr>] = &[ + &Path::new(&args.config_info.cargo_target_dir).join("mini_core_hello_world"), + &"abc", + &"bcd", + ]; + maybe_run_command_in_vm(&command, env, args)?; + Ok(()) +} + +fn build_sysroot(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[BUILD] sysroot"); + build::build_sysroot(env, &args.config_info)?; + Ok(()) +} + +// TODO(GuillaumeGomez): when rewriting in Rust, refactor with the code in tests/lang_tests_common.rs if possible. +fn maybe_run_command_in_vm( + command: &[&dyn AsRef<OsStr>], + env: &Env, + args: &TestArg, +) -> Result<(), String> { + if !args.config_info.run_in_vm { + run_command_with_output_and_env(command, None, Some(env))?; + return Ok(()); + } + let vm_parent_dir = match env.get("CG_GCC_VM_DIR") { + Some(dir) if !dir.is_empty() => PathBuf::from(dir.clone()), + _ => std::env::current_dir().unwrap(), + }; + let vm_dir = "vm"; + let exe_to_run = command.first().unwrap(); + let exe = Path::new(&exe_to_run); + let exe_filename = exe.file_name().unwrap(); + let vm_home_dir = vm_parent_dir.join(vm_dir).join("home"); + let vm_exe_path = vm_home_dir.join(exe_filename); + let inside_vm_exe_path = Path::new("/home").join(exe_filename); + + let sudo_command: &[&dyn AsRef<OsStr>] = &[&"sudo", &"cp", &exe, &vm_exe_path]; + run_command_with_env(sudo_command, None, Some(env))?; + + let mut vm_command: Vec<&dyn AsRef<OsStr>> = vec![ + &"sudo", + &"chroot", + &vm_dir, + &"qemu-m68k-static", + &inside_vm_exe_path, + ]; + vm_command.extend_from_slice(command); + run_command_with_output_and_env(&vm_command, Some(&vm_parent_dir), Some(env))?; + Ok(()) +} + +fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { + let cargo_target_dir = Path::new(&args.config_info.cargo_target_dir); + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] arbitrary_self_types_pointers_and_wrappers"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/arbitrary_self_types_pointers_and_wrappers.rs", + &"--crate-name", + &"arbitrary_self_types_pointers_and_wrappers", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm( + &[&cargo_target_dir.join("arbitrary_self_types_pointers_and_wrappers")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] alloc_system"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/alloc_system.rs", + &"--crate-type", + &"lib", + &"--target", + &args.config_info.target_triple, + ]); + if args.is_using_gcc_master_branch() { + command.extend_from_slice(&[&"--cfg", &"feature=\"master\""]); + } + run_command_with_env(&command, None, Some(env))?; + + // FIXME: doesn't work on m68k. + if args.config_info.host_triple == args.config_info.target_triple { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] alloc_example"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/alloc_example.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm(&[&cargo_target_dir.join("alloc_example")], env, args)?; + } + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] dst_field_align"); + // FIXME(antoyo): Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed. + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/dst-field-align.rs", + &"--crate-name", + &"dst_field_align", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm(&[&cargo_target_dir.join("dst_field_align")], env, args)?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] std_example"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/std_example.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + if args.is_using_gcc_master_branch() { + command.extend_from_slice(&[&"--cfg", &"feature=\"master\""]); + } + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm( + &[ + &cargo_target_dir.join("std_example"), + &"--target", + &args.config_info.target_triple, + ], + env, + args, + )?; + + let test_flags = if let Some(test_flags) = env.get("TEST_FLAGS") { + split_args(test_flags)? + } else { + Vec::new() + }; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] subslice-patterns-const-eval"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/subslice-patterns-const-eval.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + for test_flag in &test_flags { + command.push(test_flag); + } + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm( + &[&cargo_target_dir.join("subslice-patterns-const-eval")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] track-caller-attribute"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/track-caller-attribute.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + for test_flag in &test_flags { + command.push(test_flag); + } + run_command_with_env(&command, None, Some(env))?; + maybe_run_command_in_vm( + &[&cargo_target_dir.join("track-caller-attribute")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] mod_bench"); + let mut command = args.config_info.rustc_command_vec(); + command.extend_from_slice(&[ + &"example/mod_bench.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + // FIXME: the compiled binary is not run. + + Ok(()) +} + +fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<PathBuf, String> { + let toolchain = format!( + "+{channel}-{host}", + channel = get_toolchain()?, // May also include date + host = args.config_info.host_triple + ); + let rust_dir_path = Path::new(crate::BUILD_DIR).join("rust"); + // If the repository was already cloned, command will fail, so doesn't matter. + let _ = git_clone( + "https://github.com/rust-lang/rust.git", + Some(&rust_dir_path), + false, + ); + let rust_dir: Option<&Path> = Some(&rust_dir_path); + run_command(&[&"git", &"checkout", &"--", &"tests/"], rust_dir)?; + run_command_with_output_and_env(&[&"git", &"fetch"], rust_dir, Some(env))?; + let rustc_commit = match rustc_version_info(env.get("RUSTC").map(|s| s.as_str()))?.commit_hash { + Some(commit_hash) => commit_hash, + None => return Err("Couldn't retrieve rustc commit hash".to_string()), + }; + if rustc_commit != "unknown" { + run_command_with_output_and_env( + &[&"git", &"checkout", &rustc_commit], + rust_dir, + Some(env), + )?; + } else { + run_command_with_output_and_env(&[&"git", &"checkout"], rust_dir, Some(env))?; + } + let cargo = String::from_utf8( + run_command_with_env(&[&"rustup", &"which", &"cargo"], rust_dir, Some(env))?.stdout, + ) + .map_err(|error| format!("Failed to retrieve cargo path: {:?}", error)) + .and_then(|cargo| { + let cargo = cargo.trim().to_owned(); + if cargo.is_empty() { + Err(format!("`cargo` path is empty")) + } else { + Ok(cargo) + } + })?; + let rustc = String::from_utf8( + run_command_with_env( + &[&"rustup", &toolchain, &"which", &"rustc"], + rust_dir, + Some(env), + )? + .stdout, + ) + .map_err(|error| format!("Failed to retrieve rustc path: {:?}", error)) + .and_then(|rustc| { + let rustc = rustc.trim().to_owned(); + if rustc.is_empty() { + Err(format!("`rustc` path is empty")) + } else { + Ok(rustc) + } + })?; + let llvm_filecheck = match run_command_with_env( + &[ + &"bash", + &"-c", + &"which FileCheck-10 || \ + which FileCheck-11 || \ + which FileCheck-12 || \ + which FileCheck-13 || \ + which FileCheck-14", + ], + rust_dir, + Some(env), + ) { + Ok(cmd) => String::from_utf8_lossy(&cmd.stdout).to_string(), + Err(_) => { + eprintln!("Failed to retrieve LLVM FileCheck, ignoring..."); + String::new() + } + }; + let file_path = rust_dir_path.join("config.toml"); + std::fs::write( + &file_path, + &format!( + r#"change-id = 115898 + +[rust] +codegen-backends = [] +deny-warnings = false +verbose-tests = true + +[build] +cargo = "{cargo}" +local-rebuild = true +rustc = "{rustc}" + +[target.x86_64-unknown-linux-gnu] +llvm-filecheck = "{llvm_filecheck}" + +[llvm] +download-ci-llvm = false +"#, + cargo = cargo, + rustc = rustc, + llvm_filecheck = llvm_filecheck.trim(), + ), + ) + .map_err(|error| { + format!( + "Failed to write into `{}`: {:?}", + file_path.display(), + error + ) + })?; + Ok(rust_dir_path) +} + +fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { + let mut env = env.clone(); + let rust_dir = setup_rustc(&mut env, args)?; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rustc asm test suite"); + + env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); + + let extra = if args.is_using_gcc_master_branch() { + "" + } else { + " -Csymbol-mangling-version=v0" + }; + + let rustc_args = &format!( + r#"-Zpanic-abort-tests \ + -Zcodegen-backend="{pwd}/target/{channel}/librustc_codegen_gcc.{dylib_ext}" \ + --sysroot "{pwd}/build_sysroot/sysroot" -Cpanic=abort{extra}"#, + pwd = std::env::current_dir() + .map_err(|error| format!("`current_dir` failed: {:?}", error))? + .display(), + channel = args.config_info.channel.as_str(), + dylib_ext = args.config_info.dylib_ext, + extra = extra, + ); + + run_command_with_env( + &[ + &"./x.py", + &"test", + &"--run", + &"always", + &"--stage", + &"0", + &"tests/assembly/asm", + &"--rustc-args", + &rustc_args, + ], + Some(&rust_dir), + Some(&env), + )?; + Ok(()) +} + +fn run_cargo_command( + command: &[&dyn AsRef<OsStr>], + cwd: Option<&Path>, + env: &Env, + args: &TestArg, +) -> Result<(), String> { + run_cargo_command_with_callback(command, cwd, env, args, |cargo_command, cwd, env| { + run_command_with_output_and_env(cargo_command, cwd, Some(env))?; + Ok(()) + }) +} + +fn run_cargo_command_with_callback<F>( + command: &[&dyn AsRef<OsStr>], + cwd: Option<&Path>, + env: &Env, + args: &TestArg, + callback: F, +) -> Result<(), String> +where + F: Fn(&[&dyn AsRef<OsStr>], Option<&Path>, &Env) -> Result<(), String>, +{ + let toolchain = get_toolchain()?; + let toolchain_arg = format!("+{}", toolchain); + let rustc_version = String::from_utf8( + run_command_with_env(&[&args.config_info.rustc_command[0], &"-V"], cwd, Some(env))?.stdout, + ) + .map_err(|error| format!("Failed to retrieve rustc version: {:?}", error))?; + let rustc_toolchain_version = String::from_utf8( + run_command_with_env( + &[&args.config_info.rustc_command[0], &toolchain_arg, &"-V"], + cwd, + Some(env), + )? + .stdout, + ) + .map_err(|error| format!("Failed to retrieve rustc +toolchain version: {:?}", error))?; + + if rustc_version != rustc_toolchain_version { + eprintln!( + "rustc_codegen_gcc is built for `{}` but the default rustc version is `{}`.", + rustc_toolchain_version, rustc_version, + ); + eprintln!("Using `{}`.", rustc_toolchain_version); + } + let mut env = env.clone(); + let rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); + env.insert("RUSTDOCFLAGS".to_string(), rustflags); + let mut cargo_command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &toolchain_arg]; + cargo_command.extend_from_slice(&command); + callback(&cargo_command, cwd, &env) +} + +// FIXME(antoyo): linker gives multiple definitions error on Linux +// echo "[BUILD] sysroot in release mode" +// ./build_sysroot/build_sysroot.sh --release + +fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> { + let projects = [ + //"https://gitlab.gnome.org/GNOME/librsvg", // FIXME: doesn't compile in the CI since the + // version of cairo and other libraries is too old. + "https://github.com/rust-random/getrandom", + "https://github.com/BurntSushi/memchr", + "https://github.com/dtolnay/itoa", + "https://github.com/rust-lang/cfg-if", + "https://github.com/rust-lang-nursery/lazy-static.rs", + //"https://github.com/marshallpierce/rust-base64", // FIXME: one test is OOM-killed. + // TODO: ignore the base64 test that is OOM-killed. + "https://github.com/time-rs/time", + "https://github.com/rust-lang/log", + "https://github.com/bitflags/bitflags", + //"https://github.com/serde-rs/serde", // FIXME: one test fails. + //"https://github.com/rayon-rs/rayon", // TODO: very slow, only run on master? + //"https://github.com/rust-lang/cargo", // TODO: very slow, only run on master? + ]; + + let run_tests = |projects_path, iter: &mut dyn Iterator<Item = &&str>| -> Result<(), String> { + for project in iter { + let clone_result = git_clone_root_dir(project, projects_path, true)?; + let repo_path = Path::new(&clone_result.repo_dir); + run_cargo_command(&[&"build", &"--release"], Some(repo_path), env, args)?; + run_cargo_command(&[&"test"], Some(repo_path), env, args)?; + } + + Ok(()) + }; + + let projects_path = Path::new("projects"); + create_dir_all(projects_path) + .map_err(|err| format!("Failed to create directory `projects`: {}", err))?; + + let nb_parts = args.nb_parts.unwrap_or(0); + if nb_parts > 0 { + // We increment the number of tests by one because if this is an odd number, we would skip + // one test. + let count = projects.len() / nb_parts + 1; + let current_part = args.current_part.unwrap(); + let start = current_part * count; + // We remove the projects we don't want to test. + run_tests(projects_path, &mut projects.iter().skip(start).take(count))?; + } else { + run_tests(projects_path, &mut projects.iter())?; } + + Ok(()) +} + +fn test_libcore(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] libcore"); + let path = Path::new("build_sysroot/sysroot_src/library/core/tests"); + let _ = remove_dir_all(path.join("target")); + run_cargo_command(&[&"test"], Some(path), env, args)?; + Ok(()) +} + +// echo "[BENCH COMPILE] mod_bench" +// +// COMPILE_MOD_BENCH_INLINE="$RUSTC example/mod_bench.rs --crate-type bin -Zmir-opt-level=3 -O --crate-name mod_bench_inline" +// COMPILE_MOD_BENCH_LLVM_0="rustc example/mod_bench.rs --crate-type bin -Copt-level=0 -o $cargo_target_dir/mod_bench_llvm_0 -Cpanic=abort" +// COMPILE_MOD_BENCH_LLVM_1="rustc example/mod_bench.rs --crate-type bin -Copt-level=1 -o $cargo_target_dir/mod_bench_llvm_1 -Cpanic=abort" +// COMPILE_MOD_BENCH_LLVM_2="rustc example/mod_bench.rs --crate-type bin -Copt-level=2 -o $cargo_target_dir/mod_bench_llvm_2 -Cpanic=abort" +// COMPILE_MOD_BENCH_LLVM_3="rustc example/mod_bench.rs --crate-type bin -Copt-level=3 -o $cargo_target_dir/mod_bench_llvm_3 -Cpanic=abort" +// +// Use 100 runs, because a single compilations doesn't take more than ~150ms, so it isn't very slow +// hyperfine --runs ${COMPILE_RUNS:-100} "$COMPILE_MOD_BENCH_INLINE" "$COMPILE_MOD_BENCH_LLVM_0" "$COMPILE_MOD_BENCH_LLVM_1" "$COMPILE_MOD_BENCH_LLVM_2" "$COMPILE_MOD_BENCH_LLVM_3" +// echo "[BENCH RUN] mod_bench" +// hyperfine --runs ${RUN_RUNS:-10} $cargo_target_dir/mod_bench{,_inline} $cargo_target_dir/mod_bench_llvm_* + +fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { + if !args.is_using_gcc_master_branch() { + println!("Not using GCC master branch. Skipping `extended_rand_tests`."); + return Ok(()); + } + let mut env = env.clone(); + // newer aho_corasick versions throw a deprecation warning + let rustflags = format!( + "{} --cap-lints warn", + env.get("RUSTFLAGS").cloned().unwrap_or_default() + ); + env.insert("RUSTFLAGS".to_string(), rustflags); + + let path = Path::new(crate::BUILD_DIR).join("rand"); + run_cargo_command(&[&"clean"], Some(&path), &env, args)?; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-random/rand"); + run_cargo_command(&[&"test", &"--workspace"], Some(&path), &env, args)?; + Ok(()) +} + +fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> { + if !args.is_using_gcc_master_branch() { + println!("Not using GCC master branch. Skipping `extended_regex_example_tests`."); + return Ok(()); + } + let path = Path::new(crate::BUILD_DIR).join("regex"); + run_cargo_command(&[&"clean"], Some(&path), env, args)?; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-lang/regex example shootout-regex-dna"); + let mut env = env.clone(); + // newer aho_corasick versions throw a deprecation warning + let rustflags = format!( + "{} --cap-lints warn", + env.get("RUSTFLAGS").cloned().unwrap_or_default() + ); + env.insert("RUSTFLAGS".to_string(), rustflags); + // Make sure `[codegen mono items] start` doesn't poison the diff + run_cargo_command( + &[&"build", &"--example", &"shootout-regex-dna"], + Some(&path), + &env, + args, + )?; + + run_cargo_command_with_callback( + &[&"run", &"--example", &"shootout-regex-dna"], + Some(&path), + &env, + args, + |cargo_command, cwd, env| { + // FIXME: rewrite this with `child.stdin.write_all()` because + // `examples/regexdna-input.txt` is very small. + let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"bash", &"-c"]; + let cargo_args = cargo_command + .iter() + .map(|s| s.as_ref().to_str().unwrap()) + .collect::<Vec<_>>(); + let bash_command = format!( + "cat examples/regexdna-input.txt | {} | grep -v 'Spawned thread' > res.txt", + cargo_args.join(" "), + ); + command.push(&bash_command); + run_command_with_output_and_env(&command, cwd, Some(env))?; + run_command_with_output_and_env( + &[&"diff", &"-u", &"res.txt", &"examples/regexdna-output.txt"], + cwd, + Some(env), + )?; + Ok(()) + }, + )?; + + Ok(()) +} + +fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { + if !args.is_using_gcc_master_branch() { + println!("Not using GCC master branch. Skipping `extended_regex_tests`."); + return Ok(()); + } + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-lang/regex tests"); + let mut env = env.clone(); + // newer aho_corasick versions throw a deprecation warning + let rustflags = format!( + "{} --cap-lints warn", + env.get("RUSTFLAGS").cloned().unwrap_or_default() + ); + env.insert("RUSTFLAGS".to_string(), rustflags); + let path = Path::new(crate::BUILD_DIR).join("regex"); + run_cargo_command( + &[ + &"test", + &"--tests", + &"--", + // FIXME: try removing `--exclude-should-panic` argument + &"--exclude-should-panic", + &"--test-threads", + &"1", + &"-Zunstable-options", + &"-q", + ], + Some(&path), + &env, + args, + )?; + Ok(()) +} + +fn extended_sysroot_tests(env: &Env, args: &TestArg) -> Result<(), String> { + // pushd simple-raytracer + // echo "[BENCH COMPILE] ebobby/simple-raytracer" + // hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "cargo clean" \ + // "RUSTC=rustc RUSTFLAGS='' cargo build" \ + // "../y.sh cargo build" + + // echo "[BENCH RUN] ebobby/simple-raytracer" + // cp ./target/debug/main ./raytracer_cg_gcc + // hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_gcc + // popd + extended_rand_tests(env, args)?; + extended_regex_example_tests(env, args)?; + extended_regex_tests(env, args)?; + + Ok(()) +} + +fn should_not_remove_test(file: &str) -> bool { + // contains //~ERROR, but shouldn't be removed + [ + "issues/auxiliary/issue-3136-a.rs", + "type-alias-impl-trait/auxiliary/cross_crate_ice.rs", + "type-alias-impl-trait/auxiliary/cross_crate_ice2.rs", + "macros/rfc-2011-nicer-assert-messages/auxiliary/common.rs", + "imports/ambiguous-1.rs", + "imports/ambiguous-4-extern.rs", + "entry-point/auxiliary/bad_main_functions.rs", + ] + .iter() + .any(|to_ignore| file.ends_with(to_ignore)) +} + +fn should_remove_test(file_path: &Path) -> Result<bool, String> { + // Tests generating errors. + let file = File::open(file_path) + .map_err(|error| format!("Failed to read `{}`: {:?}", file_path.display(), error))?; + for line in BufReader::new(file).lines().filter_map(|line| line.ok()) { + let line = line.trim(); + if line.is_empty() { + continue; + } + if [ + "//@ error-pattern:", + "//@ build-fail", + "//@ run-fail", + "-Cllvm-args", + "//~", + "thread", + ] + .iter() + .any(|check| line.contains(check)) + { + return Ok(true); + } + if line.contains("//[") && line.contains("]~") { + return Ok(true); + } + } + if file_path + .display() + .to_string() + .contains("ambiguous-4-extern.rs") + { + eprintln!("nothing found for {file_path:?}"); + } + Ok(false) +} + +fn test_rustc_inner<F>(env: &Env, args: &TestArg, prepare_files_callback: F) -> Result<(), String> +where + F: Fn(&Path) -> Result<bool, String>, +{ + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-lang/rust"); + let mut env = env.clone(); + let rust_path = setup_rustc(&mut env, args)?; + + walk_dir( + rust_path.join("tests/ui"), + |dir| { + let dir_name = dir.file_name().and_then(|name| name.to_str()).unwrap_or(""); + if [ + "abi", + "extern", + "unsized-locals", + "proc-macro", + "threads-sendsync", + "borrowck", + "test-attrs", + ] + .iter() + .any(|name| *name == dir_name) + { + std::fs::remove_dir_all(dir).map_err(|error| { + format!("Failed to remove folder `{}`: {:?}", dir.display(), error) + })?; + } + Ok(()) + }, + |_| Ok(()), + )?; + + // These two functions are used to remove files that are known to not be working currently + // with the GCC backend to reduce noise. + fn dir_handling(dir: &Path) -> Result<(), String> { + if dir + .file_name() + .map(|name| name == "auxiliary") + .unwrap_or(true) + { + return Ok(()); + } + walk_dir(dir, dir_handling, file_handling) + } + fn file_handling(file_path: &Path) -> Result<(), String> { + if !file_path + .extension() + .map(|extension| extension == "rs") + .unwrap_or(false) + { + return Ok(()); + } + let path_str = file_path.display().to_string().replace("\\", "/"); + if should_not_remove_test(&path_str) { + return Ok(()); + } else if should_remove_test(file_path)? { + return remove_file(&file_path); + } + Ok(()) + } + + remove_file(&rust_path.join("tests/ui/consts/const_cmp_type_id.rs"))?; + remove_file(&rust_path.join("tests/ui/consts/issue-73976-monomorphic.rs"))?; + // this test is oom-killed in the CI. + remove_file(&rust_path.join("tests/ui/consts/issue-miri-1910.rs"))?; + // Tests generating errors. + remove_file(&rust_path.join("tests/ui/consts/issue-94675.rs"))?; + remove_file(&rust_path.join("tests/ui/mir/mir_heavy_promoted.rs"))?; + remove_file(&rust_path.join("tests/ui/rfcs/rfc-2632-const-trait-impl/const-drop-fail.rs"))?; + remove_file(&rust_path.join("tests/ui/rfcs/rfc-2632-const-trait-impl/const-drop.rs"))?; + + walk_dir(rust_path.join("tests/ui"), dir_handling, file_handling)?; + + if !prepare_files_callback(&rust_path)? { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("Keeping all UI tests"); + } + + let nb_parts = args.nb_parts.unwrap_or(0); + if nb_parts > 0 { + let current_part = args.current_part.unwrap(); + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!( + "Splitting ui_test into {} parts (and running part {})", + nb_parts, current_part + ); + let out = String::from_utf8( + run_command( + &[ + &"find", + &"tests/ui", + &"-type", + &"f", + &"-name", + &"*.rs", + &"-not", + &"-path", + &"*/auxiliary/*", + ], + Some(&rust_path), + )? + .stdout, + ) + .map_err(|error| format!("Failed to retrieve output of find command: {:?}", error))?; + let mut files = out + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + .collect::<Vec<_>>(); + // To ensure it'll be always the same sub files, we sort the content. + files.sort(); + // We increment the number of tests by one because if this is an odd number, we would skip + // one test. + let count = files.len() / nb_parts + 1; + let start = current_part * count; + // We remove the files we don't want to test. + for path in files.iter().skip(start).take(count) { + remove_file(&rust_path.join(path))?; + } + } + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rustc test suite"); + env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); + + let extra = if args.is_using_gcc_master_branch() { + "" + } else { + " -Csymbol-mangling-version=v0" + }; + + let rustc_args = format!( + "{} -Zcodegen-backend={} --sysroot {}{}", + env.get("TEST_FLAGS").unwrap_or(&String::new()), + args.config_info.cg_backend_path, + args.config_info.sysroot_path, + extra, + ); + + env.get_mut("RUSTFLAGS").unwrap().clear(); + run_command_with_output_and_env( + &[ + &"./x.py", + &"test", + &"--run", + &"always", + &"--stage", + &"0", + &"tests/ui", + &"--rustc-args", + &rustc_args, + ], + Some(&rust_path), + Some(&env), + )?; + Ok(()) +} + +fn test_rustc(env: &Env, args: &TestArg) -> Result<(), String> { + test_rustc_inner(env, args, |_| Ok(false)) +} + +fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { + test_rustc_inner(env, args, |rust_path| { + // Removing all tests. + run_command( + &[ + &"find", + &"tests/ui", + &"-type", + &"f", + &"-name", + &"*.rs", + &"-not", + &"-path", + &"*/auxiliary/*", + &"-delete", + ], + Some(rust_path), + )?; + // Putting back only the failing ones. + let path = "tests/failing-ui-tests.txt"; + if let Ok(files) = std::fs::read_to_string(path) { + for file in files + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + { + run_command(&[&"git", &"checkout", &"--", &file], Some(&rust_path))?; + } + } else { + println!( + "Failed to read `{}`, not putting back failing ui tests", + path + ); + } + Ok(true) + }) +} + +fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { + test_rustc_inner(env, args, |rust_path| { + // Removing the failing tests. + let path = "tests/failing-ui-tests.txt"; + if let Ok(files) = std::fs::read_to_string(path) { + for file in files + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + { + let path = rust_path.join(file); + remove_file(&path)?; + } + } else { + println!( + "Failed to read `{}`, not putting back failing ui tests", + path + ); + } + Ok(true) + }) +} + +fn run_all(env: &Env, args: &TestArg) -> Result<(), String> { + clean(env, args)?; + mini_tests(env, args)?; + build_sysroot(env, args)?; + std_tests(env, args)?; + // asm_tests(env, args)?; + test_libcore(env, args)?; + extended_sysroot_tests(env, args)?; + test_rustc(env, args)?; + Ok(()) } pub fn run() -> Result<(), String> { - let mut args: Vec<&dyn AsRef<std::ffi::OsStr>> = vec![&"bash", &"test.sh"]; - let extra_args = std::env::args().skip(2).collect::<Vec<_>>(); - get_args(&mut args, &extra_args); - let current_dir = std::env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; - run_command_with_output(args.as_slice(), Some(¤t_dir)) + let mut args = match TestArg::new()? { + Some(args) => args, + None => return Ok(()), + }; + let mut env: HashMap<String, String> = std::env::vars().collect(); + + if !args.use_system_gcc { + args.config_info.setup_gcc_path()?; + env.insert( + "LIBRARY_PATH".to_string(), + args.config_info.gcc_path.clone(), + ); + env.insert( + "LD_LIBRARY_PATH".to_string(), + args.config_info.gcc_path.clone(), + ); + } + + build_if_no_backend(&env, &args)?; + if args.build_only { + println!("Since it's build only, exiting..."); + return Ok(()); + } + + args.config_info.setup(&mut env, args.use_system_gcc)?; + + if args.runners.is_empty() { + run_all(&env, &args)?; + } else { + let runners = get_runners(); + for runner in args.runners.iter() { + runners.get(runner.as_str()).unwrap().1(&env, &args)?; + } + } + + Ok(()) } diff --git a/compiler/rustc_codegen_gcc/build_system/src/utils.rs b/compiler/rustc_codegen_gcc/build_system/src/utils.rs index 536f33a8029..d9c13fd143d 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/utils.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/utils.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::ffi::OsStr; use std::fmt::Debug; use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Command, ExitStatus, Output}; fn get_command_inner( @@ -29,22 +29,40 @@ fn check_exit_status( input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>, exit_status: ExitStatus, + output: Option<&Output>, + show_err: bool, ) -> Result<(), String> { if exit_status.success() { - Ok(()) - } else { - Err(format!( - "Command `{}`{} exited with status {:?}", - input - .iter() - .map(|s| s.as_ref().to_str().unwrap()) - .collect::<Vec<_>>() - .join(" "), - cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display())) - .unwrap_or_default(), - exit_status.code(), - )) + return Ok(()); } + let mut error = format!( + "Command `{}`{} exited with status {:?}", + input + .iter() + .map(|s| s.as_ref().to_str().unwrap()) + .collect::<Vec<_>>() + .join(" "), + cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display())) + .unwrap_or_default(), + exit_status.code() + ); + let input = input.iter().map(|i| i.as_ref()).collect::<Vec<&OsStr>>(); + if show_err { + eprintln!("Command `{:?}` failed", input); + } + if let Some(output) = output { + let stdout = String::from_utf8_lossy(&output.stdout); + if !stdout.is_empty() { + error.push_str("\n==== STDOUT ====\n"); + error.push_str(&*stdout); + } + let stderr = String::from_utf8_lossy(&output.stderr); + if !stderr.is_empty() { + error.push_str("\n==== STDERR ====\n"); + error.push_str(&*stderr); + } + } + Err(error) } fn command_error<D: Debug>(input: &[&dyn AsRef<OsStr>], cwd: &Option<&Path>, error: D) -> String { @@ -73,7 +91,7 @@ pub fn run_command_with_env( let output = get_command_inner(input, cwd, env) .output() .map_err(|e| command_error(input, &cwd, e))?; - check_exit_status(input, cwd, output.status)?; + check_exit_status(input, cwd, output.status, Some(&output), true)?; Ok(output) } @@ -86,7 +104,7 @@ pub fn run_command_with_output( .map_err(|e| command_error(input, &cwd, e))? .wait() .map_err(|e| command_error(input, &cwd, e))?; - check_exit_status(input, cwd, exit_status)?; + check_exit_status(input, cwd, exit_status, None, true)?; Ok(()) } @@ -100,7 +118,21 @@ pub fn run_command_with_output_and_env( .map_err(|e| command_error(input, &cwd, e))? .wait() .map_err(|e| command_error(input, &cwd, e))?; - check_exit_status(input, cwd, exit_status)?; + check_exit_status(input, cwd, exit_status, None, true)?; + Ok(()) +} + +pub fn run_command_with_output_and_env_no_err( + input: &[&dyn AsRef<OsStr>], + cwd: Option<&Path>, + env: Option<&HashMap<String, String>>, +) -> Result<(), String> { + let exit_status = get_command_inner(input, cwd, env) + .spawn() + .map_err(|e| command_error(input, &cwd, e))? + .wait() + .map_err(|e| command_error(input, &cwd, e))?; + check_exit_status(input, cwd, exit_status, None, false)?; Ok(()) } @@ -143,80 +175,157 @@ pub fn get_os_name() -> Result<String, String> { } } -pub fn get_rustc_host_triple() -> Result<String, String> { - let output = run_command(&[&"rustc", &"-vV"], None)?; +#[derive(Default, PartialEq)] +pub struct RustcVersionInfo { + pub short: String, + pub version: String, + pub host: Option<String>, + pub commit_hash: Option<String>, + pub commit_date: Option<String>, +} + +pub fn rustc_toolchain_version_info(toolchain: &str) -> Result<RustcVersionInfo, String> { + rustc_version_info_inner(None, Some(toolchain)) +} + +pub fn rustc_version_info(rustc: Option<&str>) -> Result<RustcVersionInfo, String> { + rustc_version_info_inner(rustc, None) +} + +fn rustc_version_info_inner( + rustc: Option<&str>, + toolchain: Option<&str>, +) -> Result<RustcVersionInfo, String> { + let output = if let Some(toolchain) = toolchain { + run_command(&[&rustc.unwrap_or("rustc"), &toolchain, &"-vV"], None) + } else { + run_command(&[&rustc.unwrap_or("rustc"), &"-vV"], None) + }?; let content = std::str::from_utf8(&output.stdout).unwrap_or(""); - for line in content.split('\n').map(|line| line.trim()) { - if !line.starts_with("host:") { - continue; + let mut info = RustcVersionInfo::default(); + let mut lines = content.split('\n'); + info.short = match lines.next() { + Some(s) => s.to_string(), + None => return Err("failed to retrieve rustc version".to_string()), + }; + + for line in lines.map(|line| line.trim()) { + match line.split_once(':') { + Some(("host", data)) => info.host = Some(data.trim().to_string()), + Some(("release", data)) => info.version = data.trim().to_string(), + Some(("commit-hash", data)) => info.commit_hash = Some(data.trim().to_string()), + Some(("commit-date", data)) => info.commit_date = Some(data.trim().to_string()), + _ => {} } - return Ok(line.split(':').nth(1).unwrap().trim().to_string()); } - Err("Cannot find host triple".to_string()) + if info.version.is_empty() { + Err("failed to retrieve rustc version".to_string()) + } else { + Ok(info) + } } -pub fn get_gcc_path() -> Result<String, String> { - let content = match fs::read_to_string("gcc_path") { +pub fn get_toolchain() -> Result<String, String> { + let content = match fs::read_to_string("rust-toolchain") { Ok(content) => content, - Err(_) => { - return Err( - "Please put the path to your custom build of libgccjit in the file \ - `gcc_path`, see Readme.md for details" - .into(), - ) - } + Err(_) => return Err("No `rust-toolchain` file found".to_string()), }; match content .split('\n') .map(|line| line.trim()) .filter(|line| !line.is_empty()) + .filter_map(|line| { + if !line.starts_with("channel") { + return None; + } + line.split('"').skip(1).next() + }) .next() { - Some(gcc_path) => { - let path = Path::new(gcc_path); - if !path.exists() { - Err(format!( - "Path `{}` contained in the `gcc_path` file doesn't exist", - gcc_path, - )) - } else { - Ok(gcc_path.into()) - } - } - None => Err("No path found in `gcc_path` file".into()), + Some(toolchain) => Ok(toolchain.to_string()), + None => Err("Couldn't find `channel` in `rust-toolchain` file".to_string()), } } pub struct CloneResult { pub ran_clone: bool, pub repo_name: String, + pub repo_dir: String, } -pub fn git_clone(to_clone: &str, dest: Option<&Path>) -> Result<CloneResult, String> { - let repo_name = to_clone.split('/').last().unwrap(); - let repo_name = match repo_name.strip_suffix(".git") { - Some(n) => n.to_string(), - None => repo_name.to_string(), - }; - - let dest = dest - .map(|dest| dest.join(&repo_name)) - .unwrap_or_else(|| Path::new(&repo_name).into()); +fn git_clone_inner( + to_clone: &str, + dest: &Path, + shallow_clone: bool, + repo_name: String, +) -> Result<CloneResult, String> { if dest.is_dir() { return Ok(CloneResult { ran_clone: false, repo_name, + repo_dir: dest.display().to_string(), }); } - run_command_with_output(&[&"git", &"clone", &to_clone, &dest], None)?; + let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"git", &"clone", &to_clone, &dest]; + if shallow_clone { + command.push(&"--depth"); + command.push(&"1"); + } + run_command_with_output(&command, None)?; Ok(CloneResult { ran_clone: true, repo_name, + repo_dir: dest.display().to_string(), }) } +fn get_repo_name(url: &str) -> String { + let repo_name = url.split('/').last().unwrap(); + match repo_name.strip_suffix(".git") { + Some(n) => n.to_string(), + None => repo_name.to_string(), + } +} + +pub fn git_clone( + to_clone: &str, + dest: Option<&Path>, + shallow_clone: bool, +) -> Result<CloneResult, String> { + let repo_name = get_repo_name(to_clone); + let tmp: PathBuf; + + let dest = match dest { + Some(dest) => dest, + None => { + tmp = repo_name.clone().into(); + &tmp + } + }; + git_clone_inner(to_clone, dest, shallow_clone, repo_name) +} + +/// This function differs from `git_clone` in how it handles *where* the repository will be cloned. +/// In `git_clone`, it is cloned in the provided path. In this function, the path you provide is +/// the parent folder. So if you pass "a" as folder and try to clone "b.git", it will be cloned into +/// `a/b`. +pub fn git_clone_root_dir( + to_clone: &str, + dest_parent_dir: &Path, + shallow_clone: bool, +) -> Result<CloneResult, String> { + let repo_name = get_repo_name(to_clone); + + git_clone_inner( + to_clone, + &dest_parent_dir.join(&repo_name), + shallow_clone, + repo_name, + ) +} + pub fn walk_dir<P, D, F>(dir: P, mut dir_cb: D, mut file_cb: F) -> Result<(), String> where P: AsRef<Path>, @@ -238,3 +347,105 @@ where } Ok(()) } + +pub fn split_args(args: &str) -> Result<Vec<String>, String> { + let mut out = Vec::new(); + let mut start = 0; + let args = args.trim(); + let mut iter = args.char_indices().peekable(); + + while let Some((pos, c)) = iter.next() { + if c == ' ' { + out.push(args[start..pos].to_string()); + let mut found_start = false; + while let Some((pos, c)) = iter.peek() { + if *c != ' ' { + start = *pos; + found_start = true; + break; + } else { + iter.next(); + } + } + if !found_start { + return Ok(out); + } + } else if c == '"' || c == '\'' { + let end = c; + let mut found_end = false; + while let Some((_, c)) = iter.next() { + if c == end { + found_end = true; + break; + } else if c == '\\' { + // We skip the escaped character. + iter.next(); + } + } + if !found_end { + return Err(format!( + "Didn't find `{}` at the end of `{}`", + end, + &args[start..] + )); + } + } else if c == '\\' { + // We skip the escaped character. + iter.next(); + } + } + let s = args[start..].trim(); + if !s.is_empty() { + out.push(s.to_string()); + } + Ok(out) +} + +pub fn remove_file<P: AsRef<Path> + ?Sized>(file_path: &P) -> Result<(), String> { + std::fs::remove_file(file_path).map_err(|error| { + format!( + "Failed to remove `{}`: {:?}", + file_path.as_ref().display(), + error + ) + }) +} + +pub fn create_symlink<P: AsRef<Path>, Q: AsRef<Path>>(original: P, link: Q) -> Result<(), String> { + #[cfg(windows)] + let symlink = std::os::windows::fs::symlink_file; + #[cfg(not(windows))] + let symlink = std::os::unix::fs::symlink; + + symlink(&original, &link).map_err(|err| { + format!( + "failed to create a symlink `{}` to `{}`: {:?}", + original.as_ref().display(), + link.as_ref().display(), + err, + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_args() { + // Missing `"` at the end. + assert!(split_args("\"tada").is_err()); + // Missing `'` at the end. + assert!(split_args("\'tada").is_err()); + + assert_eq!( + split_args("a \"b\" c"), + Ok(vec!["a".to_string(), "\"b\"".to_string(), "c".to_string()]) + ); + // Trailing whitespace characters. + assert_eq!( + split_args(" a \"b\" c "), + Ok(vec!["a".to_string(), "\"b\"".to_string(), "c".to_string()]) + ); + } +} |