diff options
| author | Guillaume Gomez <guillaume1.gomez@gmail.com> | 2025-06-18 15:11:44 +0200 |
|---|---|---|
| committer | Guillaume Gomez <guillaume1.gomez@gmail.com> | 2025-06-18 15:11:44 +0200 |
| commit | c48d8d4d800df8a59d412c5c4299c8fadcd039e9 (patch) | |
| tree | cba780e8e9940952265aa1587419e557994e3423 /compiler/rustc_codegen_gcc/build_system/src | |
| parent | 6f935a044d1ddeb6160494a6320d008d7c311aef (diff) | |
| parent | fda0bb9588912a3e0606e880ca9f6e913cf8a5a4 (diff) | |
| download | rust-c48d8d4d800df8a59d412c5c4299c8fadcd039e9.tar.gz rust-c48d8d4d800df8a59d412c5c4299c8fadcd039e9.zip | |
Merge commit 'fda0bb9588912a3e0606e880ca9f6e913cf8a5a4' into subtree-update_cg_gcc_2025-06-18
Diffstat (limited to 'compiler/rustc_codegen_gcc/build_system/src')
13 files changed, 933 insertions, 153 deletions
diff --git a/compiler/rustc_codegen_gcc/build_system/src/build.rs b/compiler/rustc_codegen_gcc/build_system/src/build.rs index e98377f15a9..ecc4c1b2fe2 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/build.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/build.rs @@ -33,7 +33,7 @@ impl BuildArg { } arg => { if !build_arg.config_info.parse_argument(arg, &mut args)? { - return Err(format!("Unknown argument `{}`", arg)); + return Err(format!("Unknown argument `{arg}`")); } } } @@ -105,14 +105,14 @@ pub fn create_build_sysroot_content(start_dir: &Path) -> Result<(), String> { if !start_dir.is_dir() { create_dir(start_dir)?; } - copy_file("build_system/build_sysroot/Cargo.toml", &start_dir.join("Cargo.toml"))?; - copy_file("build_system/build_sysroot/Cargo.lock", &start_dir.join("Cargo.lock"))?; + copy_file("build_system/build_sysroot/Cargo.toml", start_dir.join("Cargo.toml"))?; + copy_file("build_system/build_sysroot/Cargo.lock", start_dir.join("Cargo.lock"))?; let src_dir = start_dir.join("src"); if !src_dir.is_dir() { create_dir(&src_dir)?; } - copy_file("build_system/build_sysroot/lib.rs", &start_dir.join("src/lib.rs")) + copy_file("build_system/build_sysroot/lib.rs", start_dir.join("src/lib.rs")) } pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Result<(), String> { @@ -169,7 +169,7 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ()) }; walk_dir( - start_dir.join(&format!("target/{}/{}/deps", config.target_triple, channel)), + start_dir.join(format!("target/{}/{}/deps", config.target_triple, channel)), &mut copier.clone(), &mut copier, false, diff --git a/compiler/rustc_codegen_gcc/build_system/src/clean.rs b/compiler/rustc_codegen_gcc/build_system/src/clean.rs index 768a78e789e..a441ed613f9 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/clean.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/clean.rs @@ -17,12 +17,12 @@ enum CleanArg { impl CleanArg { fn new() -> Result<Self, String> { // We skip the binary and the "clean" option. - for arg in std::env::args().skip(2) { + if let Some(arg) = std::env::args().nth(2) { return match arg.as_str() { "all" => Ok(Self::All), "ui-tests" => Ok(Self::UiTests), "--help" => Ok(Self::Help), - a => Err(format!("Unknown argument `{}`", a)), + a => Err(format!("Unknown argument `{a}`")), }; } Ok(Self::default()) diff --git a/compiler/rustc_codegen_gcc/build_system/src/clone_gcc.rs b/compiler/rustc_codegen_gcc/build_system/src/clone_gcc.rs index b49dd47f352..ee683df419c 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/clone_gcc.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/clone_gcc.rs @@ -43,7 +43,7 @@ impl Args { } arg => { if !command_args.config_info.parse_argument(arg, &mut args)? { - return Err(format!("Unknown option {}", arg)); + return Err(format!("Unknown option {arg}")); } } } @@ -52,7 +52,7 @@ impl Args { Some(p) => p.into(), None => PathBuf::from("./gcc"), }; - return Ok(Some(command_args)); + Ok(Some(command_args)) } } @@ -64,7 +64,7 @@ pub fn run() -> Result<(), String> { let result = git_clone("https://github.com/rust-lang/gcc", Some(&args.out_path), false)?; if result.ran_clone { let gcc_commit = args.config_info.get_gcc_commit()?; - println!("Checking out GCC commit `{}`...", gcc_commit); + println!("Checking out GCC commit `{gcc_commit}`..."); run_command_with_output( &[&"git", &"checkout", &gcc_commit], Some(Path::new(&result.repo_dir)), diff --git a/compiler/rustc_codegen_gcc/build_system/src/config.rs b/compiler/rustc_codegen_gcc/build_system/src/config.rs index 4f9fcc97151..650c030ca53 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/config.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/config.rs @@ -66,7 +66,7 @@ impl ConfigFile { "Expected a boolean for `download-gccjit`", ); } - _ => return failed_config_parsing(config_file, &format!("Unknown key `{}`", key)), + _ => return failed_config_parsing(config_file, &format!("Unknown key `{key}`")), } } match (config.gcc_path.as_mut(), config.download_gccjit) { @@ -86,9 +86,7 @@ impl ConfigFile { let path = Path::new(gcc_path); *gcc_path = path .canonicalize() - .map_err(|err| { - format!("Failed to get absolute path of `{}`: {:?}", gcc_path, err) - })? + .map_err(|err| format!("Failed to get absolute path of `{gcc_path}`: {err:?}"))? .display() .to_string(); } @@ -175,7 +173,7 @@ impl ConfigInfo { "--sysroot-panic-abort" => self.sysroot_panic_abort = true, "--gcc-path" => match args.next() { Some(arg) if !arg.is_empty() => { - self.gcc_path = Some(arg.into()); + self.gcc_path = Some(arg); } _ => { return Err("Expected a value after `--gcc-path`, found nothing".to_string()); @@ -244,7 +242,7 @@ impl ConfigInfo { let libgccjit_so = output_dir.join(libgccjit_so_name); if !libgccjit_so.is_file() && !self.no_download { // Download time! - let tempfile_name = format!("{}.download", libgccjit_so_name); + let tempfile_name = format!("{libgccjit_so_name}.download"); let tempfile = output_dir.join(&tempfile_name); let is_in_ci = std::env::var("GITHUB_ACTIONS").is_ok(); @@ -262,14 +260,14 @@ impl ConfigInfo { ) })?; - println!("Downloaded libgccjit.so version {} successfully!", commit); + println!("Downloaded libgccjit.so version {commit} successfully!"); // We need to create a link named `libgccjit.so.0` because that's what the linker is // looking for. - create_symlink(&libgccjit_so, output_dir.join(&format!("{}.0", libgccjit_so_name)))?; + create_symlink(&libgccjit_so, output_dir.join(format!("{libgccjit_so_name}.0")))?; } let gcc_path = output_dir.display().to_string(); - println!("Using `{}` as path for libgccjit", gcc_path); + println!("Using `{gcc_path}` as path for libgccjit"); self.gcc_path = Some(gcc_path); Ok(()) } @@ -286,8 +284,7 @@ impl ConfigInfo { // since we already have everything we need. if let Some(gcc_path) = &self.gcc_path { println!( - "`--gcc-path` was provided, ignoring config file. Using `{}` as path for libgccjit", - gcc_path + "`--gcc-path` was provided, ignoring config file. Using `{gcc_path}` as path for libgccjit" ); return Ok(()); } @@ -343,7 +340,7 @@ impl ConfigInfo { self.dylib_ext = match os_name.as_str() { "Linux" => "so", "Darwin" => "dylib", - os => return Err(format!("unsupported OS `{}`", os)), + os => return Err(format!("unsupported OS `{os}`")), } .to_string(); let rustc = match env.get("RUSTC") { @@ -355,10 +352,10 @@ impl ConfigInfo { None => return Err("no host found".to_string()), }; - if self.target_triple.is_empty() { - if let Some(overwrite) = env.get("OVERWRITE_TARGET_TRIPLE") { - self.target_triple = overwrite.clone(); - } + if self.target_triple.is_empty() + && let Some(overwrite) = env.get("OVERWRITE_TARGET_TRIPLE") + { + self.target_triple = overwrite.clone(); } if self.target_triple.is_empty() { self.target_triple = self.host_triple.clone(); @@ -378,7 +375,7 @@ impl ConfigInfo { } let current_dir = - std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; + std_env::current_dir().map_err(|error| format!("`current_dir` failed: {error:?}"))?; let channel = if self.channel == Channel::Release { "release" } else if let Some(channel) = env.get("CHANNEL") { @@ -391,15 +388,15 @@ impl ConfigInfo { self.cg_backend_path = current_dir .join("target") .join(channel) - .join(&format!("librustc_codegen_gcc.{}", self.dylib_ext)) + .join(format!("librustc_codegen_gcc.{}", self.dylib_ext)) .display() .to_string(); self.sysroot_path = - current_dir.join(&get_sysroot_dir()).join("sysroot").display().to_string(); + current_dir.join(get_sysroot_dir()).join("sysroot").display().to_string(); if let Some(backend) = &self.backend { // This option is only used in the rust compiler testsuite. The sysroot is handled // by its build system directly so no need to set it ourselves. - rustflags.push(format!("-Zcodegen-backend={}", backend)); + rustflags.push(format!("-Zcodegen-backend={backend}")); } else { rustflags.extend_from_slice(&[ "--sysroot".to_string(), @@ -412,10 +409,10 @@ impl ConfigInfo { // We have a different environment variable than RUSTFLAGS to make sure those flags are // only sent to rustc_codegen_gcc and not the LLVM backend. if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { - rustflags.extend_from_slice(&split_args(&cg_rustflags)?); + rustflags.extend_from_slice(&split_args(cg_rustflags)?); } if let Some(test_flags) = env.get("TEST_FLAGS") { - rustflags.extend_from_slice(&split_args(&test_flags)?); + rustflags.extend_from_slice(&split_args(test_flags)?); } if let Some(linker) = linker { @@ -438,8 +435,8 @@ impl ConfigInfo { env.insert("RUSTC_LOG".to_string(), "warn".to_string()); let sysroot = current_dir - .join(&get_sysroot_dir()) - .join(&format!("sysroot/lib/rustlib/{}/lib", self.target_triple)); + .join(get_sysroot_dir()) + .join(format!("sysroot/lib/rustlib/{}/lib", self.target_triple)); let ld_library_path = format!( "{target}:{sysroot}:{gcc_path}", target = self.cargo_target_dir, @@ -505,7 +502,7 @@ fn download_gccjit( with_progress_bar: bool, ) -> Result<(), String> { let url = if std::env::consts::OS == "linux" && std::env::consts::ARCH == "x86_64" { - format!("https://github.com/rust-lang/gcc/releases/download/master-{}/libgccjit.so", commit) + format!("https://github.com/rust-lang/gcc/releases/download/master-{commit}/libgccjit.so") } else { eprintln!( "\ @@ -518,7 +515,7 @@ to `download-gccjit = false` and set `gcc-path` to the appropriate directory." )); }; - println!("Downloading `{}`...", url); + println!("Downloading `{url}`..."); // Try curl. If that fails and we are on windows, fallback to PowerShell. let mut ret = run_command_with_output( @@ -538,7 +535,7 @@ to `download-gccjit = false` and set `gcc-path` to the appropriate directory." if with_progress_bar { &"--progress-bar" } else { &"-s" }, &url.as_str(), ], - Some(&output_dir), + Some(output_dir), ); if ret.is_err() && cfg!(windows) { eprintln!("Fallback to PowerShell"); @@ -549,12 +546,11 @@ to `download-gccjit = false` and set `gcc-path` to the appropriate directory." &"-Command", &"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", &format!( - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, tempfile_name, + "(New-Object System.Net.WebClient).DownloadFile('{url}', '{tempfile_name}')", ) .as_str(), ], - Some(&output_dir), + Some(output_dir), ); } ret diff --git a/compiler/rustc_codegen_gcc/build_system/src/fmt.rs b/compiler/rustc_codegen_gcc/build_system/src/fmt.rs index de310a6a30f..7e6594f50f9 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/fmt.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/fmt.rs @@ -16,21 +16,21 @@ fn show_usage() { pub fn run() -> Result<(), String> { let mut check = false; // We skip binary name and the `info` command. - let mut args = std::env::args().skip(2); - while let Some(arg) = args.next() { + let args = std::env::args().skip(2); + for arg in args { match arg.as_str() { "--help" => { show_usage(); return Ok(()); } "--check" => check = true, - _ => return Err(format!("Unknown option {}", arg)), + _ => return Err(format!("Unknown option {arg}")), } } let cmd: &[&dyn AsRef<OsStr>] = if check { &[&"cargo", &"fmt", &"--check"] } else { &[&"cargo", &"fmt"] }; - run_command_with_output(cmd, Some(&Path::new(".")))?; - run_command_with_output(cmd, Some(&Path::new("build_system"))) + run_command_with_output(cmd, Some(Path::new(".")))?; + run_command_with_output(cmd, Some(Path::new("build_system"))) } diff --git a/compiler/rustc_codegen_gcc/build_system/src/fuzz.rs b/compiler/rustc_codegen_gcc/build_system/src/fuzz.rs new file mode 100644 index 00000000000..453211366b3 --- /dev/null +++ b/compiler/rustc_codegen_gcc/build_system/src/fuzz.rs @@ -0,0 +1,289 @@ +use std::ffi::OsStr; +use std::path::Path; + +mod reduce; + +use crate::utils::run_command_with_output; + +fn show_usage() { + println!( + r#" +`fuzz` command help: + --reduce : Reduces a file generated by rustlantis + --help : Show this help + --start : Start of the fuzzed range + --count : The number of cases to fuzz + -j --jobs : The number of threads to use during fuzzing"# + ); +} + +pub fn run() -> Result<(), String> { + // We skip binary name and the `fuzz` command. + let mut args = std::env::args().skip(2); + let mut start = 0; + let mut count = 100; + let mut threads = + std::thread::available_parallelism().map(|threads| threads.get()).unwrap_or(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "--reduce" => { + let Some(path) = args.next() else { + return Err("--reduce must be provided with a path".into()); + }; + if !std::fs::exists(&path).unwrap_or(false) { + return Err("--reduce must be provided with a valid path".into()); + } + reduce::reduce(&path); + return Ok(()); + } + "--help" => { + show_usage(); + return Ok(()); + } + "--start" => { + start = + str::parse(&args.next().ok_or_else(|| "Fuzz start not provided!".to_string())?) + .map_err(|err| (format!("Fuzz start not a number {err:?}!")))?; + } + "--count" => { + count = + str::parse(&args.next().ok_or_else(|| "Fuzz count not provided!".to_string())?) + .map_err(|err| (format!("Fuzz count not a number {err:?}!")))?; + } + "-j" | "--jobs" => { + threads = str::parse( + &args.next().ok_or_else(|| "Fuzz thread count not provided!".to_string())?, + ) + .map_err(|err| (format!("Fuzz thread count not a number {err:?}!")))?; + } + _ => return Err(format!("Unknown option {arg}")), + } + } + + // Ensure that we have a cloned version of rustlantis on hand. + crate::utils::git_clone( + "https://github.com/cbeuw/rustlantis.git", + Some("clones/rustlantis".as_ref()), + true, + ) + .map_err(|err| (format!("Git clone failed with message: {err:?}!")))?; + + // Ensure that we are on the newest rustlantis commit. + let cmd: &[&dyn AsRef<OsStr>] = &[&"git", &"pull", &"origin"]; + run_command_with_output(cmd, Some(Path::new("clones/rustlantis")))?; + + // Build the release version of rustlantis + let cmd: &[&dyn AsRef<OsStr>] = &[&"cargo", &"build", &"--release"]; + run_command_with_output(cmd, Some(Path::new("clones/rustlantis")))?; + // Fuzz a given range + fuzz_range(start, start + count, threads); + Ok(()) +} + +/// Fuzzes a range `start..end` with `threads`. +fn fuzz_range(start: u64, end: u64, threads: usize) { + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{Duration, Instant}; + // Total amount of files to fuzz + let total = end - start; + // Currently fuzzed element + let start = Arc::new(AtomicU64::new(start)); + // Count time during fuzzing + let start_time = Instant::now(); + let mut workers = Vec::with_capacity(threads); + // Spawn `threads`.. + for _ in 0..threads { + let start = start.clone(); + // .. which each will .. + workers.push(std::thread::spawn(move || { + // ... grab the next fuzz seed ... + while start.load(Ordering::Relaxed) < end { + let next = start.fetch_add(1, Ordering::Relaxed); + // .. test that seed . + match test(next, false) { + Err(err) => { + // If the test failed at compile-time... + println!("test({next}) failed because {err:?}"); + // ... copy that file to the directory `target/fuzz/compiletime_error`... + let mut out_path: std::path::PathBuf = + "target/fuzz/compiletime_error".into(); + std::fs::create_dir_all(&out_path).unwrap(); + // .. into a file named `fuzz{seed}.rs`. + out_path.push(format!("fuzz{next}.rs")); + std::fs::copy(err, out_path).unwrap(); + } + Ok(Err(err)) => { + // If the test failed at run-time... + println!("The LLVM and GCC results don't match for {err:?}"); + // ... generate a new file, which prints temporaries(instead of hashing them)... + let mut out_path: std::path::PathBuf = "target/fuzz/runtime_error".into(); + std::fs::create_dir_all(&out_path).unwrap(); + let Ok(Err(tmp_print_err)) = test(next, true) else { + // ... if that file does not reproduce the issue... + // ... save the original sample in a file named `fuzz{seed}.rs`... + out_path.push(format!("fuzz{next}.rs")); + std::fs::copy(err, &out_path).unwrap(); + continue; + }; + // ... if that new file still produces the issue, copy it to `fuzz{seed}.rs`.. + out_path.push(format!("fuzz{next}.rs")); + std::fs::copy(tmp_print_err, &out_path).unwrap(); + // ... and start reducing it, using some properties of `rustlantis` to speed up the process. + reduce::reduce(&out_path); + } + // If the test passed, do nothing + Ok(Ok(())) => (), + } + } + })); + } + // The "manager" thread loop. + while start.load(Ordering::Relaxed) < end || !workers.iter().all(|t| t.is_finished()) { + // Every 500 ms... + let five_hundred_millis = Duration::from_millis(500); + std::thread::sleep(five_hundred_millis); + // ... calculate the remaining fuzz iters ... + let remaining = end - start.load(Ordering::Relaxed); + // ... fix the count(the start counter counts the cases that + // begun fuzzing, and not only the ones that are done)... + let fuzzed = (total - remaining).saturating_sub(threads as u64); + // ... and the fuzz speed ... + let iter_per_sec = fuzzed as f64 / start_time.elapsed().as_secs_f64(); + // .. and use them to display fuzzing stats. + println!( + "fuzzed {fuzzed} cases({}%), at rate {iter_per_sec} iter/s, remaining ~{}s", + (100 * fuzzed) as f64 / total as f64, + (remaining as f64) / iter_per_sec + ) + } + drop(workers); +} + +/// Builds & runs a file with LLVM. +fn debug_llvm(path: &std::path::Path) -> Result<Vec<u8>, String> { + // Build a file named `llvm_elf`... + let exe_path = path.with_extension("llvm_elf"); + // ... using the LLVM backend ... + let output = std::process::Command::new("rustc") + .arg(path) + .arg("-o") + .arg(&exe_path) + .output() + .map_err(|err| format!("{err:?}"))?; + // ... check that the compilation succeeded ... + if !output.status.success() { + return Err(format!("LLVM compilation failed:{output:?}")); + } + // ... run the resulting executable ... + let output = + std::process::Command::new(&exe_path).output().map_err(|err| format!("{err:?}"))?; + // ... check it run normally ... + if !output.status.success() { + return Err(format!( + "The program at {path:?}, compiled with LLVM, exited unsuccessfully:{output:?}" + )); + } + // ... cleanup that executable ... + std::fs::remove_file(exe_path).map_err(|err| format!("{err:?}"))?; + // ... and return the output(stdout + stderr - this allows UB checks to fire). + let mut res = output.stdout; + res.extend(output.stderr); + Ok(res) +} + +/// Builds & runs a file with GCC. +fn release_gcc(path: &std::path::Path) -> Result<Vec<u8>, String> { + // Build a file named `gcc_elf`... + let exe_path = path.with_extension("gcc_elf"); + // ... using the GCC backend ... + let output = std::process::Command::new("./y.sh") + .arg("rustc") + .arg(path) + .arg("-O") + .arg("-o") + .arg(&exe_path) + .output() + .map_err(|err| format!("{err:?}"))?; + // ... check that the compilation succeeded ... + if !output.status.success() { + return Err(format!("GCC compilation failed:{output:?}")); + } + // ... run the resulting executable .. + let output = + std::process::Command::new(&exe_path).output().map_err(|err| format!("{err:?}"))?; + // ... check it run normally ... + if !output.status.success() { + return Err(format!( + "The program at {path:?}, compiled with GCC, exited unsuccessfully:{output:?}" + )); + } + // ... cleanup that executable ... + std::fs::remove_file(exe_path).map_err(|err| format!("{err:?}"))?; + // ... and return the output(stdout + stderr - this allows UB checks to fire). + let mut res = output.stdout; + res.extend(output.stderr); + Ok(res) +} +type ResultCache = Option<(Vec<u8>, Vec<u8>)>; +/// Generates a new rustlantis file, & compares the result of running it with GCC and LLVM. +fn test(seed: u64, print_tmp_vars: bool) -> Result<Result<(), std::path::PathBuf>, String> { + // Generate a Rust source... + let source_file = generate(seed, print_tmp_vars)?; + test_file(&source_file, true) +} +/// Tests a file with a cached LLVM result. Used for reduction, when it is known +/// that a given transformation should not change the execution result. +fn test_cached( + source_file: &Path, + remove_tmps: bool, + cache: &mut ResultCache, +) -> Result<Result<(), std::path::PathBuf>, String> { + // Test `source_file` with release GCC ... + let gcc_res = release_gcc(source_file)?; + if cache.is_none() { + // ...test `source_file` with debug LLVM ... + *cache = Some((debug_llvm(source_file)?, gcc_res.clone())); + } + let (llvm_res, old_gcc) = cache.as_ref().unwrap(); + // ... compare the results ... + if *llvm_res != gcc_res && gcc_res == *old_gcc { + // .. if they don't match, report an error. + Ok(Err(source_file.to_path_buf())) + } else { + if remove_tmps { + std::fs::remove_file(source_file).map_err(|err| format!("{err:?}"))?; + } + Ok(Ok(())) + } +} +fn test_file( + source_file: &Path, + remove_tmps: bool, +) -> Result<Result<(), std::path::PathBuf>, String> { + let mut uncached = None; + test_cached(source_file, remove_tmps, &mut uncached) +} + +/// Generates a new rustlantis file for us to run tests on. +fn generate(seed: u64, print_tmp_vars: bool) -> Result<std::path::PathBuf, String> { + use std::io::Write; + let mut out_path = std::env::temp_dir(); + out_path.push(format!("fuzz{seed}.rs")); + // We need to get the command output here. + let mut generate = std::process::Command::new("cargo"); + generate + .args(["run", "--release", "--bin", "generate"]) + .arg(format!("{seed}")) + .current_dir("clones/rustlantis"); + if print_tmp_vars { + generate.arg("--debug"); + } + let out = generate.output().map_err(|err| format!("{err:?}"))?; + // Stuff the rustlantis output in a source file. + std::fs::File::create(&out_path) + .map_err(|err| format!("{err:?}"))? + .write_all(&out.stdout) + .map_err(|err| format!("{err:?}"))?; + Ok(out_path) +} diff --git a/compiler/rustc_codegen_gcc/build_system/src/fuzz/reduce.rs b/compiler/rustc_codegen_gcc/build_system/src/fuzz/reduce.rs new file mode 100644 index 00000000000..20715ab0e7c --- /dev/null +++ b/compiler/rustc_codegen_gcc/build_system/src/fuzz/reduce.rs @@ -0,0 +1,432 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; + +use super::ResultCache; + +/// Saves a reduced file for a given `stage` +fn save_reduction(lines: &[String], path: &Path, stage: &str) { + let mut path = path.to_path_buf(); + path.set_extension(format!("rs.{stage}")); + let mut file = std::fs::File::create(&path).expect("Could not create the reduced example file"); + for line in lines { + file.write_all(line.as_bytes()).expect("Could not save the reduced example"); + } +} + +/// Checks if a given reduction is valid. +fn test_reduction(lines: &[String], path: &Path, cache: &mut ResultCache) -> bool { + let mut path = path.to_path_buf(); + path.set_extension("rs_reduced"); + let mut file = std::fs::File::create(&path).expect("Could not create the reduced example file"); + for line in lines { + file.write_all(line.as_bytes()).expect("Could not save the reduced example"); + } + let res = super::test_cached(&path, false, cache); + let Ok(Err(_)) = res else { + return false; + }; + true +} + +/// Removes duplicate assignments in bulk. +/// If a line A = B is followed directly by A = C, +/// then removing the first line ought to be fully sound, +/// and not change the behaviour of the program at all. Detect & remove such lines. +fn remove_dup_assign( + file: &mut Vec<String>, + path: &PathBuf, + starts: usize, + ends: usize, + cache: &mut ResultCache, +) { + let mut file_copy = file.clone(); + let mut reduction_count = 0; + // Not worth it. + if ends - starts < 8 { + return; + } + for index in starts..ends { + let Some((prefix, _)) = file_copy[index].split_once('=') else { + continue; + }; + let Some((prefix2, postifx2)) = file_copy[index + 1].split_once('=') else { + continue; + }; + let prefix = prefix.trim(); + let prefix2 = prefix2.trim(); + // FIXME: Right now, remove_dup_assign cares about assignments to the exact same place. + // However, given an assigemnt like this: + // ``` + // A.0 = 1_u32; + // A = (2_u32, 3.0); + // ``` + // The first assignment could be safely omitted. + // Additionally, we try to check if the second assignment could depend on the first one. + // In such cases, the result is likely to change, so we bail. + if prefix == prefix2 && !postifx2.contains(prefix) { + file_copy[index] = "".into(); + reduction_count += 1; + } + } + // We have removed no lines - no point in testing. + if reduction_count == 0 { + return; + } + // Check if the removed lines affected the execution result in any way, shape or form. + if test_reduction(&file_copy, path, cache) { + println!("Reduced {path:?} by {reduction_count} lines `remove_dup_assign`"); + *file = file_copy; + } else { + // The execution result changed. + // This can occur if the second assignment depended on the first one. + // Eg. + // ``` + // a = b + c; + // a = a + d; + // ``` + remove_dup_assign(file, path, starts, (starts + ends) / 2, cache); + remove_dup_assign(file, path, (starts + ends) / 2, ends, cache); + } + save_reduction(file, path, "remove_dup_assign"); +} + +/// Removes all the unneeded calls to `dump_var`. This is not something tools like `cvise` can do, +/// but it greately speeds up MIR interpretation + native execution. +fn remove_dump_var(file: &mut Vec<String>, path: &PathBuf) { + let mut curr = 0; + // ... try disabling `dump_vars` one by one, until only the necessary ones are left. + while curr < file.len() { + let Some(line) = file[curr..].iter().position(|line| line.contains("dump_var")) else { + // No more `dump_var`s to remove - exit early. + break; + }; + // Make the line absolute again. + let line = line + curr; + let mut file_copy = file.clone(); + // Try removing 3 consecutive lines(the call, block end and block beginning). This effectively removes a `dump_var`. + file_copy.remove(line); + file_copy.remove(line); + file_copy.remove(line); + // Not cached - the execution result can change. + let mut uncached = None; + // Check if this reduction is valid. + if test_reduction(&file_copy, path, &mut uncached) { + println!("Reduced {path:?} by 3 lines `remove_dump_var`"); + *file = file_copy; + curr = line; + } else { + curr = line + 1; + } + } + save_reduction(file, path, "remove_dump_var"); +} + +/// Replaces matches with gotos where possible. +/// This exploits some properties of rustlantis(match arm order), +/// and is only soundly applicable to MIR generated by it. +/// Still, it is not something `cvise` can do, but it simplifies the code a ton. +fn match_to_goto(file: &mut Vec<String>, path: &PathBuf, cache: &mut ResultCache) { + let mut curr = 0; + + while curr < file.len() { + let Some(match_starts) = file[curr..].iter().position(|line| line.contains("match")) else { + // No more `match`es to remove - exit early. + break; + }; + let match_starts = match_starts + curr; + // Find the end of the match + let Some(match_ends) = file[match_starts..].iter().position(|line| line.contains('}')) + else { + // Can't find match end - exit early. + break; + }; + let match_ends = match_ends + match_starts; + let match_body = &file[match_starts..match_ends]; + + // Find where this match should normally jump to. + // This *should* be the second-last arm of the match, as per the paper(the remaining blocks are decoys). + // If this ever changes, this reduction may not always be sound. + // This is not a problem, however: we NEED to use MIRI for reduction anwyway, + // and it will catch this issue. + let jumps_to = &match_body[match_body.len() - 2].trim(); + let Some((_, bb_ident)) = jumps_to.split_once("bb") else { + break; + }; + // We now have the number of the block we jump to at runtime. + let bb_ident = bb_ident.trim_matches(','); + // Try replacing this match with an unconditional jump. + let mut file_copy = file.clone(); + for _ in match_starts..(match_ends + 1) { + file_copy.remove(match_starts); + } + file_copy.insert(match_starts, format!("Goto(bb{bb_ident})\n")); + if test_reduction(&file_copy, path, cache) { + println!("Reduced {path:?} by {} lines `match_to_goto`", match_ends - match_starts); + *file = file_copy; + curr = match_starts; + } else { + curr = match_ends; + } + } + save_reduction(file, path, "match_to_goto"); +} + +/// At this point, we can try "killing" blocks, by replacing their bodies with calls to `abort`. +/// This is always sound(the program aborts, so no UB can occur after the block), +/// and allows us to safely remove *a lot* of unneeded blocks. +fn block_abort(file: &mut Vec<String>, path: &PathBuf, cache: &mut ResultCache) { + let mut curr = 0; + while curr < file.len() { + let Some(block_starts) = file[curr..] + .iter() + .position(|line| line.starts_with("bb") && line.trim_end().ends_with(" = {")) + else { + // No more `block`s to kill - exit early. + break; + }; + let block_starts = block_starts + curr; + // Find the beginning of the next block to find the end of this block. + let Some(block_ends) = file[(block_starts + 1)..] + .iter() + .position(|line| line.starts_with("bb") && line.trim_end().ends_with(" = {")) + else { + // No more `block`s to kill - exit early. + break; + }; + let block_ends = block_starts + block_ends; + let block_starts = block_starts + 1; + let mut file_copy = file.clone(); + // Remove the block body... + for _ in block_starts..(block_ends) { + file_copy.remove(block_starts); + } + // ..and insert an unconditional call to abort. + file_copy.insert( + block_starts, + "Call(tmp = core::intrinsics::abort(), ReturnTo(bb1), UnwindUnreachable())\n" + .to_string(), + ); + file_copy.insert(block_starts, "let tmp = ();\n".to_string()); + + if test_reduction(&file_copy, path, cache) { + println!("Reduced {path:?} by {} lines `block_abort`", block_ends - block_starts - 2); + *file = file_copy; + curr = block_starts; + } else { + curr = block_ends; + } + } + save_reduction(file, path, "block_abort"); +} + +/// Removes unreachable basic blocks +fn remove_block(file: &mut Vec<String>, path: &PathBuf, cache: &mut ResultCache) { + let mut curr = 0; + + // Next, we try to outright remove blocks. + while curr < file.len() { + let Some(block_starts) = file[curr..] + .iter() + .position(|line| line.starts_with("bb") && line.trim_end().ends_with(" = {")) + else { + // No more `block`s to remove - exit early. + break; + }; + let block_starts = block_starts + curr; + // Find the beginning of the next block to find the end of this block. + let Some(block_ends) = file[(block_starts + 1)..] + .iter() + .position(|line| line.starts_with("bb") && line.trim_end().ends_with(" = {")) + else { + // No more `block`s to remove - exit early. + break; + }; + let block_ends = block_starts + block_ends + 1; + // Large blocks are likely to be necessary. + if block_ends - block_starts > 6 { + curr = block_starts + 1; + continue; + } + let mut file_copy = file.clone(); + file_copy.drain(block_starts..block_ends); + if test_reduction(&file_copy, path, cache) { + println!("Reduced {path:?} by {} lines `remove_blocks`", block_ends - block_starts); + *file = file_copy; + curr = block_starts; + } else { + curr = block_starts + 1; + } + } + save_reduction(file, path, "remove_block"); +} + +/// Merges blocks ending with unconditional jumps. +fn linearize_cf(file: &mut Vec<String>, path: &PathBuf, cache: &mut ResultCache) { + let mut curr = 0; + + // Next, we try to linearize the control flow. What the does that mean? + // Given a sequence like this: + // Goto(bb22) + // } + // bb22 = { + // We remove those 3 lines, merging the blocks together. This is not something `cvise` can do, + // and it makes other transformations easier. + while curr < file.len() { + let Some(block_starts) = file[curr..] + .iter() + .position(|line| line.starts_with("bb") && line.trim_end().ends_with(" = {")) + else { + // No more `block`s to remove - exit early. + break; + }; + let block_starts = block_starts + curr; + // Extract the block id. + let Some((block, _)) = file[block_starts].split_once('=') else { + curr = block_starts + 1; + continue; + }; + let block = block.trim(); + if file[block_starts - 2].trim() != format!("Goto({block})") { + curr = block_starts + 1; + continue; + } + let mut file_copy = file.clone(); + // Try removing 3 consecutive lines(the goto, block end and block beginning). This effectively removes a `Goto(next)`. + file_copy.remove(block_starts - 2); + file_copy.remove(block_starts - 2); + file_copy.remove(block_starts - 2); + // Check if this reduction is valid. + if test_reduction(&file_copy, path, cache) { + println!("Reduced {path:?} by 3 lines `linearize_cf`"); + *file = file_copy; + curr = block_starts; + } else { + curr = block_starts + 1; + } + } + save_reduction(file, path, "linearize_cf"); +} + +/// Replaces a call to a given function with a 0 assignment to the destination place, and a Goto. +/// This is always sound, because: +/// 1. All the functions arguments are always initialized +/// 2. and point to initialized memory(the operand of &raw must be an initialized place in rustlantis). +fn remove_fn_calls(file: &mut Vec<String>, path: &PathBuf, cache: &mut ResultCache) { + let mut curr = 0; + + while curr < file.len() { + let Some(fn_call) = + file[curr..].iter().position(|line| line.contains("Call(") && line.contains(" = fn")) + else { + // No more calls to remove - exit early. + break; + }; + let fn_call = fn_call + curr; + let line = file[fn_call].trim(); + // Skip the Call( + let line = &line["Call(".len()..]; + // Extract the destination place + let Some((place, line)) = line.split_once('=') else { + curr = fn_call + 1; + continue; + }; + // Skip till the return block id. + let Some((_, line)) = line.split_once("ReturnTo(") else { + curr = fn_call + 1; + continue; + }; + // Extract the full return block + let Some((block, _)) = line.split_once(')') else { + curr = fn_call + 1; + continue; + }; + let mut file_copy = file.clone(); + // Remove the call. + file_copy.remove(fn_call); + file_copy.insert(fn_call, format!("Goto({block})\n")); + file_copy.insert(fn_call, format!("{place} = 0;\n")); + // Check if this reduction is valid. + if test_reduction(&file_copy, path, cache) { + println!("Reduced {path:?} using `remove_fn_calls` {cache:?}"); + *file = file_copy; + curr = fn_call; + } else { + curr = fn_call + 1; + } + } + save_reduction(file, path, "remove_fn_calls"); +} + +/// Fully removes unreachable functions. +fn remove_fns(file: &mut Vec<String>, path: &PathBuf, cache: &mut ResultCache) { + let mut curr = 0; + + while curr < file.len() { + // Find a function start + let Some(fn_start) = file[curr..].iter().position(|line| { + line.contains("#[custom_mir(dialect = \"runtime\", phase = \"initial\")]") + }) else { + // No more functions to remove - exit early. + break; + }; + // Find the next function(and use that to find the end of this one). + // FIXME: this check is flawed: it will never remove the very last function(the one before main). + // The other checks will turn that function into a single call to abort, but it is still annoying that it is kept. + let fn_start = fn_start + curr; + let Some(fn_end) = file[(fn_start + 3)..].iter().position(|line| line.contains("fn fn")) + else { + // No more functions to remove - exit early. + break; + }; + let fn_end = fn_start + 2 + fn_end; + let mut file_copy = file.clone(); + // Remove the function.\\ + file_copy.drain(fn_start..fn_end); + // Check if this reduction is valid. + if test_reduction(&file_copy, path, cache) { + println!("Reduced {path:?} by {} lines `remove_fns`", fn_end - fn_start); + *file = file_copy; + } else { + curr = fn_start + 1; + } + } + save_reduction(file, path, "remove_fns"); +} + +pub(super) fn reduce(path: impl AsRef<Path>) { + let path = path.as_ref().to_owned(); + // ... read the file to a buffer .. + let file = std::fs::read_to_string(&path).expect("Could not open the file to reduce"); + let mut file: Vec<_> = file.split_inclusive('\n').map(|s| s.to_string()).collect(); + + // ... and run reduction passes. + println!("running `remove_dump_var` on {path:?}."); + remove_dump_var(&mut file, &path); + // After `dump_var`, the execution results ought not to change. Cache them. + let mut cache = None; + // Fill the cache + assert!( + test_reduction(&file, &path, &mut cache), + "Reduction error: check that the input file is a valid reproducer." + ); + println!("cache:{cache:?}"); + println!("running `remove_fn_calls` on {path:?}."); + remove_fn_calls(&mut file, &path, &mut cache); + println!("running `remove_fns` on {path:?}."); + remove_fns(&mut file, &path, &mut cache); + let len = file.len(); + println!("running `remove_dup_assign` on {path:?}."); + remove_dup_assign(&mut file, &path, 0, len, &mut cache); + file.retain(|line| !line.is_empty()); + println!("running `match_to_goto` on {path:?}."); + match_to_goto(&mut file, &path, &mut cache); + println!("running `block_abort` on {path:?}."); + block_abort(&mut file, &path, &mut cache); + println!("running `remove_block` on {path:?}."); + remove_block(&mut file, &path, &mut cache); + println!("running `linearize_cf` on {path:?}."); + linearize_cf(&mut file, &path, &mut cache); + let mut out = std::fs::File::create(&path).expect("Could not save the reduction result."); + let file = file.into_iter().collect::<String>(); + out.write_all(file.as_bytes()).expect("failed to write into file"); +} diff --git a/compiler/rustc_codegen_gcc/build_system/src/info.rs b/compiler/rustc_codegen_gcc/build_system/src/info.rs index bd891de2eb4..66fdcf88cbb 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/info.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/info.rs @@ -15,7 +15,7 @@ pub fn run() -> Result<(), String> { config.no_download = true; config.setup_gcc_path()?; if let Some(gcc_path) = config.gcc_path { - println!("{}", gcc_path); + println!("{gcc_path}"); } Ok(()) } diff --git a/compiler/rustc_codegen_gcc/build_system/src/main.rs b/compiler/rustc_codegen_gcc/build_system/src/main.rs index c70b00e09ae..078a4726ba8 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/main.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/main.rs @@ -5,6 +5,7 @@ mod clean; mod clone_gcc; mod config; mod fmt; +mod fuzz; mod info; mod prepare; mod rust_tools; @@ -42,7 +43,8 @@ Commands: test : Runs tests for the project. info : Displays information about the build environment and project configuration. clone-gcc : Clones the GCC compiler from a specified source. - fmt : Runs rustfmt" + fmt : Runs rustfmt + fuzz : Fuzzes `cg_gcc` using rustlantis" ); } @@ -56,6 +58,7 @@ pub enum Command { Test, Info, Fmt, + Fuzz, } fn main() { @@ -75,6 +78,7 @@ fn main() { Some("info") => Command::Info, Some("clone-gcc") => Command::CloneGcc, Some("fmt") => Command::Fmt, + Some("fuzz") => Command::Fuzz, Some("--help") => { usage(); process::exit(0); @@ -97,6 +101,7 @@ fn main() { Command::Info => info::run(), Command::CloneGcc => clone_gcc::run(), Command::Fmt => fmt::run(), + Command::Fuzz => fuzz::run(), } { eprintln!("Command failed to run: {e}"); process::exit(1); diff --git a/compiler/rustc_codegen_gcc/build_system/src/prepare.rs b/compiler/rustc_codegen_gcc/build_system/src/prepare.rs index d14639afee5..35a6e20fb86 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/prepare.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/prepare.rs @@ -18,9 +18,9 @@ fn prepare_libcore( if let Some(path) = sysroot_source { rustlib_dir = Path::new(&path) .canonicalize() - .map_err(|error| format!("Failed to canonicalize path: {:?}", error))?; + .map_err(|error| format!("Failed to canonicalize path: {error:?}"))?; if !rustlib_dir.is_dir() { - return Err(format!("Custom sysroot path {:?} not found", rustlib_dir)); + return Err(format!("Custom sysroot path {rustlib_dir:?} not found")); } } else { let rustc_path = match get_rustc_path() { @@ -36,17 +36,17 @@ fn prepare_libcore( rustlib_dir = parent .join("../lib/rustlib/src/rust") .canonicalize() - .map_err(|error| format!("Failed to canonicalize path: {:?}", error))?; + .map_err(|error| format!("Failed to canonicalize path: {error:?}"))?; if !rustlib_dir.is_dir() { return Err("Please install `rust-src` component".to_string()); } } let sysroot_dir = sysroot_path.join("sysroot_src"); - if sysroot_dir.is_dir() { - if let Err(error) = fs::remove_dir_all(&sysroot_dir) { - return Err(format!("Failed to remove `{}`: {:?}", sysroot_dir.display(), error,)); - } + if sysroot_dir.is_dir() + && let Err(error) = fs::remove_dir_all(&sysroot_dir) + { + return Err(format!("Failed to remove `{}`: {:?}", sysroot_dir.display(), error,)); } let sysroot_library_dir = sysroot_dir.join("library"); @@ -122,7 +122,7 @@ fn prepare_rand() -> Result<(), String> { // Apply patch for the rand crate. let file_path = "patches/crates/0001-Remove-deny-warnings.patch"; let rand_dir = Path::new("build/rand"); - println!("[GIT] apply `{}`", file_path); + println!("[GIT] apply `{file_path}`"); let path = Path::new("../..").join(file_path); run_command_with_output(&[&"git", &"apply", &path], Some(rand_dir))?; run_command_with_output(&[&"git", &"add", &"-A"], Some(rand_dir))?; @@ -149,7 +149,7 @@ fn clone_and_setup<F>(repo_url: &str, checkout_commit: &str, extra: Option<F>) - where F: Fn(&Path) -> Result<(), String>, { - let clone_result = git_clone_root_dir(repo_url, &Path::new(crate::BUILD_DIR), false)?; + let clone_result = git_clone_root_dir(repo_url, Path::new(crate::BUILD_DIR), false)?; if !clone_result.ran_clone { println!("`{}` has already been cloned", clone_result.repo_name); } diff --git a/compiler/rustc_codegen_gcc/build_system/src/rust_tools.rs b/compiler/rustc_codegen_gcc/build_system/src/rust_tools.rs index 105f5eebe24..b1faa27acc4 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/rust_tools.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/rust_tools.rs @@ -1,24 +1,22 @@ use std::collections::HashMap; use std::ffi::OsStr; +#[cfg(unix)] +use std::os::unix::process::CommandExt; use std::path::PathBuf; use crate::config::ConfigInfo; -use crate::utils::{ - get_toolchain, run_command_with_output_and_env_no_err, rustc_toolchain_version_info, - rustc_version_info, -}; +use crate::utils::{get_toolchain, rustc_toolchain_version_info, rustc_version_info}; fn args(command: &str) -> Result<Option<Vec<String>>, String> { // We skip the binary and the "cargo"/"rustc" option. - if let Some("--help") = std::env::args().skip(2).next().as_deref() { + if let Some("--help") = std::env::args().nth(2).as_deref() { usage(command); return Ok(None); } let args = std::env::args().skip(2).collect::<Vec<_>>(); if args.is_empty() { return Err(format!( - "Expected at least one argument for `{}` subcommand, found none", - command + "Expected at least one argument for `{command}` subcommand, found none" )); } Ok(Some(args)) @@ -27,12 +25,11 @@ fn args(command: &str) -> Result<Option<Vec<String>>, String> { fn usage(command: &str) { println!( r#" -`{}` command help: +`{command}` command help: [args] : Arguments to be passed to the cargo command --help : Show this help "#, - command, ) } @@ -51,10 +48,10 @@ impl RustcTools { // expected. let current_dir = std::env::current_dir() .and_then(|path| path.canonicalize()) - .map_err(|error| format!("Failed to get current directory path: {:?}", error))?; + .map_err(|error| format!("Failed to get current directory path: {error:?}"))?; let current_exe = std::env::current_exe() .and_then(|path| path.canonicalize()) - .map_err(|error| format!("Failed to get current exe path: {:?}", error))?; + .map_err(|error| format!("Failed to get current exe path: {error:?}"))?; let mut parent_dir = current_exe.components().map(|comp| comp.as_os_str()).collect::<Vec<_>>(); // We run this script from "build_system/target/release/y", so we need to remove these elements. @@ -68,7 +65,7 @@ impl RustcTools { )); } } - let parent_dir = PathBuf::from(parent_dir.join(&OsStr::new("/"))); + let parent_dir = PathBuf::from(parent_dir.join(OsStr::new("/"))); std::env::set_current_dir(&parent_dir).map_err(|error| { format!("Failed to go to `{}` folder: {:?}", parent_dir.display(), error) })?; @@ -92,11 +89,31 @@ impl RustcTools { std::env::set_current_dir(¤t_dir).map_err(|error| { format!("Failed to go back to `{}` folder: {:?}", current_dir.display(), error) })?; - let toolchain = format!("+{}", toolchain); + let toolchain = format!("+{toolchain}"); Ok(Some(Self { toolchain, args, env, config })) } } +fn exec(input: &[&dyn AsRef<OsStr>], env: &HashMap<String, String>) -> Result<(), String> { + #[cfg(unix)] + { + // We use `exec` to call the `execvp` syscall instead of creating a new process where the + // command will be executed because very few signals can actually kill a current process, + // so if segmentation fault (SIGSEGV signal) happens and we raise to the current process, + // it will simply do nothing and we won't have the nice error message for the shell. + let error = crate::utils::get_command_inner(input, None, Some(env)).exec(); + eprintln!("execvp syscall failed: {error:?}"); + std::process::exit(1); + } + #[cfg(not(unix))] + { + if crate::utils::run_command_with_output_and_env_no_err(input, None, Some(env)).is_err() { + std::process::exit(1); + } + Ok(()) + } +} + pub fn run_cargo() -> Result<(), String> { let Some(mut tools) = RustcTools::new("cargo")? else { return Ok(()) }; let rustflags = tools.env.get("RUSTFLAGS").cloned().unwrap_or_default(); @@ -105,11 +122,7 @@ pub fn run_cargo() -> Result<(), String> { for arg in &tools.args { command.push(arg); } - if run_command_with_output_and_env_no_err(&command, None, Some(&tools.env)).is_err() { - std::process::exit(1); - } - - Ok(()) + exec(&command, &tools.env) } pub fn run_rustc() -> Result<(), String> { @@ -118,8 +131,5 @@ pub fn run_rustc() -> Result<(), String> { for arg in &tools.args { command.push(arg); } - if run_command_with_output_and_env_no_err(&command, None, Some(&tools.env)).is_err() { - std::process::exit(1); - } - Ok(()) + exec(&command, &tools.env) } diff --git a/compiler/rustc_codegen_gcc/build_system/src/test.rs b/compiler/rustc_codegen_gcc/build_system/src/test.rs index df4ac85233b..bcaab0fb526 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/test.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/test.rs @@ -9,8 +9,8 @@ use crate::build; use crate::config::{Channel, ConfigInfo}; use crate::utils::{ create_dir, get_sysroot_dir, get_toolchain, git_clone, git_clone_root_dir, remove_file, - run_command, run_command_with_env, run_command_with_output_and_env, rustc_version_info, - split_args, walk_dir, + run_command, run_command_with_env, run_command_with_output, run_command_with_output_and_env, + rustc_version_info, split_args, walk_dir, }; type Env = HashMap<String, String>; @@ -42,7 +42,7 @@ fn get_runners() -> Runners { ); runners.insert("--extended-regex-tests", ("Run extended regex tests", extended_regex_tests)); runners.insert("--mini-tests", ("Run mini tests", mini_tests)); - + runners.insert("--cargo-tests", ("Run cargo tests", cargo_tests)); runners } @@ -53,9 +53,9 @@ fn get_number_after_arg( match args.next() { Some(nb) if !nb.is_empty() => match usize::from_str(&nb) { Ok(nb) => Ok(nb), - Err(_) => Err(format!("Expected a number after `{}`, found `{}`", option, nb)), + Err(_) => Err(format!("Expected a number after `{option}`, found `{nb}`")), }, - _ => Err(format!("Expected a number after `{}`, found nothing", option)), + _ => Err(format!("Expected a number after `{option}`, found nothing")), } } @@ -76,8 +76,8 @@ fn show_usage() { for (option, (doc, _)) in get_runners() { // FIXME: Instead of using the hard-coded `23` value, better to compute it instead. let needed_spaces = 23_usize.saturating_sub(option.len()); - let spaces: String = std::iter::repeat(' ').take(needed_spaces).collect(); - println!(" {}{}: {}", option, spaces, doc); + let spaces: String = std::iter::repeat_n(' ', needed_spaces).collect(); + println!(" {option}{spaces}: {doc}"); } println!(" --help : Show this help"); } @@ -88,6 +88,8 @@ struct TestArg { use_system_gcc: bool, runners: Vec<String>, flags: Vec<String>, + /// Additional arguments, to be passed to commands like `cargo test`. + test_args: Vec<String>, nb_parts: Option<usize>, current_part: Option<usize>, sysroot_panic_abort: bool, @@ -137,13 +139,14 @@ impl TestArg { test_arg.sysroot_features.push(feature); } _ => { - return Err(format!("Expected an argument after `{}`, found nothing", arg)); + return Err(format!("Expected an argument after `{arg}`, found nothing")); } }, "--help" => { show_usage(); return Ok(None); } + "--" => test_arg.test_args.extend(&mut args), x if runners.contains_key(x) && !test_arg.runners.iter().any(|runner| runner == x) => { @@ -151,7 +154,7 @@ impl TestArg { } arg => { if !test_arg.config_info.parse_argument(arg, &mut args)? { - return Err(format!("Unknown option {}", arg)); + return Err(format!("Unknown option {arg}")); } } } @@ -189,7 +192,7 @@ fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> { command.push(&"--release"); &tmp_env } else { - &env + env }; for flag in args.flags.iter() { command.push(flag); @@ -203,6 +206,33 @@ fn clean(_env: &Env, args: &TestArg) -> Result<(), String> { create_dir(&path) } +fn cargo_tests(test_env: &Env, test_args: &TestArg) -> Result<(), String> { + // First, we call `mini_tests` to build minicore for us. This ensures we are testing with a working `minicore`, + // and that any changes we have made affect `minicore`(since it would get rebuilt). + mini_tests(test_env, test_args)?; + // Then, we copy some of the env vars from `test_env` + // We don't want to pass things like `RUSTFLAGS`, since they contain the -Zcodegen-backend flag. + // That would force `cg_gcc` to *rebuild itself* and only then run tests, which is undesirable. + let mut env = HashMap::new(); + env.insert( + "LD_LIBRARY_PATH".into(), + test_env.get("LD_LIBRARY_PATH").expect("LD_LIBRARY_PATH missing!").to_string(), + ); + env.insert( + "LIBRARY_PATH".into(), + test_env.get("LIBRARY_PATH").expect("LIBRARY_PATH missing!").to_string(), + ); + env.insert( + "CG_RUSTFLAGS".into(), + test_env.get("CG_RUSTFLAGS").map(|s| s.as_str()).unwrap_or("").to_string(), + ); + // Pass all the default args + the user-specified ones. + let mut args: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"test"]; + args.extend(test_args.test_args.iter().map(|s| s as &dyn AsRef<OsStr>)); + run_command_with_output_and_env(&args, None, Some(&env))?; + Ok(()) +} + fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[BUILD] mini_core"); @@ -222,7 +252,7 @@ fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"--target", &args.config_info.target_triple, ]); - run_command_with_output_and_env(&command, None, Some(&env))?; + run_command_with_output_and_env(&command, None, Some(env))?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[BUILD] example"); @@ -234,7 +264,7 @@ fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"--target", &args.config_info.target_triple, ]); - run_command_with_output_and_env(&command, None, Some(&env))?; + run_command_with_output_and_env(&command, None, Some(env))?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] mini_core_hello_world"); @@ -249,14 +279,14 @@ fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"--target", &args.config_info.target_triple, ]); - run_command_with_output_and_env(&command, None, Some(&env))?; + run_command_with_output_and_env(&command, None, Some(env))?; let command: &[&dyn AsRef<OsStr>] = &[ &Path::new(&args.config_info.cargo_target_dir).join("mini_core_hello_world"), &"abc", &"bcd", ]; - maybe_run_command_in_vm(&command, env, args)?; + maybe_run_command_in_vm(command, env, args)?; Ok(()) } @@ -454,22 +484,47 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<PathBuf, String> { } else { run_command_with_output_and_env(&[&"git", &"checkout"], rust_dir, Some(env))?; } + + let mut patches = Vec::new(); + walk_dir( + "patches/tests", + &mut |_| Ok(()), + &mut |file_path: &Path| { + patches.push(file_path.to_path_buf()); + Ok(()) + }, + false, + )?; + patches.sort(); + // TODO: remove duplication with prepare.rs by creating a apply_patch function in the utils + // module. + for file_path in patches { + println!("[GIT] apply `{}`", file_path.display()); + let path = Path::new("../..").join(file_path); + run_command_with_output(&[&"git", &"apply", &path], rust_dir)?; + run_command_with_output(&[&"git", &"add", &"-A"], rust_dir)?; + run_command_with_output( + &[&"git", &"commit", &"--no-gpg-sign", &"-m", &format!("Patch {}", path.display())], + rust_dir, + )?; + } + let cargo = String::from_utf8( run_command_with_env(&[&"rustup", &"which", &"cargo"], rust_dir, Some(env))?.stdout, ) - .map_err(|error| format!("Failed to retrieve cargo path: {:?}", error)) + .map_err(|error| format!("Failed to retrieve cargo path: {error:?}")) .and_then(|cargo| { let cargo = cargo.trim().to_owned(); - if cargo.is_empty() { Err(format!("`cargo` path is empty")) } else { Ok(cargo) } + if cargo.is_empty() { Err("`cargo` path is empty".to_string()) } else { Ok(cargo) } })?; let rustc = String::from_utf8( run_command_with_env(&[&"rustup", &toolchain, &"which", &"rustc"], rust_dir, Some(env))? .stdout, ) - .map_err(|error| format!("Failed to retrieve rustc path: {:?}", error)) + .map_err(|error| format!("Failed to retrieve rustc path: {error:?}")) .and_then(|rustc| { let rustc = rustc.trim().to_owned(); - if rustc.is_empty() { Err(format!("`rustc` path is empty")) } else { Ok(rustc) } + if rustc.is_empty() { Err("`rustc` path is empty".to_string()) } else { Ok(rustc) } })?; let llvm_filecheck = match run_command_with_env( &[ @@ -479,7 +534,8 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<PathBuf, String> { which FileCheck-11 || \ which FileCheck-12 || \ which FileCheck-13 || \ - which FileCheck-14", + which FileCheck-14 || \ + which FileCheck", ], rust_dir, Some(env), @@ -487,13 +543,15 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<PathBuf, String> { Ok(cmd) => String::from_utf8_lossy(&cmd.stdout).to_string(), Err(_) => { eprintln!("Failed to retrieve LLVM FileCheck, ignoring..."); + // FIXME: the test tests/run-make/no-builtins-attribute will fail if we cannot find + // FileCheck. String::new() } }; let file_path = rust_dir_path.join("config.toml"); std::fs::write( &file_path, - &format!( + format!( r#"change-id = 115898 [rust] @@ -532,7 +590,7 @@ fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { let codegen_backend_path = format!( "{pwd}/target/{channel}/librustc_codegen_gcc.{dylib_ext}", pwd = std::env::current_dir() - .map_err(|error| format!("`current_dir` failed: {:?}", error))? + .map_err(|error| format!("`current_dir` failed: {error:?}"))? .display(), channel = args.config_info.channel.as_str(), dylib_ext = args.config_info.dylib_ext, @@ -587,11 +645,11 @@ where F: Fn(&[&dyn AsRef<OsStr>], Option<&Path>, &Env) -> Result<(), String>, { let toolchain = get_toolchain()?; - let toolchain_arg = format!("+{}", toolchain); + let toolchain_arg = format!("+{toolchain}"); let rustc_version = String::from_utf8( run_command_with_env(&[&args.config_info.rustc_command[0], &"-V"], cwd, Some(env))?.stdout, ) - .map_err(|error| format!("Failed to retrieve rustc version: {:?}", error))?; + .map_err(|error| format!("Failed to retrieve rustc version: {error:?}"))?; let rustc_toolchain_version = String::from_utf8( run_command_with_env( &[&args.config_info.rustc_command[0], &toolchain_arg, &"-V"], @@ -600,20 +658,19 @@ where )? .stdout, ) - .map_err(|error| format!("Failed to retrieve rustc +toolchain version: {:?}", error))?; + .map_err(|error| format!("Failed to retrieve rustc +toolchain version: {error:?}"))?; if rustc_version != rustc_toolchain_version { eprintln!( - "rustc_codegen_gcc is built for `{}` but the default rustc version is `{}`.", - rustc_toolchain_version, rustc_version, + "rustc_codegen_gcc is built for `{rustc_toolchain_version}` but the default rustc version is `{rustc_version}`.", ); - eprintln!("Using `{}`.", rustc_toolchain_version); + eprintln!("Using `{rustc_toolchain_version}`."); } let mut env = env.clone(); let rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); env.insert("RUSTDOCFLAGS".to_string(), rustflags); let mut cargo_command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &toolchain_arg]; - cargo_command.extend_from_slice(&command); + cargo_command.extend_from_slice(command); callback(&cargo_command, cwd, &env) } @@ -680,7 +737,15 @@ fn test_libcore(env: &Env, args: &TestArg) -> Result<(), String> { println!("[TEST] libcore"); let path = get_sysroot_dir().join("sysroot_src/library/coretests"); let _ = remove_dir_all(path.join("target")); - run_cargo_command(&[&"test"], Some(&path), env, args)?; + // TODO(antoyo): run in release mode when we fix the failures. + // TODO(antoyo): remove the --skip f16::test_total_cmp when this issue is fixed: + // https://github.com/rust-lang/rust/issues/141503 + run_cargo_command( + &[&"test", &"--", &"--skip", &"f16::test_total_cmp"], + Some(&path), + env, + args, + )?; Ok(()) } @@ -818,7 +883,7 @@ fn contains_ui_error_patterns(file_path: &Path, keep_lto_tests: bool) -> Result< // Tests generating errors. let file = File::open(file_path) .map_err(|error| format!("Failed to read `{}`: {:?}", file_path.display(), error))?; - for line in BufReader::new(file).lines().filter_map(|line| line.ok()) { + for line in BufReader::new(file).lines().map_while(Result::ok) { let line = line.trim(); if line.is_empty() { continue; @@ -887,7 +952,7 @@ where if !prepare_files_callback(&rust_path)? { // FIXME: create a function "display_if_not_quiet" or something along the line. - println!("Keeping all {} tests", test_type); + println!("Keeping all {test_type} tests"); } if test_type == "ui" { @@ -919,8 +984,7 @@ where "borrowck", "test-attrs", ] - .iter() - .any(|name| *name == dir_name) + .contains(&dir_name) { remove_dir_all(dir).map_err(|error| { format!("Failed to remove folder `{}`: {:?}", dir.display(), error) @@ -975,10 +1039,7 @@ where if nb_parts > 0 { let current_part = args.current_part.unwrap(); // FIXME: create a function "display_if_not_quiet" or something along the line. - println!( - "Splitting ui_test into {} parts (and running part {})", - nb_parts, current_part - ); + println!("Splitting ui_test into {nb_parts} parts (and running part {current_part})"); let out = String::from_utf8( run_command( &[ @@ -996,7 +1057,7 @@ where )? .stdout, ) - .map_err(|error| format!("Failed to retrieve output of find command: {:?}", error))?; + .map_err(|error| format!("Failed to retrieve output of find command: {error:?}"))?; let mut files = out .split('\n') .map(|line| line.trim()) @@ -1016,7 +1077,7 @@ where } // FIXME: create a function "display_if_not_quiet" or something along the line. - println!("[TEST] rustc {} test suite", test_type); + println!("[TEST] rustc {test_type} test suite"); env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); let extra = @@ -1040,7 +1101,7 @@ where &"always", &"--stage", &"0", - &format!("tests/{}", test_type), + &format!("tests/{test_type}"), &"--compiletest-rustc-args", &rustc_args, ], @@ -1051,19 +1112,18 @@ where } fn test_rustc(env: &Env, args: &TestArg) -> Result<(), String> { - //test_rustc_inner(env, args, |_| Ok(false), false, "run-make")?; + test_rustc_inner(env, args, |_| Ok(false), false, "run-make")?; test_rustc_inner(env, args, |_| Ok(false), false, "ui") } fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { - let result1 = Ok(()); - /*test_rustc_inner( + let result1 = test_rustc_inner( env, args, retain_files_callback("tests/failing-run-make-tests.txt", "run-make"), false, "run-make", - )*/ + ); let result2 = test_rustc_inner( env, @@ -1084,14 +1144,13 @@ fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { false, "ui", )?; - Ok(()) - /*test_rustc_inner( + test_rustc_inner( env, args, remove_files_callback("tests/failing-run-make-tests.txt", "run-make"), false, "run-make", - )*/ + ) } fn test_failing_ui_pattern_tests(env: &Env, args: &TestArg) -> Result<(), String> { @@ -1118,7 +1177,7 @@ fn retain_files_callback<'a>( run_command( &[ &"find", - &format!("tests/{}", test_type), + &format!("tests/{test_type}"), &"-mindepth", &"1", &"-type", @@ -1137,7 +1196,7 @@ fn retain_files_callback<'a>( run_command( &[ &"find", - &format!("tests/{}", test_type), + &format!("tests/{test_type}"), &"-type", &"f", &"-name", @@ -1152,15 +1211,12 @@ fn retain_files_callback<'a>( } // Putting back only the failing ones. - if let Ok(files) = std::fs::read_to_string(&file_path) { + if let Ok(files) = std::fs::read_to_string(file_path) { for file in files.split('\n').map(|line| line.trim()).filter(|line| !line.is_empty()) { - run_command(&[&"git", &"checkout", &"--", &file], Some(&rust_path))?; + run_command(&[&"git", &"checkout", &"--", &file], Some(rust_path))?; } } else { - println!( - "Failed to read `{}`, not putting back failing {} tests", - file_path, test_type - ); + println!("Failed to read `{file_path}`, not putting back failing {test_type} tests"); } Ok(true) @@ -1188,8 +1244,7 @@ fn remove_files_callback<'a>( } } else { println!( - "Failed to read `{}`, not putting back failing {} tests", - file_path, test_type + "Failed to read `{file_path}`, not putting back failing {test_type} tests" ); } } else { @@ -1202,7 +1257,7 @@ fn remove_files_callback<'a>( remove_file(&path)?; } } else { - println!("Failed to read `{}`, not putting back failing ui tests", file_path); + println!("Failed to read `{file_path}`, not putting back failing ui tests"); } } Ok(true) @@ -1217,7 +1272,9 @@ fn run_all(env: &Env, args: &TestArg) -> Result<(), String> { // asm_tests(env, args)?; test_libcore(env, args)?; extended_sysroot_tests(env, args)?; + cargo_tests(env, args)?; test_rustc(env, args)?; + Ok(()) } diff --git a/compiler/rustc_codegen_gcc/build_system/src/utils.rs b/compiler/rustc_codegen_gcc/build_system/src/utils.rs index ca177a5feb8..d77707d5f17 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/utils.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/utils.rs @@ -1,7 +1,5 @@ use std::collections::HashMap; use std::ffi::OsStr; -#[cfg(unix)] -use std::ffi::c_int; use std::fmt::Debug; use std::fs; #[cfg(unix)] @@ -9,11 +7,6 @@ use std::os::unix::process::ExitStatusExt; use std::path::{Path, PathBuf}; use std::process::{Command, ExitStatus, Output}; -#[cfg(unix)] -unsafe extern "C" { - fn raise(signal: c_int) -> c_int; -} - fn exec_command( input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>, @@ -27,17 +20,14 @@ fn exec_command( #[cfg(unix)] { if let Some(signal) = status.signal() { - unsafe { - raise(signal as _); - } // In case the signal didn't kill the current process. - return Err(command_error(input, &cwd, format!("Process received signal {}", signal))); + return Err(command_error(input, &cwd, format!("Process received signal {signal}"))); } } Ok(status) } -fn get_command_inner( +pub(crate) fn get_command_inner( input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>, env: Option<&HashMap<String, String>>, @@ -75,18 +65,18 @@ fn check_exit_status( ); let input = input.iter().map(|i| i.as_ref()).collect::<Vec<&OsStr>>(); if show_err { - eprintln!("Command `{:?}` failed", input); + eprintln!("Command `{input:?}` failed"); } if let Some(output) = output { let stdout = String::from_utf8_lossy(&output.stdout); if !stdout.is_empty() { error.push_str("\n==== STDOUT ====\n"); - error.push_str(&*stdout); + error.push_str(&stdout); } let stderr = String::from_utf8_lossy(&output.stderr); if !stderr.is_empty() { error.push_str("\n==== STDERR ====\n"); - error.push_str(&*stderr); + error.push_str(&stderr); } } Err(error) @@ -136,6 +126,7 @@ pub fn run_command_with_output_and_env( Ok(()) } +#[cfg(not(unix))] pub fn run_command_with_output_and_env_no_err( input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>, @@ -242,7 +233,7 @@ pub fn get_toolchain() -> Result<String, String> { if !line.starts_with("channel") { return None; } - line.split('"').skip(1).next() + line.split('"').nth(1) }) .next() { @@ -281,7 +272,7 @@ fn git_clone_inner( } fn get_repo_name(url: &str) -> String { - let repo_name = url.split('/').last().unwrap(); + let repo_name = url.split('/').next_back().unwrap(); match repo_name.strip_suffix(".git") { Some(n) => n.to_string(), None => repo_name.to_string(), |
