combine rust files into one compilation

6 files changed, 199 insertions, 194 deletions

diff --git a/library/stdarch/crates/intrinsic-test/src/arm/mod.rs b/library/stdarch/crates/intrinsic-test/src/arm/mod.rs
index 0a64a24e731..bd0bdf53015 100644
--- a/library/stdarch/crates/intrinsic-test/src/arm/mod.rs
+++ b/library/stdarch/crates/intrinsic-test/src/arm/mod.rs
@@ -13,10 +13,9 @@ use crate::common::SupportedArchitectureTest;
 use crate::common::cli::ProcessedCli;
 use crate::common::compare::compare_outputs;
 use crate::common::gen_c::{write_main_cpp, write_mod_cpp};
-use crate::common::gen_rust::compile_rust_programs;
-use crate::common::intrinsic::{Intrinsic, IntrinsicDefinition};
+use crate::common::gen_rust::{compile_rust_programs, write_cargo_toml, write_main_rs};
+use crate::common::intrinsic::Intrinsic;
 use crate::common::intrinsic_helpers::TypeKind;
-use crate::common::write_file::write_rust_testfiles;
 use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, build_notices};
 use intrinsic::ArmIntrinsicType;
 use json_parser::get_neon_intrinsics;
@@ -118,26 +117,61 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
     }
 
     fn build_rust_file(&self) -> bool {
-        let rust_target = if self.cli_options.target.contains("v7") {
+        std::fs::create_dir_all("rust_programs/src").unwrap();
+
+        let architecture = if self.cli_options.target.contains("v7") {
             "arm"
         } else {
             "aarch64"
         };
+
+        let available_parallelism = std::thread::available_parallelism().unwrap().get();
+        let chunk_size = self.intrinsics.len().div_ceil(available_parallelism);
+
+        let mut cargo = File::create("rust_programs/Cargo.toml").unwrap();
+        write_cargo_toml(&mut cargo, &[]).unwrap();
+
+        let mut main_rs = File::create("rust_programs/src/main.rs").unwrap();
+        write_main_rs(
+            &mut main_rs,
+            available_parallelism,
+            architecture,
+            AARCH_CONFIGURATIONS,
+            F16_FORMATTING_DEF,
+            self.intrinsics.iter().map(|i| i.name.as_str()),
+        )
+        .unwrap();
+
         let target = &self.cli_options.target;
         let toolchain = self.cli_options.toolchain.as_deref();
         let linker = self.cli_options.linker.as_deref();
-        let intrinsics_name_list = write_rust_testfiles(
-            self.intrinsics
-                .iter()
-                .map(|i| i as &dyn IntrinsicDefinition<_>)
-                .collect::<Vec<_>>(),
-            rust_target,
-            &build_notices("// "),
-            F16_FORMATTING_DEF,
-            AARCH_CONFIGURATIONS,
-        );
 
-        compile_rust_programs(intrinsics_name_list, toolchain, target, linker)
+        let notice = &build_notices("// ");
+        self.intrinsics
+            .par_chunks(chunk_size)
+            .enumerate()
+            .map(|(i, chunk)| {
+                use std::io::Write;
+
+                let rust_filename = format!("rust_programs/src/mod_{i}.rs");
+                trace!("generating `{rust_filename}`");
+                let mut file = File::create(rust_filename).unwrap();
+
+                write!(file, "{notice}")?;
+
+                writeln!(file, "use core_arch::arch::{architecture}::*;")?;
+                writeln!(file, "use crate::{{debug_simd_finish, debug_f16}};")?;
+
+                for intrinsic in chunk {
+                    crate::common::gen_rust::create_rust_test_module(&mut file, intrinsic)?;
+                }
+
+                Ok(())
+            })
+            .collect::<Result<(), std::io::Error>>()
+            .unwrap();
+
+        compile_rust_programs(toolchain, target, linker)
     }
 
     fn compare_outputs(&self) -> bool {
diff --git a/library/stdarch/crates/intrinsic-test/src/common/argument.rs b/library/stdarch/crates/intrinsic-test/src/common/argument.rs
index 1df4f55995e..9cba8c90d9f 100644
--- a/library/stdarch/crates/intrinsic-test/src/common/argument.rs
+++ b/library/stdarch/crates/intrinsic-test/src/common/argument.rs
@@ -146,21 +146,25 @@ where
 
     /// Creates a line for each argument that initializes an array for Rust from which `loads` argument
     /// values can be loaded as a sliding window, e.g `const A_VALS: [u32; 20]  = [...];`
-    pub fn gen_arglists_rust(&self, indentation: Indentation, loads: u32) -> String {
-        self.iter()
-            .filter(|&arg| !arg.has_constraint())
-            .map(|arg| {
-                format!(
-                    "{indentation}{bind} {name}: [{ty}; {load_size}] = {values};",
-                    bind = arg.rust_vals_array_binding(),
-                    name = arg.rust_vals_array_name(),
-                    ty = arg.ty.rust_scalar_type(),
-                    load_size = arg.ty.num_lanes() * arg.ty.num_vectors() + loads - 1,
-                    values = arg.ty.populate_random(indentation, loads, &Language::Rust)
-                )
-            })
-            .collect::<Vec<_>>()
-            .join("\n")
+    pub fn gen_arglists_rust(
+        &self,
+        w: &mut impl std::io::Write,
+        indentation: Indentation,
+        loads: u32,
+    ) -> std::io::Result<()> {
+        for arg in self.iter().filter(|&arg| !arg.has_constraint()) {
+            writeln!(
+                w,
+                "{indentation}{bind} {name}: [{ty}; {load_size}] = {values};",
+                bind = arg.rust_vals_array_binding(),
+                name = arg.rust_vals_array_name(),
+                ty = arg.ty.rust_scalar_type(),
+                load_size = arg.ty.num_lanes() * arg.ty.num_vectors() + loads - 1,
+                values = arg.ty.populate_random(indentation, loads, &Language::Rust)
+            )?
+        }
+
+        Ok(())
     }
 
     /// Creates a line for each argument that initializes the argument from an array `[arg]_vals` at
diff --git a/library/stdarch/crates/intrinsic-test/src/common/compare.rs b/library/stdarch/crates/intrinsic-test/src/common/compare.rs
index cb55922eb19..183bb23ee0f 100644
--- a/library/stdarch/crates/intrinsic-test/src/common/compare.rs
+++ b/library/stdarch/crates/intrinsic-test/src/common/compare.rs
@@ -2,25 +2,28 @@ use super::cli::FailureReason;
 use rayon::prelude::*;
 use std::process::Command;
 
-pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target: &str) -> bool {
-    fn runner_command(runner: &str) -> Command {
-        let mut it = runner.split_whitespace();
-        let mut cmd = Command::new(it.next().unwrap());
-        cmd.args(it);
+fn runner_command(runner: &str) -> Command {
+    let mut it = runner.split_whitespace();
+    let mut cmd = Command::new(it.next().unwrap());
+    cmd.args(it);
 
-        cmd
-    }
+    cmd
+}
 
+pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target: &str) -> bool {
     let intrinsics = intrinsic_name_list
         .par_iter()
         .filter_map(|intrinsic_name| {
+
             let c = runner_command(runner)
-                .arg("./c_programs/intrinsic-test-programs")
+                .arg("intrinsic-test-programs")
                 .arg(intrinsic_name)
+                .current_dir("c_programs")
                 .output();
 
             let rust = runner_command(runner)
-                .arg(format!("target/{target}/release/{intrinsic_name}"))
+                .arg(format!("target/{target}/release/intrinsic-test-programs"))
+                .arg(intrinsic_name)
                 .output();
 
             let (c, rust) = match (c, rust) {
@@ -30,7 +33,7 @@ pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target:
 
             if !c.status.success() {
                 error!(
-                    "Failed to run C program for intrinsic {intrinsic_name}\nstdout: {stdout}\nstderr: {stderr}",
+                    "Failed to run C program for intrinsic `{intrinsic_name}`\nstdout: {stdout}\nstderr: {stderr}",
                     stdout = std::str::from_utf8(&c.stdout).unwrap_or(""),
                     stderr = std::str::from_utf8(&c.stderr).unwrap_or(""),
                 );
@@ -39,9 +42,9 @@ pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target:
 
             if !rust.status.success() {
                 error!(
-                    "Failed to run Rust program for intrinsic {intrinsic_name}\nstdout: {stdout}\nstderr: {stderr}",
-                    stdout = String::from_utf8_lossy(&rust.stdout),
-                    stderr = String::from_utf8_lossy(&rust.stderr),
+                    "Failed to run Rust program for intrinsic `{intrinsic_name}`\nstdout: {stdout}\nstderr: {stderr}",
+                    stdout = std::str::from_utf8(&rust.stdout).unwrap_or(""),
+                    stderr = std::str::from_utf8(&rust.stderr).unwrap_or(""),
                 );
                 return Some(FailureReason::RunRust(intrinsic_name.clone()));
             }
diff --git a/library/stdarch/crates/intrinsic-test/src/common/gen_rust.rs b/library/stdarch/crates/intrinsic-test/src/common/gen_rust.rs
index 0e4a95ab528..fb88e87d7e6 100644
--- a/library/stdarch/crates/intrinsic-test/src/common/gen_rust.rs
+++ b/library/stdarch/crates/intrinsic-test/src/common/gen_rust.rs
@@ -1,7 +1,4 @@
 use itertools::Itertools;
-use rayon::prelude::*;
-use std::collections::BTreeMap;
-use std::fs::File;
 use std::process::Command;
 
 use super::argument::Argument;
@@ -12,32 +9,7 @@ use super::intrinsic_helpers::IntrinsicTypeDefinition;
 // The number of times each intrinsic will be called.
 const PASSES: u32 = 20;
 
-pub fn format_rust_main_template(
-    notices: &str,
-    definitions: &str,
-    configurations: &str,
-    arch_definition: &str,
-    arglists: &str,
-    passes: &str,
-) -> String {
-    format!(
-        r#"{notices}#![feature(simd_ffi)]
-#![feature(f16)]
-#![allow(unused)]
-{configurations}
-{definitions}
-
-use core_arch::arch::{arch_definition}::*;
-
-fn main() {{
-{arglists}
-{passes}
-}}
-"#,
-    )
-}
-
-fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io::Result<()> {
+pub fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io::Result<()> {
     writeln!(
         w,
         concat!(
@@ -73,25 +45,65 @@ fn write_cargo_toml(w: &mut impl std::io::Write, binaries: &[String]) -> std::io
     Ok(())
 }
 
-pub fn compile_rust_programs(
-    binaries: Vec<String>,
-    toolchain: Option<&str>,
-    target: &str,
-    linker: Option<&str>,
-) -> bool {
-    let mut cargo = File::create("rust_programs/Cargo.toml").unwrap();
-    write_cargo_toml(&mut cargo, &binaries).unwrap();
+pub fn write_main_rs<'a>(
+    w: &mut impl std::io::Write,
+    available_parallelism: usize,
+    architecture: &str,
+    cfg: &str,
+    definitions: &str,
+    intrinsics: impl Iterator<Item = &'a str> + Clone,
+) -> std::io::Result<()> {
+    writeln!(w, "#![feature(simd_ffi)]")?;
+    writeln!(w, "#![feature(f16)]")?;
+    writeln!(w, "#![allow(unused)]")?;
+
+    // Cargo will spam the logs if these warnings are not silenced.
+    writeln!(w, "#![allow(non_upper_case_globals)]")?;
+    writeln!(w, "#![allow(non_camel_case_types)]")?;
+    writeln!(w, "#![allow(non_snake_case)]")?;
+
+    writeln!(w, "{cfg}")?;
+    writeln!(w, "{definitions}")?;
+
+    writeln!(w, "use core_arch::arch::{architecture}::*;")?;
+
+    for module in 0..Ord::min(available_parallelism, intrinsics.clone().count()) {
+        writeln!(w, "mod mod_{module};")?;
+        writeln!(w, "use mod_{module}::*;")?;
+    }
+
+    writeln!(w, "fn main() {{")?;
+
+    writeln!(w, "    match std::env::args().nth(1).unwrap().as_str() {{")?;
+
+    for binary in intrinsics {
+        writeln!(w, "        \"{binary}\" => run_{binary}(),")?;
+    }
+
+    writeln!(
+        w,
+        "        other => panic!(\"unknown intrinsic `{{}}`\", other),"
+    )?;
+
+    writeln!(w, "    }}")?;
+    writeln!(w, "}}")?;
+
+    Ok(())
+}
 
+pub fn compile_rust_programs(toolchain: Option<&str>, target: &str, linker: Option<&str>) -> bool {
     /* If there has been a linker explicitly set from the command line then
      * we want to set it via setting it in the RUSTFLAGS*/
 
+    trace!("Building cargo command");
+
     let mut cargo_command = Command::new("cargo");
     cargo_command.current_dir("rust_programs");
 
-    if let Some(toolchain) = toolchain {
-        if !toolchain.is_empty() {
-            cargo_command.arg(toolchain);
-        }
+    if let Some(toolchain) = toolchain
+        && !toolchain.is_empty()
+    {
+        cargo_command.arg(toolchain);
     }
     cargo_command.args(["build", "--target", target, "--release"]);
 
@@ -105,7 +117,16 @@ pub fn compile_rust_programs(
     }
 
     cargo_command.env("RUSTFLAGS", rust_flags);
+
+    trace!("running cargo");
+
+    if log::log_enabled!(log::Level::Trace) {
+        cargo_command.stdout(std::process::Stdio::inherit());
+        cargo_command.stderr(std::process::Stdio::inherit());
+    }
+
     let output = cargo_command.output();
+    trace!("cargo is done");
 
     if let Ok(output) = output {
         if output.status.success() {
@@ -124,26 +145,13 @@ pub fn compile_rust_programs(
     }
 }
 
-// Creates directory structure and file path mappings
-pub fn setup_rust_file_paths(identifiers: &Vec<String>) -> BTreeMap<&String, String> {
-    identifiers
-        .par_iter()
-        .map(|identifier| {
-            let rust_dir = format!("rust_programs/{identifier}");
-            let _ = std::fs::create_dir_all(&rust_dir);
-            let rust_filename = format!("{rust_dir}/main.rs");
-
-            (identifier, rust_filename)
-        })
-        .collect::<BTreeMap<&String, String>>()
-}
-
 pub fn generate_rust_test_loop<T: IntrinsicTypeDefinition>(
+    w: &mut impl std::io::Write,
     intrinsic: &dyn IntrinsicDefinition<T>,
     indentation: Indentation,
     additional: &str,
     passes: u32,
-) -> String {
+) -> std::io::Result<()> {
     let constraints = intrinsic.arguments().as_constraint_parameters_rust();
     let constraints = if !constraints.is_empty() {
         format!("::<{constraints}>")
@@ -154,7 +162,8 @@ pub fn generate_rust_test_loop<T: IntrinsicTypeDefinition>(
     let return_value = format_f16_return_value(intrinsic);
     let indentation2 = indentation.nested();
     let indentation3 = indentation2.nested();
-    format!(
+    writeln!(
+        w,
         "{indentation}for i in 0..{passes} {{\n\
             {indentation2}unsafe {{\n\
                 {loaded_args}\
@@ -169,74 +178,63 @@ pub fn generate_rust_test_loop<T: IntrinsicTypeDefinition>(
     )
 }
 
-pub fn generate_rust_constraint_blocks<T: IntrinsicTypeDefinition>(
+fn generate_rust_constraint_blocks<'a, T: IntrinsicTypeDefinition + 'a>(
+    w: &mut impl std::io::Write,
     intrinsic: &dyn IntrinsicDefinition<T>,
     indentation: Indentation,
-    constraints: &[&Argument<T>],
+    constraints: &mut (impl Iterator<Item = &'a Argument<T>> + Clone),
     name: String,
-) -> String {
-    if let Some((current, constraints)) = constraints.split_last() {
-        let range = current
-            .constraint
-            .iter()
-            .map(|c| c.to_range())
-            .flat_map(|r| r.into_iter());
-
-        let body_indentation = indentation.nested();
-        range
-            .map(|i| {
-                format!(
-                    "{indentation}{{\n\
-                        {body_indentation}const {name}: {ty} = {val};\n\
-                        {pass}\n\
-                    {indentation}}}",
-                    name = current.name,
-                    ty = current.ty.rust_type(),
-                    val = i,
-                    pass = generate_rust_constraint_blocks(
-                        intrinsic,
-                        body_indentation,
-                        constraints,
-                        format!("{name}-{i}")
-                    )
-                )
-            })
-            .join("\n")
-    } else {
-        generate_rust_test_loop(intrinsic, indentation, &name, PASSES)
+) -> std::io::Result<()> {
+    let Some(current) = constraints.next() else {
+        return generate_rust_test_loop(w, intrinsic, indentation, &name, PASSES);
+    };
+
+    let body_indentation = indentation.nested();
+    for i in current.constraint.iter().flat_map(|c| c.to_range()) {
+        let ty = current.ty.rust_type();
+
+        writeln!(w, "{indentation}{{")?;
+
+        writeln!(w, "{body_indentation}const {}: {ty} = {i};", current.name)?;
+
+        generate_rust_constraint_blocks(
+            w,
+            intrinsic,
+            body_indentation,
+            &mut constraints.clone(),
+            format!("{name}-{i}"),
+        )?;
+
+        writeln!(w, "{indentation}}}")?;
     }
+
+    Ok(())
 }
 
 // Top-level function to create complete test program
-pub fn create_rust_test_program<T: IntrinsicTypeDefinition>(
+pub fn create_rust_test_module<T: IntrinsicTypeDefinition>(
+    w: &mut impl std::io::Write,
     intrinsic: &dyn IntrinsicDefinition<T>,
-    target: &str,
-    notice: &str,
-    definitions: &str,
-    cfg: &str,
-) -> String {
+) -> std::io::Result<()> {
+    trace!("generating `{}`", intrinsic.name());
+    let indentation = Indentation::default();
+
+    writeln!(w, "pub fn run_{}() {{", intrinsic.name())?;
+
+    // Define the arrays of arguments.
     let arguments = intrinsic.arguments();
-    let constraints = arguments
-        .iter()
-        .filter(|i| i.has_constraint())
-        .collect_vec();
+    arguments.gen_arglists_rust(w, indentation.nested(), PASSES)?;
 
-    let indentation = Indentation::default();
-    format_rust_main_template(
-        notice,
-        definitions,
-        cfg,
-        target,
-        intrinsic
-            .arguments()
-            .gen_arglists_rust(indentation.nested(), PASSES)
-            .as_str(),
-        generate_rust_constraint_blocks(
-            intrinsic,
-            indentation.nested(),
-            &constraints,
-            Default::default(),
-        )
-        .as_str(),
-    )
+    // Define any const generics as `const` items, then generate the actual test loop.
+    generate_rust_constraint_blocks(
+        w,
+        intrinsic,
+        indentation.nested(),
+        &mut arguments.iter().rev().filter(|i| i.has_constraint()),
+        Default::default(),
+    )?;
+
+    writeln!(w, "}}")?;
+
+    Ok(())
 }
diff --git a/library/stdarch/crates/intrinsic-test/src/common/mod.rs b/library/stdarch/crates/intrinsic-test/src/common/mod.rs
index 5d51d3460ec..6c3154af385 100644
--- a/library/stdarch/crates/intrinsic-test/src/common/mod.rs
+++ b/library/stdarch/crates/intrinsic-test/src/common/mod.rs
@@ -11,7 +11,6 @@ pub mod indentation;
 pub mod intrinsic;
 pub mod intrinsic_helpers;
 pub mod values;
-pub mod write_file;
 
 /// Architectures must support this trait
 /// to be successfully tested.
diff --git a/library/stdarch/crates/intrinsic-test/src/common/write_file.rs b/library/stdarch/crates/intrinsic-test/src/common/write_file.rs
deleted file mode 100644
index 92dd70b7c57..00000000000
--- a/library/stdarch/crates/intrinsic-test/src/common/write_file.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-use super::gen_rust::{create_rust_test_program, setup_rust_file_paths};
-use super::intrinsic::IntrinsicDefinition;
-use super::intrinsic_helpers::IntrinsicTypeDefinition;
-use std::fs::File;
-use std::io::Write;
-
-pub fn write_file(filename: &String, code: String) {
-    let mut file = File::create(filename).unwrap();
-    file.write_all(code.into_bytes().as_slice()).unwrap();
-}
-
-pub fn write_rust_testfiles<T: IntrinsicTypeDefinition>(
-    intrinsics: Vec<&dyn IntrinsicDefinition<T>>,
-    rust_target: &str,
-    notice: &str,
-    definitions: &str,
-    cfg: &str,
-) -> Vec<String> {
-    let intrinsics_name_list = intrinsics
-        .iter()
-        .map(|i| i.name().clone())
-        .collect::<Vec<_>>();
-    let filename_mapping = setup_rust_file_paths(&intrinsics_name_list);
-
-    intrinsics.iter().for_each(|&i| {
-        let rust_code = create_rust_test_program(i, rust_target, notice, definitions, cfg);
-        if let Some(filename) = filename_mapping.get(&i.name()) {
-            write_file(filename, rust_code)
-        }
-    });
-
-    intrinsics_name_list
-}