`intrinsic-test`: combine C files for more efficient compilation

6 files changed, 259 insertions, 230 deletions

diff --git a/library/stdarch/crates/intrinsic-test/src/arm/mod.rs b/library/stdarch/crates/intrinsic-test/src/arm/mod.rs
index 18530e7e6bd..0a64a24e731 100644
--- a/library/stdarch/crates/intrinsic-test/src/arm/mod.rs
+++ b/library/stdarch/crates/intrinsic-test/src/arm/mod.rs
@@ -4,15 +4,20 @@ mod intrinsic;
 mod json_parser;
 mod types;
 
+use std::fs::File;
+
+use rayon::prelude::*;
+
+use crate::arm::config::POLY128_OSTREAM_DEF;
 use crate::common::SupportedArchitectureTest;
 use crate::common::cli::ProcessedCli;
 use crate::common::compare::compare_outputs;
-use crate::common::gen_c::compile_c_programs;
+use crate::common::gen_c::{write_main_cpp, write_mod_cpp};
 use crate::common::gen_rust::compile_rust_programs;
 use crate::common::intrinsic::{Intrinsic, IntrinsicDefinition};
 use crate::common::intrinsic_helpers::TypeKind;
-use crate::common::write_file::{write_c_testfiles, write_rust_testfiles};
-use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, POLY128_OSTREAM_DEF, build_notices};
+use crate::common::write_file::write_rust_testfiles;
+use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, build_notices};
 use intrinsic::ArmIntrinsicType;
 use json_parser::get_neon_intrinsics;
 
@@ -21,6 +26,13 @@ pub struct ArmArchitectureTest {
     cli_options: ProcessedCli,
 }
 
+fn chunk_info(intrinsic_count: usize) -> (usize, usize) {
+    let available_parallelism = std::thread::available_parallelism().unwrap().get();
+    let chunk_size = intrinsic_count.div_ceil(Ord::min(available_parallelism, intrinsic_count));
+
+    (chunk_size, intrinsic_count.div_ceil(chunk_size))
+}
+
 impl SupportedArchitectureTest for ArmArchitectureTest {
     fn create(cli_options: ProcessedCli) -> Box<Self> {
         let a32 = cli_options.target.contains("v7");
@@ -51,24 +63,58 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
     }
 
     fn build_c_file(&self) -> bool {
-        let target = &self.cli_options.target;
         let c_target = "aarch64";
+        let platform_headers = &["arm_neon.h", "arm_acle.h", "arm_fp16.h"];
 
-        let intrinsics_name_list = write_c_testfiles(
-            &self
-                .intrinsics
-                .iter()
-                .map(|i| i as &dyn IntrinsicDefinition<_>)
-                .collect::<Vec<_>>(),
-            target,
+        let (chunk_size, chunk_count) = chunk_info(self.intrinsics.len());
+
+        let cpp_compiler = compile::build_cpp_compilation(&self.cli_options).unwrap();
+
+        let notice = &build_notices("// ");
+        self.intrinsics
+            .par_chunks(chunk_size)
+            .enumerate()
+            .map(|(i, chunk)| {
+                let c_filename = format!("c_programs/mod_{i}.cpp");
+                let mut file = File::create(&c_filename).unwrap();
+                write_mod_cpp(&mut file, notice, c_target, platform_headers, chunk).unwrap();
+
+                // compile this cpp file into a .o file
+                let output = cpp_compiler
+                    .compile_object_file(&format!("mod_{i}.cpp"), &format!("mod_{i}.o"))?;
+                assert!(output.status.success(), "{output:?}");
+
+                Ok(())
+            })
+            .collect::<Result<(), std::io::Error>>()
+            .unwrap();
+
+        let mut file = File::create("c_programs/main.cpp").unwrap();
+        write_main_cpp(
+            &mut file,
             c_target,
-            &["arm_neon.h", "arm_acle.h", "arm_fp16.h"],
-            &build_notices("// "),
-            &[POLY128_OSTREAM_DEF],
-        );
+            POLY128_OSTREAM_DEF,
+            self.intrinsics.iter().map(|i| i.name.as_str()),
+        )
+        .unwrap();
+
+        // compile this cpp file into a .o file
+        info!("compiling main.cpp");
+        let output = cpp_compiler
+            .compile_object_file("main.cpp", "intrinsic-test-programs.o")
+            .unwrap();
+        assert!(output.status.success(), "{output:?}");
+
+        let object_files = (0..chunk_count)
+            .map(|i| format!("mod_{i}.o"))
+            .chain(["intrinsic-test-programs.o".to_owned()]);
+
+        let output = cpp_compiler
+            .link_executable(object_files, "intrinsic-test-programs")
+            .unwrap();
+        assert!(output.status.success(), "{output:?}");
 
-        let pipeline = compile::build_cpp_compilation(&self.cli_options).unwrap();
-        compile_c_programs(&pipeline, &intrinsics_name_list)
+        true
     }
 
     fn build_rust_file(&self) -> bool {
diff --git a/library/stdarch/crates/intrinsic-test/src/common/argument.rs b/library/stdarch/crates/intrinsic-test/src/common/argument.rs
index 443ccb919f4..338f0d344a7 100644
--- a/library/stdarch/crates/intrinsic-test/src/common/argument.rs
+++ b/library/stdarch/crates/intrinsic-test/src/common/argument.rs
@@ -125,19 +125,23 @@ where
     /// Creates a line for each argument that initializes an array for C from which `loads` argument
     /// values can be loaded  as a sliding window.
     /// e.g `const int32x2_t a_vals = {0x3effffff, 0x3effffff, 0x3f7fffff}`, if loads=2.
-    pub fn gen_arglists_c(&self, indentation: Indentation, loads: u32) -> String {
-        self.iter()
-            .filter(|&arg| !arg.has_constraint())
-            .map(|arg| {
-                format!(
-                    "{indentation}const {ty} {name}_vals[] = {values};",
-                    ty = arg.ty.c_scalar_type(),
-                    name = arg.name,
-                    values = arg.ty.populate_random(indentation, loads, &Language::C)
-                )
-            })
-            .collect::<Vec<_>>()
-            .join("\n")
+    pub fn gen_arglists_c(
+        &self,
+        w: &mut impl std::io::Write,
+        indentation: Indentation,
+        loads: u32,
+    ) -> std::io::Result<()> {
+        for arg in self.iter().filter(|&arg| !arg.has_constraint()) {
+            writeln!(
+                w,
+                "{indentation}const {ty} {name}_vals[] = {values};",
+                ty = arg.ty.c_scalar_type(),
+                name = arg.name,
+                values = arg.ty.populate_random(indentation, loads, &Language::C)
+            )?
+        }
+
+        Ok(())
     }
 
     /// Creates a line for each argument that initializes an array for Rust from which `loads` argument
diff --git a/library/stdarch/crates/intrinsic-test/src/common/compare.rs b/library/stdarch/crates/intrinsic-test/src/common/compare.rs
index f16d83b0264..cb55922eb19 100644
--- a/library/stdarch/crates/intrinsic-test/src/common/compare.rs
+++ b/library/stdarch/crates/intrinsic-test/src/common/compare.rs
@@ -15,12 +15,12 @@ pub fn compare_outputs(intrinsic_name_list: &Vec<String>, runner: &str, target:
         .par_iter()
         .filter_map(|intrinsic_name| {
             let c = runner_command(runner)
-                .arg(format!("./c_programs/{intrinsic_name}"))
+                .arg("./c_programs/intrinsic-test-programs")
+                .arg(intrinsic_name)
                 .output();
 
             let rust = runner_command(runner)
                 .arg(format!("target/{target}/release/{intrinsic_name}"))
-                .env("RUSTFLAGS", "-Cdebuginfo=0")
                 .output();
 
             let (c, rust) = match (c, rust) {
diff --git a/library/stdarch/crates/intrinsic-test/src/common/compile_c.rs b/library/stdarch/crates/intrinsic-test/src/common/compile_c.rs
index 8e2eb8e0f40..0c905a149e4 100644
--- a/library/stdarch/crates/intrinsic-test/src/common/compile_c.rs
+++ b/library/stdarch/crates/intrinsic-test/src/common/compile_c.rs
@@ -112,11 +112,24 @@ impl CppCompilation {
         &mut self.0
     }
 
-    pub fn run(&self, inputs: &[String], output: &str) -> std::io::Result<std::process::Output> {
+    pub fn compile_object_file(
+        &self,
+        input: &str,
+        output: &str,
+    ) -> std::io::Result<std::process::Output> {
+        let mut cmd = clone_command(&self.0);
+        cmd.args([input, "-c", "-o", output]);
+        cmd.output()
+    }
+
+    pub fn link_executable(
+        &self,
+        inputs: impl Iterator<Item = String>,
+        output: &str,
+    ) -> std::io::Result<std::process::Output> {
         let mut cmd = clone_command(&self.0);
         cmd.args(inputs);
         cmd.args(["-o", output]);
-
         cmd.output()
     }
 }
diff --git a/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs b/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs
index 84167f2f4ae..905efb6d890 100644
--- a/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs
+++ b/library/stdarch/crates/intrinsic-test/src/common/gen_c.rs
@@ -1,9 +1,3 @@
-use itertools::Itertools;
-use rayon::prelude::*;
-use std::collections::BTreeMap;
-
-use crate::common::compile_c::CppCompilation;
-
 use super::argument::Argument;
 use super::indentation::Indentation;
 use super::intrinsic::IntrinsicDefinition;
@@ -12,105 +6,16 @@ use super::intrinsic_helpers::IntrinsicTypeDefinition;
 // The number of times each intrinsic will be called.
 const PASSES: u32 = 20;
 
-// Formats the main C program template with placeholders
-pub fn format_c_main_template(
-    notices: &str,
-    header_files: &[&str],
-    arch_identifier: &str,
-    arch_specific_definitions: &[&str],
-    arglists: &str,
-    passes: &str,
-) -> String {
-    format!(
-        r#"{notices}{header_files}
-#include <iostream>
-#include <cstring>
-#include <iomanip>
-#include <sstream>
-
-template<typename T1, typename T2> T1 cast(T2 x) {{
-  static_assert(sizeof(T1) == sizeof(T2), "sizeof T1 and T2 must be the same");
-  T1 ret{{}};
-  memcpy(&ret, &x, sizeof(T1));
-  return ret;
-}}
-
-std::ostream& operator<<(std::ostream& os, float16_t value) {{
-    uint16_t temp = 0;
-    memcpy(&temp, &value, sizeof(float16_t));
-    std::stringstream ss;
-    ss << "0x" << std::setfill('0') << std::setw(4) << std::hex << temp;
-    os << ss.str();
-    return os;
-}}
-
-#ifdef __{arch_identifier}__
-{arch_specific_definitions}
-#endif
-
-{arglists}
-
-int main(int argc, char **argv) {{
-{passes}
-    return 0;
-}}"#,
-        header_files = header_files
-            .iter()
-            .map(|header| format!("#include <{header}>"))
-            .collect::<Vec<_>>()
-            .join("\n"),
-        arch_specific_definitions = arch_specific_definitions.join("\n"),
-    )
-}
-
-pub fn compile_c_programs(pipeline: &CppCompilation, intrinsics: &[String]) -> bool {
-    intrinsics
-        .par_iter()
-        .map(
-            |intrinsic| match pipeline.run(&[format!("{intrinsic}.cpp")], intrinsic) {
-                Ok(output) if output.status.success() => Ok(()),
-                Ok(output) => {
-                    let msg = format!(
-                        "Failed to compile code for intrinsic `{intrinsic}`: \n\nstdout:\n{}\n\nstderr:\n{}",
-                        std::str::from_utf8(&output.stdout).unwrap_or(""),
-                        std::str::from_utf8(&output.stderr).unwrap_or("")
-                    );
-                    error!("{msg}");
-
-                    Err(msg)
-                }
-                Err(e) => {
-                    error!("command for `{intrinsic}` failed with IO error: {e:?}");
-                    Err(e.to_string())
-                }
-            },
-        )
-        .collect::<Result<(), String>>()
-        .is_ok()
-}
-
-// Creates directory structure and file path mappings
-pub fn setup_c_file_paths(identifiers: &Vec<String>) -> BTreeMap<&String, String> {
-    let _ = std::fs::create_dir("c_programs");
-    identifiers
-        .par_iter()
-        .map(|identifier| {
-            let c_filename = format!(r#"c_programs/{identifier}.cpp"#);
-
-            (identifier, c_filename)
-        })
-        .collect::<BTreeMap<&String, String>>()
-}
-
 pub fn generate_c_test_loop<T: IntrinsicTypeDefinition + Sized>(
+    w: &mut impl std::io::Write,
     intrinsic: &dyn IntrinsicDefinition<T>,
     indentation: Indentation,
     additional: &str,
     passes: u32,
-    _target: &str,
-) -> String {
+) -> std::io::Result<()> {
     let body_indentation = indentation.nested();
-    format!(
+    writeln!(
+        w,
         "{indentation}for (int i=0; i<{passes}; i++) {{\n\
             {loaded_args}\
             {body_indentation}auto __return_value = {intrinsic_call}({args});\n\
@@ -123,78 +28,172 @@ pub fn generate_c_test_loop<T: IntrinsicTypeDefinition + Sized>(
     )
 }
 
-pub fn generate_c_constraint_blocks<T: IntrinsicTypeDefinition>(
+pub fn generate_c_constraint_blocks<'a, T: IntrinsicTypeDefinition + 'a>(
+    w: &mut impl std::io::Write,
     intrinsic: &dyn IntrinsicDefinition<T>,
     indentation: Indentation,
-    constraints: &[&Argument<T>],
+    constraints: &mut (impl Iterator<Item = &'a Argument<T>> + Clone),
     name: String,
-    target: &str,
-) -> String {
-    if let Some((current, constraints)) = constraints.split_last() {
-        let range = current
-            .constraint
-            .iter()
-            .map(|c| c.to_range())
-            .flat_map(|r| r.into_iter());
-
-        let body_indentation = indentation.nested();
-        range
-            .map(|i| {
-                format!(
-                    "{indentation}{{\n\
-                        {body_indentation}{ty} {name} = {val};\n\
-                        {pass}\n\
-                    {indentation}}}",
-                    name = current.name,
-                    ty = current.ty.c_type(),
-                    val = i,
-                    pass = generate_c_constraint_blocks(
-                        intrinsic,
-                        body_indentation,
-                        constraints,
-                        format!("{name}-{i}"),
-                        target,
-                    )
-                )
-            })
-            .join("\n")
-    } else {
-        generate_c_test_loop(intrinsic, indentation, &name, PASSES, target)
+) -> std::io::Result<()> {
+    let Some(current) = constraints.next() else {
+        return generate_c_test_loop(w, intrinsic, indentation, &name, PASSES);
+    };
+
+    let body_indentation = indentation.nested();
+    for i in current.constraint.iter().flat_map(|c| c.to_range()) {
+        let ty = current.ty.c_type();
+
+        writeln!(w, "{indentation}{{")?;
+        writeln!(w, "{body_indentation}{ty} {} = {i};", current.name)?;
+
+        generate_c_constraint_blocks(
+            w,
+            intrinsic,
+            body_indentation,
+            &mut constraints.clone(),
+            format!("{name}-{i}"),
+        )?;
+
+        writeln!(w, "{indentation}}}")?;
     }
+
+    Ok(())
 }
 
 // Compiles C test programs using specified compiler
-pub fn create_c_test_program<T: IntrinsicTypeDefinition>(
+pub fn create_c_test_function<T: IntrinsicTypeDefinition>(
+    w: &mut impl std::io::Write,
     intrinsic: &dyn IntrinsicDefinition<T>,
-    header_files: &[&str],
-    target: &str,
-    c_target: &str,
-    notices: &str,
-    arch_specific_definitions: &[&str],
-) -> String {
+) -> std::io::Result<()> {
+    let indentation = Indentation::default();
+
+    writeln!(w, "int run_{}() {{", intrinsic.name())?;
+
+    // Define the arrays of arguments.
     let arguments = intrinsic.arguments();
-    let constraints = arguments
-        .iter()
-        .filter(|&i| i.has_constraint())
-        .collect_vec();
+    arguments.gen_arglists_c(w, indentation.nested(), PASSES)?;
 
-    let indentation = Indentation::default();
-    format_c_main_template(
-        notices,
-        header_files,
-        c_target,
-        arch_specific_definitions,
-        intrinsic
-            .arguments()
-            .gen_arglists_c(indentation, PASSES)
-            .as_str(),
-        generate_c_constraint_blocks(
-            intrinsic,
-            indentation.nested(),
-            constraints.as_slice(),
-            Default::default(),
-            target,
-        )
-        .as_str(),
-    )
+    generate_c_constraint_blocks(
+        w,
+        intrinsic,
+        indentation.nested(),
+        &mut arguments.iter().rev().filter(|&i| i.has_constraint()),
+        Default::default(),
+    )?;
+
+    writeln!(w, "    return 0;")?;
+    writeln!(w, "}}")?;
+
+    Ok(())
+}
+
+pub fn write_mod_cpp<T: IntrinsicTypeDefinition>(
+    w: &mut impl std::io::Write,
+    notice: &str,
+    architecture: &str,
+    platform_headers: &[&str],
+    intrinsics: &[impl IntrinsicDefinition<T>],
+) -> std::io::Result<()> {
+    write!(w, "{notice}")?;
+
+    for header in platform_headers {
+        writeln!(w, "#include <{header}>")?;
+    }
+
+    writeln!(
+        w,
+        r#"
+#include <iostream>
+#include <cstring>
+#include <iomanip>
+#include <sstream>
+
+template<typename T1, typename T2> T1 cast(T2 x) {{
+  static_assert(sizeof(T1) == sizeof(T2), "sizeof T1 and T2 must be the same");
+  T1 ret{{}};
+  memcpy(&ret, &x, sizeof(T1));
+  return ret;
+}}
+
+std::ostream& operator<<(std::ostream& os, float16_t value);
+
+
+
+"#
+    )?;
+
+    writeln!(w, "#ifdef __{architecture}__")?;
+    writeln!(
+        w,
+        "std::ostream& operator<<(std::ostream& os, poly128_t value);"
+    )?;
+    writeln!(w, "#endif")?;
+
+    for intrinsic in intrinsics {
+        create_c_test_function(w, intrinsic)?;
+    }
+
+    Ok(())
+}
+
+pub fn write_main_cpp<'a>(
+    w: &mut impl std::io::Write,
+    architecture: &str,
+    arch_specific_definitions: &str,
+    intrinsics: impl Iterator<Item = &'a str> + Clone,
+) -> std::io::Result<()> {
+    writeln!(w, "#include <iostream>")?;
+    writeln!(w, "#include <string>")?;
+
+    for header in ["arm_neon.h", "arm_acle.h", "arm_fp16.h"] {
+        writeln!(w, "#include <{header}>")?;
+    }
+
+    writeln!(
+        w,
+        r#"
+#include <cstring>
+#include <iomanip>
+#include <sstream>
+
+std::ostream& operator<<(std::ostream& os, float16_t value) {{
+    uint16_t temp = 0;
+    memcpy(&temp, &value, sizeof(float16_t));
+    std::stringstream ss;
+    ss << "0x" << std::setfill('0') << std::setw(4) << std::hex << temp;
+    os << ss.str();
+    return os;
+}}
+"#
+    )?;
+
+    writeln!(w, "#ifdef __{architecture}__")?;
+    writeln!(w, "{arch_specific_definitions }")?;
+    writeln!(w, "#endif")?;
+
+    for intrinsic in intrinsics.clone() {
+        writeln!(w, "extern int run_{intrinsic}(void);")?;
+    }
+
+    writeln!(w, "int main(int argc, char **argv) {{")?;
+    writeln!(w, "    std::string intrinsic_name = argv[1];")?;
+
+    writeln!(w, "    if (false) {{")?;
+
+    for intrinsic in intrinsics {
+        writeln!(w, "    }} else if (intrinsic_name == \"{intrinsic}\") {{")?;
+        writeln!(w, "        return run_{intrinsic}();")?;
+    }
+
+    writeln!(w, "    }} else {{")?;
+    writeln!(
+        w,
+        "        std::cerr << \"Unknown command: \" << intrinsic_name << \"\\n\";"
+    )?;
+    writeln!(w, "        return -1;")?;
+    writeln!(w, "    }}")?;
+
+    writeln!(w, "}}")?;
+
+    Ok(())
 }
diff --git a/library/stdarch/crates/intrinsic-test/src/common/write_file.rs b/library/stdarch/crates/intrinsic-test/src/common/write_file.rs
index 0ba3e829a6b..92dd70b7c57 100644
--- a/library/stdarch/crates/intrinsic-test/src/common/write_file.rs
+++ b/library/stdarch/crates/intrinsic-test/src/common/write_file.rs
@@ -1,5 +1,3 @@
-use super::gen_c::create_c_test_program;
-use super::gen_c::setup_c_file_paths;
 use super::gen_rust::{create_rust_test_program, setup_rust_file_paths};
 use super::intrinsic::IntrinsicDefinition;
 use super::intrinsic_helpers::IntrinsicTypeDefinition;
@@ -11,37 +9,6 @@ pub fn write_file(filename: &String, code: String) {
     file.write_all(code.into_bytes().as_slice()).unwrap();
 }
 
-pub fn write_c_testfiles<T: IntrinsicTypeDefinition + Sized>(
-    intrinsics: &Vec<&dyn IntrinsicDefinition<T>>,
-    target: &str,
-    c_target: &str,
-    headers: &[&str],
-    notice: &str,
-    arch_specific_definitions: &[&str],
-) -> Vec<String> {
-    let intrinsics_name_list = intrinsics
-        .iter()
-        .map(|i| i.name().clone())
-        .collect::<Vec<_>>();
-    let filename_mapping = setup_c_file_paths(&intrinsics_name_list);
-
-    intrinsics.iter().for_each(|&i| {
-        let c_code = create_c_test_program(
-            i,
-            headers,
-            target,
-            c_target,
-            notice,
-            arch_specific_definitions,
-        );
-        if let Some(filename) = filename_mapping.get(&i.name()) {
-            write_file(filename, c_code)
-        };
-    });
-
-    intrinsics_name_list
-}
-
 pub fn write_rust_testfiles<T: IntrinsicTypeDefinition>(
     intrinsics: Vec<&dyn IntrinsicDefinition<T>>,
     rust_target: &str,