Auto merge of #132362 - mustartt:aix-dylib-detection, r=jieyouxu

[AIX] change system dynamic library format

Historically on AIX, almost all dynamic libraries are distributed in `.a` Big Archive Format which can consists of both static and shared objects in the same archive (e.g. `libc++abi.a(libc++abi.so.1)`). During the initial porting process, the dynamic libraries are kept as `.a` to simplify the migration, but semantically having an XCOFF object under the archive extension is wrong. For crate type `cdylib` we want to be able to distribute the libraries as archives as well.

We are migrating to archives with the following format:
```
$ ar -t lib<name>.a
lib<name>.so
```
where each archive contains a single member that is a shared XCOFF object that can be loaded.

10 files changed, 127 insertions, 22 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 51e21f6dc82..e4dcf13a84b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1965,9 +1965,9 @@ checksum = "baff4b617f7df3d896f97fe922b64817f6cd9a756bb81d40f8883f2f66dcb401"
 
 [[package]]
 name = "libc"
-version = "0.2.161"
+version = "0.2.164"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
+checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f"
 
 [[package]]
 name = "libdbus-sys"
@@ -3990,6 +3990,7 @@ name = "rustc_metadata"
 version = "0.0.0"
 dependencies = [
  "bitflags 2.6.0",
+ "libc",
  "libloading",
  "odht",
  "rustc_abi",
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index fd1126e8528..5149e3a12f2 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -777,6 +777,16 @@ fn link_natively(
     info!("preparing {:?} to {:?}", crate_type, out_filename);
     let (linker_path, flavor) = linker_and_flavor(sess);
     let self_contained_components = self_contained_components(sess, crate_type);
+
+    // On AIX, we ship all libraries as .a big_af archive
+    // the expected format is lib<name>.a(libname.so) for the actual
+    // dynamic library. So we link to a temporary .so file to be archived
+    // at the final out_filename location
+    let should_archive = crate_type != CrateType::Executable && sess.target.is_like_aix;
+    let archive_member =
+        should_archive.then(|| tmpdir.join(out_filename.file_name().unwrap()).with_extension("so"));
+    let temp_filename = archive_member.as_deref().unwrap_or(out_filename);
+
     let mut cmd = linker_with_args(
         &linker_path,
         flavor,
@@ -784,7 +794,7 @@ fn link_natively(
         archive_builder_builder,
         crate_type,
         tmpdir,
-        out_filename,
+        temp_filename,
         codegen_results,
         self_contained_components,
     )?;
@@ -1158,6 +1168,12 @@ fn link_natively(
         }
     }
 
+    if should_archive {
+        let mut ab = archive_builder_builder.new_archive_builder(sess);
+        ab.add_file(temp_filename);
+        ab.build(out_filename);
+    }
+
     Ok(())
 }
 
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index bdf7030f946..ba7b53321a8 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -5,6 +5,7 @@ use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 
+use itertools::Itertools;
 use object::write::{self, StandardSegment, Symbol, SymbolSection};
 use object::{
     Architecture, BinaryFormat, Endianness, FileFlags, Object, ObjectSection, ObjectSymbol,
@@ -21,6 +22,7 @@ use rustc_middle::bug;
 use rustc_session::Session;
 use rustc_span::sym;
 use rustc_target::spec::{RelocModel, Target, ef_avr_arch};
+use tracing::debug;
 
 use super::apple;
 
@@ -53,6 +55,7 @@ fn load_metadata_with(
 
 impl MetadataLoader for DefaultMetadataLoader {
     fn get_rlib_metadata(&self, target: &Target, path: &Path) -> Result<OwnedSlice, String> {
+        debug!("getting rlib metadata for {}", path.display());
         load_metadata_with(path, |data| {
             let archive = object::read::archive::ArchiveFile::parse(&*data)
                 .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?;
@@ -77,8 +80,26 @@ impl MetadataLoader for DefaultMetadataLoader {
     }
 
     fn get_dylib_metadata(&self, target: &Target, path: &Path) -> Result<OwnedSlice, String> {
+        debug!("getting dylib metadata for {}", path.display());
         if target.is_like_aix {
-            load_metadata_with(path, |data| get_metadata_xcoff(path, data))
+            load_metadata_with(path, |data| {
+                let archive = object::read::archive::ArchiveFile::parse(&*data).map_err(|e| {
+                    format!("failed to parse aix dylib '{}': {}", path.display(), e)
+                })?;
+
+                match archive.members().exactly_one() {
+                    Ok(lib) => {
+                        let lib = lib.map_err(|e| {
+                            format!("failed to parse aix dylib '{}': {}", path.display(), e)
+                        })?;
+                        let data = lib.data(data).map_err(|e| {
+                            format!("failed to parse aix dylib '{}': {}", path.display(), e)
+                        })?;
+                        get_metadata_xcoff(path, data)
+                    }
+                    Err(e) => Err(format!("failed to parse aix dylib '{}': {}", path.display(), e)),
+                }
+            })
         } else {
             load_metadata_with(path, |data| search_for_section(path, data, ".rustc"))
         }
diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml
index cece700b4dd..12519be9870 100644
--- a/compiler/rustc_metadata/Cargo.toml
+++ b/compiler/rustc_metadata/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 # tidy-alphabetical-start
 bitflags = "2.4.1"
+libc = "0.2"
 libloading = "0.8.0"
 odht = { version = "0.3.1", features = ["nightly"] }
 rustc_abi = { path = "../rustc_abi" }
diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs
index ca16a66763a..a611e8010be 100644
--- a/compiler/rustc_metadata/src/creader.rs
+++ b/compiler/rustc_metadata/src/creader.rs
@@ -540,6 +540,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
                 Some(cnum)
             }
             Err(err) => {
+                debug!("failed to resolve crate {} {:?}", name, dep_kind);
                 let missing_core =
                     self.maybe_resolve_crate(sym::core, CrateDepKind::Explicit, None).is_err();
                 err.report(self.sess, span, missing_core);
@@ -588,6 +589,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
             match self.load(&mut locator)? {
                 Some(res) => (res, None),
                 None => {
+                    info!("falling back to loading proc_macro");
                     dep_kind = CrateDepKind::MacrosOnly;
                     match self.load_proc_macro(&mut locator, path_kind, host_hash)? {
                         Some(res) => res,
@@ -599,6 +601,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
 
         match result {
             (LoadResult::Previous(cnum), None) => {
+                info!("library for `{}` was loaded previously", name);
                 // When `private_dep` is none, it indicates the directly dependent crate. If it is
                 // not specified by `--extern` on command line parameters, it may be
                 // `private-dependency` when `register_crate` is called for the first time. Then it must be updated to
@@ -613,6 +616,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
                 Ok(cnum)
             }
             (LoadResult::Loaded(library), host_library) => {
+                info!("register newly loaded library for `{}`", name);
                 self.register_crate(host_library, root, library, dep_kind, name, private_dep)
             }
             _ => panic!(),
@@ -696,7 +700,25 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
         stable_crate_id: StableCrateId,
     ) -> Result<&'static [ProcMacro], CrateError> {
         let sym_name = self.sess.generate_proc_macro_decls_symbol(stable_crate_id);
-        Ok(unsafe { *load_symbol_from_dylib::<*const &[ProcMacro]>(path, &sym_name)? })
+        debug!("trying to dlsym proc_macros {} for symbol `{}`", path.display(), sym_name);
+
+        unsafe {
+            let result = load_symbol_from_dylib::<*const &[ProcMacro]>(path, &sym_name);
+            match result {
+                Ok(result) => {
+                    debug!("loaded dlsym proc_macros {} for symbol `{}`", path.display(), sym_name);
+                    Ok(*result)
+                }
+                Err(err) => {
+                    debug!(
+                        "failed to dlsym proc_macros {} for symbol `{}`",
+                        path.display(),
+                        sym_name
+                    );
+                    Err(err.into())
+                }
+            }
+        }
     }
 
     fn inject_panic_runtime(&mut self, krate: &ast::Crate) {
@@ -1141,6 +1163,29 @@ fn format_dlopen_err(e: &(dyn std::error::Error + 'static)) -> String {
     e.sources().map(|e| format!(": {e}")).collect()
 }
 
+fn attempt_load_dylib(path: &Path) -> Result<libloading::Library, libloading::Error> {
+    #[cfg(target_os = "aix")]
+    if let Some(ext) = path.extension()
+        && ext.eq("a")
+    {
+        // On AIX, we ship all libraries as .a big_af archive
+        // the expected format is lib<name>.a(libname.so) for the actual
+        // dynamic library
+        let library_name = path.file_stem().expect("expect a library name");
+        let mut archive_member = OsString::from("a(");
+        archive_member.push(library_name);
+        archive_member.push(".so)");
+        let new_path = path.with_extension(archive_member);
+
+        // On AIX, we need RTLD_MEMBER to dlopen an archived shared
+        let flags = libc::RTLD_LAZY | libc::RTLD_LOCAL | libc::RTLD_MEMBER;
+        return unsafe { libloading::os::unix::Library::open(Some(&new_path), flags) }
+            .map(|lib| lib.into());
+    }
+
+    unsafe { libloading::Library::new(&path) }
+}
+
 // On Windows the compiler would sometimes intermittently fail to open the
 // proc-macro DLL with `Error::LoadLibraryExW`. It is suspected that something in the
 // system still holds a lock on the file, so we retry a few times before calling it
@@ -1151,7 +1196,8 @@ fn load_dylib(path: &Path, max_attempts: usize) -> Result<libloading::Library, S
     let mut last_error = None;
 
     for attempt in 0..max_attempts {
-        match unsafe { libloading::Library::new(&path) } {
+        debug!("Attempt to load proc-macro `{}`.", path.display());
+        match attempt_load_dylib(path) {
             Ok(lib) => {
                 if attempt > 0 {
                     debug!(
@@ -1165,6 +1211,7 @@ fn load_dylib(path: &Path, max_attempts: usize) -> Result<libloading::Library, S
             Err(err) => {
                 // Only try to recover from this specific error.
                 if !matches!(err, libloading::Error::LoadLibraryExW { .. }) {
+                    debug!("Failed to load proc-macro `{}`. Not retrying", path.display());
                     let err = format_dlopen_err(&err);
                     // We include the path of the dylib in the error ourselves, so
                     // if it's in the error, we strip it.
diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs
index 0b53e5eeaa8..d59ec7af6ec 100644
--- a/compiler/rustc_metadata/src/locator.rs
+++ b/compiler/rustc_metadata/src/locator.rs
@@ -847,7 +847,10 @@ fn get_metadata_section<'p>(
         )));
     };
     match blob.check_compatibility(cfg_version) {
-        Ok(()) => Ok(blob),
+        Ok(()) => {
+            debug!("metadata blob read okay");
+            Ok(blob)
+        }
         Err(None) => Err(MetadataError::LoadFailure(format!(
             "invalid metadata version found: {}",
             filename.display()
diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs
index 24be705f481..8e088682f92 100644
--- a/src/bootstrap/src/core/build_steps/compile.rs
+++ b/src/bootstrap/src/core/build_steps/compile.rs
@@ -1457,7 +1457,7 @@ impl Step for CodegenBackend {
         }
         let mut files = files.into_iter().filter(|f| {
             let filename = f.file_name().unwrap().to_str().unwrap();
-            is_dylib(filename) && filename.contains("rustc_codegen_")
+            is_dylib(f) && filename.contains("rustc_codegen_")
         });
         let codegen_backend = match files.next() {
             Some(f) => f,
@@ -1936,7 +1936,7 @@ impl Step for Assemble {
             let filename = f.file_name().into_string().unwrap();
 
             let is_proc_macro = proc_macros.contains(&filename);
-            let is_dylib_or_debug = is_dylib(&filename) || is_debug_info(&filename);
+            let is_dylib_or_debug = is_dylib(&f.path()) || is_debug_info(&filename);
 
             // If we link statically to stdlib, do not copy the libstd dynamic library file
             // FIXME: Also do this for Windows once incremental post-optimization stage0 tests
@@ -2089,7 +2089,7 @@ pub fn run_cargo(
             if filename.ends_with(".lib")
                 || filename.ends_with(".a")
                 || is_debug_info(&filename)
-                || is_dylib(&filename)
+                || is_dylib(Path::new(&*filename))
             {
                 // Always keep native libraries, rust dylibs and debuginfo
                 keep = true;
@@ -2189,7 +2189,7 @@ pub fn run_cargo(
             Some(triple) => triple.0.to_str().unwrap(),
             None => panic!("no output generated for {prefix:?} {extension:?}"),
         };
-        if is_dylib(path_to_add) {
+        if is_dylib(Path::new(path_to_add)) {
             let candidate = format!("{path_to_add}.lib");
             let candidate = PathBuf::from(candidate);
             if candidate.exists() {
diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index c022285211f..0216ac81162 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -436,13 +436,10 @@ impl Step for Rustc {
             if libdir_relative.to_str() != Some("bin") {
                 let libdir = builder.rustc_libdir(compiler);
                 for entry in builder.read_dir(&libdir) {
-                    let name = entry.file_name();
-                    if let Some(s) = name.to_str() {
-                        if is_dylib(s) {
-                            // Don't use custom libdir here because ^lib/ will be resolved again
-                            // with installer
-                            builder.install(&entry.path(), &image.join("lib"), 0o644);
-                        }
+                    if is_dylib(&entry.path()) {
+                        // Don't use custom libdir here because ^lib/ will be resolved again
+                        // with installer
+                        builder.install(&entry.path(), &image.join("lib"), 0o644);
                     }
                 }
             }
diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index 532c8f767eb..dcea9f5f7d1 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -2949,8 +2949,7 @@ impl Step for RemoteCopyLibs {
         // Push all our dylibs to the emulator
         for f in t!(builder.sysroot_target_libdir(compiler, target).read_dir()) {
             let f = t!(f);
-            let name = f.file_name().into_string().unwrap();
-            if helpers::is_dylib(&name) {
+            if helpers::is_dylib(&f.path()) {
                 command(&tool).arg("push").arg(f.path()).run(builder);
             }
         }
diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs
index 3fb2330469a..9ca036a2afd 100644
--- a/src/bootstrap/src/utils/helpers.rs
+++ b/src/bootstrap/src/utils/helpers.rs
@@ -11,6 +11,7 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH};
 use std::{env, fs, io, str};
 
 use build_helper::util::fail;
+use object::read::archive::ArchiveFile;
 
 use crate::LldMode;
 use crate::core::builder::Builder;
@@ -53,8 +54,27 @@ pub fn exe(name: &str, target: TargetSelection) -> String {
 }
 
 /// Returns `true` if the file name given looks like a dynamic library.
-pub fn is_dylib(name: &str) -> bool {
-    name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll")
+pub fn is_dylib(path: &Path) -> bool {
+    path.extension().and_then(|ext| ext.to_str()).map_or(false, |ext| {
+        ext == "dylib" || ext == "so" || ext == "dll" || (ext == "a" && is_aix_shared_archive(path))
+    })
+}
+
+fn is_aix_shared_archive(path: &Path) -> bool {
+    // FIXME(#133268): reading the entire file as &[u8] into memory seems excessive
+    // look into either mmap it or use the ReadCache
+    let data = match fs::read(path) {
+        Ok(data) => data,
+        Err(_) => return false,
+    };
+    let file = match ArchiveFile::parse(&*data) {
+        Ok(file) => file,
+        Err(_) => return false,
+    };
+
+    file.members()
+        .filter_map(Result::ok)
+        .any(|entry| String::from_utf8_lossy(entry.name()).contains(".so"))
 }
 
 /// Returns `true` if the file name given looks like a debug info file