Auto merge of #137535 - Kobzol:split-metadata, r=petrochenkov

Introduce `-Zembed-metadata` to allow omitting full metadata from rlibs and dylibs

This is a continuation of https://github.com/rust-lang/rust/pull/120855 (I was mentored by `@bjorn3` to move it forward). Most of the original code was written by bjorn3, I tried to clean it up a bit and add some documentation and tests.

This PR introduces a new unstable compiler flag called `-Zembed-metadata=[no|yes]`, with the default being `yes` (see https://github.com/rust-lang/rust/issues/57076 for context). When set to `no`, rustc will only store a small metadata stub inside rlibs/dylibs instead of the full metadata, to keep their size smaller. It should be used in combination with `--emit=metadata`, so that the users of such a compiled library can still read the metadata from the corresponding `.rmeta` file. [This comment](https://github.com/rust-lang/rust/pull/120855#issuecomment-1937018169) shows an example of binary/artifact size wins that can be achieved using this approach.

Contrary to https://github.com/rust-lang/rust/pull/120855, this PR only introduces the new flag, along with a couple of run-make tests and documentation, but does not yet use it in bootstrap to actually compile rustc. I plan to do that as a follow-up step (along with integration in Cargo, which should ideally just always pass this flag to reduce the size of target directories).

Fixes https://github.com/rust-lang/rust/issues/23366
Closes https://github.com/rust-lang/rust/issues/29511
Fixes https://github.com/rust-lang/rust/issues/57076

Another attempt of https://github.com/rust-lang/rust/pull/93945 and https://github.com/rust-lang/rust/pull/120855.

r? `@petrochenkov`

3 files changed, 92 insertions, 0 deletions

diff --git a/tests/run-make/embed-metadata/dep1.rs b/tests/run-make/embed-metadata/dep1.rs
new file mode 100644
index 00000000000..be70c3933e0
--- /dev/null
+++ b/tests/run-make/embed-metadata/dep1.rs
@@ -0,0 +1 @@
+pub fn func_dep1() {}
diff --git a/tests/run-make/embed-metadata/foo.rs b/tests/run-make/embed-metadata/foo.rs
new file mode 100644
index 00000000000..0cc9cede860
--- /dev/null
+++ b/tests/run-make/embed-metadata/foo.rs
@@ -0,0 +1,5 @@
+extern crate dep1;
+
+fn main() {
+    dep1::func_dep1();
+}
diff --git a/tests/run-make/embed-metadata/rmake.rs b/tests/run-make/embed-metadata/rmake.rs
new file mode 100644
index 00000000000..acefb186484
--- /dev/null
+++ b/tests/run-make/embed-metadata/rmake.rs
@@ -0,0 +1,86 @@
+// Tests the -Zembed-metadata compiler flag.
+// Tracking issue: https://github.com/rust-lang/rust/issues/139165
+
+use run_make_support::rfs::{create_dir, remove_file, rename};
+use run_make_support::{Rustc, dynamic_lib_name, path, run_in_tmpdir, rust_lib_name, rustc};
+
+#[derive(Debug, Copy, Clone)]
+enum LibraryKind {
+    Rlib,
+    Dylib,
+}
+
+impl LibraryKind {
+    fn crate_type(&self) -> &str {
+        match self {
+            LibraryKind::Rlib => "rlib",
+            LibraryKind::Dylib => "dylib",
+        }
+    }
+
+    fn add_extern(&self, rustc: &mut Rustc, dep_name: &str, dep_path: &str) {
+        let dep_path = match self {
+            LibraryKind::Dylib => format!("{dep_path}/{}", dynamic_lib_name(dep_name)),
+            LibraryKind::Rlib => format!("{dep_path}/{}", rust_lib_name(dep_name)),
+        };
+        rustc.extern_(dep_name, dep_path);
+    }
+}
+
+fn main() {
+    // The compiler takes different paths based on if --extern is passed or not, so we test all
+    // combinations (`rlib`/`dylib` x `--extern`/`no --extern`).
+    for kind in [LibraryKind::Rlib, LibraryKind::Dylib] {
+        eprintln!("Testing library kind {kind:?}");
+        lookup_rmeta_in_lib_dir(kind);
+        lookup_rmeta_through_extern(kind);
+        lookup_rmeta_missing(kind);
+    }
+}
+
+// Lookup .rmeta file in the same directory as a rlib/dylib with stub metadata.
+fn lookup_rmeta_in_lib_dir(kind: LibraryKind) {
+    run_in_tmpdir(|| {
+        build_dep_rustc(kind).run();
+        rustc().input("foo.rs").run();
+    });
+}
+
+// Lookup .rmeta file when specifying the dependency using --extern.
+fn lookup_rmeta_through_extern(kind: LibraryKind) {
+    run_in_tmpdir(|| {
+        // Generate libdep1.rlib and libdep1.rmeta in deps
+        create_dir("deps");
+        build_dep_rustc(kind).out_dir("deps").run();
+
+        let mut rustc = rustc();
+        kind.add_extern(&mut rustc, "dep1", "deps");
+        rustc.extern_("dep1", path("deps").join("libdep1.rmeta"));
+        rustc.input("foo.rs").run();
+    });
+}
+
+// Check the error message when the .rmeta file is missing.
+fn lookup_rmeta_missing(kind: LibraryKind) {
+    run_in_tmpdir(|| {
+        create_dir("deps");
+        build_dep_rustc(kind).out_dir("deps").run();
+
+        let mut rustc = rustc();
+        kind.add_extern(&mut rustc, "dep1", "deps");
+        rustc.input("foo.rs").run_fail().assert_stderr_contains("only metadata stub found");
+    });
+}
+
+fn build_dep_rustc(kind: LibraryKind) -> Rustc {
+    let mut dep_rustc = rustc();
+    dep_rustc
+        .arg("-Zembed-metadata=no")
+        .crate_type(kind.crate_type())
+        .input("dep1.rs")
+        .emit("metadata,link");
+    if matches!(kind, LibraryKind::Dylib) {
+        dep_rustc.arg("-Cprefer-dynamic");
+    }
+    dep_rustc
+}