Auto merge of #105609 - bjorn3:shrink_rustc_dev, r=jyn514

Only include metadata for non-dynamic libraries in rustc-dev

The actual object code should be linked from librustc_driver.so, which is still included in rustc-dev. This saves on download time and disk usage.

Fixes https://github.com/rust-lang/rust/issues/103538

2 files changed, 56 insertions, 10 deletions

diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index 32e5d414061..b203ecd3844 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -105,7 +105,7 @@ impl Step for Std {
             "Checking stage{} library artifacts ({} -> {})",
             builder.top_stage, &compiler.host, target
         ));
-        run_cargo(builder, cargo, &libstd_stamp(builder, compiler, target), vec![], true);
+        run_cargo(builder, cargo, &libstd_stamp(builder, compiler, target), vec![], true, false);
 
         // We skip populating the sysroot in non-zero stage because that'll lead
         // to rlib/rmeta conflicts if std gets built during this session.
@@ -155,7 +155,14 @@ impl Step for Std {
             "Checking stage{} library test/bench/example targets ({} -> {})",
             builder.top_stage, &compiler.host, target
         ));
-        run_cargo(builder, cargo, &libstd_test_stamp(builder, compiler, target), vec![], true);
+        run_cargo(
+            builder,
+            cargo,
+            &libstd_test_stamp(builder, compiler, target),
+            vec![],
+            true,
+            false,
+        );
     }
 }
 
@@ -225,7 +232,7 @@ impl Step for Rustc {
             "Checking stage{} compiler artifacts ({} -> {})",
             builder.top_stage, &compiler.host, target
         ));
-        run_cargo(builder, cargo, &librustc_stamp(builder, compiler, target), vec![], true);
+        run_cargo(builder, cargo, &librustc_stamp(builder, compiler, target), vec![], true, false);
 
         let libdir = builder.sysroot_libdir(compiler, target);
         let hostdir = builder.sysroot_libdir(compiler, compiler.host);
@@ -285,6 +292,7 @@ impl Step for CodegenBackend {
             &codegen_backend_stamp(builder, compiler, target, backend),
             vec![],
             true,
+            false,
         );
     }
 }
@@ -343,7 +351,7 @@ impl Step for RustAnalyzer {
             "Checking stage{} {} artifacts ({} -> {})",
             compiler.stage, "rust-analyzer", &compiler.host.triple, target.triple
         ));
-        run_cargo(builder, cargo, &stamp(builder, compiler, target), vec![], true);
+        run_cargo(builder, cargo, &stamp(builder, compiler, target), vec![], true, false);
 
         /// Cargo's output path in a given stage, compiled by a particular
         /// compiler for the specified target.
@@ -417,6 +425,7 @@ macro_rules! tool_check_step {
                     &stamp(builder, compiler, target),
                     vec![],
                     true,
+                    false,
                 );
 
                 /// Cargo's output path in a given stage, compiled by a particular
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index f9a04f2e91d..147ded3a9ee 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -141,7 +141,14 @@ impl Step for Std {
             &compiler.host,
             target,
         ));
-        run_cargo(builder, cargo, &libstd_stamp(builder, compiler, target), target_deps, false);
+        run_cargo(
+            builder,
+            cargo,
+            &libstd_stamp(builder, compiler, target),
+            target_deps,
+            false,
+            false,
+        );
 
         builder.ensure(StdLink::from_std(
             self,
@@ -728,7 +735,14 @@ impl Step for Rustc {
             &compiler.host,
             target,
         ));
-        run_cargo(builder, cargo, &librustc_stamp(builder, compiler, target), vec![], false);
+        run_cargo(
+            builder,
+            cargo,
+            &librustc_stamp(builder, compiler, target),
+            vec![],
+            false,
+            true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files.
+        );
 
         builder.ensure(RustcLink::from_rustc(
             self,
@@ -984,7 +998,7 @@ impl Step for CodegenBackend {
             "Building stage{} codegen backend {} ({} -> {})",
             compiler.stage, backend, &compiler.host, target
         ));
-        let files = run_cargo(builder, cargo, &tmp_stamp, vec![], false);
+        let files = run_cargo(builder, cargo, &tmp_stamp, vec![], false, false);
         if builder.config.dry_run() {
             return;
         }
@@ -1411,6 +1425,7 @@ pub fn run_cargo(
     stamp: &Path,
     additional_target_deps: Vec<(PathBuf, DependencyType)>,
     is_check: bool,
+    rlib_only_metadata: bool,
 ) -> Vec<PathBuf> {
     if builder.config.dry_run() {
         return Vec::new();
@@ -1444,13 +1459,35 @@ pub fn run_cargo(
         };
         for filename in filenames {
             // Skip files like executables
-            if !(filename.ends_with(".rlib")
-                || filename.ends_with(".lib")
+            let mut keep = false;
+            if filename.ends_with(".lib")
                 || filename.ends_with(".a")
                 || is_debug_info(&filename)
                 || is_dylib(&filename)
-                || (is_check && filename.ends_with(".rmeta")))
             {
+                // Always keep native libraries, rust dylibs and debuginfo
+                keep = true;
+            }
+            if is_check && filename.ends_with(".rmeta") {
+                // During check builds we need to keep crate metadata
+                keep = true;
+            } else if rlib_only_metadata {
+                if filename.contains("jemalloc_sys") || filename.contains("rustc_smir") {
+                    // jemalloc_sys and rustc_smir are not linked into librustc_driver.so,
+                    // so we need to distribute them as rlib to be able to use them.
+                    keep |= filename.ends_with(".rlib");
+                } else {
+                    // Distribute the rest of the rustc crates as rmeta files only to reduce
+                    // the tarball sizes by about 50%. The object files are linked into
+                    // librustc_driver.so, so it is still possible to link against them.
+                    keep |= filename.ends_with(".rmeta");
+                }
+            } else {
+                // In all other cases keep all rlibs
+                keep |= filename.ends_with(".rlib");
+            }
+
+            if !keep {
                 continue;
             }