Auto merge of #105609 - bjorn3:shrink_rustc_dev, r=jyn514

Only include metadata for non-dynamic libraries in rustc-dev

The actual object code should be linked from librustc_driver.so, which is still included in rustc-dev. This saves on download time and disk usage.

Fixes https://github.com/rust-lang/rust/issues/103538

18 files changed, 161 insertions, 10 deletions

diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index 32e5d414061..b203ecd3844 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -105,7 +105,7 @@ impl Step for Std {
             "Checking stage{} library artifacts ({} -> {})",
             builder.top_stage, &compiler.host, target
         ));
-        run_cargo(builder, cargo, &libstd_stamp(builder, compiler, target), vec![], true);
+        run_cargo(builder, cargo, &libstd_stamp(builder, compiler, target), vec![], true, false);
 
         // We skip populating the sysroot in non-zero stage because that'll lead
         // to rlib/rmeta conflicts if std gets built during this session.
@@ -155,7 +155,14 @@ impl Step for Std {
             "Checking stage{} library test/bench/example targets ({} -> {})",
             builder.top_stage, &compiler.host, target
         ));
-        run_cargo(builder, cargo, &libstd_test_stamp(builder, compiler, target), vec![], true);
+        run_cargo(
+            builder,
+            cargo,
+            &libstd_test_stamp(builder, compiler, target),
+            vec![],
+            true,
+            false,
+        );
     }
 }
 
@@ -225,7 +232,7 @@ impl Step for Rustc {
             "Checking stage{} compiler artifacts ({} -> {})",
             builder.top_stage, &compiler.host, target
         ));
-        run_cargo(builder, cargo, &librustc_stamp(builder, compiler, target), vec![], true);
+        run_cargo(builder, cargo, &librustc_stamp(builder, compiler, target), vec![], true, false);
 
         let libdir = builder.sysroot_libdir(compiler, target);
         let hostdir = builder.sysroot_libdir(compiler, compiler.host);
@@ -285,6 +292,7 @@ impl Step for CodegenBackend {
             &codegen_backend_stamp(builder, compiler, target, backend),
             vec![],
             true,
+            false,
         );
     }
 }
@@ -343,7 +351,7 @@ impl Step for RustAnalyzer {
             "Checking stage{} {} artifacts ({} -> {})",
             compiler.stage, "rust-analyzer", &compiler.host.triple, target.triple
         ));
-        run_cargo(builder, cargo, &stamp(builder, compiler, target), vec![], true);
+        run_cargo(builder, cargo, &stamp(builder, compiler, target), vec![], true, false);
 
         /// Cargo's output path in a given stage, compiled by a particular
         /// compiler for the specified target.
@@ -417,6 +425,7 @@ macro_rules! tool_check_step {
                     &stamp(builder, compiler, target),
                     vec![],
                     true,
+                    false,
                 );
 
                 /// Cargo's output path in a given stage, compiled by a particular
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index f9a04f2e91d..147ded3a9ee 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -141,7 +141,14 @@ impl Step for Std {
             &compiler.host,
             target,
         ));
-        run_cargo(builder, cargo, &libstd_stamp(builder, compiler, target), target_deps, false);
+        run_cargo(
+            builder,
+            cargo,
+            &libstd_stamp(builder, compiler, target),
+            target_deps,
+            false,
+            false,
+        );
 
         builder.ensure(StdLink::from_std(
             self,
@@ -728,7 +735,14 @@ impl Step for Rustc {
             &compiler.host,
             target,
         ));
-        run_cargo(builder, cargo, &librustc_stamp(builder, compiler, target), vec![], false);
+        run_cargo(
+            builder,
+            cargo,
+            &librustc_stamp(builder, compiler, target),
+            vec![],
+            false,
+            true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files.
+        );
 
         builder.ensure(RustcLink::from_rustc(
             self,
@@ -984,7 +998,7 @@ impl Step for CodegenBackend {
             "Building stage{} codegen backend {} ({} -> {})",
             compiler.stage, backend, &compiler.host, target
         ));
-        let files = run_cargo(builder, cargo, &tmp_stamp, vec![], false);
+        let files = run_cargo(builder, cargo, &tmp_stamp, vec![], false, false);
         if builder.config.dry_run() {
             return;
         }
@@ -1411,6 +1425,7 @@ pub fn run_cargo(
     stamp: &Path,
     additional_target_deps: Vec<(PathBuf, DependencyType)>,
     is_check: bool,
+    rlib_only_metadata: bool,
 ) -> Vec<PathBuf> {
     if builder.config.dry_run() {
         return Vec::new();
@@ -1444,13 +1459,35 @@ pub fn run_cargo(
         };
         for filename in filenames {
             // Skip files like executables
-            if !(filename.ends_with(".rlib")
-                || filename.ends_with(".lib")
+            let mut keep = false;
+            if filename.ends_with(".lib")
                 || filename.ends_with(".a")
                 || is_debug_info(&filename)
                 || is_dylib(&filename)
-                || (is_check && filename.ends_with(".rmeta")))
             {
+                // Always keep native libraries, rust dylibs and debuginfo
+                keep = true;
+            }
+            if is_check && filename.ends_with(".rmeta") {
+                // During check builds we need to keep crate metadata
+                keep = true;
+            } else if rlib_only_metadata {
+                if filename.contains("jemalloc_sys") || filename.contains("rustc_smir") {
+                    // jemalloc_sys and rustc_smir are not linked into librustc_driver.so,
+                    // so we need to distribute them as rlib to be able to use them.
+                    keep |= filename.ends_with(".rlib");
+                } else {
+                    // Distribute the rest of the rustc crates as rmeta files only to reduce
+                    // the tarball sizes by about 50%. The object files are linked into
+                    // librustc_driver.so, so it is still possible to link against them.
+                    keep |= filename.ends_with(".rmeta");
+                }
+            } else {
+                // In all other cases keep all rlibs
+                keep |= filename.ends_with(".rlib");
+            }
+
+            if !keep {
                 continue;
             }
 
diff --git a/src/test/run-make-fulldeps/save-analysis/foo.rs b/src/test/run-make-fulldeps/save-analysis/foo.rs
index 74aaabfbf1b..384589de3b4 100644
--- a/src/test/run-make-fulldeps/save-analysis/foo.rs
+++ b/src/test/run-make-fulldeps/save-analysis/foo.rs
@@ -5,6 +5,11 @@
 extern crate rustc_graphviz;
 // A simple rust project
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 extern crate krate2;
 extern crate krate2 as krate3;
 
diff --git a/src/test/ui-fulldeps/deriving-encodable-decodable-box.rs b/src/test/ui-fulldeps/deriving-encodable-decodable-box.rs
index a4b911878e0..1c376f59e51 100644
--- a/src/test/ui-fulldeps/deriving-encodable-decodable-box.rs
+++ b/src/test/ui-fulldeps/deriving-encodable-decodable-box.rs
@@ -6,6 +6,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 use rustc_serialize::opaque::{MemDecoder, MemEncoder};
 use rustc_serialize::{Decodable, Encodable, Encoder};
diff --git a/src/test/ui-fulldeps/deriving-encodable-decodable-cell-refcell.rs b/src/test/ui-fulldeps/deriving-encodable-decodable-cell-refcell.rs
index 580c85f9b78..844d40f2ecd 100644
--- a/src/test/ui-fulldeps/deriving-encodable-decodable-cell-refcell.rs
+++ b/src/test/ui-fulldeps/deriving-encodable-decodable-cell-refcell.rs
@@ -8,6 +8,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 use rustc_serialize::opaque::{MemDecoder, MemEncoder};
 use rustc_serialize::{Decodable, Encodable, Encoder};
diff --git a/src/test/ui-fulldeps/deriving-global.rs b/src/test/ui-fulldeps/deriving-global.rs
index 921767af981..214bb4368ff 100644
--- a/src/test/ui-fulldeps/deriving-global.rs
+++ b/src/test/ui-fulldeps/deriving-global.rs
@@ -5,6 +5,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 mod submod {
     use rustc_macros::{Decodable, Encodable};
 
diff --git a/src/test/ui-fulldeps/deriving-hygiene.rs b/src/test/ui-fulldeps/deriving-hygiene.rs
index 8486b8b6e48..e1084a08fec 100644
--- a/src/test/ui-fulldeps/deriving-hygiene.rs
+++ b/src/test/ui-fulldeps/deriving-hygiene.rs
@@ -7,6 +7,11 @@ extern crate rustc_serialize;
 
 use rustc_macros::{Decodable, Encodable};
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 pub const other: u8 = 1;
 pub const f: u8 = 1;
 pub const d: u8 = 1;
diff --git a/src/test/ui-fulldeps/dropck_tarena_sound_drop.rs b/src/test/ui-fulldeps/dropck_tarena_sound_drop.rs
index 187f9a24a90..ffad80171da 100644
--- a/src/test/ui-fulldeps/dropck_tarena_sound_drop.rs
+++ b/src/test/ui-fulldeps/dropck_tarena_sound_drop.rs
@@ -14,6 +14,11 @@
 
 extern crate rustc_arena;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_arena::TypedArena;
 
 trait HasId { fn count(&self) -> usize; }
diff --git a/src/test/ui-fulldeps/empty-struct-braces-derive.rs b/src/test/ui-fulldeps/empty-struct-braces-derive.rs
index 6e5eb54629c..10e8beaa7b1 100644
--- a/src/test/ui-fulldeps/empty-struct-braces-derive.rs
+++ b/src/test/ui-fulldeps/empty-struct-braces-derive.rs
@@ -6,6 +6,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Debug, Encodable, Decodable)]
diff --git a/src/test/ui-fulldeps/issue-14021.rs b/src/test/ui-fulldeps/issue-14021.rs
index 215dfaed7ab..309b5c4a03d 100644
--- a/src/test/ui-fulldeps/issue-14021.rs
+++ b/src/test/ui-fulldeps/issue-14021.rs
@@ -7,6 +7,11 @@
 extern crate rustc_macros;
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 use rustc_serialize::opaque::{MemDecoder, MemEncoder};
 use rustc_serialize::{Decodable, Encodable, Encoder};
diff --git a/src/test/ui-fulldeps/missing-rustc-driver-error.rs b/src/test/ui-fulldeps/missing-rustc-driver-error.rs
new file mode 100644
index 00000000000..654cd6f6dc9
--- /dev/null
+++ b/src/test/ui-fulldeps/missing-rustc-driver-error.rs
@@ -0,0 +1,11 @@
+// Test that we get the following hint when trying to use a compiler crate without rustc_driver.
+// error-pattern: try adding `extern crate rustc_driver;` at the top level of this crate
+// compile-flags: --emit link
+// The exactly list of required crates depends on the target. as such only test Unix targets.
+// only-unix
+
+#![feature(rustc_private)]
+
+extern crate rustc_serialize;
+
+fn main() {}
diff --git a/src/test/ui-fulldeps/missing-rustc-driver-error.stderr b/src/test/ui-fulldeps/missing-rustc-driver-error.stderr
new file mode 100644
index 00000000000..ad03ba0103c
--- /dev/null
+++ b/src/test/ui-fulldeps/missing-rustc-driver-error.stderr
@@ -0,0 +1,24 @@
+error: crate `rustc_serialize` required to be available in rlib format, but was not found in this form
+   |
+   = help: try adding `extern crate rustc_driver;` at the top level of this crate
+
+error: crate `smallvec` required to be available in rlib format, but was not found in this form
+
+error: crate `thin_vec` required to be available in rlib format, but was not found in this form
+
+error: crate `indexmap` required to be available in rlib format, but was not found in this form
+
+error: crate `hashbrown` required to be available in rlib format, but was not found in this form
+
+error: crate `ahash` required to be available in rlib format, but was not found in this form
+
+error: crate `once_cell` required to be available in rlib format, but was not found in this form
+
+error: crate `getrandom` required to be available in rlib format, but was not found in this form
+
+error: crate `cfg_if` required to be available in rlib format, but was not found in this form
+
+error: crate `libc` required to be available in rlib format, but was not found in this form
+
+error: aborting due to 10 previous errors
+
diff --git a/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs b/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs
index bb246de0e57..ff1be080415 100644
--- a/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs
+++ b/src/test/ui-fulldeps/mod_dir_path_canonicalized.rs
@@ -10,6 +10,11 @@ extern crate rustc_parse;
 extern crate rustc_session;
 extern crate rustc_span;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_parse::new_parser_from_file;
 use rustc_session::parse::ParseSess;
 use rustc_span::source_map::FilePathMapping;
diff --git a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
index a93ba87470a..6dbabc8eb34 100644
--- a/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
+++ b/src/test/ui-fulldeps/pprust-expr-roundtrip.rs
@@ -27,6 +27,11 @@ extern crate rustc_session;
 extern crate rustc_span;
 extern crate thin_vec;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_ast::mut_visit::{self, visit_clobber, MutVisitor};
 use rustc_ast::ptr::P;
 use rustc_ast::*;
diff --git a/src/test/ui-fulldeps/regions-mock-tcx.rs b/src/test/ui-fulldeps/regions-mock-tcx.rs
index 30e62723240..63975ef62c5 100644
--- a/src/test/ui-fulldeps/regions-mock-tcx.rs
+++ b/src/test/ui-fulldeps/regions-mock-tcx.rs
@@ -14,6 +14,11 @@
 extern crate rustc_arena;
 extern crate libc;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use TypeStructure::{TypeInt, TypeFunction};
 use AstKind::{ExprInt, ExprVar, ExprLambda};
 use rustc_arena::TypedArena;
diff --git a/src/test/ui-fulldeps/rustc_encodable_hygiene.rs b/src/test/ui-fulldeps/rustc_encodable_hygiene.rs
index 452110a65e4..509a6b1d22c 100644
--- a/src/test/ui-fulldeps/rustc_encodable_hygiene.rs
+++ b/src/test/ui-fulldeps/rustc_encodable_hygiene.rs
@@ -6,6 +6,11 @@ extern crate rustc_macros;
 #[allow(dead_code)]
 extern crate rustc_serialize;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use rustc_macros::{Decodable, Encodable};
 
 #[derive(Decodable, Encodable, Debug)]
diff --git a/src/tools/miri/src/lib.rs b/src/tools/miri/src/lib.rs
index 97750cb78cd..7024927b205 100644
--- a/src/tools/miri/src/lib.rs
+++ b/src/tools/miri/src/lib.rs
@@ -54,6 +54,11 @@ extern crate rustc_session;
 extern crate rustc_span;
 extern crate rustc_target;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 mod borrow_tracker;
 mod clock;
 mod concurrency;
diff --git a/src/tools/rustfmt/src/lib.rs b/src/tools/rustfmt/src/lib.rs
index 1d1ef525f23..0c27bcacfb8 100644
--- a/src/tools/rustfmt/src/lib.rs
+++ b/src/tools/rustfmt/src/lib.rs
@@ -24,6 +24,11 @@ extern crate rustc_parse;
 extern crate rustc_session;
 extern crate rustc_span;
 
+// Necessary to pull in object code as the rest of the rustc crates are shipped only as rmeta
+// files.
+#[allow(unused_extern_crates)]
+extern crate rustc_driver;
+
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::fmt;