Auto merge of #106560 - bjorn3:support_staticlib_dylib_linking, r=pnkfelix

Support linking to rust dylib with --crate-type staticlib

This allows for example dynamically linking libstd, while statically linking the user crate into an executable or C dynamic library. For this two unstable flags (`-Z staticlib-allow-rdylib-deps` and `-Z staticlib-prefer-dynamic`) are introduced. Without the former you get an error. The latter is the equivalent to `-C prefer-dynamic` for the staticlib crate type to indicate that dynamically linking is preferred when both options are available, like for libstd. Care must be taken to ensure that no crate ends up being merged into two distinct staticlibs that are linked together. Doing so will cause a linker error at best and undefined behavior at worst. In addition two distinct staticlibs compiled by different rustc may not be combined under any circumstances due to some rustc private symbols not being mangled.

To successfully link a staticlib, `--print native-static-libs` can be used while compiling to ask rustc for the linker flags necessary when linking the staticlib. This is an existing flag which previously only listed native libraries. It has been extended to list rust dylibs too. Trying to locate libstd yourself to link against it is not supported and may break if for example the libstd of multiple rustc versions are put in the same directory.

For an example on how to use this see the `src/test/run-make-fulldeps/staticlib-dylib-linkage/` test.

7 files changed, 144 insertions, 15 deletions

diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 8a7809a1468..ea06cb02d8b 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -546,12 +546,38 @@ fn link_staticlib<'a>(
 
     ab.build(out_filename);
 
-    if !all_native_libs.is_empty() {
-        if sess.opts.prints.contains(&PrintRequest::NativeStaticLibs) {
-            print_native_static_libs(sess, &all_native_libs);
+    let crates = codegen_results.crate_info.used_crates.iter();
+
+    let fmts = codegen_results
+        .crate_info
+        .dependency_formats
+        .iter()
+        .find_map(|&(ty, ref list)| if ty == CrateType::Staticlib { Some(list) } else { None })
+        .expect("no dependency formats for staticlib");
+
+    let mut all_rust_dylibs = vec![];
+    for &cnum in crates {
+        match fmts.get(cnum.as_usize() - 1) {
+            Some(&Linkage::Dynamic) => {}
+            _ => continue,
+        }
+        let crate_name = codegen_results.crate_info.crate_name[&cnum];
+        let used_crate_source = &codegen_results.crate_info.used_crate_source[&cnum];
+        if let Some((path, _)) = &used_crate_source.dylib {
+            all_rust_dylibs.push(&**path);
+        } else {
+            if used_crate_source.rmeta.is_some() {
+                sess.emit_fatal(errors::LinkRlibError::OnlyRmetaFound { crate_name });
+            } else {
+                sess.emit_fatal(errors::LinkRlibError::NotFound { crate_name });
+            }
         }
     }
 
+    if sess.opts.prints.contains(&PrintRequest::NativeStaticLibs) {
+        print_native_static_libs(sess, &all_native_libs, &all_rust_dylibs);
+    }
+
     Ok(())
 }
 
@@ -1370,8 +1396,12 @@ enum RlibFlavor {
     StaticlibBase,
 }
 
-fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) {
-    let lib_args: Vec<_> = all_native_libs
+fn print_native_static_libs(
+    sess: &Session,
+    all_native_libs: &[NativeLib],
+    all_rust_dylibs: &[&Path],
+) {
+    let mut lib_args: Vec<_> = all_native_libs
         .iter()
         .filter(|l| relevant_lib(sess, l))
         .filter_map(|lib| {
@@ -1401,6 +1431,41 @@ fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) {
             }
         })
         .collect();
+    for path in all_rust_dylibs {
+        // FIXME deduplicate with add_dynamic_crate
+
+        // Just need to tell the linker about where the library lives and
+        // what its name is
+        let parent = path.parent();
+        if let Some(dir) = parent {
+            let dir = fix_windows_verbatim_for_gcc(dir);
+            if sess.target.is_like_msvc {
+                let mut arg = String::from("/LIBPATH:");
+                arg.push_str(&dir.display().to_string());
+                lib_args.push(arg);
+            } else {
+                lib_args.push("-L".to_owned());
+                lib_args.push(dir.display().to_string());
+            }
+        }
+        let stem = path.file_stem().unwrap().to_str().unwrap();
+        // Convert library file-stem into a cc -l argument.
+        let prefix = if stem.starts_with("lib") && !sess.target.is_like_windows { 3 } else { 0 };
+        let lib = &stem[prefix..];
+        let path = parent.unwrap_or_else(|| Path::new(""));
+        if sess.target.is_like_msvc {
+            // When producing a dll, the MSVC linker may not actually emit a
+            // `foo.lib` file if the dll doesn't actually export any symbols, so we
+            // check to see if the file is there and just omit linking to it if it's
+            // not present.
+            let name = format!("{}.dll.lib", lib);
+            if path.join(&name).exists() {
+                lib_args.push(name);
+            }
+        } else {
+            lib_args.push(format!("-l{}", lib));
+        }
+    }
     if !lib_args.is_empty() {
         sess.emit_note(errors::StaticLibraryNativeArtifacts);
         // Prefix for greppability
diff --git a/compiler/rustc_metadata/src/dependency_format.rs b/compiler/rustc_metadata/src/dependency_format.rs
index 39ef4276faf..72b208a7132 100644
--- a/compiler/rustc_metadata/src/dependency_format.rs
+++ b/compiler/rustc_metadata/src/dependency_format.rs
@@ -89,11 +89,25 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
         // to try to eagerly statically link all dependencies. This is normally
         // done for end-product dylibs, not intermediate products.
         //
-        // Treat cdylibs similarly. If `-C prefer-dynamic` is set, the caller may
-        // be code-size conscious, but without it, it makes sense to statically
-        // link a cdylib.
-        CrateType::Dylib | CrateType::Cdylib if !sess.opts.cg.prefer_dynamic => Linkage::Static,
-        CrateType::Dylib | CrateType::Cdylib => Linkage::Dynamic,
+        // Treat cdylibs and staticlibs similarly. If `-C prefer-dynamic` is set,
+        // the caller may be code-size conscious, but without it, it makes sense
+        // to statically link a cdylib or staticlib. For staticlibs we use
+        // `-Z staticlib-prefer-dynamic` for now. This may be merged into
+        // `-C prefer-dynamic` in the future.
+        CrateType::Dylib | CrateType::Cdylib => {
+            if sess.opts.cg.prefer_dynamic {
+                Linkage::Dynamic
+            } else {
+                Linkage::Static
+            }
+        }
+        CrateType::Staticlib => {
+            if sess.opts.unstable_opts.staticlib_prefer_dynamic {
+                Linkage::Dynamic
+            } else {
+                Linkage::Static
+            }
+        }
 
         // If the global prefer_dynamic switch is turned off, or the final
         // executable will be statically linked, prefer static crate linkage.
@@ -108,9 +122,6 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
         // No linkage happens with rlibs, we just needed the metadata (which we
         // got long ago), so don't bother with anything.
         CrateType::Rlib => Linkage::NotLinked,
-
-        // staticlibs must have all static dependencies.
-        CrateType::Staticlib => Linkage::Static,
     };
 
     match preferred_linkage {
@@ -123,9 +134,9 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
                 return v;
             }
 
-            // Staticlibs and static executables must have all static dependencies.
+            // Static executables must have all static dependencies.
             // If any are not found, generate some nice pretty errors.
-            if ty == CrateType::Staticlib
+            if (ty == CrateType::Staticlib && !sess.opts.unstable_opts.staticlib_allow_rdylib_deps)
                 || (ty == CrateType::Executable
                     && sess.crt_static(Some(ty))
                     && !sess.target.crt_static_allows_dylibs)
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 5cbd7f98b6b..5976b9aa3e7 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1720,6 +1720,10 @@ options! {
     #[rustc_lint_opt_deny_field_access("use `Session::stack_protector` instead of this field")]
     stack_protector: StackProtector = (StackProtector::None, parse_stack_protector, [TRACKED],
         "control stack smash protection strategy (`rustc --print stack-protector-strategies` for details)"),
+    staticlib_allow_rdylib_deps: bool = (false, parse_bool, [TRACKED],
+        "allow staticlibs to have rust dylib dependencies"),
+    staticlib_prefer_dynamic: bool = (false, parse_bool, [TRACKED],
+        "prefer dynamic linking to static linking for staticlibs (default: no)"),
     strict_init_checks: bool = (false, parse_bool, [TRACKED],
         "control if mem::uninitialized and mem::zeroed panic on more UB"),
     strip: Strip = (Strip::None, parse_strip, [UNTRACKED],
diff --git a/tests/run-make/staticlib-dylib-linkage/Makefile b/tests/run-make/staticlib-dylib-linkage/Makefile
new file mode 100644
index 00000000000..a1e86a7ce4b
--- /dev/null
+++ b/tests/run-make/staticlib-dylib-linkage/Makefile
@@ -0,0 +1,21 @@
+include ../tools.mk
+
+# ignore-cross-compile
+# ignore-msvc FIXME(bjorn3) can't figure out how to link with the MSVC toolchain
+# ignore-wasm wasm doesn't support dynamic libraries
+
+all:
+	$(RUSTC) -C prefer-dynamic bar.rs
+	$(RUSTC) foo.rs --crate-type staticlib --print native-static-libs \
+		-Z staticlib-allow-rdylib-deps 2>&1 | grep 'note: native-static-libs: ' \
+		| sed 's/note: native-static-libs: \(.*\)/\1/' > $(TMPDIR)/libs.txt
+	cat $(TMPDIR)/libs.txt
+
+ifdef IS_MSVC
+	$(CC) $(CFLAGS) /c foo.c /Fo:$(TMPDIR)/foo.o
+	$(RUSTC_LINKER) $(TMPDIR)/foo.o $(TMPDIR)/foo.lib $$(cat $(TMPDIR)/libs.txt) $(call OUT_EXE,foo)
+else
+	$(CC) $(CFLAGS) foo.c -L $(TMPDIR) -lfoo $$(cat $(TMPDIR)/libs.txt) -o $(call RUN_BINFILE,foo)
+endif
+
+	$(call RUN,foo)
diff --git a/tests/run-make/staticlib-dylib-linkage/bar.rs b/tests/run-make/staticlib-dylib-linkage/bar.rs
new file mode 100644
index 00000000000..b3a7539abae
--- /dev/null
+++ b/tests/run-make/staticlib-dylib-linkage/bar.rs
@@ -0,0 +1,5 @@
+#![crate_type = "dylib"]
+
+pub fn bar() {
+    println!("hello!");
+}
diff --git a/tests/run-make/staticlib-dylib-linkage/foo.c b/tests/run-make/staticlib-dylib-linkage/foo.c
new file mode 100644
index 00000000000..154f9682ef8
--- /dev/null
+++ b/tests/run-make/staticlib-dylib-linkage/foo.c
@@ -0,0 +1,10 @@
+#include <assert.h>
+
+extern void foo();
+extern unsigned bar(unsigned a, unsigned b);
+
+int main() {
+  foo();
+  assert(bar(1, 2) == 3);
+  return 0;
+}
diff --git a/tests/run-make/staticlib-dylib-linkage/foo.rs b/tests/run-make/staticlib-dylib-linkage/foo.rs
new file mode 100644
index 00000000000..af439391c75
--- /dev/null
+++ b/tests/run-make/staticlib-dylib-linkage/foo.rs
@@ -0,0 +1,13 @@
+#![crate_type = "staticlib"]
+
+extern crate bar;
+
+#[no_mangle]
+pub extern "C" fn foo() {
+    bar::bar();
+}
+
+#[no_mangle]
+pub extern "C" fn bar(a: u32, b: u32) -> u32 {
+    a + b
+}