Rollup merge of #60385 - nnethercote:earlier-metadata, r=alexcrichton

Emit metadata files earlier

This will make cargo pipelining much more effective.

3 files changed, 69 insertions, 110 deletions

diff --git a/src/librustc_codegen_ssa/back/link.rs b/src/librustc_codegen_ssa/back/link.rs
index 4cae20b698a..f25891d77ce 100644
--- a/src/librustc_codegen_ssa/back/link.rs
+++ b/src/librustc_codegen_ssa/back/link.rs
@@ -7,7 +7,7 @@ use rustc::session::config::{
 };
 use rustc::session::search_paths::PathKind;
 use rustc::middle::dependency_format::Linkage;
-use rustc::middle::cstore::{LibSource, NativeLibrary, NativeLibraryKind};
+use rustc::middle::cstore::{EncodedMetadata, LibSource, NativeLibrary, NativeLibraryKind};
 use rustc::util::common::{time, time_ext};
 use rustc::hir::def_id::CrateNum;
 use rustc_data_structures::fx::FxHashSet;
@@ -50,9 +50,9 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
                                               outputs: &OutputFilenames,
                                               crate_name: &str,
                                               target_cpu: &str) {
+    let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
     for &crate_type in sess.crate_types.borrow().iter() {
         // Ignore executable crates if we have -Z no-codegen, as they will error.
-        let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
         if (sess.opts.debugging_opts.no_codegen || !sess.opts.output_types.should_codegen()) &&
            !output_metadata &&
            crate_type == config::CrateType::Executable {
@@ -63,12 +63,43 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
            bug!("invalid output type `{:?}` for target os `{}`",
                 crate_type, sess.opts.target_triple);
         }
-        link_binary_output::<B>(sess,
-                                codegen_results,
-                                crate_type,
-                                outputs,
-                                crate_name,
-                                target_cpu);
+
+        for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) {
+            check_file_is_writeable(obj, sess);
+        }
+
+        let tmpdir = TempFileBuilder::new().prefix("rustc").tempdir().unwrap_or_else(|err|
+            sess.fatal(&format!("couldn't create a temp dir: {}", err)));
+
+        if outputs.outputs.should_codegen() {
+            let out_filename = out_filename(sess, crate_type, outputs, crate_name);
+            match crate_type {
+                config::CrateType::Rlib => {
+                    link_rlib::<B>(sess,
+                              codegen_results,
+                              RlibFlavor::Normal,
+                              &out_filename,
+                              &tmpdir).build();
+                }
+                config::CrateType::Staticlib => {
+                    link_staticlib::<B>(sess, codegen_results, &out_filename, &tmpdir);
+                }
+                _ => {
+                    link_natively::<B>(
+                        sess,
+                        crate_type,
+                        &out_filename,
+                        codegen_results,
+                        tmpdir.path(),
+                        target_cpu,
+                    );
+                }
+            }
+        }
+
+        if sess.opts.cg.save_temps {
+            let _ = tmpdir.into_path();
+        }
     }
 
     // Remove the temporary object file and metadata if we aren't saving temps
@@ -85,7 +116,7 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
             if let Some(ref obj) = metadata_module.object {
                 remove(sess, obj);
             }
-         }
+        }
         if let Some(ref allocator_module) = codegen_results.allocator_module {
             if let Some(ref obj) = allocator_module.object {
                 remove(sess, obj);
@@ -97,73 +128,6 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
     }
 }
 
-fn link_binary_output<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
-                                                 codegen_results: &CodegenResults,
-                                                 crate_type: config::CrateType,
-                                                 outputs: &OutputFilenames,
-                                                 crate_name: &str,
-                                                 target_cpu: &str) {
-    for obj in codegen_results.modules.iter().filter_map(|m| m.object.as_ref()) {
-        check_file_is_writeable(obj, sess);
-    }
-
-    if outputs.outputs.contains_key(&OutputType::Metadata) {
-        let out_filename = filename_for_metadata(sess, crate_name, outputs);
-        // To avoid races with another rustc process scanning the output directory,
-        // we need to write the file somewhere else and atomically move it to its
-        // final destination, with a `fs::rename` call. In order for the rename to
-        // always succeed, the temporary file needs to be on the same filesystem,
-        // which is why we create it inside the output directory specifically.
-        let metadata_tmpdir = TempFileBuilder::new()
-            .prefix("rmeta")
-            .tempdir_in(out_filename.parent().unwrap())
-            .unwrap_or_else(|err| sess.fatal(&format!("couldn't create a temp dir: {}", err)));
-        let metadata = emit_metadata(sess, codegen_results, &metadata_tmpdir);
-        match fs::rename(&metadata, &out_filename) {
-            Ok(_) => {
-                if sess.opts.debugging_opts.emit_directives {
-                    sess.parse_sess.span_diagnostic.maybe_emit_json_directive(
-                        format!("metadata file written: {}", out_filename.display()));
-                }
-            }
-            Err(e) => sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)),
-        }
-    }
-
-    let tmpdir = TempFileBuilder::new().prefix("rustc").tempdir().unwrap_or_else(|err|
-        sess.fatal(&format!("couldn't create a temp dir: {}", err)));
-
-    if outputs.outputs.should_codegen() {
-        let out_filename = out_filename(sess, crate_type, outputs, crate_name);
-        match crate_type {
-            config::CrateType::Rlib => {
-                link_rlib::<B>(sess,
-                          codegen_results,
-                          RlibFlavor::Normal,
-                          &out_filename,
-                          &tmpdir).build();
-            }
-            config::CrateType::Staticlib => {
-                link_staticlib::<B>(sess, codegen_results, &out_filename, &tmpdir);
-            }
-            _ => {
-                link_natively::<B>(
-                    sess,
-                    crate_type,
-                    &out_filename,
-                    codegen_results,
-                    tmpdir.path(),
-                    target_cpu,
-                );
-            }
-        }
-    }
-
-    if sess.opts.cg.save_temps {
-        let _ = tmpdir.into_path();
-    }
-}
-
 // The third parameter is for env vars, used on windows to set up the
 // path for MSVC to find its DLLs, and gcc to find its bundled
 // toolchain
@@ -261,13 +225,13 @@ pub fn each_linked_rlib(sess: &Session,
 /// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a
 /// directory being searched for `extern crate` (observing an incomplete file).
 /// The returned path is the temporary file containing the complete metadata.
-fn emit_metadata<'a>(
+pub fn emit_metadata<'a>(
     sess: &'a Session,
-    codegen_results: &CodegenResults,
+    metadata: &EncodedMetadata,
     tmpdir: &TempDir
 ) -> PathBuf {
     let out_filename = tmpdir.path().join(METADATA_FILENAME);
-    let result = fs::write(&out_filename, &codegen_results.metadata.raw_data);
+    let result = fs::write(&out_filename, &metadata.raw_data);
 
     if let Err(e) = result {
         sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
@@ -351,7 +315,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
         RlibFlavor::Normal => {
             // Instead of putting the metadata in an object file section, rlibs
             // contain the metadata in a separate file.
-            ab.add_file(&emit_metadata(sess, codegen_results, tmpdir));
+            ab.add_file(&emit_metadata(sess, &codegen_results.metadata, tmpdir));
 
             // For LTO purposes, the bytecode of this library is also inserted
             // into the archive.
diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs
index 3046c069981..3cd47dfbb29 100644
--- a/src/librustc_codegen_ssa/base.rs
+++ b/src/librustc_codegen_ssa/base.rs
@@ -17,6 +17,7 @@ use crate::{ModuleCodegen, ModuleKind, CachedModuleCodegen};
 
 use rustc::dep_graph::cgu_reuse_tracker::CguReuse;
 use rustc::hir::def_id::{DefId, LOCAL_CRATE};
+use rustc::middle::cstore::EncodedMetadata;
 use rustc::middle::lang_items::StartFnLangItem;
 use rustc::middle::weak_lang_items;
 use rustc::mir::mono::{Stats, CodegenUnitNameBuilder};
@@ -25,7 +26,7 @@ use rustc::ty::layout::{self, Align, TyLayout, LayoutOf, VariantIdx, HasTyCtxt};
 use rustc::ty::query::Providers;
 use rustc::middle::cstore::{self, LinkagePreference};
 use rustc::util::common::{time, print_time_passes_entry};
-use rustc::session::config::{self, CrateType, EntryFnType, Lto};
+use rustc::session::config::{self, EntryFnType, Lto};
 use rustc::session::Session;
 use rustc_mir::monomorphize::item::DefPathBasedNames;
 use rustc_mir::monomorphize::Instance;
@@ -530,26 +531,13 @@ pub const CODEGEN_WORKER_ID: usize = ::std::usize::MAX;
 pub fn codegen_crate<B: ExtraBackendMethods>(
     backend: B,
     tcx: TyCtxt<'a, 'tcx, 'tcx>,
+    metadata: EncodedMetadata,
+    need_metadata_module: bool,
     rx: mpsc::Receiver<Box<dyn Any + Send>>
 ) -> OngoingCodegen<B> {
 
     check_for_rustc_errors_attr(tcx);
 
-    let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
-
-    // Codegen the metadata.
-    tcx.sess.profiler(|p| p.start_activity("codegen crate metadata"));
-
-    let metadata_cgu_name = cgu_name_builder.build_cgu_name(LOCAL_CRATE,
-                                                            &["crate"],
-                                                            Some("metadata")).as_str()
-                                                                             .to_string();
-    let mut metadata_llvm_module = backend.new_metadata(tcx, &metadata_cgu_name);
-    let metadata = time(tcx.sess, "write metadata", || {
-        backend.write_metadata(tcx, &mut metadata_llvm_module)
-    });
-    tcx.sess.profiler(|p| p.end_activity("codegen crate metadata"));
-
     // Skip crate items and just output metadata in -Z no-codegen mode.
     if tcx.sess.opts.debugging_opts.no_codegen ||
        !tcx.sess.opts.output_types.should_codegen() {
@@ -569,6 +557,8 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
         return ongoing_codegen;
     }
 
+    let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
+
     // Run the monomorphization collector and partition the collected items into
     // codegen units.
     let codegen_units = tcx.collect_and_partition_mono_items(LOCAL_CRATE).1;
@@ -632,17 +622,21 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
         ongoing_codegen.submit_pre_codegened_module_to_llvm(tcx, allocator_module);
     }
 
-    let needs_metadata_module = tcx.sess.crate_types.borrow().iter().any(|ct| {
-        match *ct {
-            CrateType::Dylib |
-            CrateType::ProcMacro => true,
-            CrateType::Executable |
-            CrateType::Rlib |
-            CrateType::Staticlib |
-            CrateType::Cdylib => false,
-        }
-    });
-    if needs_metadata_module {
+    if need_metadata_module {
+        // Codegen the encoded metadata.
+        tcx.sess.profiler(|p| p.start_activity("codegen crate metadata"));
+
+        let metadata_cgu_name = cgu_name_builder.build_cgu_name(LOCAL_CRATE,
+                                                                &["crate"],
+                                                                Some("metadata")).as_str()
+                                                                                 .to_string();
+        let mut metadata_llvm_module = backend.new_metadata(tcx, &metadata_cgu_name);
+        time(tcx.sess, "write compressed metadata", || {
+            backend.write_compressed_metadata(tcx, &ongoing_codegen.metadata,
+                                              &mut metadata_llvm_module);
+        });
+        tcx.sess.profiler(|p| p.end_activity("codegen crate metadata"));
+
         let metadata_module = ModuleCodegen {
             name: metadata_cgu_name,
             module_llvm: metadata_llvm_module,
diff --git a/src/librustc_codegen_ssa/traits/backend.rs b/src/librustc_codegen_ssa/traits/backend.rs
index a9e0eadb198..530eba516a6 100644
--- a/src/librustc_codegen_ssa/traits/backend.rs
+++ b/src/librustc_codegen_ssa/traits/backend.rs
@@ -33,11 +33,12 @@ impl<'tcx, T> Backend<'tcx> for T where
 
 pub trait ExtraBackendMethods: CodegenBackend + WriteBackendMethods + Sized + Send {
     fn new_metadata(&self, sess: TyCtxt<'_, '_, '_>, mod_name: &str) -> Self::Module;
-    fn write_metadata<'b, 'gcx>(
+    fn write_compressed_metadata<'b, 'gcx>(
         &self,
         tcx: TyCtxt<'b, 'gcx, 'gcx>,
-        metadata: &mut Self::Module,
-    ) -> EncodedMetadata;
+        metadata: &EncodedMetadata,
+        llvm_module: &mut Self::Module,
+    );
     fn codegen_allocator<'b, 'gcx>(
         &self,
         tcx: TyCtxt<'b, 'gcx, 'gcx>,