speed up static linking by combining `ar` invocations

2 files changed, 161 insertions, 67 deletions

diff --git a/src/librustc/back/link.rs b/src/librustc/back/link.rs
index c7dca1b93ef..1f02a348be7 100644
--- a/src/librustc/back/link.rs
+++ b/src/librustc/back/link.rs
@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use super::archive::{Archive, ArchiveConfig, METADATA_FILENAME};
+use super::archive::{Archive, ArchiveBuilder, ArchiveConfig, METADATA_FILENAME};
 use super::rpath;
 use super::rpath::RPathConfig;
 use super::svh::Svh;
@@ -983,7 +983,7 @@ fn link_binary_output(sess: &Session,
 
     match crate_type {
         config::CrateTypeRlib => {
-            link_rlib(sess, Some(trans), &obj_filename, &out_filename);
+            link_rlib(sess, Some(trans), &obj_filename, &out_filename).build();
         }
         config::CrateTypeStaticlib => {
             link_staticlib(sess, &obj_filename, &out_filename);
@@ -1019,7 +1019,7 @@ fn archive_search_paths(sess: &Session) -> Vec<Path> {
 fn link_rlib<'a>(sess: &'a Session,
                  trans: Option<&CrateTranslation>, // None == no metadata/bytecode
                  obj_filename: &Path,
-                 out_filename: &Path) -> Archive<'a> {
+                 out_filename: &Path) -> ArchiveBuilder<'a> {
     let handler = &sess.diagnostic().handler;
     let config = ArchiveConfig {
         handler: handler,
@@ -1028,17 +1028,30 @@ fn link_rlib<'a>(sess: &'a Session,
         os: sess.targ_cfg.os,
         maybe_ar_prog: sess.opts.cg.ar.clone()
     };
-    let mut a = Archive::create(config, obj_filename);
+    let mut ab = ArchiveBuilder::create(config);
+    ab.add_file(obj_filename).unwrap();
 
     for &(ref l, kind) in sess.cstore.get_used_libraries().borrow().iter() {
         match kind {
             cstore::NativeStatic => {
-                a.add_native_library(l.as_slice()).unwrap();
+                ab.add_native_library(l.as_slice()).unwrap();
             }
             cstore::NativeFramework | cstore::NativeUnknown => {}
         }
     }
 
+    // After adding all files to the archive, we need to update the
+    // symbol table of the archive.
+    ab.update_symbols();
+
+    let mut ab = match sess.targ_cfg.os {
+        // For OSX/iOS, we must be careful to update symbols only when adding
+        // object files.  We're about to start adding non-object files, so run
+        // `ar` now to process the object files.
+        abi::OsMacos | abi::OsiOS => ab.build().extend(),
+        _ => ab,
+    };
+
     // Note that it is important that we add all of our non-object "magical
     // files" *after* all of the object files in the archive. The reason for
     // this is as follows:
@@ -1078,7 +1091,7 @@ fn link_rlib<'a>(sess: &'a Session,
                     sess.abort_if_errors();
                 }
             }
-            a.add_file(&metadata, false);
+            ab.add_file(&metadata).unwrap();
             remove(sess, &metadata);
 
             // For LTO purposes, the bytecode of this library is also inserted
@@ -1105,25 +1118,18 @@ fn link_rlib<'a>(sess: &'a Session,
                     sess.abort_if_errors()
                 }
             }
-            a.add_file(&bc_deflated, false);
+            ab.add_file(&bc_deflated).unwrap();
             remove(sess, &bc_deflated);
             if !sess.opts.cg.save_temps &&
                !sess.opts.output_types.contains(&OutputTypeBitcode) {
                 remove(sess, &bc);
             }
-
-            // After adding all files to the archive, we need to update the
-            // symbol table of the archive. This currently dies on OSX (see
-            // #11162), and isn't necessary there anyway
-            match sess.targ_cfg.os {
-                abi::OsMacos | abi::OsiOS => {}
-                _ => { a.update_symbols(); }
-            }
         }
 
         None => {}
     }
-    return a;
+
+    ab
 }
 
 // Create a static archive
@@ -1139,9 +1145,13 @@ fn link_rlib<'a>(sess: &'a Session,
 // link in the metadata object file (and also don't prepare the archive with a
 // metadata file).
 fn link_staticlib(sess: &Session, obj_filename: &Path, out_filename: &Path) {
-    let mut a = link_rlib(sess, None, obj_filename, out_filename);
-    a.add_native_library("morestack").unwrap();
-    a.add_native_library("compiler-rt").unwrap();
+    let ab = link_rlib(sess, None, obj_filename, out_filename);
+    let mut ab = match sess.targ_cfg.os {
+        abi::OsMacos | abi::OsiOS => ab.build().extend(),
+        _ => ab,
+    };
+    ab.add_native_library("morestack").unwrap();
+    ab.add_native_library("compiler-rt").unwrap();
 
     let crates = sess.cstore.get_used_crates(cstore::RequireStatic);
     let mut all_native_libs = vec![];
@@ -1155,12 +1165,15 @@ fn link_staticlib(sess: &Session, obj_filename: &Path, out_filename: &Path) {
                 continue
             }
         };
-        a.add_rlib(&p, name.as_slice(), sess.lto()).unwrap();
+        ab.add_rlib(&p, name.as_slice(), sess.lto()).unwrap();
 
         let native_libs = csearch::get_native_libraries(&sess.cstore, cnum);
         all_native_libs.extend(native_libs.move_iter());
     }
 
+    ab.update_symbols();
+    let _ = ab.build();
+
     if !all_native_libs.is_empty() {
         sess.warn("link against the following native artifacts when linking against \
                   this static library");
diff --git a/src/librustc_back/archive.rs b/src/librustc_back/archive.rs
index c4a9d9c80ef..e2cadf817d5 100644
--- a/src/librustc_back/archive.rs
+++ b/src/librustc_back/archive.rs
@@ -36,6 +36,17 @@ pub struct Archive<'a> {
     maybe_ar_prog: Option<String>
 }
 
+/// Helper for adding many files to an archive with a single invocation of
+/// `ar`.
+#[must_use = "must call build() to finish building the archive"]
+pub struct ArchiveBuilder<'a> {
+    archive: Archive<'a>,
+    work_dir: TempDir,
+    /// Filename of each member that should be added to the archive.
+    members: Vec<Path>,
+    should_update_symbols: bool,
+}
+
 fn run_ar(handler: &ErrorHandler, maybe_ar_prog: &Option<String>,
           args: &str, cwd: Option<&Path>,
           paths: &[&Path]) -> ProcessOutput {
@@ -85,10 +96,8 @@ fn run_ar(handler: &ErrorHandler, maybe_ar_prog: &Option<String>,
 }
 
 impl<'a> Archive<'a> {
-    /// Initializes a new static archive with the given object file
-    pub fn create<'b>(config: ArchiveConfig<'a>, initial_object: &'b Path) -> Archive<'a> {
+    fn new(config: ArchiveConfig<'a>) -> Archive<'a> {
         let ArchiveConfig { handler, dst, lib_search_paths, os, maybe_ar_prog } = config;
-        run_ar(handler, &maybe_ar_prog, "crus", None, [&dst, initial_object]);
         Archive {
             handler: handler,
             dst: dst,
@@ -100,17 +109,47 @@ impl<'a> Archive<'a> {
 
     /// Opens an existing static archive
     pub fn open(config: ArchiveConfig<'a>) -> Archive<'a> {
-        let ArchiveConfig { handler, dst, lib_search_paths, os, maybe_ar_prog } = config;
-        assert!(dst.exists());
-        Archive {
-            handler: handler,
-            dst: dst,
-            lib_search_paths: lib_search_paths,
-            os: os,
-            maybe_ar_prog: maybe_ar_prog
+        let archive = Archive::new(config);
+        assert!(archive.dst.exists());
+        archive
+    }
+
+    /// Removes a file from this archive
+    pub fn remove_file(&mut self, file: &str) {
+        run_ar(self.handler, &self.maybe_ar_prog, "d", None, [&self.dst, &Path::new(file)]);
+    }
+
+    /// Lists all files in an archive
+    pub fn files(&self) -> Vec<String> {
+        let output = run_ar(self.handler, &self.maybe_ar_prog, "t", None, [&self.dst]);
+        let output = str::from_utf8(output.output.as_slice()).unwrap();
+        // use lines_any because windows delimits output with `\r\n` instead of
+        // just `\n`
+        output.lines_any().map(|s| s.to_string()).collect()
+    }
+
+    /// Creates an `ArchiveBuilder` for adding files to this archive.
+    pub fn extend(self) -> ArchiveBuilder<'a> {
+        ArchiveBuilder::new(self)
+    }
+}
+
+impl<'a> ArchiveBuilder<'a> {
+    fn new(archive: Archive<'a>) -> ArchiveBuilder<'a> {
+        ArchiveBuilder {
+            archive: archive,
+            work_dir: TempDir::new("rsar").unwrap(),
+            members: vec![],
+            should_update_symbols: false,
         }
     }
 
+    /// Create a new static archive, ready for adding files.
+    pub fn create(config: ArchiveConfig<'a>) -> ArchiveBuilder<'a> {
+        let archive = Archive::new(config);
+        ArchiveBuilder::new(archive)
+    }
+
     /// Adds all of the contents of a native library to this archive. This will
     /// search in the relevant locations for a library named `name`.
     pub fn add_native_library(&mut self, name: &str) -> io::IoResult<()> {
@@ -135,48 +174,96 @@ impl<'a> Archive<'a> {
     }
 
     /// Adds an arbitrary file to this archive
-    pub fn add_file(&mut self, file: &Path, has_symbols: bool) {
-        let cmd = if has_symbols {"r"} else {"rS"};
-        run_ar(self.handler, &self.maybe_ar_prog, cmd, None, [&self.dst, file]);
-    }
-
-    /// Removes a file from this archive
-    pub fn remove_file(&mut self, file: &str) {
-        run_ar(self.handler, &self.maybe_ar_prog, "d", None, [&self.dst, &Path::new(file)]);
+    pub fn add_file(&mut self, file: &Path) -> io::IoResult<()> {
+        let filename = Path::new(file.filename().unwrap());
+        let new_file = self.work_dir.path().join(&filename);
+        try!(fs::copy(file, &new_file));
+        self.members.push(filename);
+        Ok(())
     }
 
-    /// Updates all symbols in the archive (runs 'ar s' over it)
+    /// Indicate that the next call to `build` should updates all symbols in
+    /// the archive (run 'ar s' over it).
     pub fn update_symbols(&mut self) {
-        run_ar(self.handler, &self.maybe_ar_prog, "s", None, [&self.dst]);
+        self.should_update_symbols = true;
     }
 
-    /// Lists all files in an archive
-    pub fn files(&self) -> Vec<String> {
-        let output = run_ar(self.handler, &self.maybe_ar_prog, "t", None, [&self.dst]);
-        let output = str::from_utf8(output.output.as_slice()).unwrap();
-        // use lines_any because windows delimits output with `\r\n` instead of
-        // just `\n`
-        output.lines_any().map(|s| s.to_string()).collect()
+    /// Combine the provided files, rlibs, and native libraries into a single
+    /// `Archive`.
+    pub fn build(self) -> Archive<'a> {
+        // Get an absolute path to the destination, so `ar` will work even
+        // though we run it from `self.work_dir`.
+        let abs_dst = os::getcwd().join(&self.archive.dst);
+        assert!(!abs_dst.is_relative());
+        let mut args = vec![&abs_dst];
+        let mut total_len = abs_dst.as_vec().len();
+
+        if self.members.is_empty() {
+            // OSX `ar` does not allow using `r` with no members, but it does
+            // allow running `ar s file.a` to update symbols only.
+            if self.should_update_symbols {
+                run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
+                       "s", Some(self.work_dir.path()), args.as_slice());
+            }
+            return self.archive;
+        }
+
+        // Don't allow the total size of `args` to grow beyond 32,000 bytes.
+        // Windows will raise an error if the argument string is longer than
+        // 32,768, and we leave a bit of extra space for the program name.
+        static ARG_LENGTH_LIMIT: uint = 32000;
+
+        for member_name in self.members.iter() {
+            let len = member_name.as_vec().len();
+
+            // `len + 1` to account for the space that's inserted before each
+            // argument.  (Windows passes command-line arguments as a single
+            // string, not an array of strings.)
+            if total_len + len + 1 > ARG_LENGTH_LIMIT {
+                // Add the archive members seen so far, without updating the
+                // symbol table (`S`).
+                run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
+                       "cruS", Some(self.work_dir.path()), args.as_slice());
+
+                args.clear();
+                args.push(&abs_dst);
+                total_len = abs_dst.as_vec().len();
+            }
+
+            args.push(member_name);
+            total_len += len + 1;
+        }
+
+        // Add the remaining archive members, and update the symbol table if
+        // necessary.
+        let flags = if self.should_update_symbols { "crus" } else { "cruS" };
+        run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
+               flags, Some(self.work_dir.path()), args.as_slice());
+
+        self.archive
     }
 
     fn add_archive(&mut self, archive: &Path, name: &str,
                    skip: &[&str]) -> io::IoResult<()> {
         let loc = TempDir::new("rsar").unwrap();
 
-        // First, extract the contents of the archive to a temporary directory
+        // First, extract the contents of the archive to a temporary directory.
+        // We don't unpack directly into `self.work_dir` due to the possibility
+        // of filename collisions.
         let archive = os::make_absolute(archive);
-        run_ar(self.handler, &self.maybe_ar_prog, "x", Some(loc.path()), [&archive]);
+        run_ar(self.archive.handler, &self.archive.maybe_ar_prog,
+               "x", Some(loc.path()), [&archive]);
 
         // Next, we must rename all of the inputs to "guaranteed unique names".
-        // The reason for this is that archives are keyed off the name of the
-        // files, so if two files have the same name they will override one
-        // another in the archive (bad).
+        // We move each file into `self.work_dir` under its new unique name.
+        // The reason for this renaming is that archives are keyed off the name
+        // of the files, so if two files have the same name they will override
+        // one another in the archive (bad).
         //
         // We skip any files explicitly desired for skipping, and we also skip
         // all SYMDEF files as these are just magical placeholders which get
         // re-created when we make a new archive anyway.
         let files = try!(fs::readdir(loc.path()));
-        let mut inputs = Vec::new();
         for file in files.iter() {
             let filename = file.filename_str().unwrap();
             if skip.iter().any(|s| *s == filename) { continue }
@@ -192,21 +279,15 @@ impl<'a> Archive<'a> {
             } else {
                 filename
             };
-            let new_filename = file.with_filename(filename);
+            let new_filename = self.work_dir.path().join(filename.as_slice());
             try!(fs::rename(file, &new_filename));
-            inputs.push(new_filename);
+            self.members.push(Path::new(filename));
         }
-        if inputs.len() == 0 { return Ok(()) }
-
-        // Finally, add all the renamed files to this archive
-        let mut args = vec!(&self.dst);
-        args.extend(inputs.iter());
-        run_ar(self.handler, &self.maybe_ar_prog, "r", None, args.as_slice());
         Ok(())
     }
 
     fn find_library(&self, name: &str) -> Path {
-        let (osprefix, osext) = match self.os {
+        let (osprefix, osext) = match self.archive.os {
             abi::OsWin32 => ("", "lib"), _ => ("lib", "a"),
         };
         // On Windows, static libraries sometimes show up as libfoo.a and other
@@ -214,7 +295,7 @@ impl<'a> Archive<'a> {
         let oslibname = format!("{}{}.{}", osprefix, name, osext);
         let unixlibname = format!("lib{}.a", name);
 
-        for path in self.lib_search_paths.iter() {
+        for path in self.archive.lib_search_paths.iter() {
             debug!("looking for {} inside {}", name, path.display());
             let test = path.join(oslibname.as_slice());
             if test.exists() { return test }
@@ -223,9 +304,9 @@ impl<'a> Archive<'a> {
                 if test.exists() { return test }
             }
         }
-        self.handler.fatal(format!("could not find native static library `{}`, \
-                                 perhaps an -L flag is missing?",
-                                name).as_slice());
+        self.archive.handler.fatal(format!("could not find native static library `{}`, \
+                                            perhaps an -L flag is missing?",
+                                           name).as_slice());
     }
 }