From 4a824275b9b057f43a7dbab84583ad7c2a1a2e63 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 9 Jul 2015 00:14:20 -0700 Subject: trans: Use LLVM's writeArchive to modify archives We have previously always relied upon an external tool, `ar`, to modify archives that the compiler produces (staticlibs, rlibs, etc). This approach, however, has a number of downsides: * Spawning a process is relatively expensive for small compilations * Encoding arguments across process boundaries often incurs unnecessary overhead or lossiness. For example `ar` has a tough time dealing with files that have the same name in archives, and the compiler copies many files around to ensure they can be passed to `ar` in a reasonable fashion. * Most `ar` programs found do **not** have the ability to target arbitrary platforms, so this is an extra tool which needs to be found/specified when cross compiling. The LLVM project has had a tool called `llvm-ar` for quite some time now, but it wasn't available in the standard LLVM libraries (it was just a standalone program). Recently, however, in LLVM 3.7, this functionality has been moved to a library and is now accessible by consumers of LLVM via the `writeArchive` function. This commit migrates our archive bindings to no longer invoke `ar` by default but instead make a library call to LLVM to do various operations. This solves all of the downsides listed above: * Archive management is now much faster, for example creating a "hello world" staticlib is now 6x faster (50ms => 8ms). Linking dynamic libraries also recently started requiring modification of rlibs, and linking a hello world dynamic library is now 2x faster. * The compiler is now one step closer to "hassle free" cross compilation because no external tool is needed for managing archives, LLVM does the right thing! This commit does not remove support for calling a system `ar` utility currently. We will continue to maintain compatibility with LLVM 3.5 and 3.6 looking forward (so the system LLVM can be used wherever possible), and in these cases we must shell out to a system utility. All nightly builds of Rust, however, will stop needing a system `ar`. --- src/rustllvm/ArchiveWrapper.cpp | 168 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 src/rustllvm/ArchiveWrapper.cpp (limited to 'src/rustllvm/ArchiveWrapper.cpp') diff --git a/src/rustllvm/ArchiveWrapper.cpp b/src/rustllvm/ArchiveWrapper.cpp new file mode 100644 index 00000000000..2e94c196935 --- /dev/null +++ b/src/rustllvm/ArchiveWrapper.cpp @@ -0,0 +1,168 @@ +// Copyright 2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#include "rustllvm.h" + +#include "llvm/Object/Archive.h" + +#if LLVM_VERSION_MINOR >= 7 +#include "llvm/Object/ArchiveWriter.h" +#endif + +using namespace llvm; +using namespace llvm::object; + +struct LLVMRustArchiveMember { + const char *filename; + const char *name; + Archive::Child child; + + LLVMRustArchiveMember(): filename(NULL), name(NULL), child(NULL, NULL) {} + ~LLVMRustArchiveMember() {} +}; + +#if LLVM_VERSION_MINOR >= 6 +typedef OwningBinary RustArchive; +#define GET_ARCHIVE(a) ((a)->getBinary()) +#else +typedef Archive RustArchive; +#define GET_ARCHIVE(a) (a) +#endif + +extern "C" void* +LLVMRustOpenArchive(char *path) { + ErrorOr> buf_or = MemoryBuffer::getFile(path, + -1, + false); + if (!buf_or) { + LLVMRustSetLastError(buf_or.getError().message().c_str()); + return nullptr; + } + +#if LLVM_VERSION_MINOR >= 6 + ErrorOr> archive_or = + Archive::create(buf_or.get()->getMemBufferRef()); + + if (!archive_or) { + LLVMRustSetLastError(archive_or.getError().message().c_str()); + return nullptr; + } + + OwningBinary *ret = new OwningBinary( + std::move(archive_or.get()), std::move(buf_or.get())); +#else + std::error_code err; + Archive *ret = new Archive(std::move(buf_or.get()), err); + if (err) { + LLVMRustSetLastError(err.message().c_str()); + return nullptr; + } +#endif + + return ret; +} + +extern "C" void +LLVMRustDestroyArchive(RustArchive *ar) { + delete ar; +} + +struct RustArchiveIterator { + Archive::child_iterator cur; + Archive::child_iterator end; +}; + +extern "C" RustArchiveIterator* +LLVMRustArchiveIteratorNew(RustArchive *ra) { + Archive *ar = GET_ARCHIVE(ra); + RustArchiveIterator *rai = new RustArchiveIterator(); + rai->cur = ar->child_begin(); + rai->end = ar->child_end(); + return rai; +} + +extern "C" const Archive::Child* +LLVMRustArchiveIteratorNext(RustArchiveIterator *rai) { + if (rai->cur == rai->end) + return NULL; + const Archive::Child *cur = rai->cur.operator->(); + Archive::Child *ret = new Archive::Child(*cur); + ++rai->cur; + return ret; +} + +extern "C" void +LLVMRustArchiveChildFree(Archive::Child *child) { + delete child; +} + +extern "C" void +LLVMRustArchiveIteratorFree(RustArchiveIterator *rai) { + delete rai; +} + +extern "C" const char* +LLVMRustArchiveChildName(const Archive::Child *child, size_t *size) { + ErrorOr name_or_err = child->getName(); + if (name_or_err.getError()) + return NULL; + StringRef name = name_or_err.get(); + *size = name.size(); + return name.data(); +} + +extern "C" const char* +LLVMRustArchiveChildData(Archive::Child *child, size_t *size) { + StringRef buf = child->getBuffer(); + *size = buf.size(); + return buf.data(); +} + +extern "C" LLVMRustArchiveMember* +LLVMRustArchiveMemberNew(char *Filename, char *Name, Archive::Child *child) { + LLVMRustArchiveMember *Member = new LLVMRustArchiveMember; + Member->filename = Filename; + Member->name = Name; + if (child) + Member->child = *child; + return Member; +} + +extern "C" void +LLVMRustArchiveMemberFree(LLVMRustArchiveMember *Member) { + delete Member; +} + +extern "C" int +LLVMRustWriteArchive(char *Dst, + size_t NumMembers, + const LLVMRustArchiveMember **NewMembers, + bool WriteSymbtab) { +#if LLVM_VERSION_MINOR >= 7 + std::vector Members; + + for (size_t i = 0; i < NumMembers; i++) { + auto Member = NewMembers[i]; + assert(Member->name); + if (Member->filename) { + Members.push_back(NewArchiveIterator(Member->filename, Member->name)); + } else { + Members.push_back(NewArchiveIterator(Member->child, Member->name)); + } + } + auto pair = writeArchive(Dst, Members, WriteSymbtab); + if (!pair.second) + return 0; + LLVMRustSetLastError(pair.second.message().c_str()); +#else + LLVMRustSetLastError("writing archives not supported with this LLVM version"); +#endif + return -1; +} -- cgit 1.4.1-3-g733a5 From 74e198126b19efb7871aa673ae17483753f067b0 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 16 Jul 2015 00:11:09 -0700 Subject: trans: Add kind to writeArchive Updates our LLVM bindings to be able to write out multiple kinds of archives. This commit also enables using LLVM instead of the system ar on all current targets. --- src/librustc_back/target/apple_base.rs | 1 + src/librustc_back/target/bitrig_base.rs | 1 + src/librustc_back/target/dragonfly_base.rs | 1 + src/librustc_back/target/freebsd_base.rs | 1 + src/librustc_back/target/netbsd_base.rs | 1 + src/librustc_back/target/openbsd_base.rs | 1 + src/librustc_llvm/archive_ro.rs | 3 +++ src/librustc_llvm/lib.rs | 12 +++++++++++- src/librustc_trans/back/archive.rs | 29 ++++++++++++++++++----------- src/rustllvm/ArchiveWrapper.cpp | 17 ++++++++++++++--- 10 files changed, 52 insertions(+), 15 deletions(-) (limited to 'src/rustllvm/ArchiveWrapper.cpp') diff --git a/src/librustc_back/target/apple_base.rs b/src/librustc_back/target/apple_base.rs index 795a2c18bc6..f34ba40a8b2 100644 --- a/src/librustc_back/target/apple_base.rs +++ b/src/librustc_back/target/apple_base.rs @@ -23,6 +23,7 @@ pub fn opts() -> TargetOptions { has_rpath: true, dll_prefix: "lib".to_string(), dll_suffix: ".dylib".to_string(), + archive_format: "bsd".to_string(), pre_link_args: Vec::new(), .. Default::default() } diff --git a/src/librustc_back/target/bitrig_base.rs b/src/librustc_back/target/bitrig_base.rs index 6e5a48c0ea1..9f6a1f1e530 100644 --- a/src/librustc_back/target/bitrig_base.rs +++ b/src/librustc_back/target/bitrig_base.rs @@ -22,6 +22,7 @@ pub fn opts() -> TargetOptions { position_independent_executables: true, pre_link_args: vec!( ), + archive_format: "bsd".to_string(), .. Default::default() } diff --git a/src/librustc_back/target/dragonfly_base.rs b/src/librustc_back/target/dragonfly_base.rs index a56621ff97e..51a371db724 100644 --- a/src/librustc_back/target/dragonfly_base.rs +++ b/src/librustc_back/target/dragonfly_base.rs @@ -29,6 +29,7 @@ pub fn opts() -> TargetOptions { "-Wl,--as-needed".to_string(), ), position_independent_executables: true, + archive_format: "bsd".to_string(), .. Default::default() } } diff --git a/src/librustc_back/target/freebsd_base.rs b/src/librustc_back/target/freebsd_base.rs index 3ec6307c72f..2c3d240dbf3 100644 --- a/src/librustc_back/target/freebsd_base.rs +++ b/src/librustc_back/target/freebsd_base.rs @@ -18,6 +18,7 @@ pub fn opts() -> TargetOptions { executables: true, morestack: true, has_rpath: true, + archive_format: "bsd".to_string(), .. Default::default() } diff --git a/src/librustc_back/target/netbsd_base.rs b/src/librustc_back/target/netbsd_base.rs index 0f2ab32be24..9b20bd927cb 100644 --- a/src/librustc_back/target/netbsd_base.rs +++ b/src/librustc_back/target/netbsd_base.rs @@ -27,6 +27,7 @@ pub fn opts() -> TargetOptions { "-Wl,--as-needed".to_string(), ), position_independent_executables: true, + archive_format: "bsd".to_string(), .. Default::default() } } diff --git a/src/librustc_back/target/openbsd_base.rs b/src/librustc_back/target/openbsd_base.rs index 0f2ab32be24..9b20bd927cb 100644 --- a/src/librustc_back/target/openbsd_base.rs +++ b/src/librustc_back/target/openbsd_base.rs @@ -27,6 +27,7 @@ pub fn opts() -> TargetOptions { "-Wl,--as-needed".to_string(), ), position_independent_executables: true, + archive_format: "bsd".to_string(), .. Default::default() } } diff --git a/src/librustc_llvm/archive_ro.rs b/src/librustc_llvm/archive_ro.rs index 2c6022bc614..85c0c721114 100644 --- a/src/librustc_llvm/archive_ro.rs +++ b/src/librustc_llvm/archive_ro.rs @@ -118,6 +118,9 @@ impl<'a> Child<'a> { unsafe { let mut data_len = 0; let data_ptr = ::LLVMRustArchiveChildData(self.ptr, &mut data_len); + if data_ptr.is_null() { + panic!("failed to read data from archive child"); + } slice::from_raw_parts(data_ptr as *const u8, data_len as usize) } } diff --git a/src/librustc_llvm/lib.rs b/src/librustc_llvm/lib.rs index 05f82b86ebb..051cc1c5bb2 100644 --- a/src/librustc_llvm/lib.rs +++ b/src/librustc_llvm/lib.rs @@ -452,6 +452,15 @@ pub enum DiagnosticKind { DK_OptimizationFailure, } +#[repr(C)] +#[derive(Copy, Clone)] +pub enum ArchiveKind { + K_GNU, + K_MIPS64, + K_BSD, + K_COFF, +} + // Opaque pointer types #[allow(missing_copy_implementations)] pub enum Module_opaque {} @@ -2119,7 +2128,8 @@ extern { pub fn LLVMRustWriteArchive(Dst: *const c_char, NumMembers: size_t, Members: *const RustArchiveMemberRef, - WriteSymbtab: bool) -> c_int; + WriteSymbtab: bool, + Kind: ArchiveKind) -> c_int; pub fn LLVMRustArchiveMemberNew(Filename: *const c_char, Name: *const c_char, Child: ArchiveChildRef) -> RustArchiveMemberRef; diff --git a/src/librustc_trans/back/archive.rs b/src/librustc_trans/back/archive.rs index cc3d1d842fa..02f4bc83b75 100644 --- a/src/librustc_trans/back/archive.rs +++ b/src/librustc_trans/back/archive.rs @@ -22,7 +22,7 @@ use std::str; use libc; use llvm::archive_ro::{ArchiveRO, Child}; -use llvm; +use llvm::{self, ArchiveKind}; use rustc::metadata::loader::METADATA_FILENAME; use rustc::session::Session; use rustc_back::tempdir::TempDir; @@ -208,28 +208,34 @@ impl<'a> ArchiveBuilder<'a> { /// Combine the provided files, rlibs, and native libraries into a single /// `Archive`. pub fn build(&mut self) { - let res = if self.using_llvm() { - self.build_with_llvm() - } else { - self.build_with_ar_cmd() + let res = match self.llvm_archive_kind() { + Some(kind) => self.build_with_llvm(kind), + None => self.build_with_ar_cmd(), }; if let Err(e) = res { self.config.sess.fatal(&format!("failed to build archive: {}", e)); } } - pub fn using_llvm(&self) -> bool { + pub fn llvm_archive_kind(&self) -> Option { if unsafe { llvm::LLVMVersionMinor() < 7 } { - return false + return None } // Currently LLVM only supports writing archives in the 'gnu' format. match &self.config.sess.target.target.options.archive_format[..] { - "gnu" => true, - _ => false, + "gnu" => Some(ArchiveKind::K_GNU), + "mips64" => Some(ArchiveKind::K_MIPS64), + "bsd" => Some(ArchiveKind::K_BSD), + "coff" => Some(ArchiveKind::K_COFF), + _ => None, } } + pub fn using_llvm(&self) -> bool { + self.llvm_archive_kind().is_some() + } + fn build_with_ar_cmd(&mut self) -> io::Result<()> { let removals = mem::replace(&mut self.removals, Vec::new()); let additions = mem::replace(&mut self.additions, Vec::new()); @@ -425,7 +431,7 @@ impl<'a> ArchiveBuilder<'a> { } } - fn build_with_llvm(&mut self) -> io::Result<()> { + fn build_with_llvm(&mut self, kind: ArchiveKind) -> io::Result<()> { let mut archives = Vec::new(); let mut strings = Vec::new(); let mut members = Vec::new(); @@ -482,7 +488,8 @@ impl<'a> ArchiveBuilder<'a> { let r = llvm::LLVMRustWriteArchive(dst.as_ptr(), members.len() as libc::size_t, members.as_ptr(), - self.should_update_symbols); + self.should_update_symbols, + kind); let ret = if r != 0 { let err = llvm::LLVMRustGetLastError(); let msg = if err.is_null() { diff --git a/src/rustllvm/ArchiveWrapper.cpp b/src/rustllvm/ArchiveWrapper.cpp index 2e94c196935..86225874df7 100644 --- a/src/rustllvm/ArchiveWrapper.cpp +++ b/src/rustllvm/ArchiveWrapper.cpp @@ -120,7 +120,17 @@ LLVMRustArchiveChildName(const Archive::Child *child, size_t *size) { extern "C" const char* LLVMRustArchiveChildData(Archive::Child *child, size_t *size) { - StringRef buf = child->getBuffer(); + StringRef buf; +#if LLVM_VERSION_MINOR >= 7 + ErrorOr buf_or_err = child->getBuffer(); + if (buf_or_err.getError()) { + LLVMRustSetLastError(buf_or_err.getError().message().c_str()); + return NULL; + } + buf = buf_or_err.get(); +#else + buf = child->getBuffer(); +#endif *size = buf.size(); return buf.data(); } @@ -144,7 +154,8 @@ extern "C" int LLVMRustWriteArchive(char *Dst, size_t NumMembers, const LLVMRustArchiveMember **NewMembers, - bool WriteSymbtab) { + bool WriteSymbtab, + Archive::Kind Kind) { #if LLVM_VERSION_MINOR >= 7 std::vector Members; @@ -157,7 +168,7 @@ LLVMRustWriteArchive(char *Dst, Members.push_back(NewArchiveIterator(Member->child, Member->name)); } } - auto pair = writeArchive(Dst, Members, WriteSymbtab); + auto pair = writeArchive(Dst, Members, WriteSymbtab, Kind, false); if (!pair.second) return 0; LLVMRustSetLastError(pair.second.message().c_str()); -- cgit 1.4.1-3-g733a5 From 4e67f9c611e31691aa1be1f2b1e09dc523fa7056 Mon Sep 17 00:00:00 2001 From: eternaleye Date: Wed, 22 Jul 2015 23:54:59 -0700 Subject: Write deterministic archives Currently, `rustc` generates nondeterministic archives, which contain system timestamps. These don't really serve any useful purpose, and enabling deterministic archives moves us a little closer to completely deterministic builds. For a small toy library using `std::ops::{Deref,DerefMut}`, this change actually results in a bit-for-bit identical build every time. --- src/rustllvm/ArchiveWrapper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rustllvm/ArchiveWrapper.cpp') diff --git a/src/rustllvm/ArchiveWrapper.cpp b/src/rustllvm/ArchiveWrapper.cpp index 86225874df7..a40f0a245d1 100644 --- a/src/rustllvm/ArchiveWrapper.cpp +++ b/src/rustllvm/ArchiveWrapper.cpp @@ -168,7 +168,7 @@ LLVMRustWriteArchive(char *Dst, Members.push_back(NewArchiveIterator(Member->child, Member->name)); } } - auto pair = writeArchive(Dst, Members, WriteSymbtab, Kind, false); + auto pair = writeArchive(Dst, Members, WriteSymbtab, Kind, true); if (!pair.second) return 0; LLVMRustSetLastError(pair.second.message().c_str()); -- cgit 1.4.1-3-g733a5