Auto merge of #53245 - michaelwoerister:thinlto-rust-llvm, r=alexcrichton

[experimental]: Build LLVM with ThinLTO enabled (2nd attempt)

This is https://github.com/rust-lang/rust/pull/51207 revived. This time, I'd like to run actual performance tests to see if it improves compile times.

11 files changed, 144 insertions, 23 deletions

diff --git a/config.toml.example b/config.toml.example
index 107375ac5cc..35f69cd05b6 100644
--- a/config.toml.example
+++ b/config.toml.example
@@ -21,6 +21,12 @@
 # Indicates whether the LLVM build is a Release or Debug build
 #optimize = true
 
+# Indicates whether LLVM should be built with ThinLTO. Note that this will
+# only succeed if you use clang, lld, llvm-ar, and llvm-ranlib in your C/C++
+# toolchain (see the `cc`, `cxx`, `linker`, `ar`, and `ranlib` options below).
+# More info at: https://clang.llvm.org/docs/ThinLTO.html#clang-bootstrap
+#thin-lto = false
+
 # Indicates whether an LLVM Release build should include debug info
 #release-debuginfo = false
 
@@ -388,6 +394,10 @@
 # Note: an absolute path should be used, otherwise LLVM build will break.
 #ar = "ar"
 
+# Ranlib to be used to assemble static libraries compiled from C/C++ code.
+# Note: an absolute path should be used, otherwise LLVM build will break.
+#ranlib = "ranlib"
+
 # Linker to be used to link Rust code. Note that the
 # default value is platform specific, and if not specified it may also depend on
 # what platform is crossing to what platform.
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 12c1972c220..5c287f25e26 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -827,7 +827,7 @@ impl<'a> Builder<'a> {
         if let Some(ref error_format) = self.config.rustc_error_format {
             cargo.env("RUSTC_ERROR_FORMAT", error_format);
         }
-        if cmd != "build" && cmd != "check" && want_rustdoc {
+        if cmd != "build" && cmd != "check" && cmd != "rustc" && want_rustdoc {
             cargo.env("RUSTDOC_LIBDIR", self.sysroot_libdir(compiler, self.config.build));
         }
 
@@ -988,7 +988,7 @@ impl<'a> Builder<'a> {
             }
         }
 
-        if cmd == "build"
+        if (cmd == "build" || cmd == "rustc")
             && mode == Mode::Std
             && self.config.extended
             && compiler.is_final_stage(self)
diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs
index 133e5aa37a7..20cdfcb3d29 100644
--- a/src/bootstrap/check.rs
+++ b/src/bootstrap/check.rs
@@ -50,6 +50,7 @@ impl Step for Std {
         println!("Checking std artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libstd_stamp(builder, compiler, target),
                   true);
 
@@ -98,6 +99,7 @@ impl Step for Rustc {
         println!("Checking compiler artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &librustc_stamp(builder, compiler, target),
                   true);
 
@@ -149,6 +151,7 @@ impl Step for CodegenBackend {
         let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &codegen_backend_stamp(builder, compiler, target, backend),
                   true);
     }
@@ -187,6 +190,7 @@ impl Step for Test {
         println!("Checking test artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libtest_stamp(builder, compiler, target),
                   true);
 
@@ -236,6 +240,7 @@ impl Step for Rustdoc {
         println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &rustdoc_stamp(builder, compiler, target),
                   true);
 
diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs
index 2f8816d111a..da0ccf5e177 100644
--- a/src/bootstrap/compile.rs
+++ b/src/bootstrap/compile.rs
@@ -117,6 +117,7 @@ impl Step for Std {
                 &compiler.host, target));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libstd_stamp(builder, compiler, target),
                   false);
 
@@ -396,6 +397,7 @@ impl Step for Test {
                 &compiler.host, target));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &libtest_stamp(builder, compiler, target),
                   false);
 
@@ -529,6 +531,7 @@ impl Step for Rustc {
                  compiler.stage, &compiler.host, target));
         run_cargo(builder,
                   &mut cargo,
+                  vec![],
                   &librustc_stamp(builder, compiler, target),
                   false);
 
@@ -673,18 +676,47 @@ impl Step for CodegenBackend {
         let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
         builder.clear_if_dirty(&out_dir, &librustc_stamp(builder, compiler, target));
 
-        let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "build");
+        let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "rustc");
         cargo.arg("--manifest-path")
             .arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml"));
         rustc_cargo_env(builder, &mut cargo);
 
         let features = build_codegen_backend(&builder, &mut cargo, &compiler, target, backend);
 
+        let mut cargo_tails_args = vec![];
+
+        if builder.config.llvm_thin_lto {
+            cargo_tails_args.push("--".to_string());
+
+            let num_jobs = builder.jobs();
+
+            if !target.contains("msvc") {
+                // Here we assume that the linker is clang. If it's not, there'll
+                // be linker errors.
+                cargo_tails_args.push("-Clink-arg=-fuse-ld=lld".to_string());
+                cargo_tails_args.push("-Clink-arg=-flto=thin".to_string());
+
+                if builder.config.llvm_optimize {
+                    cargo_tails_args.push("-Clink-arg=-O2".to_string());
+                }
+
+                // Let's make LLD respect the `-j` option.
+                let num_jobs_arg = format!("-Clink-arg=-Wl,--thinlto-jobs={}", num_jobs);
+                cargo_tails_args.push(num_jobs_arg);
+            } else {
+                // Here we assume that the linker is lld-link.exe. lld-link.exe
+                // does not need the extra arguments except for num_jobs
+                let num_jobs_arg = format!("-Clink-arg=/opt:lldltojobs={}", num_jobs);
+                cargo_tails_args.push(num_jobs_arg);
+            }
+        }
+
         let tmp_stamp = out_dir.join(".tmp.stamp");
 
         let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
         let files = run_cargo(builder,
                               cargo.arg("--features").arg(features),
+                              cargo_tails_args,
                               &tmp_stamp,
                               false);
         if builder.config.dry_run {
@@ -1045,7 +1077,11 @@ fn stderr_isatty() -> bool {
     }
 }
 
-pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check: bool)
+pub fn run_cargo(builder: &Builder,
+                 cargo: &mut Command,
+                 tail_args: Vec<String>,
+                 stamp: &Path,
+                 is_check: bool)
     -> Vec<PathBuf>
 {
     if builder.config.dry_run {
@@ -1066,7 +1102,7 @@ pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check:
     // files we need to probe for later.
     let mut deps = Vec::new();
     let mut toplevel = Vec::new();
-    let ok = stream_cargo(builder, cargo, &mut |msg| {
+    let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| {
         let filenames = match msg {
             CargoMessage::CompilerArtifact { filenames, .. } => filenames,
             _ => return,
@@ -1191,6 +1227,7 @@ pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check:
 pub fn stream_cargo(
     builder: &Builder,
     cargo: &mut Command,
+    tail_args: Vec<String>,
     cb: &mut dyn FnMut(CargoMessage),
 ) -> bool {
     if builder.config.dry_run {
@@ -1210,6 +1247,10 @@ pub fn stream_cargo(
         cargo.env("RUSTC_COLOR", "1");
     }
 
+    for arg in tail_args {
+        cargo.arg(arg);
+    }
+
     builder.verbose(&format!("running: {:?}", cargo));
     let mut child = match cargo.spawn() {
         Ok(child) => child,
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 43650332d3b..bf4d39c4947 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -77,6 +77,7 @@ pub struct Config {
     pub llvm_enabled: bool,
     pub llvm_assertions: bool,
     pub llvm_optimize: bool,
+    pub llvm_thin_lto: bool,
     pub llvm_release_debuginfo: bool,
     pub llvm_version_check: bool,
     pub llvm_static_stdcpp: bool,
@@ -163,6 +164,7 @@ pub struct Target {
     pub cc: Option<PathBuf>,
     pub cxx: Option<PathBuf>,
     pub ar: Option<PathBuf>,
+    pub ranlib: Option<PathBuf>,
     pub linker: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
     pub crt_static: Option<bool>,
@@ -246,6 +248,7 @@ struct Llvm {
     ninja: Option<bool>,
     assertions: Option<bool>,
     optimize: Option<bool>,
+    thin_lto: Option<bool>,
     release_debuginfo: Option<bool>,
     version_check: Option<bool>,
     static_libstdcpp: Option<bool>,
@@ -327,6 +330,7 @@ struct TomlTarget {
     cc: Option<String>,
     cxx: Option<String>,
     ar: Option<String>,
+    ranlib: Option<String>,
     linker: Option<String>,
     android_ndk: Option<String>,
     crt_static: Option<bool>,
@@ -503,6 +507,7 @@ impl Config {
             set(&mut config.llvm_enabled, llvm.enabled);
             llvm_assertions = llvm.assertions;
             set(&mut config.llvm_optimize, llvm.optimize);
+            set(&mut config.llvm_thin_lto, llvm.thin_lto);
             set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
             set(&mut config.llvm_version_check, llvm.version_check);
             set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
@@ -581,6 +586,7 @@ impl Config {
                 target.cc = cfg.cc.clone().map(PathBuf::from);
                 target.cxx = cfg.cxx.clone().map(PathBuf::from);
                 target.ar = cfg.ar.clone().map(PathBuf::from);
+                target.ranlib = cfg.ranlib.clone().map(PathBuf::from);
                 target.linker = cfg.linker.clone().map(PathBuf::from);
                 target.crt_static = cfg.crt_static.clone();
                 target.musl_root = cfg.musl_root.clone().map(PathBuf::from);
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 6e473fae3be..c6ff63ad71b 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -1885,6 +1885,34 @@ impl Step for HashSign {
     }
 }
 
+// Maybe add libLLVM.so to the lib-dir. It will only have been built if
+// LLVM tools are linked dynamically.
+// Note: This function does no yet support Windows but we also don't support
+//       linking LLVM tools dynamically on Windows yet.
+fn maybe_install_llvm_dylib(builder: &Builder,
+                            target: Interned<String>,
+                            image: &Path) {
+    let src_libdir = builder
+        .llvm_out(target)
+        .join("lib");
+
+    // Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
+    // Since tools link to the latter rather than the former, we have to
+    // follow the symlink to find out what to distribute.
+    let llvm_dylib_path = src_libdir.join("libLLVM.so");
+    if llvm_dylib_path.exists() {
+        let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
+            panic!("dist: Error calling canonicalize path `{}`: {}",
+                   llvm_dylib_path.display(), e);
+        });
+
+        let dst_libdir = image.join("lib");
+        t!(fs::create_dir_all(&dst_libdir));
+
+        builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
+    }
+}
+
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]
 pub struct LlvmTools {
     pub stage: u32,
@@ -1929,18 +1957,18 @@ impl Step for LlvmTools {
         drop(fs::remove_dir_all(&image));
 
         // Prepare the image directory
-        let bindir = builder
+        let src_bindir = builder
             .llvm_out(target)
             .join("bin");
-        let dst = image.join("lib/rustlib")
-            .join(target)
-            .join("bin");
-        t!(fs::create_dir_all(&dst));
+        let dst_bindir = image.join("bin");
+        t!(fs::create_dir_all(&dst_bindir));
         for tool in LLVM_TOOLS {
-            let exe = bindir.join(exe(tool, &target));
-            builder.install(&exe, &dst, 0o755);
+            let exe = src_bindir.join(exe(tool, &target));
+            builder.install(&exe, &dst_bindir, 0o755);
         }
 
+        maybe_install_llvm_dylib(builder, target, &image);
+
         // Prepare the overlay
         let overlay = tmp.join("llvm-tools-overlay");
         drop(fs::remove_dir_all(&overlay));
@@ -2025,7 +2053,6 @@ impl Step for Lldb {
         let dst = image.join("lib");
         t!(fs::create_dir_all(&dst));
         for entry in t!(fs::read_dir(&libdir)) {
-            // let entry = t!(entry);
             let entry = entry.unwrap();
             if let Ok(name) = entry.file_name().into_string() {
                 if name.starts_with("liblldb.") && !name.ends_with(".a") {
@@ -2060,6 +2087,9 @@ impl Step for Lldb {
             }
         }
 
+        // Copy libLLVM.so to the lib dir as well, if needed.
+        maybe_install_llvm_dylib(builder, target, &image);
+
         // Prepare the overlay
         let overlay = tmp.join("lldb-overlay");
         drop(fs::remove_dir_all(&overlay));
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index 5bb475e07ba..b6a89e1c18f 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -281,6 +281,7 @@ pub struct Build {
     cc: HashMap<Interned<String>, cc::Tool>,
     cxx: HashMap<Interned<String>, cc::Tool>,
     ar: HashMap<Interned<String>, PathBuf>,
+    ranlib: HashMap<Interned<String>, PathBuf>,
     // Misc
     crates: HashMap<Interned<String>, Crate>,
     is_sudo: bool,
@@ -406,6 +407,7 @@ impl Build {
             cc: HashMap::new(),
             cxx: HashMap::new(),
             ar: HashMap::new(),
+            ranlib: HashMap::new(),
             crates: HashMap::new(),
             lldb_version: None,
             lldb_python_dir: None,
@@ -772,6 +774,11 @@ impl Build {
         self.ar.get(&target).map(|p| &**p)
     }
 
+    /// Returns the path to the `ranlib` utility for the target specified.
+    fn ranlib(&self, target: Interned<String>) -> Option<&Path> {
+        self.ranlib.get(&target).map(|p| &**p)
+    }
+
     /// Returns the path to the C++ compiler for the target specified.
     fn cxx(&self, target: Interned<String>) -> Result<&Path, String> {
         match self.cxx.get(&target) {
@@ -1018,6 +1025,10 @@ impl Build {
         self.rust_version()
     }
 
+    fn llvm_link_tools_dynamically(&self, target: Interned<String>) -> bool {
+        (target.contains("linux-gnu") || target.contains("apple-darwin"))
+    }
+
     /// Returns the `version` string associated with this compiler for Rust
     /// itself.
     ///
diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index c99347aa94e..c28b467df50 100644
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -153,6 +153,11 @@ impl Step for Llvm {
            .define("LLVM_TARGET_ARCH", target.split('-').next().unwrap())
            .define("LLVM_DEFAULT_TARGET_TRIPLE", target);
 
+        if builder.config.llvm_thin_lto && !emscripten {
+            cfg.define("LLVM_ENABLE_LTO", "Thin")
+               .define("LLVM_ENABLE_LLD", "ON");
+        }
+
         // By default, LLVM will automatically find OCaml and, if it finds it,
         // install the LLVM bindings in LLVM_OCAML_INSTALL_PATH, which defaults
         // to /usr/bin/ocaml.
@@ -166,14 +171,10 @@ impl Step for Llvm {
 
         // This setting makes the LLVM tools link to the dynamic LLVM library,
         // which saves both memory during parallel links and overall disk space
-        // for the tools.  We don't distribute any of those tools, so this is
-        // just a local concern.  However, it doesn't work well everywhere.
-        //
-        // If we are shipping llvm tools then we statically link them LLVM
-        if (target.contains("linux-gnu") || target.contains("apple-darwin")) &&
-            !builder.config.llvm_tools_enabled &&
-            !want_lldb {
-                cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
+        // for the tools. We don't do this on every platform as it doesn't work
+        // equally well everywhere.
+        if builder.llvm_link_tools_dynamically(target) && !emscripten {
+            cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
         }
 
         // For distribution we want the LLVM tools to be *statically* linked to libstdc++
@@ -379,6 +380,14 @@ fn configure_cmake(builder: &Builder,
         }
     }
 
+    if let Some(ranlib) = builder.ranlib(target) {
+        if ranlib.is_absolute() {
+            // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it
+            // tries to resolve this path in the LLVM build directory.
+            cfg.define("CMAKE_RANLIB", sanitize_cc(ranlib));
+        }
+    }
+
     if env::var_os("SCCACHE_ERROR_LOG").is_some() {
         cfg.env("RUST_LOG", "sccache=warn");
     }
diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs
index 23ef031dcb7..04aaa970654 100644
--- a/src/bootstrap/tool.rs
+++ b/src/bootstrap/tool.rs
@@ -136,7 +136,7 @@ impl Step for ToolBuild {
         let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool));
         builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
         let mut duplicates = Vec::new();
-        let is_expected = compile::stream_cargo(builder, &mut cargo, &mut |msg| {
+        let is_expected = compile::stream_cargo(builder, &mut cargo, vec![], &mut |msg| {
             // Only care about big things like the RLS/Cargo for now
             match tool {
                 | "rls"
diff --git a/src/ci/docker/dist-x86_64-linux/Dockerfile b/src/ci/docker/dist-x86_64-linux/Dockerfile
index 5726fab7524..01f6db03e8e 100644
--- a/src/ci/docker/dist-x86_64-linux/Dockerfile
+++ b/src/ci/docker/dist-x86_64-linux/Dockerfile
@@ -93,7 +93,10 @@ ENV RUST_CONFIGURE_ARGS \
       --enable-sanitizers \
       --enable-profiler \
       --enable-compiler-docs \
-      --set target.x86_64-unknown-linux-gnu.linker=clang
+      --set target.x86_64-unknown-linux-gnu.linker=clang \
+      --set target.x86_64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \
+      --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
+      --set llvm.thin-lto=true
 ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
 ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
 
diff --git a/src/ci/docker/dist-x86_64-linux/build-clang.sh b/src/ci/docker/dist-x86_64-linux/build-clang.sh
index b0c27aa45bf..4595eacb310 100755
--- a/src/ci/docker/dist-x86_64-linux/build-clang.sh
+++ b/src/ci/docker/dist-x86_64-linux/build-clang.sh
@@ -30,6 +30,12 @@ curl https://releases.llvm.org/$LLVM/cfe-$LLVM.src.tar.xz | \
   xz -d | \
   tar xf - -C tools/clang --strip-components=1
 
+mkdir -p tools/lld
+
+curl https://releases.llvm.org/$LLVM/lld-$LLVM.src.tar.xz | \
+  xz -d | \
+  tar xf - -C tools/lld --strip-components=1
+
 mkdir ../clang-build
 cd ../clang-build