Auto merge of #40199 - alexcrichton:doc-proc-macro, r=brson

rustbuild: Build documentation for `proc_macro`

This commit fixes #38749 by building documentation for the `proc_macro` crate by
default for configured hosts. Unfortunately did not turn out to be a trivial
fix. Currently rustbuild generates documentation into multiple locations: one
for std, one for test, and one for rustc. The initial fix for this issue simply
actually executed `cargo doc -p proc_macro` which was otherwise completely
elided before.

Unfortunately rustbuild was the left to merge two documentation trees together.
One for the standard library and one for the rustc tree (which only had docs for
the `proc_macro` crate). Rustdoc itself knows how to merge documentation files
(specifically around search indexes, etc) but rustbuild was unaware of this, so
an initial fix ended up destroying the sidebar and the search bar from the
libstd docs.

To solve this issue the method of documentation has been tweaked slightly in
rustbuild. The build system will not use symlinks (or directory junctions on
Windows) to generate all documentation into the same location initially. This'll
rely on rustdoc's logic to weave together all the output and ensure that it ends
up all consistent.

Closes #38749

1 files changed, 58 insertions, 9 deletions

diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs
index d19e5b1b884..5a5cfe0c682 100644
--- a/src/bootstrap/doc.rs
+++ b/src/bootstrap/doc.rs
@@ -19,10 +19,12 @@
 
 use std::fs::{self, File};
 use std::io::prelude::*;
+use std::io;
+use std::path::Path;
 use std::process::Command;
 
 use {Build, Compiler, Mode};
-use util::cp_r;
+use util::{cp_r, symlink_dir};
 use build_helper::up_to_date;
 
 /// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
@@ -141,7 +143,22 @@ pub fn std(build: &Build, stage: u32, target: &str) {
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
 
-    build.clear_if_dirty(&out_dir, &rustdoc);
+    // Here what we're doing is creating a *symlink* (directory junction on
+    // Windows) to the final output location. This is not done as an
+    // optimization but rather for correctness. We've got three trees of
+    // documentation, one for std, one for test, and one for rustc. It's then
+    // our job to merge them all together.
+    //
+    // Unfortunately rustbuild doesn't know nearly as well how to merge doc
+    // trees as rustdoc does itself, so instead of actually having three
+    // separate trees we just have rustdoc output to the same location across
+    // all of them.
+    //
+    // This way rustdoc generates output directly into the output, and rustdoc
+    // will also directly handle merging.
+    let my_out = build.crate_doc_out(target);
+    build.clear_if_dirty(&my_out, &rustdoc);
+    t!(symlink_dir_force(&my_out, &out_dir));
 
     let mut cargo = build.cargo(&compiler, Mode::Libstd, target, "doc");
     cargo.arg("--manifest-path")
@@ -166,7 +183,7 @@ pub fn std(build: &Build, stage: u32, target: &str) {
 
 
     build.run(&mut cargo);
-    cp_r(&out_dir, &out)
+    cp_r(&my_out, &out);
 }
 
 /// Compile all libtest documentation.
@@ -187,13 +204,16 @@ pub fn test(build: &Build, stage: u32, target: &str) {
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
 
-    build.clear_if_dirty(&out_dir, &rustdoc);
+    // See docs in std above for why we symlink
+    let my_out = build.crate_doc_out(target);
+    build.clear_if_dirty(&my_out, &rustdoc);
+    t!(symlink_dir_force(&my_out, &out_dir));
 
     let mut cargo = build.cargo(&compiler, Mode::Libtest, target, "doc");
     cargo.arg("--manifest-path")
          .arg(build.src.join("src/libtest/Cargo.toml"));
     build.run(&mut cargo);
-    cp_r(&out_dir, &out)
+    cp_r(&my_out, &out);
 }
 
 /// Generate all compiler documentation.
@@ -213,15 +233,28 @@ pub fn rustc(build: &Build, stage: u32, target: &str) {
     let out_dir = build.stage_out(&compiler, Mode::Librustc)
                        .join(target).join("doc");
     let rustdoc = build.rustdoc(&compiler);
-    if !up_to_date(&rustdoc, &out_dir.join("rustc/index.html")) && out_dir.exists() {
-        t!(fs::remove_dir_all(&out_dir));
-    }
+
+    // See docs in std above for why we symlink
+    let my_out = build.crate_doc_out(target);
+    build.clear_if_dirty(&my_out, &rustdoc);
+    t!(symlink_dir_force(&my_out, &out_dir));
+
     let mut cargo = build.cargo(&compiler, Mode::Librustc, target, "doc");
     cargo.arg("--manifest-path")
          .arg(build.src.join("src/rustc/Cargo.toml"))
          .arg("--features").arg(build.rustc_features());
+
+    // Like with libstd above if compiler docs aren't enabled then we're not
+    // documenting internal dependencies, so we have a whitelist.
+    if !build.config.compiler_docs {
+        cargo.arg("--no-deps");
+        for krate in &["proc_macro"] {
+            cargo.arg("-p").arg(krate);
+        }
+    }
+
     build.run(&mut cargo);
-    cp_r(&out_dir, &out)
+    cp_r(&my_out, &out);
 }
 
 /// Generates the HTML rendered error-index by running the
@@ -240,3 +273,19 @@ pub fn error_index(build: &Build, target: &str) {
 
     build.run(&mut index);
 }
+
+fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> {
+    if let Ok(m) = fs::symlink_metadata(dst) {
+        if m.file_type().is_dir() {
+            try!(fs::remove_dir_all(dst));
+        } else {
+            // handle directory junctions on windows by falling back to
+            // `remove_dir`.
+            try!(fs::remove_file(dst).or_else(|_| {
+                fs::remove_dir(dst)
+            }));
+        }
+    }
+
+    symlink_dir(src, dst)
+}