Auto merge of #42612 - est31:master, r=nagisa

Autogenerate stubs and SUMMARY.md in the unstable book

Removes a speed bump in compiler development by autogenerating stubs for features in the unstable book. See #42454 for discussion.

The PR contains three commits, separated in order to make review easy:

* The first commit converts the tidy tool from a binary crate to a crate that contains both a library and a binary. In the second commit, we'll use the tidy library
* The second and main commit introduces autogeneration of SUMMARY.md and feature stub files
* The third commit turns off the tidy lint that checks for features without a stub, and removes the stub files. A separate commit due to the large number of files touched

Members of the doc team who wish to document some features can either do this (where `$rustsrc` is the root of the rust repo git checkout):

1. cd to `$rustsrc/src/tools/unstable-book-gen` and then do `cargo run $rustsrc/src $rustsrc/src/doc/unstable-book` to put the stubs into the unstable book
2. cd to `$rustsrc` and run `git ls-files --others --exclude-standard` to list the newly added stubs
3. choose a file to edit, then `git add` it and `git commit`
4. afterwards, remove all changes by the tool by doing `git --reset hard` and `git clean -f`

Or they can do this:

1. remove the comment marker in `src/tools/tidy/src/unstable_book.rs` line 122
2. run `./x.py test src/tools/tidy` to list the unstable features which only have stubs
3. revert the change in 1
3. document one of the chosen unstable features

The changes done by this PR also allow for further development:

* tidy obtains information about tracking issues. We can now forbid differing tracking issues between differing `#![unstable]` annotations. I haven't done this but plan to in a future PR
* we now have a general framework for generating stuff for the unstable book at build time. Further changes can autogenerate a list of the API a given library feature exposes.

The old way to simply click through the documentation after it has been uploaded to rust-lang.org works as well.

r? @nagisa

Fixes #42454

3 files changed, 49 insertions, 4 deletions

diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs
index fc75b6ff5c3..23a38f6a896 100644
--- a/src/bootstrap/doc.rs
+++ b/src/bootstrap/doc.rs
@@ -27,18 +27,26 @@ use {Build, Compiler, Mode};
 use util::{cp_r, symlink_dir};
 use build_helper::up_to_date;
 
-/// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
-/// `name` into the `out` path.
+/// Invoke `rustbook` for `target` for the doc book `name`.
 ///
 /// This will not actually generate any documentation if the documentation has
 /// already been generated.
 pub fn rustbook(build: &Build, target: &str, name: &str) {
+    let src = build.src.join("src/doc");
+    rustbook_src(build, target, name, &src);
+}
+
+/// Invoke `rustbook` for `target` for the doc book `name` from the `src` path.
+///
+/// This will not actually generate any documentation if the documentation has
+/// already been generated.
+pub fn rustbook_src(build: &Build, target: &str, name: &str, src: &Path) {
     let out = build.doc_out(target);
     t!(fs::create_dir_all(&out));
 
     let out = out.join(name);
     let compiler = Compiler::new(0, &build.config.build);
-    let src = build.src.join("src/doc").join(name);
+    let src = src.join(name);
     let index = out.join("index.html");
     let rustbook = build.tool(&compiler, "rustbook");
     if up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
@@ -354,6 +362,19 @@ pub fn error_index(build: &Build, target: &str) {
     build.run(&mut index);
 }
 
+pub fn unstable_book_gen(build: &Build, target: &str) {
+    println!("Generating unstable book md files ({})", target);
+    let out = build.md_doc_out(target).join("unstable-book");
+    t!(fs::create_dir_all(&out));
+    t!(fs::remove_dir_all(&out));
+    let compiler = Compiler::new(0, &build.config.build);
+    let mut cmd = build.tool_cmd(&compiler, "unstable-book-gen");
+    cmd.arg(build.src.join("src"));
+    cmd.arg(out);
+
+    build.run(&mut cmd);
+}
+
 fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> {
     if let Ok(m) = fs::symlink_metadata(dst) {
         if m.file_type().is_dir() {
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index c8cd71f8f28..e0e6583b935 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -680,6 +680,11 @@ impl Build {
         self.out.join(target).join("doc")
     }
 
+    /// Output directory for some generated md crate documentation for a target (temporary)
+    fn md_doc_out(&self, target: &str) -> PathBuf {
+        self.out.join(target).join("md-doc")
+    }
+
     /// Output directory for all crate documentation for a target (temporary)
     ///
     /// The artifacts here are then copied into `doc_out` above.
diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs
index 8c366200267..5f0724c6577 100644
--- a/src/bootstrap/step.rs
+++ b/src/bootstrap/step.rs
@@ -548,6 +548,10 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .dep(|s| s.name("maybe-clean-tools"))
          .dep(|s| s.name("librustc-tool"))
          .run(move |s| compile::tool(build, s.stage, s.target, "error_index_generator"));
+    rules.build("tool-unstable-book-gen", "src/tools/unstable-book-gen")
+         .dep(|s| s.name("maybe-clean-tools"))
+         .dep(|s| s.name("libstd-tool"))
+         .run(move |s| compile::tool(build, s.stage, s.target, "unstable-book-gen"));
     rules.build("tool-tidy", "src/tools/tidy")
          .dep(|s| s.name("maybe-clean-tools"))
          .dep(|s| s.name("libstd-tool"))
@@ -662,8 +666,12 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
               .target(&build.config.build)
               .stage(0)
          })
+         .dep(move |s| s.name("doc-unstable-book-gen"))
          .default(build.config.docs)
-         .run(move |s| doc::rustbook(build, s.target, "unstable-book"));
+         .run(move |s| doc::rustbook_src(build,
+                                         s.target,
+                                         "unstable-book",
+                                         &build.md_doc_out(s.target)));
     rules.doc("doc-standalone", "src/doc")
          .dep(move |s| {
              s.name("rustc")
@@ -679,6 +687,17 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .default(build.config.docs)
          .host(true)
          .run(move |s| doc::error_index(build, s.target));
+    rules.doc("doc-unstable-book-gen", "src/tools/unstable-book-gen")
+         .dep(move |s| {
+             s.name("tool-unstable-book-gen")
+              .host(&build.config.build)
+              .target(&build.config.build)
+              .stage(0)
+         })
+         .dep(move |s| s.name("libstd-link"))
+         .default(build.config.docs)
+         .host(true)
+         .run(move |s| doc::unstable_book_gen(build, s.target));
     for (krate, path, default) in krates("std") {
         rules.doc(&krate.doc_step, path)
              .dep(|s| s.name("libstd-link"))