Auto merge of #54543 - GuillaumeGomez:top-level-index, r=QuietMisdreavus

Add index page argument

@Mark-Simulacrum: I might need some help from you: in bootstrap, I want to add an argument (a new flag added into `rustdoc`) in order to generate the current index directly when `rustdoc` is documenting the `std` lib. However, my change in `bootstrap` didn't do it and I assume it must be moved inside the `Std` struct. But there, I don't see how to pass it to `rustdoc` through `cargo`. Did I miss anything?

r? @QuietMisdreavus

7 files changed, 179 insertions, 54 deletions

diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 5abc0455b58..c7121cb0369 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -713,7 +713,7 @@ impl<'a> Builder<'a> {
             "build" => self.cargo_out(compiler, mode, target),
 
             // This is the intended out directory for crate documentation.
-            "doc" =>  self.crate_doc_out(target),
+            "doc" | "rustdoc" =>  self.crate_doc_out(target),
 
             _ => self.stage_out(compiler, mode),
         };
@@ -742,7 +742,7 @@ impl<'a> Builder<'a> {
             _ => compile::librustc_stamp(self, cmp, target),
         };
 
-        if cmd == "doc" {
+        if cmd == "doc" || cmd == "rustdoc" {
             if mode == Mode::Rustc || mode == Mode::ToolRustc || mode == Mode::Codegen {
                 // This is the intended out directory for compiler documentation.
                 my_out = self.compiler_doc_out(target);
@@ -882,7 +882,7 @@ impl<'a> Builder<'a> {
             .env("RUSTDOC", self.out.join("bootstrap/debug/rustdoc"))
             .env(
                 "RUSTDOC_REAL",
-                if cmd == "doc" || (cmd == "test" && want_rustdoc) {
+                if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) {
                     self.rustdoc(compiler.host)
                 } else {
                     PathBuf::from("/path/to/nowhere/rustdoc/not/required")
diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs
index ed4805b8ea3..7623ca1e27e 100644
--- a/src/bootstrap/doc.rs
+++ b/src/bootstrap/doc.rs
@@ -405,14 +405,15 @@ impl Step for Standalone {
             cmd.arg("--html-after-content").arg(&footer)
                .arg("--html-before-content").arg(&version_info)
                .arg("--html-in-header").arg(&favicon)
+               .arg("--markdown-no-toc")
+               .arg("--index-page").arg(&builder.src.join("src/doc/index.md"))
                .arg("--markdown-playground-url")
                .arg("https://play.rust-lang.org/")
                .arg("-o").arg(&out)
                .arg(&path);
 
             if filename == "not_found.md" {
-                cmd.arg("--markdown-no-toc")
-                   .arg("--markdown-css")
+                cmd.arg("--markdown-css")
                    .arg("https://doc.rust-lang.org/rust.css");
             } else {
                 cmd.arg("--markdown-css").arg("rust.css");
@@ -480,23 +481,31 @@ impl Step for Std {
         // will also directly handle merging.
         let my_out = builder.crate_doc_out(target);
         t!(symlink_dir_force(&builder.config, &my_out, &out_dir));
+        t!(fs::copy(builder.src.join("src/doc/rust.css"), out.join("rust.css")));
 
-        let mut cargo = builder.cargo(compiler, Mode::Std, target, "doc");
-        compile::std_cargo(builder, &compiler, target, &mut cargo);
+        let run_cargo_rustdoc_for = |package: &str| {
+            let mut cargo = builder.cargo(compiler, Mode::Std, target, "rustdoc");
+            compile::std_cargo(builder, &compiler, target, &mut cargo);
 
-        // Keep a whitelist so we do not build internal stdlib crates, these will be
-        // build by the rustc step later if enabled.
-        cargo.arg("--no-deps");
-        for krate in &["alloc", "core", "std"] {
-            cargo.arg("-p").arg(krate);
+            // Keep a whitelist so we do not build internal stdlib crates, these will be
+            // build by the rustc step later if enabled.
+            cargo.arg("-Z").arg("unstable-options")
+                 .arg("-p").arg(package);
             // Create all crate output directories first to make sure rustdoc uses
             // relative links.
             // FIXME: Cargo should probably do this itself.
-            t!(fs::create_dir_all(out_dir.join(krate)));
+            t!(fs::create_dir_all(out_dir.join(package)));
+            cargo.arg("--")
+                 .arg("--markdown-css").arg("rust.css")
+                 .arg("--markdown-no-toc")
+                 .arg("--index-page").arg(&builder.src.join("src/doc/index.md"));
+
+            builder.run(&mut cargo);
+            builder.cp_r(&my_out, &out);
+        };
+        for krate in &["alloc", "core", "std"] {
+            run_cargo_rustdoc_for(krate);
         }
-
-        builder.run(&mut cargo);
-        builder.cp_r(&my_out, &out);
     }
 }
 
diff --git a/src/doc/rustdoc/src/unstable-features.md b/src/doc/rustdoc/src/unstable-features.md
index 071575b1fc0..43cdab27e9d 100644
--- a/src/doc/rustdoc/src/unstable-features.md
+++ b/src/doc/rustdoc/src/unstable-features.md
@@ -197,6 +197,22 @@ issue][issue-include].
 [unstable-include]: ../unstable-book/language-features/external-doc.html
 [issue-include]: https://github.com/rust-lang/rust/issues/44732
 
+### Add aliases for an item in documentation search
+
+This feature allows you to add alias(es) to an item when using the `rustdoc` search through the
+`doc(alias)` attribute. Example:
+
+```rust,no_run
+#![feature(doc_alias)]
+
+#[doc(alias = "x")]
+#[doc(alias = "big")]
+pub struct BigX;
+```
+
+Then, when looking for it through the `rustdoc` search, if you enter "x" or
+"big", search will show the `BigX` struct first.
+
 ## Unstable command-line arguments
 
 These features are enabled by passing a command-line flag to Rustdoc, but the flags in question are
@@ -374,18 +390,15 @@ This is an internal flag intended for the standard library and compiler that app
 allows `rustdoc` to be able to generate documentation for the compiler crates and the standard
 library, as an equivalent command-line argument is provided to `rustc` when building those crates.
 
-### `doc_alias` feature
+### `--index-page`: provide a top-level landing page for docs
 
-This feature allows you to add alias(es) to an item when using the `rustdoc` search through the
-`doc(alias)` attribute. Example:
+This feature allows you to generate an index-page with a given markdown file. A good example of it
+is the [rust documentation index](https://doc.rust-lang.org/index.html).
 
-```rust,no_run
-#![feature(doc_alias)]
+With this, you'll have a page which you can custom as much as you want at the top of your crates.
 
-#[doc(alias = "x")]
-#[doc(alias = "big")]
-pub struct BigX;
-```
+Using `index-page` option enables `enable-index-page` option as well.
 
-Then, when looking for it through the `rustdoc` search, if you enter "x" or
-"big", search will show the `BigX` struct first.
+### `--enable-index-page`: generate a default index page for docs
+
+This feature allows the generation of a default index-page which lists the generated crates.
diff --git a/src/librustdoc/html/render.rs b/src/librustdoc/html/render.rs
index 0fc2473725a..3f0ef61f375 100644
--- a/src/librustdoc/html/render.rs
+++ b/src/librustdoc/html/render.rs
@@ -54,6 +54,9 @@ use std::rc::Rc;
 
 use externalfiles::ExternalHtml;
 
+use errors;
+use getopts;
+
 use serialize::json::{ToJson, Json, as_json};
 use syntax::ast;
 use syntax::ext::base::MacroKind;
@@ -106,6 +109,8 @@ struct Context {
     /// The map used to ensure all generated 'id=' attributes are unique.
     id_map: Rc<RefCell<IdMap>>,
     pub shared: Arc<SharedContext>,
+    pub enable_index_page: bool,
+    pub index_page: Option<PathBuf>,
 }
 
 struct SharedContext {
@@ -501,7 +506,12 @@ pub fn run(mut krate: clean::Crate,
            sort_modules_alphabetically: bool,
            themes: Vec<PathBuf>,
            enable_minification: bool,
-           id_map: IdMap) -> Result<(), Error> {
+           id_map: IdMap,
+           enable_index_page: bool,
+           index_page: Option<PathBuf>,
+           matches: &getopts::Matches,
+           diag: &errors::Handler,
+) -> Result<(), Error> {
     let src_root = match krate.src {
         FileName::Real(ref p) => match p.parent() {
             Some(p) => p.to_path_buf(),
@@ -572,6 +582,8 @@ pub fn run(mut krate: clean::Crate,
         codes: ErrorCodes::from(UnstableFeatures::from_environment().is_nightly_build()),
         id_map: Rc::new(RefCell::new(id_map)),
         shared: Arc::new(scx),
+        enable_index_page,
+        index_page,
     };
 
     // Crawl the crate to build various caches used for the output
@@ -666,7 +678,7 @@ pub fn run(mut krate: clean::Crate,
     CACHE_KEY.with(|v| *v.borrow_mut() = cache.clone());
     CURRENT_LOCATION_KEY.with(|s| s.borrow_mut().clear());
 
-    write_shared(&cx, &krate, &*cache, index, enable_minification)?;
+    write_shared(&cx, &krate, &*cache, index, enable_minification, matches, diag)?;
 
     // And finally render the whole crate's documentation
     cx.krate(krate)
@@ -742,11 +754,15 @@ fn build_index(krate: &clean::Crate, cache: &mut Cache) -> String {
             Json::Object(crate_data))
 }
 
-fn write_shared(cx: &Context,
-                krate: &clean::Crate,
-                cache: &Cache,
-                search_index: String,
-                enable_minification: bool) -> Result<(), Error> {
+fn write_shared(
+    cx: &Context,
+    krate: &clean::Crate,
+    cache: &Cache,
+    search_index: String,
+    enable_minification: bool,
+    matches: &getopts::Matches,
+    diag: &errors::Handler,
+) -> Result<(), Error> {
     // Write out the shared files. Note that these are shared among all rustdoc
     // docs placed in the output directory, so this needs to be a synchronized
     // operation with respect to all other rustdocs running around.
@@ -902,8 +918,9 @@ themePicker.onblur = handleThemeButtonsBlur;
     write(cx.dst.join("COPYRIGHT.txt"),
           include_bytes!("static/COPYRIGHT.txt"))?;
 
-    fn collect(path: &Path, krate: &str, key: &str) -> io::Result<Vec<String>> {
+    fn collect(path: &Path, krate: &str, key: &str) -> io::Result<(Vec<String>, Vec<String>)> {
         let mut ret = Vec::new();
+        let mut krates = Vec::new();
         if path.exists() {
             for line in BufReader::new(File::open(path)?).lines() {
                 let line = line?;
@@ -914,9 +931,13 @@ themePicker.onblur = handleThemeButtonsBlur;
                     continue;
                 }
                 ret.push(line.to_string());
+                krates.push(line[key.len() + 2..].split('"')
+                                                 .next()
+                                                 .map(|s| s.to_owned())
+                                                 .unwrap_or_else(|| String::new()));
             }
         }
-        Ok(ret)
+        Ok((ret, krates))
     }
 
     fn show_item(item: &IndexItem, krate: &str) -> String {
@@ -931,7 +952,7 @@ themePicker.onblur = handleThemeButtonsBlur;
 
     let dst = cx.dst.join("aliases.js");
     {
-        let mut all_aliases = try_err!(collect(&dst, &krate.name, "ALIASES"), &dst);
+        let (mut all_aliases, _) = try_err!(collect(&dst, &krate.name, "ALIASES"), &dst);
         let mut w = try_err!(File::create(&dst), &dst);
         let mut output = String::with_capacity(100);
         for (alias, items) in &cache.aliases {
@@ -955,7 +976,7 @@ themePicker.onblur = handleThemeButtonsBlur;
 
     // Update the search index
     let dst = cx.dst.join("search-index.js");
-    let mut all_indexes = try_err!(collect(&dst, &krate.name, "searchIndex"), &dst);
+    let (mut all_indexes, mut krates) = try_err!(collect(&dst, &krate.name, "searchIndex"), &dst);
     all_indexes.push(search_index);
     // Sort the indexes by crate so the file will be generated identically even
     // with rustdoc running in parallel.
@@ -969,6 +990,46 @@ themePicker.onblur = handleThemeButtonsBlur;
     }
     try_err!(writeln!(&mut w, "initSearch(searchIndex);"), &dst);
 
+    if cx.enable_index_page == true {
+        if let Some(ref index_page) = cx.index_page {
+            ::markdown::render(index_page,
+                               cx.dst.clone(),
+                               &matches, &(*cx.shared).layout.external_html,
+                               !matches.opt_present("markdown-no-toc"),
+                               diag);
+        } else {
+            let dst = cx.dst.join("index.html");
+            let mut w = BufWriter::new(try_err!(File::create(&dst), &dst));
+            let page = layout::Page {
+                title: "Index of crates",
+                css_class: "mod",
+                root_path: "./",
+                description: "List of crates",
+                keywords: BASIC_KEYWORDS,
+                resource_suffix: &cx.shared.resource_suffix,
+            };
+            krates.push(krate.name.clone());
+            krates.sort();
+            krates.dedup();
+
+            let content = format!(
+"<h1 class='fqn'>\
+     <span class='in-band'>List of all crates</span>\
+</h1><ul class='mod'>{}</ul>",
+                                  krates
+                                    .iter()
+                                    .map(|s| {
+                                        format!("<li><a href=\"{}/index.html\">{}</li>", s, s)
+                                    })
+                                    .collect::<String>());
+            try_err!(layout::render(&mut w, &cx.shared.layout,
+                                    &page, &(""), &content,
+                                    cx.shared.css_file_extension.is_some(),
+                                    &cx.shared.themes), &dst);
+            try_err!(w.flush(), &dst);
+        }
+    }
+
     // Update the list of all implementors for traits
     let dst = cx.dst.join("implementors");
     for (&did, imps) in &cache.implementors {
@@ -1022,7 +1083,8 @@ themePicker.onblur = handleThemeButtonsBlur;
                             remote_item_type.css_class(),
                             remote_path[remote_path.len() - 1]));
 
-        let mut all_implementors = try_err!(collect(&mydst, &krate.name, "implementors"), &mydst);
+        let (mut all_implementors, _) = try_err!(collect(&mydst, &krate.name, "implementors"),
+                                                 &mydst);
         all_implementors.push(implementors);
         // Sort the implementors by crate so the file will be generated
         // identically even with rustdoc running in parallel.
diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs
index 45a0494849b..e1cb96edd48 100644
--- a/src/librustdoc/lib.rs
+++ b/src/librustdoc/lib.rs
@@ -334,6 +334,17 @@ fn opts() -> Vec<RustcOptGroup> {
                 "LEVEL",
             )
         }),
+        unstable("index-page", |o| {
+             o.optopt("",
+                      "index-page",
+                      "Markdown file to be used as index page",
+                      "PATH")
+        }),
+        unstable("enable-index-page", |o| {
+             o.optflag("",
+                       "enable-index-page",
+                       "To enable generation of the index page")
+        }),
     ]
 }
 
@@ -451,7 +462,7 @@ fn main_args(args: &[String]) -> isize {
         diag.struct_err("too many file operands").emit();
         return 1;
     }
-    let input = &matches.free[0];
+    let input = matches.free[0].clone();
 
     let mut libs = SearchPaths::new();
     for s in &matches.opt_strs("L") {
@@ -479,7 +490,7 @@ fn main_args(args: &[String]) -> isize {
                                           .collect();
 
     let should_test = matches.opt_present("test");
-    let markdown_input = Path::new(input).extension()
+    let markdown_input = Path::new(&input).extension()
         .map_or(false, |e| e == "md" || e == "markdown");
 
     let output = matches.opt_str("o").map(|s| PathBuf::from(&s));
@@ -534,6 +545,8 @@ fn main_args(args: &[String]) -> isize {
     let linker = matches.opt_str("linker").map(PathBuf::from);
     let sort_modules_alphabetically = !matches.opt_present("sort-modules-by-appearance");
     let resource_suffix = matches.opt_str("resource-suffix");
+    let index_page = matches.opt_str("index-page").map(|s| PathBuf::from(&s));
+    let enable_index_page = matches.opt_present("enable-index-page") || index_page.is_some();
     let enable_minification = !matches.opt_present("disable-minification");
 
     let edition = matches.opt_str("edition").unwrap_or("2015".to_string());
@@ -544,19 +557,25 @@ fn main_args(args: &[String]) -> isize {
             return 1;
         }
     };
+    if let Some(ref index_page) = index_page {
+        if !index_page.is_file() {
+            diag.struct_err("option `--index-page` argument must be a file").emit();
+            return 1;
+        }
+    }
 
     let cg = build_codegen_options(&matches, ErrorOutputType::default());
 
     match (should_test, markdown_input) {
         (true, true) => {
-            return markdown::test(input, cfgs, libs, externs, test_args, maybe_sysroot,
+            return markdown::test(&input, cfgs, libs, externs, test_args, maybe_sysroot,
                                   display_warnings, linker, edition, cg, &diag)
         }
         (true, false) => {
-            return test::run(Path::new(input), cfgs, libs, externs, test_args, crate_name,
+            return test::run(Path::new(&input), cfgs, libs, externs, test_args, crate_name,
                              maybe_sysroot, display_warnings, linker, edition, cg)
         }
-        (false, true) => return markdown::render(Path::new(input),
+        (false, true) => return markdown::render(Path::new(&input),
                                                  output.unwrap_or(PathBuf::from("doc")),
                                                  &matches, &external_html,
                                                  !matches.opt_present("markdown-no-toc"), &diag),
@@ -565,8 +584,8 @@ fn main_args(args: &[String]) -> isize {
 
     let output_format = matches.opt_str("w");
 
-    let res = acquire_input(PathBuf::from(input), externs, edition, cg, &matches, error_format,
-                            move |out| {
+    let res = acquire_input(PathBuf::from(input), externs, edition, cg, matches, error_format,
+                            move |out, matches| {
         let Output { krate, passes, renderinfo } = out;
         let diag = core::new_handler(error_format, None, treat_err_as_bug, ui_testing);
         info!("going to format");
@@ -580,7 +599,10 @@ fn main_args(args: &[String]) -> isize {
                                   renderinfo,
                                   sort_modules_alphabetically,
                                   themes,
-                                  enable_minification, id_map)
+                                  enable_minification, id_map,
+                                  enable_index_page, index_page,
+                                  &matches,
+                                  &diag)
                     .expect("failed to generate documentation");
                 0
             }
@@ -602,11 +624,11 @@ fn acquire_input<R, F>(input: PathBuf,
                        externs: Externs,
                        edition: Edition,
                        cg: CodegenOptions,
-                       matches: &getopts::Matches,
+                       matches: getopts::Matches,
                        error_format: ErrorOutputType,
                        f: F)
                        -> Result<R, String>
-where R: 'static + Send, F: 'static + Send + FnOnce(Output) -> R {
+where R: 'static + Send, F: 'static + Send + FnOnce(Output, &getopts::Matches) -> R {
     match matches.opt_str("r").as_ref().map(|s| &**s) {
         Some("rust") => Ok(rust_input(input, externs, edition, cg, matches, error_format, f)),
         Some(s) => Err(format!("unknown input format: {}", s)),
@@ -660,11 +682,11 @@ fn rust_input<R, F>(cratefile: PathBuf,
                     externs: Externs,
                     edition: Edition,
                     cg: CodegenOptions,
-                    matches: &getopts::Matches,
+                    matches: getopts::Matches,
                     error_format: ErrorOutputType,
                     f: F) -> R
 where R: 'static + Send,
-      F: 'static + Send + FnOnce(Output) -> R
+      F: 'static + Send + FnOnce(Output, &getopts::Matches) -> R
 {
     let default_passes = if matches.opt_present("no-defaults") {
         passes::DefaultPassOption::None
@@ -709,7 +731,7 @@ where R: 'static + Send,
         *x == "ui-testing"
     });
 
-    let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(matches, error_format);
+    let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(&matches, error_format);
 
     let (tx, rx) = channel();
 
@@ -761,7 +783,8 @@ where R: 'static + Send,
             krate = pass(krate);
         }
 
-        tx.send(f(Output { krate: krate, renderinfo: renderinfo, passes: passes })).unwrap();
+        tx.send(f(Output { krate: krate, renderinfo: renderinfo, passes: passes },
+                  &matches)).unwrap();
     }));
 
     match result {
diff --git a/src/librustdoc/markdown.rs b/src/librustdoc/markdown.rs
index a3ae953e6ee..0084c0f8592 100644
--- a/src/librustdoc/markdown.rs
+++ b/src/librustdoc/markdown.rs
@@ -77,7 +77,7 @@ pub fn render(input: &Path, mut output: PathBuf, matches: &getopts::Matches,
             diag.struct_err(&format!("{}: {}", output.display(), e)).emit();
             return 4;
         }
-        Ok(f) => f
+        Ok(f) => f,
     };
 
     let (metadata, text) = extract_leading_metadata(&input_str);
diff --git a/src/test/rustdoc/index-page.rs b/src/test/rustdoc/index-page.rs
new file mode 100644
index 00000000000..9d35f8adeac
--- /dev/null
+++ b/src/test/rustdoc/index-page.rs
@@ -0,0 +1,18 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-flags: -Z unstable-options --enable-index-page
+
+#![crate_name = "foo"]
+
+// @has foo/../index.html
+// @has - '//span[@class="in-band"]' 'List of all crates'
+// @has - '//ul[@class="mod"]//a[@href="foo/index.html"]' 'foo'
+pub struct Foo;