Sync all unstable features with Unstable Book; add tidy lint.

Add a tidy lint that checks for...

* Unstable Book sections with no corresponding SUMMARY.md links
* unstable features that don't have Unstable Book sections
* Unstable Book sections that don't have corresponding unstable features

10 files changed, 243 insertions, 85 deletions

diff --git a/src/Cargo.lock b/src/Cargo.lock
index a9021dc34e2..1fa256197ce 100644
--- a/src/Cargo.lock
+++ b/src/Cargo.lock
@@ -926,6 +926,9 @@ dependencies = [
 [[package]]
 name = "tidy"
 version = "0.1.0"
+dependencies = [
+ "regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "toml"
diff --git a/src/doc/unstable-book/src/SUMMARY.md b/src/doc/unstable-book/src/SUMMARY.md
index fe491d7f901..292f5a1ec81 100644
--- a/src/doc/unstable-book/src/SUMMARY.md
+++ b/src/doc/unstable-book/src/SUMMARY.md
@@ -146,7 +146,6 @@
 - [proc_macro](proc-macro.md)
 - [proc_macro_internals](proc-macro-internals.md)
 - [process_try_wait](process-try-wait.md)
-- [pub_restricted](pub-restricted.md)
 - [question_mark_carrier](question-mark-carrier.md)
 - [quote](quote.md)
 - [rand](rand.md)
@@ -156,11 +155,11 @@
 - [relaxed_adts](relaxed-adts.md)
 - [repr_simd](repr-simd.md)
 - [retain_hash_collection](retain-hash-collection.md)
+- [reverse_cmp_key](reverse-cmp-key.md)
 - [rt](rt.md)
 - [rustc_attrs](rustc-attrs.md)
 - [rustc_diagnostic_macros](rustc-diagnostic-macros.md)
 - [rustc_private](rustc-private.md)
-- [rustdoc](rustdoc.md)
 - [rvalue_static_promotion](rvalue-static-promotion.md)
 - [sanitizer_runtime](sanitizer-runtime.md)
 - [sanitizer_runtime_lib](sanitizer-runtime-lib.md)
@@ -181,6 +180,7 @@
 - [step_by](step-by.md)
 - [step_trait](step-trait.md)
 - [stmt_expr_attributes](stmt-expr-attributes.md)
+- [str_checked_slicing](str-checked-slicing.md)
 - [str_escape](str-escape.md)
 - [str_internals](str-internals.md)
 - [struct_field_attributes](struct-field-attributes.md)
diff --git a/src/doc/unstable-book/src/pub-restricted.md b/src/doc/unstable-book/src/pub-restricted.md
deleted file mode 100644
index 6b1e88b8603..00000000000
--- a/src/doc/unstable-book/src/pub-restricted.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# `pub_restricted`
-
-The tracking issue for this feature is: [#32409]
-
-[#38356]: https://github.com/rust-lang/rust/issues/32409
-
-------------------------
diff --git a/src/doc/unstable-book/src/reverse-cmp-key.md b/src/doc/unstable-book/src/reverse-cmp-key.md
new file mode 100644
index 00000000000..a1a851d6ed6
--- /dev/null
+++ b/src/doc/unstable-book/src/reverse-cmp-key.md
@@ -0,0 +1,7 @@
+# `reverse_cmp_key`
+
+The tracking issue for this feature is: [#40893]
+
+[#40893]: https://github.com/rust-lang/rust/issues/40893
+
+------------------------
diff --git a/src/doc/unstable-book/src/rustdoc.md b/src/doc/unstable-book/src/rustdoc.md
deleted file mode 100644
index c7491ab034b..00000000000
--- a/src/doc/unstable-book/src/rustdoc.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# `rustdoc`
-
-The tracking issue for this feature is: [#27812]
-
-[#27812]: https://github.com/rust-lang/rust/issues/27812
-
-------------------------
diff --git a/src/doc/unstable-book/src/str-checked-slicing.md b/src/doc/unstable-book/src/str-checked-slicing.md
new file mode 100644
index 00000000000..d390139a6be
--- /dev/null
+++ b/src/doc/unstable-book/src/str-checked-slicing.md
@@ -0,0 +1,7 @@
+# `str_checked_slicing`
+
+The tracking issue for this feature is: [#39932]
+
+[#39932]: https://github.com/rust-lang/rust/issues/39932
+
+------------------------
diff --git a/src/tools/tidy/Cargo.toml b/src/tools/tidy/Cargo.toml
index e900bd47fb7..371922c9e6b 100644
--- a/src/tools/tidy/Cargo.toml
+++ b/src/tools/tidy/Cargo.toml
@@ -4,3 +4,4 @@ version = "0.1.0"
 authors = ["Alex Crichton <alex@alexcrichton.com>"]
 
 [dependencies]
+regex = "0.2"
\ No newline at end of file
diff --git a/src/tools/tidy/src/features.rs b/src/tools/tidy/src/features.rs
index 9b323c95fc3..e1fdc19c27d 100644
--- a/src/tools/tidy/src/features.rs
+++ b/src/tools/tidy/src/features.rs
@@ -24,8 +24,8 @@ use std::fs::File;
 use std::io::prelude::*;
 use std::path::Path;
 
-#[derive(PartialEq)]
-enum Status {
+#[derive(Debug, PartialEq)]
+pub enum Status {
     Stable,
     Removed,
     Unstable,
@@ -42,78 +42,21 @@ impl fmt::Display for Status {
     }
 }
 
-struct Feature {
-    level: Status,
-    since: String,
-    has_gate_test: bool,
+#[derive(Debug)]
+pub struct Feature {
+    pub level: Status,
+    pub since: String,
+    pub has_gate_test: bool,
 }
 
 pub fn check(path: &Path, bad: &mut bool) {
-    let mut features = collect_lang_features(&path.join("libsyntax/feature_gate.rs"));
+    let mut features = collect_lang_features(path);
     assert!(!features.is_empty());
-    let mut lib_features = HashMap::<String, Feature>::new();
-
-    let mut contents = String::new();
-    super::walk(path,
-                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
-                &mut |file| {
-        let filename = file.file_name().unwrap().to_string_lossy();
-        if !filename.ends_with(".rs") || filename == "features.rs" ||
-           filename == "diagnostic_list.rs" {
-            return;
-        }
-
-        contents.truncate(0);
-        t!(t!(File::open(&file), &file).read_to_string(&mut contents));
 
-        for (i, line) in contents.lines().enumerate() {
-            let mut err = |msg: &str| {
-                println!("{}:{}: {}", file.display(), i + 1, msg);
-                *bad = true;
-            };
-            let level = if line.contains("[unstable(") {
-                Status::Unstable
-            } else if line.contains("[stable(") {
-                Status::Stable
-            } else {
-                continue;
-            };
-            let feature_name = match find_attr_val(line, "feature") {
-                Some(name) => name,
-                None => {
-                    err("malformed stability attribute");
-                    continue;
-                }
-            };
-            let since = match find_attr_val(line, "since") {
-                Some(name) => name,
-                None if level == Status::Stable => {
-                    err("malformed stability attribute");
-                    continue;
-                }
-                None => "None",
-            };
+    let lib_features = collect_lib_features(path, bad, &features);
+    assert!(!lib_features.is_empty());
 
-            if features.contains_key(feature_name) {
-                err("duplicating a lang feature");
-            }
-            if let Some(ref s) = lib_features.get(feature_name) {
-                if s.level != level {
-                    err("different stability level than before");
-                }
-                if s.since != since {
-                    err("different `since` than before");
-                }
-                continue;
-            }
-            lib_features.insert(feature_name.to_owned(),
-                                Feature {
-                                    level: level,
-                                    since: since.to_owned(),
-                                    has_gate_test: false,
-                                });
-        }
-    });
+    let mut contents = String::new();
 
     super::walk_many(&[&path.join("test/compile-fail"),
                        &path.join("test/compile-fail-fulldeps"),
@@ -233,8 +176,9 @@ fn test_filen_gate(filen_underscore: &str,
     return false;
 }
 
-fn collect_lang_features(path: &Path) -> HashMap<String, Feature> {
+pub fn collect_lang_features(base_src_path: &Path) -> HashMap<String, Feature> {
     let mut contents = String::new();
+    let path = base_src_path.join("libsyntax/feature_gate.rs");
     t!(t!(File::open(path)).read_to_string(&mut contents));
 
     contents.lines()
@@ -257,3 +201,71 @@ fn collect_lang_features(path: &Path) -> HashMap<String, Feature> {
         })
         .collect()
 }
+
+pub fn collect_lib_features(base_src_path: &Path,
+                            bad: &mut bool,
+                            features: &HashMap<String, Feature>) -> HashMap<String, Feature> {
+    let mut lib_features = HashMap::<String, Feature>::new();
+    let mut contents = String::new();
+    super::walk(base_src_path,
+                &mut |path| super::filter_dirs(path) || path.ends_with("src/test"),
+                &mut |file| {
+        let filename = file.file_name().unwrap().to_string_lossy();
+        if !filename.ends_with(".rs") || filename == "features.rs" ||
+           filename == "diagnostic_list.rs" {
+            return;
+        }
+
+        contents.truncate(0);
+        t!(t!(File::open(&file), &file).read_to_string(&mut contents));
+
+        for (i, line) in contents.lines().enumerate() {
+            let mut err = |msg: &str| {
+                println!("{}:{}: {}", file.display(), i + 1, msg);
+                *bad = true;
+            };
+            let level = if line.contains("[unstable(") {
+                Status::Unstable
+            } else if line.contains("[stable(") {
+                Status::Stable
+            } else {
+                continue;
+            };
+            let feature_name = match find_attr_val(line, "feature") {
+                Some(name) => name,
+                None => {
+                    err("malformed stability attribute");
+                    continue;
+                }
+            };
+            let since = match find_attr_val(line, "since") {
+                Some(name) => name,
+                None if level == Status::Stable => {
+                    err("malformed stability attribute");
+                    continue;
+                }
+                None => "None",
+            };
+
+            if features.contains_key(feature_name) {
+                err("duplicating a lang feature");
+            }
+            if let Some(ref s) = lib_features.get(feature_name) {
+                if s.level != level {
+                    err("different stability level than before");
+                }
+                if s.since != since {
+                    err("different `since` than before");
+                }
+                continue;
+            }
+            lib_features.insert(feature_name.to_owned(),
+                                Feature {
+                                    level: level,
+                                    since: since.to_owned(),
+                                    has_gate_test: false,
+                                });
+        }
+    });
+    lib_features
+}
\ No newline at end of file
diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs
index 2af891b5b85..501e35e03e8 100644
--- a/src/tools/tidy/src/main.rs
+++ b/src/tools/tidy/src/main.rs
@@ -14,6 +14,8 @@
 //! etc. This is run by default on `make check` and as part of the auto
 //! builders.
 
+extern crate regex;
+
 use std::fs;
 use std::path::{PathBuf, Path};
 use std::env;
@@ -37,6 +39,7 @@ mod features;
 mod cargo;
 mod pal;
 mod deps;
+mod unstable_book;
 
 fn main() {
     let path = env::args_os().skip(1).next().expect("need an argument");
@@ -51,6 +54,7 @@ fn main() {
     cargo::check(&path, &mut bad);
     features::check(&path, &mut bad);
     pal::check(&path, &mut bad);
+    unstable_book::check(&path, &mut bad);
     if !args.iter().any(|s| *s == "--no-vendor") {
         deps::check(&path, &mut bad);
     }
diff --git a/src/tools/tidy/src/unstable_book.rs b/src/tools/tidy/src/unstable_book.rs
new file mode 100644
index 00000000000..c10e3107794
--- /dev/null
+++ b/src/tools/tidy/src/unstable_book.rs
@@ -0,0 +1,138 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::collections::HashSet;
+use std::fs;
+use std::io::{self, BufRead, Write};
+use std::path;
+use features::{collect_lang_features, collect_lib_features, Status};
+
+const PATH_STR: &'static str = "doc/unstable-book/src";
+
+const SUMMARY_FILE_NAME: &'static str = "SUMMARY.md";
+
+static EXCLUDE: &'static [&'static str; 2] = &[SUMMARY_FILE_NAME, "the-unstable-book.md"];
+
+/// Build the path to the Unstable Book source directory from the Rust 'src' directory
+fn unstable_book_path(base_src_path: &path::Path) -> path::PathBuf {
+    base_src_path.join(PATH_STR)
+}
+
+/// Build the path to the Unstable Book SUMMARY file from the Rust 'src' directory
+fn unstable_book_summary_path(base_src_path: &path::Path) -> path::PathBuf {
+    unstable_book_path(base_src_path).join(SUMMARY_FILE_NAME)
+}
+
+/// Open the Unstable Book SUMMARY file
+fn open_unstable_book_summary_file(base_src_path: &path::Path) -> fs::File {
+    fs::File::open(unstable_book_summary_path(base_src_path))
+        .expect("could not open Unstable Book SUMMARY.md")
+}
+
+/// Test to determine if DirEntry is a file
+fn dir_entry_is_file(dir_entry: &fs::DirEntry) -> bool {
+    dir_entry.file_type().expect("could not determine file type of directory entry").is_file()
+}
+
+/// Retrieve names of all lang-related unstable features
+fn collect_unstable_lang_feature_names(base_src_path: &path::Path) -> HashSet<String> {
+    collect_lang_features(base_src_path)
+        .into_iter()
+        .filter(|&(_, ref f)| f.level == Status::Unstable)
+        .map(|(ref name, _)| name.to_owned())
+        .collect()
+}
+
+/// Retrieve names of all lib-related unstable features
+fn collect_unstable_lib_feature_names(base_src_path: &path::Path) -> HashSet<String> {
+    let mut bad = true;
+    let lang_features = collect_lang_features(base_src_path);
+    collect_lib_features(base_src_path, &mut bad, &lang_features)
+        .into_iter()
+        .filter(|&(_, ref f)| f.level == Status::Unstable)
+        .map(|(ref name, _)| name.to_owned())
+        .collect()
+}
+
+/// Retrieve names of all unstable features
+fn collect_unstable_feature_names(base_src_path: &path::Path) -> HashSet<String> {
+    collect_unstable_lib_feature_names(base_src_path)
+        .union(&collect_unstable_lang_feature_names(base_src_path))
+        .map(|n| n.to_owned())
+        .collect::<HashSet<_, _>>()
+}
+
+/// Retrieve file names of all sections in the Unstable Book with:
+///
+/// * hyphens replaced by underscores
+/// * the markdown suffix ('.md') removed
+fn collect_unstable_book_section_file_names(base_src_path: &path::Path) -> HashSet<String> {
+    fs::read_dir(unstable_book_path(base_src_path))
+        .expect("could not read directory")
+        .into_iter()
+        .map(|entry| entry.expect("could not read directory entry"))
+        .filter(dir_entry_is_file)
+        .map(|entry| entry.file_name().into_string().unwrap())
+        .filter(|n| EXCLUDE.iter().all(|e| n != e))
+        .map(|n| n.trim_right_matches(".md").replace('-', "_"))
+        .collect()
+}
+
+/// Retrieve unstable feature names that are in the Unstable Book SUMMARY file
+fn collect_unstable_book_summary_links(base_src_path: &path::Path) -> HashSet<String> {
+    let summary_link_regex =
+        ::regex::Regex::new(r"^- \[(\S+)\]\(\S+\.md\)").expect("invalid regex");
+    io::BufReader::new(open_unstable_book_summary_file(base_src_path))
+        .lines()
+        .map(|l| l.expect("could not read line from file"))
+        .filter_map(|line| {
+            summary_link_regex.captures(&line).map(|c| {
+                                                       c.get(1)
+                                                           .unwrap()
+                                                           .as_str()
+                                                           .to_owned()
+                                                   })
+        })
+        .collect()
+}
+
+pub fn check(path: &path::Path, bad: &mut bool) {
+    let unstable_feature_names = collect_unstable_feature_names(path);
+    let unstable_book_section_file_names = collect_unstable_book_section_file_names(path);
+    let unstable_book_links = collect_unstable_book_summary_links(path);
+
+    // Check for Unstable Book section names with no corresponding SUMMARY.md link
+    for feature_name in &unstable_book_section_file_names - &unstable_book_links {
+        *bad = true;
+        writeln!(io::stderr(),
+                 "The Unstable Book section '{}' needs to have a link in SUMMARY.md",
+                 feature_name)
+                .expect("could not write to stderr")
+    }
+
+    // Check for unstable features that don't have Unstable Book sections
+    for feature_name in &unstable_feature_names - &unstable_book_section_file_names {
+        *bad = true;
+        writeln!(io::stderr(),
+                 "Unstable feature '{}' needs to have a section in The Unstable Book",
+                 feature_name)
+                .expect("could not write to stderr")
+    }
+
+    // Check for Unstable Book sections that don't have a corresponding unstable feature
+    for feature_name in &unstable_book_section_file_names - &unstable_feature_names {
+        *bad = true;
+        writeln!(io::stderr(),
+                 "The Unstable Book has a section '{}' which doesn't correspond \
+                  to an unstable feature",
+                 feature_name)
+                .expect("could not write to stderr")
+    }
+}