Add tests for rustdoc json

Move rustdoc/rustdoc-json to rustdoc-json

Scaffold rustdoc-json test mode

Implement run_rustdoc_json_test

Fix up python

Make tidy happy

10 files changed, 115 insertions, 25 deletions

diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index b5bbb6372ee..6d97943548d 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -425,6 +425,7 @@ impl<'a> Builder<'a> {
                 test::RustdocJSNotStd,
                 test::RustdocTheme,
                 test::RustdocUi,
+                test::RustdocJson,
                 // Run bootstrap close to the end as it's unlikely to fail
                 test::Bootstrap,
                 // Run run-make last, since these won't pass without make on Windows
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index 1df50322a07..78b5de7897d 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -904,6 +904,12 @@ host_test!(UiFullDeps { path: "src/test/ui-fulldeps", mode: "ui", suite: "ui-ful
 host_test!(Rustdoc { path: "src/test/rustdoc", mode: "rustdoc", suite: "rustdoc" });
 host_test!(RustdocUi { path: "src/test/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" });
 
+host_test!(RustdocJson {
+    path: "src/test/rustdoc-json",
+    mode: "rustdoc-json",
+    suite: "rustdoc-json"
+});
+
 host_test!(Pretty { path: "src/test/pretty", mode: "pretty", suite: "pretty" });
 
 default_test!(RunMake { path: "src/test/run-make", mode: "run-make", suite: "run-make" });
@@ -1001,6 +1007,7 @@ note: if you're sure you want to do this, please open an issue as to why. In the
             || (mode == "run-make" && suite.ends_with("fulldeps"))
             || (mode == "ui" && is_rustdoc)
             || mode == "js-doc-test"
+            || mode == "rustdoc-json"
         {
             cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
         }
diff --git a/src/test/rustdoc/rustdoc-json/Makefile b/src/test/rustdoc-json/Makefile
index ad517ae95eb..ad517ae95eb 100644
--- a/src/test/rustdoc/rustdoc-json/Makefile
+++ b/src/test/rustdoc-json/Makefile
diff --git a/src/test/rustdoc/rustdoc-json/check_missing_items.py b/src/test/rustdoc-json/check_missing_items.py
index 0004dc8fb14..3a3bf7fa3ed 100644
--- a/src/test/rustdoc/rustdoc-json/check_missing_items.py
+++ b/src/test/rustdoc-json/check_missing_items.py
@@ -130,12 +130,16 @@ while work_list:
         work_list |= set(item["inner"]["items"]) - visited
     elif item["kind"] == "struct":
         check_generics(item["inner"]["generics"])
-        work_list |= (set(item["inner"]["fields"]) | set(item["inner"]["impls"])) - visited
+        work_list |= (
+            set(item["inner"]["fields"]) | set(item["inner"]["impls"])
+        ) - visited
     elif item["kind"] == "struct_field":
         check_type(item["inner"])
     elif item["kind"] == "enum":
         check_generics(item["inner"]["generics"])
-        work_list |= (set(item["inner"]["variants"]) | set(item["inner"]["impls"])) - visited
+        work_list |= (
+            set(item["inner"]["variants"]) | set(item["inner"]["impls"])
+        ) - visited
     elif item["kind"] == "variant":
         if item["inner"]["variant_kind"] == "tuple":
             for ty in item["inner"]["variant_inner"]:
@@ -162,7 +166,9 @@ while work_list:
         check_generics(item["inner"]["generics"])
         for bound in item["inner"]["bounds"]:
             check_generic_bound(bound)
-        work_list |= (set(item["inner"]["items"]) | set(item["inner"]["implementors"])) - visited
+        work_list |= (
+            set(item["inner"]["items"]) | set(item["inner"]["implementors"])
+        ) - visited
     elif item["kind"] == "impl":
         check_generics(item["inner"]["generics"])
         if item["inner"]["trait"]:
diff --git a/src/test/rustdoc/rustdoc-json/compare.py b/src/test/rustdoc-json/compare.py
index 5daf8903e20..afc8066685c 100644
--- a/src/test/rustdoc/rustdoc-json/compare.py
+++ b/src/test/rustdoc-json/compare.py
@@ -24,7 +24,7 @@ class SubsetException(Exception):
         super().__init__("{}: {}".format(trace, msg))
 
 
-def check_subset(expected_main, actual_main):
+def check_subset(expected_main, actual_main, base_dir):
     expected_index = expected_main["index"]
     expected_paths = expected_main["paths"]
     actual_index = actual_main["index"]
@@ -39,11 +39,24 @@ def check_subset(expected_main, actual_main):
                 "expected type `{}`, got `{}`".format(expected_type, actual_type), trace
             )
         if expected_type in (str, int, bool) and expected != actual:
-            raise SubsetException("expected `{}`, got: `{}`".format(expected, actual), trace)
+            if expected_type == str and actual.startswith(base_dir):
+                if actual.replace(base_dir + "/", "") != expected:
+                    raise SubsetException(
+                        "expected `{}`, got: `{}`".format(
+                            expected, actual.replace(base_dir + "/", "")
+                        ),
+                        trace,
+                    )
+            else:
+                raise SubsetException(
+                    "expected `{}`, got: `{}`".format(expected, actual), trace
+                )
         if expected_type is dict:
             for key in expected:
                 if key not in actual:
-                    raise SubsetException("Key `{}` not found in output".format(key), trace)
+                    raise SubsetException(
+                        "Key `{}` not found in output".format(key), trace
+                    )
                 new_trace = copy.deepcopy(trace)
                 new_trace.append(key)
                 _check_subset(expected[key], actual[key], new_trace)
@@ -52,7 +65,10 @@ def check_subset(expected_main, actual_main):
             actual_elements = len(actual)
             if expected_elements != actual_elements:
                 raise SubsetException(
-                    "Found {} items, expected {}".format(expected_elements, actual_elements), trace
+                    "Found {} items, expected {}".format(
+                        expected_elements, actual_elements
+                    ),
+                    trace,
                 )
             for expected, actual in zip(expected, actual):
                 new_trace = copy.deepcopy(trace)
@@ -60,8 +76,12 @@ def check_subset(expected_main, actual_main):
                 _check_subset(expected, actual, new_trace)
         elif expected_type is ID and expected not in already_checked:
             already_checked.add(expected)
-            _check_subset(expected_index.get(expected, {}), actual_index.get(actual, {}), trace)
-            _check_subset(expected_paths.get(expected, {}), actual_paths.get(actual, {}), trace)
+            _check_subset(
+                expected_index.get(expected, {}), actual_index.get(actual, {}), trace
+            )
+            _check_subset(
+                expected_paths.get(expected, {}), actual_paths.get(actual, {}), trace
+            )
 
     _check_subset(expected_main["root"], actual_main["root"], [])
 
@@ -90,18 +110,22 @@ def rustdoc_object_hook(obj):
     return obj
 
 
-def main(expected_fpath, actual_fpath):
-    print("checking that {} is a logical subset of {}".format(expected_fpath, actual_fpath))
+def main(expected_fpath, actual_fpath, base_dir):
+    print(
+        "checking that {} is a logical subset of {}".format(
+            expected_fpath, actual_fpath
+        )
+    )
     with open(expected_fpath) as expected_file:
         expected_main = json.load(expected_file, object_hook=rustdoc_object_hook)
     with open(actual_fpath) as actual_file:
         actual_main = json.load(actual_file, object_hook=rustdoc_object_hook)
-    check_subset(expected_main, actual_main)
+    check_subset(expected_main, actual_main, base_dir)
     print("all checks passed")
 
 
 if __name__ == "__main__":
-    if len(sys.argv) < 3:
-        print("Usage: `compare.py expected.json actual.json`")
+    if len(sys.argv) < 4:
+        print("Usage: `compare.py expected.json actual.json test-dir`")
     else:
-        main(sys.argv[1], sys.argv[2])
+        main(sys.argv[1], sys.argv[2], sys.argv[3])
diff --git a/src/test/rustdoc/rustdoc-json/structs.expected b/src/test/rustdoc-json/structs.expected
index 45b23534bc7..45b23534bc7 100644
--- a/src/test/rustdoc/rustdoc-json/structs.expected
+++ b/src/test/rustdoc-json/structs.expected
diff --git a/src/test/rustdoc/rustdoc-json/structs.rs b/src/test/rustdoc-json/structs.rs
index 43fc4743503..43fc4743503 100644
--- a/src/test/rustdoc/rustdoc-json/structs.rs
+++ b/src/test/rustdoc-json/structs.rs
diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
index 24ef98cd784..eba02333c8c 100644
--- a/src/tools/compiletest/src/common.rs
+++ b/src/tools/compiletest/src/common.rs
@@ -17,6 +17,7 @@ pub enum Mode {
     DebugInfo,
     Codegen,
     Rustdoc,
+    RustdocJson,
     CodegenUnits,
     Incremental,
     RunMake,
@@ -48,6 +49,7 @@ impl FromStr for Mode {
             "debuginfo" => Ok(DebugInfo),
             "codegen" => Ok(Codegen),
             "rustdoc" => Ok(Rustdoc),
+            "rustdoc-json" => Ok(RustdocJson),
             "codegen-units" => Ok(CodegenUnits),
             "incremental" => Ok(Incremental),
             "run-make" => Ok(RunMake),
@@ -70,6 +72,7 @@ impl fmt::Display for Mode {
             DebugInfo => "debuginfo",
             Codegen => "codegen",
             Rustdoc => "rustdoc",
+            RustdocJson => "rustdoc-json",
             CodegenUnits => "codegen-units",
             Incremental => "incremental",
             RunMake => "run-make",
diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs
index 32347db5dbb..0541548aefd 100644
--- a/src/tools/compiletest/src/main.rs
+++ b/src/tools/compiletest/src/main.rs
@@ -68,7 +68,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
             "mode",
             "which sort of compile tests to run",
             "compile-fail | run-fail | run-pass-valgrind | pretty | debug-info | codegen | rustdoc \
-             codegen-units | incremental | run-make | ui | js-doc-test | mir-opt | assembly",
+            | rustdoc-json | codegen-units | incremental | run-make | ui | js-doc-test | mir-opt | assembly",
         )
         .reqopt(
             "",
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index 1b9f0089dce..88cb8544e47 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -2,7 +2,7 @@
 
 use crate::common::{expected_output_path, UI_EXTENSIONS, UI_FIXED, UI_STDERR, UI_STDOUT};
 use crate::common::{output_base_dir, output_base_name, output_testname_unique};
-use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, Ui};
+use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJson, Ui};
 use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
 use crate::common::{CompareMode, FailMode, PassMode};
 use crate::common::{CompileFail, Pretty, RunFail, RunPassValgrind};
@@ -342,6 +342,7 @@ impl<'test> TestCx<'test> {
             DebugInfo => self.run_debuginfo_test(),
             Codegen => self.run_codegen_test(),
             Rustdoc => self.run_rustdoc_test(),
+            RustdocJson => self.run_rustdoc_json_test(),
             CodegenUnits => self.run_codegen_units_test(),
             Incremental => self.run_incremental_test(),
             RunMake => self.run_rmake_test(),
@@ -1564,7 +1565,7 @@ impl<'test> TestCx<'test> {
         self.compose_and_run_compiler(rustc, None)
     }
 
-    fn document(&self, out_dir: &Path) -> ProcRes {
+    fn document(&self, out_dir: &Path, json: bool) -> ProcRes {
         if self.props.build_aux_docs {
             for rel_ab in &self.props.aux_builds {
                 let aux_testpaths = self.compute_aux_test_paths(rel_ab);
@@ -1578,7 +1579,7 @@ impl<'test> TestCx<'test> {
                 };
                 // Create the directory for the stdout/stderr files.
                 create_dir_all(aux_cx.output_base_dir()).unwrap();
-                let auxres = aux_cx.document(out_dir);
+                let auxres = aux_cx.document(out_dir, json);
                 if !auxres.status.success() {
                     return auxres;
                 }
@@ -1600,6 +1601,10 @@ impl<'test> TestCx<'test> {
             .arg(&self.testpaths.file)
             .args(&self.props.compile_flags);
 
+        if json {
+            rustdoc.arg("--output-format").arg("json");
+        }
+
         if let Some(ref linker) = self.config.linker {
             rustdoc.arg(format!("-Clinker={}", linker));
         }
@@ -1887,7 +1892,9 @@ impl<'test> TestCx<'test> {
     }
 
     fn is_rustdoc(&self) -> bool {
-        self.config.src_base.ends_with("rustdoc-ui") || self.config.src_base.ends_with("rustdoc-js")
+        self.config.src_base.ends_with("rustdoc-ui")
+            || self.config.src_base.ends_with("rustdoc-js")
+            || self.config.src_base.ends_with("rustdoc-json")
     }
 
     fn make_compile_args(
@@ -1968,8 +1975,8 @@ impl<'test> TestCx<'test> {
 
                 rustc.arg(dir_opt);
             }
-            RunFail | RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RunMake
-            | CodegenUnits | JsDocTest | Assembly => {
+            RunFail | RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RustdocJson
+            | RunMake | CodegenUnits | JsDocTest | Assembly => {
                 // do not use JSON output
             }
         }
@@ -2329,7 +2336,7 @@ impl<'test> TestCx<'test> {
         let _ = fs::remove_dir_all(&out_dir);
         create_dir_all(&out_dir).unwrap();
 
-        let proc_res = self.document(&out_dir);
+        let proc_res = self.document(&out_dir, false);
         if !proc_res.status.success() {
             self.fatal_proc_rec("rustdoc failed!", &proc_res);
         }
@@ -2385,7 +2392,7 @@ impl<'test> TestCx<'test> {
         rustc.arg("-L").arg(&new_rustdoc.aux_output_dir_name());
         new_rustdoc.build_all_auxiliary(&mut rustc);
 
-        let proc_res = new_rustdoc.document(&compare_dir);
+        let proc_res = new_rustdoc.document(&compare_dir, false);
         if !proc_res.status.success() {
             proc_res.fatal(Some("failed to run nightly rustdoc"), || ());
         }
@@ -2466,6 +2473,48 @@ impl<'test> TestCx<'test> {
         eprintln!("{}", String::from_utf8_lossy(&output.stderr));
     }
 
+    fn run_rustdoc_json_test(&self) {
+        //FIXME: Add bless option.
+
+        assert!(self.revision.is_none(), "revisions not relevant here");
+
+        let out_dir = self.output_base_dir();
+        let _ = fs::remove_dir_all(&out_dir);
+        create_dir_all(&out_dir).unwrap();
+
+        let proc_res = self.document(&out_dir, true);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("rustdoc failed!", &proc_res);
+        }
+
+        let root = self.config.find_rust_src_root().unwrap();
+        let mut json_out = out_dir.join(self.testpaths.file.file_stem().unwrap());
+        json_out.set_extension("json");
+        let res = self.cmd2procres(
+            Command::new(&self.config.docck_python)
+                .arg(root.join("src/test/rustdoc-json/check_missing_items.py"))
+                .arg(&json_out),
+        );
+
+        if !res.status.success() {
+            self.fatal_proc_rec("check_missing_items failed!", &res);
+        }
+
+        let mut expected = self.testpaths.file.clone();
+        expected.set_extension("expected");
+        let res = self.cmd2procres(
+            Command::new(&self.config.docck_python)
+                .arg(root.join("src/test/rustdoc-json/compare.py"))
+                .arg(&expected)
+                .arg(&json_out)
+                .arg(&expected.parent().unwrap()),
+        );
+
+        if !res.status.success() {
+            self.fatal_proc_rec("compare failed!", &res);
+        }
+    }
+
     fn get_lines<P: AsRef<Path>>(
         &self,
         path: &P,
@@ -3003,7 +3052,7 @@ impl<'test> TestCx<'test> {
         if let Some(nodejs) = &self.config.nodejs {
             let out_dir = self.output_base_dir();
 
-            self.document(&out_dir);
+            self.document(&out_dir, false);
 
             let root = self.config.find_rust_src_root().unwrap();
             let file_stem =