Auto merge of #84863 - ABouttefeux:libtest, r=m-ou-se

Show test type during prints

Test output can sometimes be confusing. For example doctest with the no_run argument are displayed the same way than test that are run.

During #83857 I got the feedback that test output can be confusing.

For the moment test output is
```
test $DIR/test-type.rs - f (line 12) ... ignored
test $DIR/test-type.rs - f (line 15) ... ok
test $DIR/test-type.rs - f (line 21) ... ok
test $DIR/test-type.rs - f (line 6) ... ok
```

I propose to change output by indicating the test type as
```
test $DIR/test-type.rs - f (line 12) ... ignored
test $DIR/test-type.rs - f (line 15) - compile ... ok
test $DIR/test-type.rs - f (line 21) - compile fail ... ok
test $DIR/test-type.rs - f (line 6) ... ok
```
by indicating the test type after the test name (and in the case of doctest after the function name and line) and before the "...".

------------

Note: this is a proof of concept, the implementation is probably not optimal as the properties added in `TestDesc` are only use in the display and does not represent actual change of behavior, maybe `TestType::DocTest` could have fields

18 files changed, 219 insertions, 18 deletions

diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index e845f9ec55a..99544ddb66e 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -254,6 +254,10 @@ pub fn expand_test_or_bench(
                                         "allow_fail",
                                         cx.expr_bool(sp, should_fail(&cx.sess, &item)),
                                     ),
+                                    // compile_fail: true | false
+                                    field("compile_fail", cx.expr_bool(sp, false)),
+                                    // no_run: true | false
+                                    field("no_run", cx.expr_bool(sp, false)),
                                     // should_panic: ...
                                     field(
                                         "should_panic",
diff --git a/library/test/src/formatters/pretty.rs b/library/test/src/formatters/pretty.rs
index 5e41d6d9692..e17fc08a9ae 100644
--- a/library/test/src/formatters/pretty.rs
+++ b/library/test/src/formatters/pretty.rs
@@ -169,7 +169,11 @@ impl<T: Write> PrettyFormatter<T> {
 
     fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> {
         let name = desc.padded_name(self.max_name_len, desc.name.padding());
-        self.write_plain(&format!("test {} ... ", name))?;
+        if let Some(test_mode) = desc.test_mode() {
+            self.write_plain(&format!("test {} - {} ... ", name, test_mode))?;
+        } else {
+            self.write_plain(&format!("test {} ... ", name))?;
+        }
 
         Ok(())
     }
diff --git a/library/test/src/formatters/terse.rs b/library/test/src/formatters/terse.rs
index 6f46f7255a4..a2c223c494c 100644
--- a/library/test/src/formatters/terse.rs
+++ b/library/test/src/formatters/terse.rs
@@ -158,7 +158,11 @@ impl<T: Write> TerseFormatter<T> {
 
     fn write_test_name(&mut self, desc: &TestDesc) -> io::Result<()> {
         let name = desc.padded_name(self.max_name_len, desc.name.padding());
-        self.write_plain(&format!("test {} ... ", name))?;
+        if let Some(test_mode) = desc.test_mode() {
+            self.write_plain(&format!("test {} - {} ... ", name, test_mode))?;
+        } else {
+            self.write_plain(&format!("test {} ... ", name))?;
+        }
 
         Ok(())
     }
diff --git a/library/test/src/tests.rs b/library/test/src/tests.rs
index 6a3f31b74ea..5a4a540b04e 100644
--- a/library/test/src/tests.rs
+++ b/library/test/src/tests.rs
@@ -61,6 +61,10 @@ fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
                 ignore: true,
                 should_panic: ShouldPanic::No,
                 allow_fail: false,
+                #[cfg(not(bootstrap))]
+                compile_fail: false,
+                #[cfg(not(bootstrap))]
+                no_run: false,
                 test_type: TestType::Unknown,
             },
             testfn: DynTestFn(Box::new(move || {})),
@@ -71,6 +75,10 @@ fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
                 ignore: false,
                 should_panic: ShouldPanic::No,
                 allow_fail: false,
+                #[cfg(not(bootstrap))]
+                compile_fail: false,
+                #[cfg(not(bootstrap))]
+                no_run: false,
                 test_type: TestType::Unknown,
             },
             testfn: DynTestFn(Box::new(move || {})),
@@ -89,6 +97,10 @@ pub fn do_not_run_ignored_tests() {
             ignore: true,
             should_panic: ShouldPanic::No,
             allow_fail: false,
+            #[cfg(not(bootstrap))]
+            compile_fail: false,
+            #[cfg(not(bootstrap))]
+            no_run: false,
             test_type: TestType::Unknown,
         },
         testfn: DynTestFn(Box::new(f)),
@@ -108,6 +120,10 @@ pub fn ignored_tests_result_in_ignored() {
             ignore: true,
             should_panic: ShouldPanic::No,
             allow_fail: false,
+            #[cfg(not(bootstrap))]
+            compile_fail: false,
+            #[cfg(not(bootstrap))]
+            no_run: false,
             test_type: TestType::Unknown,
         },
         testfn: DynTestFn(Box::new(f)),
@@ -131,6 +147,10 @@ fn test_should_panic() {
             ignore: false,
             should_panic: ShouldPanic::Yes,
             allow_fail: false,
+            #[cfg(not(bootstrap))]
+            compile_fail: false,
+            #[cfg(not(bootstrap))]
+            no_run: false,
             test_type: TestType::Unknown,
         },
         testfn: DynTestFn(Box::new(f)),
@@ -154,6 +174,10 @@ fn test_should_panic_good_message() {
             ignore: false,
             should_panic: ShouldPanic::YesWithMessage("error message"),
             allow_fail: false,
+            #[cfg(not(bootstrap))]
+            compile_fail: false,
+            #[cfg(not(bootstrap))]
+            no_run: false,
             test_type: TestType::Unknown,
         },
         testfn: DynTestFn(Box::new(f)),
@@ -182,6 +206,10 @@ fn test_should_panic_bad_message() {
             ignore: false,
             should_panic: ShouldPanic::YesWithMessage(expected),
             allow_fail: false,
+            #[cfg(not(bootstrap))]
+            compile_fail: false,
+            #[cfg(not(bootstrap))]
+            no_run: false,
             test_type: TestType::Unknown,
         },
         testfn: DynTestFn(Box::new(f)),
@@ -214,6 +242,10 @@ fn test_should_panic_non_string_message_type() {
             ignore: false,
             should_panic: ShouldPanic::YesWithMessage(expected),
             allow_fail: false,
+            #[cfg(not(bootstrap))]
+            compile_fail: false,
+            #[cfg(not(bootstrap))]
+            no_run: false,
             test_type: TestType::Unknown,
         },
         testfn: DynTestFn(Box::new(f)),
@@ -238,6 +270,10 @@ fn test_should_panic_but_succeeds() {
                 ignore: false,
                 should_panic,
                 allow_fail: false,
+                #[cfg(not(bootstrap))]
+                compile_fail: false,
+                #[cfg(not(bootstrap))]
+                no_run: false,
                 test_type: TestType::Unknown,
             },
             testfn: DynTestFn(Box::new(f)),
@@ -270,6 +306,10 @@ fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
             ignore: false,
             should_panic: ShouldPanic::No,
             allow_fail: false,
+            #[cfg(not(bootstrap))]
+            compile_fail: false,
+            #[cfg(not(bootstrap))]
+            no_run: false,
             test_type: TestType::Unknown,
         },
         testfn: DynTestFn(Box::new(f)),
@@ -303,6 +343,10 @@ fn time_test_failure_template(test_type: TestType) -> TestResult {
             ignore: false,
             should_panic: ShouldPanic::No,
             allow_fail: false,
+            #[cfg(not(bootstrap))]
+            compile_fail: false,
+            #[cfg(not(bootstrap))]
+            no_run: false,
             test_type,
         },
         testfn: DynTestFn(Box::new(f)),
@@ -340,6 +384,10 @@ fn typed_test_desc(test_type: TestType) -> TestDesc {
         ignore: false,
         should_panic: ShouldPanic::No,
         allow_fail: false,
+        #[cfg(not(bootstrap))]
+        compile_fail: false,
+        #[cfg(not(bootstrap))]
+        no_run: false,
         test_type,
     }
 }
@@ -451,6 +499,10 @@ pub fn exclude_should_panic_option() {
             ignore: false,
             should_panic: ShouldPanic::Yes,
             allow_fail: false,
+            #[cfg(not(bootstrap))]
+            compile_fail: false,
+            #[cfg(not(bootstrap))]
+            no_run: false,
             test_type: TestType::Unknown,
         },
         testfn: DynTestFn(Box::new(move || {})),
@@ -473,6 +525,10 @@ pub fn exact_filter_match() {
                     ignore: false,
                     should_panic: ShouldPanic::No,
                     allow_fail: false,
+                    #[cfg(not(bootstrap))]
+                    compile_fail: false,
+                    #[cfg(not(bootstrap))]
+                    no_run: false,
                     test_type: TestType::Unknown,
                 },
                 testfn: DynTestFn(Box::new(move || {})),
@@ -565,6 +621,10 @@ pub fn sort_tests() {
                     ignore: false,
                     should_panic: ShouldPanic::No,
                     allow_fail: false,
+                    #[cfg(not(bootstrap))]
+                    compile_fail: false,
+                    #[cfg(not(bootstrap))]
+                    no_run: false,
                     test_type: TestType::Unknown,
                 },
                 testfn: DynTestFn(Box::new(testfn)),
@@ -642,6 +702,10 @@ pub fn test_bench_no_iter() {
         ignore: false,
         should_panic: ShouldPanic::No,
         allow_fail: false,
+        #[cfg(not(bootstrap))]
+        compile_fail: false,
+        #[cfg(not(bootstrap))]
+        no_run: false,
         test_type: TestType::Unknown,
     };
 
@@ -662,6 +726,10 @@ pub fn test_bench_iter() {
         ignore: false,
         should_panic: ShouldPanic::No,
         allow_fail: false,
+        #[cfg(not(bootstrap))]
+        compile_fail: false,
+        #[cfg(not(bootstrap))]
+        no_run: false,
         test_type: TestType::Unknown,
     };
 
@@ -676,6 +744,10 @@ fn should_sort_failures_before_printing_them() {
         ignore: false,
         should_panic: ShouldPanic::No,
         allow_fail: false,
+        #[cfg(not(bootstrap))]
+        compile_fail: false,
+        #[cfg(not(bootstrap))]
+        no_run: false,
         test_type: TestType::Unknown,
     };
 
@@ -684,6 +756,10 @@ fn should_sort_failures_before_printing_them() {
         ignore: false,
         should_panic: ShouldPanic::No,
         allow_fail: false,
+        #[cfg(not(bootstrap))]
+        compile_fail: false,
+        #[cfg(not(bootstrap))]
+        no_run: false,
         test_type: TestType::Unknown,
     };
 
diff --git a/library/test/src/types.rs b/library/test/src/types.rs
index c5d91f653b3..63907c71ea7 100644
--- a/library/test/src/types.rs
+++ b/library/test/src/types.rs
@@ -124,6 +124,10 @@ pub struct TestDesc {
     pub ignore: bool,
     pub should_panic: options::ShouldPanic,
     pub allow_fail: bool,
+    #[cfg(not(bootstrap))]
+    pub compile_fail: bool,
+    #[cfg(not(bootstrap))]
+    pub no_run: bool,
     pub test_type: TestType,
 }
 
@@ -140,6 +144,36 @@ impl TestDesc {
             }
         }
     }
+
+    /// Returns None for ignored test or that that are just run, otherwise give a description of the type of test.
+    /// Descriptions include "should panic", "compile fail" and "compile".
+    #[cfg(not(bootstrap))]
+    pub fn test_mode(&self) -> Option<&'static str> {
+        if self.ignore {
+            return None;
+        }
+        match self.should_panic {
+            options::ShouldPanic::Yes | options::ShouldPanic::YesWithMessage(_) => {
+                return Some("should panic");
+            }
+            options::ShouldPanic::No => {}
+        }
+        if self.allow_fail {
+            return Some("allow fail");
+        }
+        if self.compile_fail {
+            return Some("compile fail");
+        }
+        if self.no_run {
+            return Some("compile");
+        }
+        None
+    }
+
+    #[cfg(bootstrap)]
+    pub fn test_mode(&self) -> Option<&'static str> {
+        None
+    }
 }
 
 #[derive(Debug)]
diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs
index 3a4d39e1d7f..88e2f6048e9 100644
--- a/src/librustdoc/doctest.rs
+++ b/src/librustdoc/doctest.rs
@@ -880,6 +880,7 @@ impl Tester for Collector {
         let target = self.options.target.clone();
         let target_str = target.to_string();
         let unused_externs = self.unused_extern_reports.clone();
+        let no_run = config.no_run || options.no_run;
         if !config.compile_fail {
             self.compiling_test_count.fetch_add(1, Ordering::SeqCst);
         }
@@ -941,13 +942,16 @@ impl Tester for Collector {
                 // compiler failures are test failures
                 should_panic: testing::ShouldPanic::No,
                 allow_fail: config.allow_fail,
+                #[cfg(not(bootstrap))]
+                compile_fail: config.compile_fail,
+                #[cfg(not(bootstrap))]
+                no_run,
                 test_type: testing::TestType::DocTest,
             },
             testfn: testing::DynTestFn(box move || {
                 let report_unused_externs = |uext| {
                     unused_externs.lock().unwrap().push(uext);
                 };
-                let no_run = config.no_run || options.no_run;
                 let res = run_test(
                     &test,
                     &cratename,
diff --git a/src/test/rustdoc-ui/failed-doctest-compile-fail.stdout b/src/test/rustdoc-ui/failed-doctest-compile-fail.stdout
index b8bb5ccb403..af3a90a7410 100644
--- a/src/test/rustdoc-ui/failed-doctest-compile-fail.stdout
+++ b/src/test/rustdoc-ui/failed-doctest-compile-fail.stdout
@@ -1,6 +1,6 @@
 
 running 1 test
-test $DIR/failed-doctest-compile-fail.rs - Foo (line 9) ... FAILED
+test $DIR/failed-doctest-compile-fail.rs - Foo (line 9) - compile fail ... FAILED
 
 failures:
 
diff --git a/src/test/rustdoc-ui/failed-doctest-missing-codes.stdout b/src/test/rustdoc-ui/failed-doctest-missing-codes.stdout
index 7367a7d6519..bacbb47b5f9 100644
--- a/src/test/rustdoc-ui/failed-doctest-missing-codes.stdout
+++ b/src/test/rustdoc-ui/failed-doctest-missing-codes.stdout
@@ -1,6 +1,6 @@
 
 running 1 test
-test $DIR/failed-doctest-missing-codes.rs - Foo (line 9) ... FAILED
+test $DIR/failed-doctest-missing-codes.rs - Foo (line 9) - compile fail ... FAILED
 
 failures:
 
diff --git a/src/test/rustdoc-ui/issue-80992.stdout b/src/test/rustdoc-ui/issue-80992.stdout
index 1dd19f46827..d2b1cd1d550 100644
--- a/src/test/rustdoc-ui/issue-80992.stdout
+++ b/src/test/rustdoc-ui/issue-80992.stdout
@@ -1,6 +1,6 @@
 
 running 1 test
-test $DIR/issue-80992.rs - test (line 7) ... ok
+test $DIR/issue-80992.rs - test (line 7) - compile fail ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
 
diff --git a/src/test/rustdoc-ui/no-run-flag.stdout b/src/test/rustdoc-ui/no-run-flag.stdout
index d92f5da8335..02f28aaf60d 100644
--- a/src/test/rustdoc-ui/no-run-flag.stdout
+++ b/src/test/rustdoc-ui/no-run-flag.stdout
@@ -1,12 +1,12 @@
 
 running 7 tests
-test $DIR/no-run-flag.rs - f (line 11) ... ok
+test $DIR/no-run-flag.rs - f (line 11) - compile ... ok
 test $DIR/no-run-flag.rs - f (line 14) ... ignored
-test $DIR/no-run-flag.rs - f (line 17) ... ok
-test $DIR/no-run-flag.rs - f (line 23) ... ok
-test $DIR/no-run-flag.rs - f (line 28) ... ok
-test $DIR/no-run-flag.rs - f (line 32) ... ok
-test $DIR/no-run-flag.rs - f (line 8) ... ok
+test $DIR/no-run-flag.rs - f (line 17) - compile ... ok
+test $DIR/no-run-flag.rs - f (line 23) - compile fail ... ok
+test $DIR/no-run-flag.rs - f (line 28) - compile ... ok
+test $DIR/no-run-flag.rs - f (line 32) - compile ... ok
+test $DIR/no-run-flag.rs - f (line 8) - compile ... ok
 
 test result: ok. 6 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in $TIME
 
diff --git a/src/test/rustdoc-ui/test-type.rs b/src/test/rustdoc-ui/test-type.rs
new file mode 100644
index 00000000000..882da5c2503
--- /dev/null
+++ b/src/test/rustdoc-ui/test-type.rs
@@ -0,0 +1,26 @@
+// compile-flags: --test --test-args=--test-threads=1
+// check-pass
+// normalize-stdout-test: "src/test/rustdoc-ui" -> "$$DIR"
+// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME"
+
+/// ```
+/// let a = true;
+/// ```
+/// ```should_panic
+/// panic!()
+/// ```
+/// ```ignore (incomplete-code)
+/// fn foo() {
+/// ```
+/// ```no_run
+/// loop {
+///     println!("Hello, world");
+/// }
+/// ```
+/// fails to compile
+/// ```compile_fail
+/// let x = 5;
+/// x += 2; // shouldn't compile!
+/// ```
+
+pub fn f() {}
diff --git a/src/test/rustdoc-ui/test-type.stdout b/src/test/rustdoc-ui/test-type.stdout
new file mode 100644
index 00000000000..a66fd240d34
--- /dev/null
+++ b/src/test/rustdoc-ui/test-type.stdout
@@ -0,0 +1,10 @@
+
+running 5 tests
+test $DIR/test-type.rs - f (line 12) ... ignored
+test $DIR/test-type.rs - f (line 15) - compile ... ok
+test $DIR/test-type.rs - f (line 21) - compile fail ... ok
+test $DIR/test-type.rs - f (line 6) ... ok
+test $DIR/test-type.rs - f (line 9) ... ok
+
+test result: ok. 4 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/src/test/ui/test-attrs/test-type.rs b/src/test/ui/test-attrs/test-type.rs
new file mode 100644
index 00000000000..3f0fa81373f
--- /dev/null
+++ b/src/test/ui/test-attrs/test-type.rs
@@ -0,0 +1,28 @@
+// compile-flags: --test
+// run-flags: --test-threads=1
+// check-run-results
+// normalize-stdout-test "finished in \d+\.\d+s" -> "finished in $$TIME"
+// ignore-emscripten no threads support
+// run-pass
+
+
+#[test]
+fn test_ok() {
+    let _a = true;
+}
+
+#[test]
+#[should_panic]
+fn test_panic() {
+    panic!();
+}
+
+#[test]
+#[ignore]
+fn test_no_run() {
+    loop{
+        println!("Hello, world");
+    }
+}
+
+fn main() {}
diff --git a/src/test/ui/test-attrs/test-type.run.stdout b/src/test/ui/test-attrs/test-type.run.stdout
new file mode 100644
index 00000000000..be2fd8ae68c
--- /dev/null
+++ b/src/test/ui/test-attrs/test-type.run.stdout
@@ -0,0 +1,8 @@
+
+running 3 tests
+test test_no_run ... ignored
+test test_ok ... ok
+test test_panic - should panic ... ok
+
+test result: ok. 2 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in $TIME
+
diff --git a/src/test/ui/test-panic-abort-nocapture.run.stdout b/src/test/ui/test-panic-abort-nocapture.run.stdout
index 15b19676a7c..8a91732a754 100644
--- a/src/test/ui/test-panic-abort-nocapture.run.stdout
+++ b/src/test/ui/test-panic-abort-nocapture.run.stdout
@@ -2,7 +2,7 @@
 running 4 tests
 test it_fails ... about to fail
 FAILED
-test it_panics ... about to panic
+test it_panics - should panic ... about to panic
 ok
 test it_works ... about to succeed
 ok
diff --git a/src/test/ui/test-panic-abort.run.stdout b/src/test/ui/test-panic-abort.run.stdout
index 467f834afec..f608a8cdc55 100644
--- a/src/test/ui/test-panic-abort.run.stdout
+++ b/src/test/ui/test-panic-abort.run.stdout
@@ -2,7 +2,7 @@
 running 5 tests
 test it_exits ... FAILED
 test it_fails ... FAILED
-test it_panics ... ok
+test it_panics - should panic ... ok
 test it_works ... ok
 test no_residual_environment ... ok
 
diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs
index d53e19f2908..08ee8fc984d 100644
--- a/src/tools/compiletest/src/main.rs
+++ b/src/tools/compiletest/src/main.rs
@@ -664,6 +664,10 @@ fn make_test(config: &Config, testpaths: &TestPaths, inputs: &Stamp) -> Vec<test
                     ignore,
                     should_panic,
                     allow_fail: false,
+                    #[cfg(not(bootstrap))]
+                    compile_fail: false,
+                    #[cfg(not(bootstrap))]
+                    no_run: false,
                     test_type: test::TestType::Unknown,
                 },
                 testfn: make_test_closure(config, testpaths, revision),
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index e3e54894038..931c822ffe2 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -2676,12 +2676,11 @@ impl<'test> TestCx<'test> {
 
         let mut tested = 0;
         for _ in res.stdout.split('\n').filter(|s| s.starts_with("test ")).inspect(|s| {
-            let tmp: Vec<&str> = s.split(" - ").collect();
-            if tmp.len() == 2 {
-                let path = tmp[0].rsplit("test ").next().unwrap();
+            if let Some((left, right)) = s.split_once(" - ") {
+                let path = left.rsplit("test ").next().unwrap();
                 if let Some(ref mut v) = files.get_mut(&path.replace('\\', "/")) {
                     tested += 1;
-                    let mut iter = tmp[1].split("(line ");
+                    let mut iter = right.split("(line ");
                     iter.next();
                     let line = iter
                         .next()