Introduce 'run-pass' header to 'ui' tests in compiletest. Fix issue #36516.

The 'run-pass' header cause a 'ui' test to execute the result. It is used
to test the lint output, at the same time ensure those lints won't cause
the source code to become compile-fail.

12 run-pass/run-pass-fulldeps tests gained the header and are moved to
ui/ui-fulldeps. After this move, no run-pass/run-pass-fulldeps tests should
rely on the compiler's JSON message. This allows us to stop passing
`--error-format json` in run-pass tests, thus fixing #36516.

28 files changed, 216 insertions, 5 deletions

diff --git a/src/test/run-pass-fulldeps/auxiliary/lint_group_plugin_test.rs b/src/test/ui-fulldeps/auxiliary/lint_group_plugin_test.rs
index 490aa0d4693..490aa0d4693 100644
--- a/src/test/run-pass-fulldeps/auxiliary/lint_group_plugin_test.rs
+++ b/src/test/ui-fulldeps/auxiliary/lint_group_plugin_test.rs
diff --git a/src/test/run-pass-fulldeps/auxiliary/lint_plugin_test.rs b/src/test/ui-fulldeps/auxiliary/lint_plugin_test.rs
index 8647797270f..8647797270f 100644
--- a/src/test/run-pass-fulldeps/auxiliary/lint_plugin_test.rs
+++ b/src/test/ui-fulldeps/auxiliary/lint_plugin_test.rs
diff --git a/src/test/run-pass-fulldeps/deprecated-derive.rs b/src/test/ui-fulldeps/deprecated-derive.rs
index be2a45ca79a..a728523d832 100644
--- a/src/test/run-pass-fulldeps/deprecated-derive.rs
+++ b/src/test/ui-fulldeps/deprecated-derive.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
+
 #![feature(rustc_private)]
 #![allow(dead_code)]
 
diff --git a/src/test/ui-fulldeps/deprecated-derive.stderr b/src/test/ui-fulldeps/deprecated-derive.stderr
new file mode 100644
index 00000000000..3ab7567f8df
--- /dev/null
+++ b/src/test/ui-fulldeps/deprecated-derive.stderr
@@ -0,0 +1,6 @@
+warning: derive(Encodable) is deprecated in favor of derive(RustcEncodable)
+  --> $DIR/deprecated-derive.rs:18:10
+   |
+18 | #[derive(Encodable)]
+   |          ^^^^^^^^^
+
diff --git a/src/test/run-pass-fulldeps/lint-group-plugin.rs b/src/test/ui-fulldeps/lint-group-plugin.rs
index 978a78cee4f..fee9f45972d 100644
--- a/src/test/run-pass-fulldeps/lint-group-plugin.rs
+++ b/src/test/ui-fulldeps/lint-group-plugin.rs
@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
 // aux-build:lint_group_plugin_test.rs
 // ignore-stage1
 #![feature(plugin)]
diff --git a/src/test/ui-fulldeps/lint-group-plugin.stderr b/src/test/ui-fulldeps/lint-group-plugin.stderr
new file mode 100644
index 00000000000..1d68e78de5e
--- /dev/null
+++ b/src/test/ui-fulldeps/lint-group-plugin.stderr
@@ -0,0 +1,16 @@
+warning: item is named 'lintme'
+  --> $DIR/lint-group-plugin.rs:18:1
+   |
+18 | fn lintme() { } //~ WARNING item is named 'lintme'
+   | ^^^^^^^^^^^^^^^
+   |
+   = note: #[warn(test_lint)] on by default
+
+warning: item is named 'pleaselintme'
+  --> $DIR/lint-group-plugin.rs:19:1
+   |
+19 | fn pleaselintme() { } //~ WARNING item is named 'pleaselintme'
+   | ^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: #[warn(please_lint)] on by default
+
diff --git a/src/test/run-pass-fulldeps/lint-plugin-cmdline-allow.rs b/src/test/ui-fulldeps/lint-plugin-cmdline-allow.rs
index 204312064b5..1e428629cc2 100644
--- a/src/test/run-pass-fulldeps/lint-plugin-cmdline-allow.rs
+++ b/src/test/ui-fulldeps/lint-plugin-cmdline-allow.rs
@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
 // aux-build:lint_plugin_test.rs
 // ignore-stage1
 // compile-flags: -A test-lint
diff --git a/src/test/ui-fulldeps/lint-plugin-cmdline-allow.stderr b/src/test/ui-fulldeps/lint-plugin-cmdline-allow.stderr
new file mode 100644
index 00000000000..b8513b95d36
--- /dev/null
+++ b/src/test/ui-fulldeps/lint-plugin-cmdline-allow.stderr
@@ -0,0 +1,8 @@
+warning: function is never used: `lintme`
+  --> $DIR/lint-plugin-cmdline-allow.rs:19:1
+   |
+19 | fn lintme() { }
+   | ^^^^^^^^^^^^^^^
+   |
+   = note: #[warn(dead_code)] on by default
+
diff --git a/src/test/run-pass-fulldeps/lint-plugin-cmdline-load.rs b/src/test/ui-fulldeps/lint-plugin-cmdline-load.rs
index 2e86e11bd6a..27ef9649cf3 100644
--- a/src/test/run-pass-fulldeps/lint-plugin-cmdline-load.rs
+++ b/src/test/ui-fulldeps/lint-plugin-cmdline-load.rs
@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
 // aux-build:lint_plugin_test.rs
 // ignore-stage1
 // compile-flags: -Z extra-plugins=lint_plugin_test
diff --git a/src/test/ui-fulldeps/lint-plugin-cmdline-load.stderr b/src/test/ui-fulldeps/lint-plugin-cmdline-load.stderr
new file mode 100644
index 00000000000..42ececc93bd
--- /dev/null
+++ b/src/test/ui-fulldeps/lint-plugin-cmdline-load.stderr
@@ -0,0 +1,8 @@
+warning: item is named 'lintme'
+  --> $DIR/lint-plugin-cmdline-load.rs:18:1
+   |
+18 | fn lintme() { } //~ WARNING item is named 'lintme'
+   | ^^^^^^^^^^^^^^^
+   |
+   = note: #[warn(test_lint)] on by default
+
diff --git a/src/test/run-pass-fulldeps/lint-plugin.rs b/src/test/ui-fulldeps/lint-plugin.rs
index 753ad33bd01..304cfc3eb6d 100644
--- a/src/test/run-pass-fulldeps/lint-plugin.rs
+++ b/src/test/ui-fulldeps/lint-plugin.rs
@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
 // aux-build:lint_plugin_test.rs
 // ignore-stage1
 #![feature(plugin)]
diff --git a/src/test/ui-fulldeps/lint-plugin.stderr b/src/test/ui-fulldeps/lint-plugin.stderr
new file mode 100644
index 00000000000..1fe821d3115
--- /dev/null
+++ b/src/test/ui-fulldeps/lint-plugin.stderr
@@ -0,0 +1,8 @@
+warning: item is named 'lintme'
+  --> $DIR/lint-plugin.rs:18:1
+   |
+18 | fn lintme() { } //~ WARNING item is named 'lintme'
+   | ^^^^^^^^^^^^^^^
+   |
+   = note: #[warn(test_lint)] on by default
+
diff --git a/src/test/run-pass/deprecated-macro_escape-inner.rs b/src/test/ui/deprecated-macro_escape-inner.rs
index 1a2be7a719e..3d253ba49da 100644
--- a/src/test/run-pass/deprecated-macro_escape-inner.rs
+++ b/src/test/ui/deprecated-macro_escape-inner.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
+
 mod foo {
     #![macro_escape] //~ WARNING macro_escape is a deprecated synonym for macro_use
     //~^ HELP consider an outer attribute
diff --git a/src/test/ui/deprecated-macro_escape-inner.stderr b/src/test/ui/deprecated-macro_escape-inner.stderr
new file mode 100644
index 00000000000..c91db6c3365
--- /dev/null
+++ b/src/test/ui/deprecated-macro_escape-inner.stderr
@@ -0,0 +1,8 @@
+warning: macro_escape is a deprecated synonym for macro_use
+  --> $DIR/deprecated-macro_escape-inner.rs:14:5
+   |
+14 |     #![macro_escape] //~ WARNING macro_escape is a deprecated synonym for macro_use
+   |     ^^^^^^^^^^^^^^^^
+   |
+   = help: consider an outer attribute, #[macro_use] mod ...
+
diff --git a/src/test/run-pass/deprecated-macro_escape.rs b/src/test/ui/deprecated-macro_escape.rs
index b9f756cc79c..a234572343c 100644
--- a/src/test/run-pass/deprecated-macro_escape.rs
+++ b/src/test/ui/deprecated-macro_escape.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
+
 #[macro_escape] //~ WARNING macro_escape is a deprecated synonym for macro_use
 mod foo {
 }
diff --git a/src/test/ui/deprecated-macro_escape.stderr b/src/test/ui/deprecated-macro_escape.stderr
new file mode 100644
index 00000000000..aa771295281
--- /dev/null
+++ b/src/test/ui/deprecated-macro_escape.stderr
@@ -0,0 +1,6 @@
+warning: macro_escape is a deprecated synonym for macro_use
+  --> $DIR/deprecated-macro_escape.rs:13:1
+   |
+13 | #[macro_escape] //~ WARNING macro_escape is a deprecated synonym for macro_use
+   | ^^^^^^^^^^^^^^^
+
diff --git a/src/test/run-pass/deriving-meta-empty-trait-list.rs b/src/test/ui/deriving-meta-empty-trait-list.rs
index ed8a50998da..cf4dfb947f4 100644
--- a/src/test/run-pass/deriving-meta-empty-trait-list.rs
+++ b/src/test/ui/deriving-meta-empty-trait-list.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
+
 #![allow(dead_code)]
 
 #[derive]   //~ WARNING empty trait list in `derive`
diff --git a/src/test/ui/deriving-meta-empty-trait-list.stderr b/src/test/ui/deriving-meta-empty-trait-list.stderr
new file mode 100644
index 00000000000..58f871413f1
--- /dev/null
+++ b/src/test/ui/deriving-meta-empty-trait-list.stderr
@@ -0,0 +1,12 @@
+warning: empty trait list in `derive`
+  --> $DIR/deriving-meta-empty-trait-list.rs:15:1
+   |
+15 | #[derive]   //~ WARNING empty trait list in `derive`
+   | ^^^^^^^^^
+
+warning: empty trait list in `derive`
+  --> $DIR/deriving-meta-empty-trait-list.rs:18:1
+   |
+18 | #[derive()] //~ WARNING empty trait list in `derive`
+   | ^^^^^^^^^^^
+
diff --git a/src/test/run-pass/enum-size-variance.rs b/src/test/ui/enum-size-variance.rs
index a3e95a15341..075bd9acf5f 100644
--- a/src/test/run-pass/enum-size-variance.rs
+++ b/src/test/ui/enum-size-variance.rs
@@ -7,7 +7,13 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
-//
+
+// run-pass
+// ignore-x86
+// ignore-arm
+// ignore-emscripten
+// ^ ignore 32-bit targets, as the error message is target-dependent. see PR #41968.
+
 #![warn(variant_size_differences)]
 #![allow(dead_code)]
 
diff --git a/src/test/ui/enum-size-variance.stderr b/src/test/ui/enum-size-variance.stderr
new file mode 100644
index 00000000000..5745b9344b4
--- /dev/null
+++ b/src/test/ui/enum-size-variance.stderr
@@ -0,0 +1,12 @@
+warning: enum variant is more than three times larger (32 bytes) than the next largest
+  --> $DIR/enum-size-variance.rs:32:5
+   |
+32 |     L(isize, isize, isize, isize), //~ WARNING three times larger
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: lint level defined here
+  --> $DIR/enum-size-variance.rs:17:9
+   |
+17 | #![warn(variant_size_differences)]
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^
+
diff --git a/src/test/run-pass/issue-19100.rs b/src/test/ui/issue-19100.rs
index 7ff9ae996f5..bcadd94eb6f 100644
--- a/src/test/run-pass/issue-19100.rs
+++ b/src/test/ui/issue-19100.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
+
 #![allow(non_snake_case)]
 #![allow(dead_code)]
 #![allow(unused_variables)]
diff --git a/src/test/ui/issue-19100.stderr b/src/test/ui/issue-19100.stderr
new file mode 100644
index 00000000000..45e2ef652d5
--- /dev/null
+++ b/src/test/ui/issue-19100.stderr
@@ -0,0 +1,16 @@
+warning[E0170]: pattern binding `Bar` is named the same as one of the variants of the type `Foo`
+  --> $DIR/issue-19100.rs:27:1
+   |
+27 | Bar if true
+   | ^^^
+   |
+   = help: if you meant to match on a variant, consider making the path in the pattern qualified: `Foo::Bar`
+
+warning[E0170]: pattern binding `Baz` is named the same as one of the variants of the type `Foo`
+  --> $DIR/issue-19100.rs:32:1
+   |
+32 | Baz if false
+   | ^^^
+   |
+   = help: if you meant to match on a variant, consider making the path in the pattern qualified: `Foo::Baz`
+
diff --git a/src/test/run-pass/path-lookahead.rs b/src/test/ui/path-lookahead.rs
index 5c195c8a4c6..c43f2229cdc 100644
--- a/src/test/run-pass/path-lookahead.rs
+++ b/src/test/ui/path-lookahead.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
+
 // Parser test for #37765
 
 fn with_parens<T: ToString>(arg: T) -> String { //~WARN function is never used: `with_parens`
diff --git a/src/test/ui/path-lookahead.stderr b/src/test/ui/path-lookahead.stderr
new file mode 100644
index 00000000000..1e19977e84a
--- /dev/null
+++ b/src/test/ui/path-lookahead.stderr
@@ -0,0 +1,28 @@
+warning: unnecessary parentheses around `return` value
+  --> $DIR/path-lookahead.rs:16:10
+   |
+16 |   return (<T as ToString>::to_string(&arg)); //~WARN unnecessary parentheses around `return` value
+   |          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: #[warn(unused_parens)] on by default
+
+warning: function is never used: `with_parens`
+  --> $DIR/path-lookahead.rs:15:1
+   |
+15 | / fn with_parens<T: ToString>(arg: T) -> String { //~WARN function is never used: `with_parens`
+16 | |   return (<T as ToString>::to_string(&arg)); //~WARN unnecessary parentheses around `return` value
+17 | | }
+   | |_^
+   |
+   = note: #[warn(dead_code)] on by default
+
+warning: function is never used: `no_parens`
+  --> $DIR/path-lookahead.rs:19:1
+   |
+19 | / fn no_parens<T: ToString>(arg: T) -> String { //~WARN function is never used: `no_parens`
+20 | |   return <T as ToString>::to_string(&arg);
+21 | | }
+   | |_^
+   |
+   = note: #[warn(dead_code)] on by default
+
diff --git a/src/test/run-pass/test-should-panic-attr.rs b/src/test/ui/test-should-panic-attr.rs
index 2d068872a4d..8986732a269 100644
--- a/src/test/run-pass/test-should-panic-attr.rs
+++ b/src/test/ui/test-should-panic-attr.rs
@@ -8,6 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// run-pass
 // compile-flags: --test
 
 #[test]
diff --git a/src/test/ui/test-should-panic-attr.stderr b/src/test/ui/test-should-panic-attr.stderr
new file mode 100644
index 00000000000..6f143b6cbee
--- /dev/null
+++ b/src/test/ui/test-should-panic-attr.stderr
@@ -0,0 +1,40 @@
+warning: attribute must be of the form: `#[should_panic]` or `#[should_panic(expected = "error message")]`
+  --> $DIR/test-should-panic-attr.rs:15:1
+   |
+15 | #[should_panic = "foo"]
+   | ^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: Errors in this attribute were erroneously allowed and will become a hard error in a future release.
+
+warning: argument must be of the form: `expected = "error message"`
+  --> $DIR/test-should-panic-attr.rs:22:1
+   |
+22 | #[should_panic(expected)]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: Errors in this attribute were erroneously allowed and will become a hard error in a future release.
+
+warning: argument must be of the form: `expected = "error message"`
+  --> $DIR/test-should-panic-attr.rs:29:1
+   |
+29 | #[should_panic(expect)]
+   | ^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: Errors in this attribute were erroneously allowed and will become a hard error in a future release.
+
+warning: argument must be of the form: `expected = "error message"`
+  --> $DIR/test-should-panic-attr.rs:36:1
+   |
+36 | #[should_panic(expected(foo, bar))]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: Errors in this attribute were erroneously allowed and will become a hard error in a future release.
+
+warning: argument must be of the form: `expected = "error message"`
+  --> $DIR/test-should-panic-attr.rs:43:1
+   |
+43 | #[should_panic(expected = "foo", bar)]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: Errors in this attribute were erroneously allowed and will become a hard error in a future release.
+
diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs
index 0f653dfbcf0..e0999ae7793 100644
--- a/src/tools/compiletest/src/header.rs
+++ b/src/tools/compiletest/src/header.rs
@@ -233,6 +233,9 @@ pub struct TestProps {
     pub must_compile_successfully: bool,
     // rustdoc will test the output of the `--test` option
     pub check_test_line_numbers_match: bool,
+    // The test must be compiled and run successfully. Only used in UI tests for
+    // now.
+    pub run_pass: bool,
 }
 
 impl TestProps {
@@ -258,6 +261,7 @@ impl TestProps {
             incremental_dir: None,
             must_compile_successfully: false,
             check_test_line_numbers_match: false,
+            run_pass: false,
         }
     }
 
@@ -368,6 +372,10 @@ impl TestProps {
             if !self.check_test_line_numbers_match {
                 self.check_test_line_numbers_match = config.parse_check_test_line_numbers_match(ln);
             }
+
+            if !self.run_pass {
+                self.run_pass = config.parse_run_pass(ln);
+            }
         });
 
         for key in vec!["RUST_TEST_NOCAPTURE", "RUST_TEST_THREADS"] {
@@ -485,6 +493,10 @@ impl Config {
         self.parse_name_directive(line, "check-test-line-numbers-match")
     }
 
+    fn parse_run_pass(&self, line: &str) -> bool {
+        self.parse_name_directive(line, "run-pass")
+    }
+
     fn parse_env(&self, line: &str, name: &str) -> Option<(String, String)> {
         self.parse_name_value_directive(line, name).map(|nv| {
             // nv is either FOO or FOO=BAR
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index 1bec6f6af83..01419c42570 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -214,10 +214,10 @@ impl<'test> TestCx<'test> {
             self.fatal_proc_rec("compilation failed!", &proc_res);
         }
 
+        // FIXME(#41968): Move this check to tidy?
         let expected_errors = errors::load_errors(&self.testpaths.file, self.revision);
-        if !expected_errors.is_empty() {
-            self.check_expected_errors(expected_errors, &proc_res);
-        }
+        assert!(expected_errors.is_empty(),
+                "run-pass tests with expected warnings should be moved to ui/");
 
         let proc_res = self.exec_compiled_test();
 
@@ -1394,7 +1394,6 @@ actual:\n\
         match self.config.mode {
             CompileFail |
             ParseFail |
-            RunPass |
             Incremental => {
                 // If we are extracting and matching errors in the new
                 // fashion, then you want JSON mode. Old-skool error
@@ -1422,6 +1421,7 @@ actual:\n\
 
                 args.push(dir_opt);
             }
+            RunPass |
             RunFail |
             RunPassValgrind |
             Pretty |
@@ -2254,6 +2254,14 @@ actual:\n\
             self.fatal_proc_rec(&format!("{} errors occurred comparing output.", errors),
                                 &proc_res);
         }
+
+        if self.props.run_pass {
+            let proc_res = self.exec_compiled_test();
+
+            if !proc_res.status.success() {
+                self.fatal_proc_rec("test run failed!", &proc_res);
+            }
+        }
     }
 
     fn run_mir_opt_test(&self) {