Auto merge of #122997 - matthiaskrgr:compiletest_ices, r=oli-obk

compiletest ice tracking

see https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/where.20to.20mass-add.20known.20ices.20.2F.20merging.20glacier.20into.20rust/near/429082963

This will allow us to sunset most of https://github.com/rust-lang/glacier
The rustc ices will be tracked directly inside the rust testsuite
There are a couple of .sh tests remaining that I have not ported over yet.

This adds `tests/crashes`, a file inside this directory MUST ice, otherwise it is considered test-fail.
This will be used to track ICEs from glacier and the bugtracker.
When someones pr accidentally fixes one of these ICEs, they can move the test from `crashes` into `ui` for example.

I also added a new tidy lint that warns when a test inside `tests/crashes` does not have a `//@ known-bug: ` line

the env var `COMPILETEST_VERBOSE_CRASHES` can be set to get exit code, stderr and stdout of a crash-test to aid debugging/adding tests.

10 files changed, 78 insertions, 7 deletions

diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index e3dde89aefc..1980980b6d0 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -1400,6 +1400,8 @@ impl Step for RunMakeSupport {
 
 default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" });
 
+default_test!(Crashes { path: "tests/crashes", mode: "crashes", suite: "crashes" });
+
 default_test!(RunPassValgrind {
     path: "tests/run-pass-valgrind",
     mode: "run-pass-valgrind",
diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs
index e3b27600c5e..224f5350e40 100644
--- a/src/bootstrap/src/core/builder.rs
+++ b/src/bootstrap/src/core/builder.rs
@@ -745,6 +745,7 @@ impl<'a> Builder<'a> {
                 test::ExpandYamlAnchors,
                 test::Tidy,
                 test::Ui,
+                test::Crashes,
                 test::RunPassValgrind,
                 test::Coverage,
                 test::CoverageMap,
diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
index 78246136f2a..aa69791b3b4 100644
--- a/src/tools/compiletest/src/common.rs
+++ b/src/tools/compiletest/src/common.rs
@@ -69,6 +69,7 @@ string_enum! {
         Assembly => "assembly",
         CoverageMap => "coverage-map",
         CoverageRun => "coverage-run",
+        Crashes => "crashes",
     }
 }
 
diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs
index e6f4accfd7b..f78e0363f55 100644
--- a/src/tools/compiletest/src/header.rs
+++ b/src/tools/compiletest/src/header.rs
@@ -581,6 +581,16 @@ impl TestProps {
             self.incremental = true;
         }
 
+        if config.mode == Mode::Crashes {
+            // we don't want to pollute anything with backtrace-files
+            // also turn off backtraces in order to save some execution
+            // time on the tests; we only need to know IF it crashes
+            self.rustc_env = vec![
+                ("RUST_BACKTRACE".to_string(), "0".to_string()),
+                ("RUSTC_ICE".to_string(), "0".to_string()),
+            ];
+        }
+
         for key in &["RUST_TEST_NOCAPTURE", "RUST_TEST_THREADS"] {
             if let Ok(val) = env::var(key) {
                 if self.exec_env.iter().find(|&&(ref x, _)| x == key).is_none() {
@@ -596,7 +606,8 @@ impl TestProps {
 
     fn update_fail_mode(&mut self, ln: &str, config: &Config) {
         let check_ui = |mode: &str| {
-            if config.mode != Mode::Ui {
+            // Mode::Crashes may need build-fail in order to trigger llvm errors or stack overflows
+            if config.mode != Mode::Ui && config.mode != Mode::Crashes {
                 panic!("`{}-fail` header is only supported in UI tests", mode);
             }
         };
@@ -625,6 +636,7 @@ impl TestProps {
     fn update_pass_mode(&mut self, ln: &str, revision: Option<&str>, config: &Config) {
         let check_no_run = |s| match (config.mode, s) {
             (Mode::Ui, _) => (),
+            (Mode::Crashes, _) => (),
             (Mode::Codegen, "build-pass") => (),
             (Mode::Incremental, _) => {
                 if revision.is_some() && !self.revisions.iter().all(|r| r.starts_with("cfail")) {
@@ -757,6 +769,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "ignore-mode-codegen-units",
     "ignore-mode-coverage-map",
     "ignore-mode-coverage-run",
+    "ignore-mode-crashes",
     "ignore-mode-debuginfo",
     "ignore-mode-incremental",
     "ignore-mode-js-doc-test",
diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs
index 1624e2a6084..c8a8b79921e 100644
--- a/src/tools/compiletest/src/lib.rs
+++ b/src/tools/compiletest/src/lib.rs
@@ -65,7 +65,8 @@ pub fn parse_config(args: Vec<String>) -> Config {
             "mode",
             "which sort of compile tests to run",
             "run-pass-valgrind | pretty | debug-info | codegen | rustdoc \
-            | rustdoc-json | codegen-units | incremental | run-make | ui | js-doc-test | mir-opt | assembly",
+            | rustdoc-json | codegen-units | incremental | run-make | ui \
+            | js-doc-test | mir-opt | assembly | crashes",
         )
         .reqopt(
             "",
@@ -82,7 +83,12 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .optopt("", "run", "whether to execute run-* tests", "auto | always | never")
         .optflag("", "ignored", "run tests marked as ignored")
         .optflag("", "with-debug-assertions", "whether to run tests with `ignore-debug` header")
-        .optmulti("", "skip", "skip tests matching SUBSTRING. Can be passed multiple times", "SUBSTRING")
+        .optmulti(
+            "",
+            "skip",
+            "skip tests matching SUBSTRING. Can be passed multiple times",
+            "SUBSTRING",
+        )
         .optflag("", "exact", "filters match exactly")
         .optopt(
             "",
@@ -145,7 +151,11 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .optflag("", "profiler-support", "is the profiler runtime enabled for this target")
         .optflag("h", "help", "show this message")
         .reqopt("", "channel", "current Rust channel", "CHANNEL")
-        .optflag("", "git-hash", "run tests which rely on commit version being compiled into the binaries")
+        .optflag(
+            "",
+            "git-hash",
+            "run tests which rely on commit version being compiled into the binaries",
+        )
         .optopt("", "edition", "default Rust edition", "EDITION")
         .reqopt("", "git-repository", "name of the git repository", "ORG/REPO")
         .reqopt("", "nightly-branch", "name of the git branch for nightly", "BRANCH");
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index 5212f1430d8..38d22fef113 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -4,7 +4,7 @@ use crate::common::{
     expected_output_path, UI_EXTENSIONS, UI_FIXED, UI_STDERR, UI_STDOUT, UI_SVG, UI_WINDOWS_SVG,
 };
 use crate::common::{incremental_dir, output_base_dir, output_base_name, output_testname_unique};
-use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJson, Ui};
+use crate::common::{Assembly, Crashes, Incremental, JsDocTest, MirOpt, RunMake, RustdocJson, Ui};
 use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
 use crate::common::{CompareMode, FailMode, PassMode};
 use crate::common::{Config, TestPaths};
@@ -244,7 +244,7 @@ impl<'test> TestCx<'test> {
     /// Code executed for each revision in turn (or, if there are no
     /// revisions, exactly once, with revision == None).
     fn run_revision(&self) {
-        if self.props.should_ice && self.config.mode != Incremental {
+        if self.props.should_ice && self.config.mode != Incremental && self.config.mode != Crashes {
             self.fatal("cannot use should-ice in a test that is not cfail");
         }
         match self.config.mode {
@@ -263,6 +263,7 @@ impl<'test> TestCx<'test> {
             JsDocTest => self.run_js_doc_test(),
             CoverageMap => self.run_coverage_map_test(),
             CoverageRun => self.run_coverage_run_test(),
+            Crashes => self.run_crash_test(),
         }
     }
 
@@ -295,6 +296,7 @@ impl<'test> TestCx<'test> {
         match self.config.mode {
             JsDocTest => true,
             Ui => pm.is_some() || self.props.fail_mode > Some(FailMode::Build),
+            Crashes => false,
             Incremental => {
                 let revision =
                     self.revision.expect("incremental tests require a list of revisions");
@@ -359,6 +361,27 @@ impl<'test> TestCx<'test> {
         self.check_forbid_output(&output_to_check, &proc_res);
     }
 
+    fn run_crash_test(&self) {
+        let pm = self.pass_mode();
+        let proc_res = self.compile_test(WillExecute::No, self.should_emit_metadata(pm));
+
+        if std::env::var("COMPILETEST_VERBOSE_CRASHES").is_ok() {
+            eprintln!("{}", proc_res.status);
+            eprintln!("{}", proc_res.stdout);
+            eprintln!("{}", proc_res.stderr);
+            eprintln!("{}", proc_res.cmdline);
+        }
+
+        // if a test does not crash, consider it an error
+        if proc_res.status.success() || matches!(proc_res.status.code(), Some(1 | 0)) {
+            self.fatal(&format!(
+                "test no longer crashes/triggers ICE! Please give it a mearningful name, \
+            add a doc-comment to the start of the test explaining why it exists and \
+            move it to tests/ui or wherever you see fit."
+            ));
+        }
+    }
+
     fn run_rfail_test(&self) {
         let pm = self.pass_mode();
         let should_run = self.run_if_enabled();
@@ -2517,7 +2540,7 @@ impl<'test> TestCx<'test> {
                 rustc.arg("-Cdebug-assertions=no");
             }
             RunPassValgrind | Pretty | DebugInfo | Rustdoc | RustdocJson | RunMake
-            | CodegenUnits | JsDocTest => {
+            | CodegenUnits | JsDocTest | Crashes => {
                 // do not use JSON output
             }
         }
diff --git a/src/tools/opt-dist/src/tests.rs b/src/tools/opt-dist/src/tests.rs
index 8a8f98a5eda..46b0a543802 100644
--- a/src/tools/opt-dist/src/tests.rs
+++ b/src/tools/opt-dist/src/tests.rs
@@ -96,6 +96,7 @@ llvm-config = "{llvm_config}"
         "tests/pretty",
         "tests/run-pass-valgrind",
         "tests/ui",
+        "tests/crashes",
     ];
     for test_path in env.skipped_tests() {
         args.extend(["--skip", test_path]);
diff --git a/src/tools/tidy/src/known_bug.rs b/src/tools/tidy/src/known_bug.rs
new file mode 100644
index 00000000000..a62556f762b
--- /dev/null
+++ b/src/tools/tidy/src/known_bug.rs
@@ -0,0 +1,17 @@
+//! Tidy check to ensure that tests inside 'tests/crashes' have a '@known-bug' directive.
+
+use crate::walk::*;
+use std::path::Path;
+
+pub fn check(filepath: &Path, bad: &mut bool) {
+    walk(filepath, |path, _is_dir| filter_not_rust(path), &mut |entry, contents| {
+        let file = entry.path();
+        if !contents.lines().any(|line| line.starts_with("//@ known-bug: ")) {
+            tidy_error!(
+                bad,
+                "{} crash/ice test does not have a \"//@ known-bug: \" directive",
+                file.display()
+            );
+        }
+    });
+}
diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs
index 57436e8d7fe..23f303276aa 100644
--- a/src/tools/tidy/src/lib.rs
+++ b/src/tools/tidy/src/lib.rs
@@ -67,6 +67,7 @@ pub mod features;
 pub mod fluent_alphabetical;
 mod fluent_used;
 pub(crate) mod iter_header;
+pub mod known_bug;
 pub mod mir_opt_tests;
 pub mod pal;
 pub mod run_make_tests;
diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs
index 93be4d61a9a..77691815830 100644
--- a/src/tools/tidy/src/main.rs
+++ b/src/tools/tidy/src/main.rs
@@ -35,6 +35,7 @@ fn main() {
     let library_path = root_path.join("library");
     let compiler_path = root_path.join("compiler");
     let librustdoc_path = src_path.join("librustdoc");
+    let crashes_path = tests_path.join("crashes");
 
     let args: Vec<String> = env::args().skip(1).collect();
     let (cfg_args, pos_args) = match args.iter().position(|arg| arg == "--") {
@@ -108,6 +109,7 @@ fn main() {
         check!(mir_opt_tests, &tests_path, bless);
         check!(rustdoc_gui_tests, &tests_path);
         check!(rustdoc_css_themes, &librustdoc_path);
+        check!(known_bug, &crashes_path);
 
         // Checks that only make sense for the compiler.
         check!(error_codes, &root_path, &[&compiler_path, &librustdoc_path], verbose);