Rollup merge of #59398 - phansch:rustfix_coverage, r=oli-obk

Add a way to track Rustfix UI test coverage

This came out of the first Rustfix WG meeting.

One of the goals is to enable Rustfix tests for all UI tests that
trigger lints with `MachineApplicable` suggestions. In order to do that
we first want to create a tracking issue that lists all files with
missing `// run-rustfix` headers.

This PR adds a `--rustfix-coverage` flag to `./x.py` and compiletest to
list the files with the missing headers in `/tmp/rustfix_missing_coverage.txt`.

From that file we can create the tracking issue and at some point also
enforce the `// run-rustfix` flag on UI tests with `MachineApplicable`
lints.

6 files changed, 72 insertions, 1 deletions

diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index f93f3e72f83..7851ea3e363 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -1856,6 +1856,7 @@ mod __test {
             doc_tests: DocTests::No,
             bless: false,
             compare_mode: None,
+            rustfix_coverage: false,
         };
 
         let build = Build::new(config);
@@ -1897,6 +1898,7 @@ mod __test {
             doc_tests: DocTests::No,
             bless: false,
             compare_mode: None,
+            rustfix_coverage: false,
         };
 
         let build = Build::new(config);
diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs
index 0f9a4271ac0..a1f89d6c86f 100644
--- a/src/bootstrap/flags.rs
+++ b/src/bootstrap/flags.rs
@@ -56,6 +56,7 @@ pub enum Subcommand {
         rustc_args: Vec<String>,
         fail_fast: bool,
         doc_tests: DocTests,
+        rustfix_coverage: bool,
     },
     Bench {
         paths: Vec<PathBuf>,
@@ -188,6 +189,12 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`"
                     "mode describing what file the actual ui output will be compared to",
                     "COMPARE MODE",
                 );
+                opts.optflag(
+                    "",
+                    "rustfix-coverage",
+                    "enable this to generate a Rustfix coverage file, which is saved in \
+                        `/<build_base>/rustfix_missing_coverage.txt`",
+                );
             }
             "bench" => {
                 opts.optmulti("", "test-args", "extra arguments", "ARGS");
@@ -363,6 +370,7 @@ Arguments:
                 test_args: matches.opt_strs("test-args"),
                 rustc_args: matches.opt_strs("rustc-args"),
                 fail_fast: !matches.opt_present("no-fail-fast"),
+                rustfix_coverage: matches.opt_present("rustfix-coverage"),
                 doc_tests: if matches.opt_present("doc") {
                     DocTests::Only
                 } else if matches.opt_present("no-doc") {
@@ -467,6 +475,13 @@ impl Subcommand {
         }
     }
 
+    pub fn rustfix_coverage(&self) -> bool {
+        match *self {
+            Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage,
+            _ => false,
+        }
+    }
+
     pub fn compare_mode(&self) -> Option<&str> {
         match *self {
             Subcommand::Test {
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index bbe1872d395..41c73f307b6 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -1284,6 +1284,10 @@ impl Step for Compiletest {
             cmd.arg("--android-cross-path").arg("");
         }
 
+        if builder.config.cmd.rustfix_coverage() {
+            cmd.arg("--rustfix-coverage");
+        }
+
         builder.ci_env.force_coloring_in_ci(&mut cmd);
 
         let _folder = builder.fold_output(|| format!("test_{}", suite));
diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
index c1920dde5b1..089cbc7b78a 100644
--- a/src/tools/compiletest/src/common.rs
+++ b/src/tools/compiletest/src/common.rs
@@ -246,6 +246,11 @@ pub struct Config {
     /// mode describing what file the actual ui output will be compared to
     pub compare_mode: Option<CompareMode>,
 
+    /// If true, this will generate a coverage file with UI test files that run `MachineApplicable`
+    /// diagnostics but are missing `run-rustfix` annotations. The generated coverage file is
+    /// created in `/<build_base>/rustfix_missing_coverage.txt`
+    pub rustfix_coverage: bool,
+
     // Configuration for various run-make tests frobbing things like C compilers
     // or querying about various LLVM component information.
     pub cc: String,
diff --git a/src/tools/compiletest/src/main.rs b/src/tools/compiletest/src/main.rs
index d91710dda52..dfc023da973 100644
--- a/src/tools/compiletest/src/main.rs
+++ b/src/tools/compiletest/src/main.rs
@@ -233,6 +233,12 @@ pub fn parse_config(args: Vec<String>) -> Config {
             "mode describing what file the actual ui output will be compared to",
             "COMPARE MODE",
         )
+        .optflag(
+            "",
+            "rustfix-coverage",
+            "enable this to generate a Rustfix coverage file, which is saved in \
+                `./<build_base>/rustfix_missing_coverage.txt`",
+        )
         .optflag("h", "help", "show this message");
 
     let (argv0, args_) = args.split_first().unwrap();
@@ -336,6 +342,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         color,
         remote_test_client: matches.opt_str("remote-test-client").map(PathBuf::from),
         compare_mode: matches.opt_str("compare-mode").map(CompareMode::parse),
+        rustfix_coverage: matches.opt_present("rustfix-coverage"),
 
         cc: matches.opt_str("cc").unwrap(),
         cxx: matches.opt_str("cxx").unwrap(),
@@ -475,6 +482,19 @@ pub fn run_tests(config: &Config) {
         let _ = fs::remove_dir_all("tmp/partitioning-tests");
     }
 
+    // If we want to collect rustfix coverage information,
+    // we first make sure that the coverage file does not exist.
+    // It will be created later on.
+    if config.rustfix_coverage {
+        let mut coverage_file_path = config.build_base.clone();
+        coverage_file_path.push("rustfix_missing_coverage.txt");
+        if coverage_file_path.exists() {
+            if let Err(e) = fs::remove_file(&coverage_file_path) {
+                panic!("Could not delete {} due to {}", coverage_file_path.display(), e)
+            }
+        }
+    }
+
     let opts = test_opts(config);
     let tests = make_tests(config);
     // sadly osx needs some file descriptor limits raised for running tests in
diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
index 3e3499edf60..c18b6db9a01 100644
--- a/src/tools/compiletest/src/runtest.rs
+++ b/src/tools/compiletest/src/runtest.rs
@@ -19,7 +19,7 @@ use std::collections::{HashMap, HashSet, VecDeque};
 use std::env;
 use std::ffi::{OsStr, OsString};
 use std::fmt;
-use std::fs::{self, create_dir_all, File};
+use std::fs::{self, create_dir_all, File, OpenOptions};
 use std::hash::{Hash, Hasher};
 use std::io::prelude::*;
 use std::io::{self, BufReader};
@@ -2818,6 +2818,31 @@ impl<'test> TestCx<'test> {
 
         if self.config.compare_mode.is_some() {
             // don't test rustfix with nll right now
+        } else if self.config.rustfix_coverage {
+            // Find out which tests have `MachineApplicable` suggestions but are missing
+            // `run-rustfix` or `run-rustfix-only-machine-applicable` headers
+            let suggestions = get_suggestions_from_json(
+                &proc_res.stderr,
+                &HashSet::new(),
+                Filter::MachineApplicableOnly
+            ).unwrap();
+            if suggestions.len() > 0
+                && !self.props.run_rustfix
+                && !self.props.rustfix_only_machine_applicable {
+                    let mut coverage_file_path = self.config.build_base.clone();
+                    coverage_file_path.push("rustfix_missing_coverage.txt");
+                    debug!("coverage_file_path: {}", coverage_file_path.display());
+
+                    let mut file = OpenOptions::new()
+                        .create(true)
+                        .append(true)
+                        .open(coverage_file_path.as_path())
+                        .expect("could not create or open file");
+
+                    if let Err(_) = writeln!(file, "{}", self.testpaths.file.display()) {
+                        panic!("couldn't write to {}", coverage_file_path.display());
+                    }
+            }
         } else if self.props.run_rustfix {
             // Apply suggestions from rustc to the code itself
             let unfixed_code = self