Auto merge of #112482 - tgross35:ci-non-rust-linters, r=pietroalbini

Add support for tidy linting via external tools for non-rust files

This change adds the flag `--check-extras` to `tidy`. It accepts a comma separated list of any of the options:

* py (test everything applicable for python files)
* py:lint (lint python files using `ruff`)
* py:fmt (check formatting for python files using `black`)
* shell or shell:lint (lint shell files using `shellcheck`)

Specific files to check can also be specified via positional args. Examples:

* `./x test tidy --check-extras=shell,py`
* `./x test tidy --check-extras=py:fmt -- src/bootstrap/bootstrap.py`
* `./x test tidy --check-extras=shell -- src/ci/*.sh`
* Python formatting can be applied with bless: `./x test tidy --ckeck-extras=py:fmt --bless`

`ruff` and `black` need to be installed via pip; this tool manages these within a virtual environment at `build/venv`. `shellcheck` needs to be installed on the system already.

---

This PR doesn't fix any of the errors that show up (I will likely go through those at some point) and it doesn't enforce anything new in CI. Relevant zulip discussion: https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/Other.20linters.20in.20CI

5 files changed, 39 insertions, 6 deletions

diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index fe66b1d41bf..70079106689 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -472,7 +472,9 @@ class FakeArgs:
 
 class RustBuild(object):
     """Provide all the methods required to build Rust"""
-    def __init__(self, config_toml="", args=FakeArgs()):
+    def __init__(self, config_toml="", args=None):
+        if args is None:
+            args = FakeArgs()
         self.git_version = None
         self.nix_deps_dir = None
         self._should_fix_bins_and_dylibs = None
diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py
index 3c91e403df3..dc06a4c9734 100644
--- a/src/bootstrap/bootstrap_test.py
+++ b/src/bootstrap/bootstrap_test.py
@@ -4,7 +4,6 @@ Run these with `x test bootstrap`, or `python -m unittest src/bootstrap/bootstra
 
 from __future__ import absolute_import, division, print_function
 import os
-import doctest
 import unittest
 import tempfile
 import hashlib
@@ -16,12 +15,15 @@ from shutil import rmtree
 bootstrap_dir = os.path.dirname(os.path.abspath(__file__))
 # For the import below, have Python search in src/bootstrap first.
 sys.path.insert(0, bootstrap_dir)
-import bootstrap
-import configure
+import bootstrap # noqa: E402
+import configure # noqa: E402
 
-def serialize_and_parse(configure_args, bootstrap_args=bootstrap.FakeArgs()):
+def serialize_and_parse(configure_args, bootstrap_args=None):
     from io import StringIO
 
+    if bootstrap_args is None:
+        bootstrap_args = bootstrap.FakeArgs()
+
     section_order, sections, targets = configure.parse_args(configure_args)
     buffer = StringIO()
     configure.write_config_toml(buffer, section_order, targets, sections)
@@ -129,7 +131,14 @@ class GenerateAndParseConfig(unittest.TestCase):
 class BuildBootstrap(unittest.TestCase):
     """Test that we generate the appropriate arguments when building bootstrap"""
 
-    def build_args(self, configure_args=[], args=[], env={}):
+    def build_args(self, configure_args=None, args=None, env=None):
+        if configure_args is None:
+            configure_args = []
+        if args is None:
+            args = []
+        if env is None:
+            env = {}
+
         env = env.copy()
         env["PATH"] = os.environ["PATH"]
 
diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs
index d9540946c2e..43b4a34fe5b 100644
--- a/src/bootstrap/builder/tests.rs
+++ b/src/bootstrap/builder/tests.rs
@@ -587,6 +587,7 @@ mod dist {
             run: None,
             only_modified: false,
             skip: vec![],
+            extra_checks: None,
         };
 
         let build = Build::new(config);
@@ -658,6 +659,7 @@ mod dist {
             pass: None,
             run: None,
             only_modified: false,
+            extra_checks: None,
         };
         // Make sure rustfmt binary not being found isn't an error.
         config.channel = "beta".to_string();
diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs
index f5022b74fb2..e0291e407b3 100644
--- a/src/bootstrap/flags.rs
+++ b/src/bootstrap/flags.rs
@@ -339,6 +339,10 @@ pub enum Subcommand {
         /// whether to automatically update stderr/stdout files
         bless: bool,
         #[arg(long)]
+        /// comma-separated list of other files types to check (accepts py, py:lint,
+        /// py:fmt, shell)
+        extra_checks: Option<String>,
+        #[arg(long)]
         /// rerun tests even if the inputs are unchanged
         force_rerun: bool,
         #[arg(long)]
@@ -476,6 +480,13 @@ impl Subcommand {
         }
     }
 
+    pub fn extra_checks(&self) -> Option<&str> {
+        match *self {
+            Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str),
+            _ => None,
+        }
+    }
+
     pub fn only_modified(&self) -> bool {
         match *self {
             Subcommand::Test { only_modified, .. } => only_modified,
diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs
index 3462700759a..d0d62db0807 100644
--- a/src/bootstrap/test.rs
+++ b/src/bootstrap/test.rs
@@ -4,6 +4,7 @@
 //! our CI.
 
 use std::env;
+use std::ffi::OsStr;
 use std::ffi::OsString;
 use std::fs;
 use std::iter;
@@ -1094,6 +1095,14 @@ impl Step for Tidy {
         if builder.config.cmd.bless() {
             cmd.arg("--bless");
         }
+        if let Some(s) = builder.config.cmd.extra_checks() {
+            cmd.arg(format!("--extra-checks={s}"));
+        }
+        let mut args = std::env::args_os();
+        if let Some(_) = args.find(|arg| arg == OsStr::new("--")) {
+            cmd.arg("--");
+            cmd.args(args);
+        }
 
         if builder.config.channel == "dev" || builder.config.channel == "nightly" {
             builder.info("fmt check");