From b96f238308aa892825d6d75e359232e99f61b4a4 Mon Sep 17 00:00:00 2001 From: binarycat Date: Thu, 24 Jul 2025 19:16:15 -0500 Subject: rename ext_tool_checks to extra_checks and use mod.rs this makes the triagebot pings for this module simpler --- src/tools/tidy/src/ext_tool_checks.rs | 834 ----------------------- src/tools/tidy/src/ext_tool_checks/rustdoc_js.rs | 113 --- src/tools/tidy/src/extra_checks/mod.rs | 834 +++++++++++++++++++++++ src/tools/tidy/src/extra_checks/rustdoc_js.rs | 113 +++ src/tools/tidy/src/lib.rs | 2 +- src/tools/tidy/src/main.rs | 2 +- 6 files changed, 949 insertions(+), 949 deletions(-) delete mode 100644 src/tools/tidy/src/ext_tool_checks.rs delete mode 100644 src/tools/tidy/src/ext_tool_checks/rustdoc_js.rs create mode 100644 src/tools/tidy/src/extra_checks/mod.rs create mode 100644 src/tools/tidy/src/extra_checks/rustdoc_js.rs (limited to 'src') diff --git a/src/tools/tidy/src/ext_tool_checks.rs b/src/tools/tidy/src/ext_tool_checks.rs deleted file mode 100644 index 8121eb057db..00000000000 --- a/src/tools/tidy/src/ext_tool_checks.rs +++ /dev/null @@ -1,834 +0,0 @@ -//! Optional checks for file types other than Rust source -//! -//! Handles python tool version management via a virtual environment in -//! `build/venv`. -//! -//! # Functional outline -//! -//! 1. Run tidy with an extra option: `--extra-checks=py,shell`, -//! `--extra-checks=py:lint`, or similar. Optionally provide specific -//! configuration after a double dash (`--extra-checks=py -- foo.py`) -//! 2. Build configuration based on args/environment: -//! - Formatters by default are in check only mode -//! - If in CI (TIDY_PRINT_DIFF=1 is set), check and print the diff -//! - If `--bless` is provided, formatters may run -//! - Pass any additional config after the `--`. If no files are specified, -//! use a default. -//! 3. Print the output of the given command. If it fails and `TIDY_PRINT_DIFF` -//! is set, rerun the tool to print a suggestion diff (for e.g. CI) - -use std::ffi::OsStr; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::str::FromStr; -use std::{fmt, fs, io}; - -use crate::CiInfo; - -mod rustdoc_js; - -const MIN_PY_REV: (u32, u32) = (3, 9); -const MIN_PY_REV_STR: &str = "≥3.9"; - -/// Path to find the python executable within a virtual environment -#[cfg(target_os = "windows")] -const REL_PY_PATH: &[&str] = &["Scripts", "python3.exe"]; -#[cfg(not(target_os = "windows"))] -const REL_PY_PATH: &[&str] = &["bin", "python3"]; - -const RUFF_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "ruff.toml"]; -/// Location within build directory -const RUFF_CACHE_PATH: &[&str] = &["cache", "ruff_cache"]; -const PIP_REQ_PATH: &[&str] = &["src", "tools", "tidy", "config", "requirements.txt"]; - -// this must be kept in sync with with .github/workflows/spellcheck.yml -const SPELLCHECK_DIRS: &[&str] = &["compiler", "library", "src/bootstrap", "src/librustdoc"]; - -pub fn check( - root_path: &Path, - outdir: &Path, - ci_info: &CiInfo, - librustdoc_path: &Path, - tools_path: &Path, - npm: &Path, - bless: bool, - extra_checks: Option<&str>, - pos_args: &[String], - bad: &mut bool, -) { - if let Err(e) = check_impl( - root_path, - outdir, - ci_info, - librustdoc_path, - tools_path, - npm, - bless, - extra_checks, - pos_args, - ) { - tidy_error!(bad, "{e}"); - } -} - -fn check_impl( - root_path: &Path, - outdir: &Path, - ci_info: &CiInfo, - librustdoc_path: &Path, - tools_path: &Path, - npm: &Path, - bless: bool, - extra_checks: Option<&str>, - pos_args: &[String], -) -> Result<(), Error> { - let show_diff = - std::env::var("TIDY_PRINT_DIFF").is_ok_and(|v| v.eq_ignore_ascii_case("true") || v == "1"); - - // Split comma-separated args up - let lint_args = match extra_checks { - Some(s) => s - .strip_prefix("--extra-checks=") - .unwrap() - .split(',') - .map(|s| { - if s == "spellcheck:fix" { - eprintln!("warning: `spellcheck:fix` is no longer valid, use `--extra-checks=spellcheck --bless`"); - } - (ExtraCheckArg::from_str(s), s) - }) - .filter_map(|(res, src)| match res { - Ok(arg) => { - if arg.is_inactive_auto(ci_info) { - None - } else { - Some(arg) - } - } - Err(err) => { - // only warn because before bad extra checks would be silently ignored. - eprintln!("warning: bad extra check argument {src:?}: {err:?}"); - None - } - }) - .collect(), - None => vec![], - }; - - macro_rules! extra_check { - ($lang:ident, $kind:ident) => { - lint_args.iter().any(|arg| arg.matches(ExtraCheckLang::$lang, ExtraCheckKind::$kind)) - }; - } - - let python_lint = extra_check!(Py, Lint); - let python_fmt = extra_check!(Py, Fmt); - let shell_lint = extra_check!(Shell, Lint); - let cpp_fmt = extra_check!(Cpp, Fmt); - let spellcheck = extra_check!(Spellcheck, None); - let js_lint = extra_check!(Js, Lint); - let js_typecheck = extra_check!(Js, Typecheck); - - let mut py_path = None; - - let (cfg_args, file_args): (Vec<_>, Vec<_>) = pos_args - .iter() - .map(OsStr::new) - .partition(|arg| arg.to_str().is_some_and(|s| s.starts_with('-'))); - - if python_lint || python_fmt || cpp_fmt { - let venv_path = outdir.join("venv"); - let mut reqs_path = root_path.to_owned(); - reqs_path.extend(PIP_REQ_PATH); - py_path = Some(get_or_create_venv(&venv_path, &reqs_path)?); - } - - if python_lint { - eprintln!("linting python files"); - let py_path = py_path.as_ref().unwrap(); - let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, &["check".as_ref()]); - - if res.is_err() && show_diff { - eprintln!("\npython linting failed! Printing diff suggestions:"); - - let _ = run_ruff( - root_path, - outdir, - py_path, - &cfg_args, - &file_args, - &["check".as_ref(), "--diff".as_ref()], - ); - } - // Rethrow error - res?; - } - - if python_fmt { - let mut args: Vec<&OsStr> = vec!["format".as_ref()]; - if bless { - eprintln!("formatting python files"); - } else { - eprintln!("checking python file formatting"); - args.push("--check".as_ref()); - } - - let py_path = py_path.as_ref().unwrap(); - let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, &args); - - if res.is_err() && !bless { - if show_diff { - eprintln!("\npython formatting does not match! Printing diff:"); - - let _ = run_ruff( - root_path, - outdir, - py_path, - &cfg_args, - &file_args, - &["format".as_ref(), "--diff".as_ref()], - ); - } - eprintln!("rerun tidy with `--extra-checks=py:fmt --bless` to reformat Python code"); - } - - // Rethrow error - res?; - } - - if cpp_fmt { - let mut cfg_args_clang_format = cfg_args.clone(); - let mut file_args_clang_format = file_args.clone(); - let config_path = root_path.join(".clang-format"); - let config_file_arg = format!("file:{}", config_path.display()); - cfg_args_clang_format.extend(&["--style".as_ref(), config_file_arg.as_ref()]); - if bless { - eprintln!("formatting C++ files"); - cfg_args_clang_format.push("-i".as_ref()); - } else { - eprintln!("checking C++ file formatting"); - cfg_args_clang_format.extend(&["--dry-run".as_ref(), "--Werror".as_ref()]); - } - let files; - if file_args_clang_format.is_empty() { - let llvm_wrapper = root_path.join("compiler/rustc_llvm/llvm-wrapper"); - files = find_with_extension( - root_path, - Some(llvm_wrapper.as_path()), - &[OsStr::new("h"), OsStr::new("cpp")], - )?; - file_args_clang_format.extend(files.iter().map(|p| p.as_os_str())); - } - let args = merge_args(&cfg_args_clang_format, &file_args_clang_format); - let res = py_runner(py_path.as_ref().unwrap(), false, None, "clang-format", &args); - - if res.is_err() && show_diff { - eprintln!("\nclang-format linting failed! Printing diff suggestions:"); - - let mut cfg_args_clang_format_diff = cfg_args.clone(); - cfg_args_clang_format_diff.extend(&["--style".as_ref(), config_file_arg.as_ref()]); - for file in file_args_clang_format { - let mut formatted = String::new(); - let mut diff_args = cfg_args_clang_format_diff.clone(); - diff_args.push(file); - let _ = py_runner( - py_path.as_ref().unwrap(), - false, - Some(&mut formatted), - "clang-format", - &diff_args, - ); - if formatted.is_empty() { - eprintln!( - "failed to obtain the formatted content for '{}'", - file.to_string_lossy() - ); - continue; - } - let actual = std::fs::read_to_string(file).unwrap_or_else(|e| { - panic!( - "failed to read the C++ file at '{}' due to '{e}'", - file.to_string_lossy() - ) - }); - if formatted != actual { - let diff = similar::TextDiff::from_lines(&actual, &formatted); - eprintln!( - "{}", - diff.unified_diff().context_radius(4).header( - &format!("{} (actual)", file.to_string_lossy()), - &format!("{} (formatted)", file.to_string_lossy()) - ) - ); - } - } - } - // Rethrow error - res?; - } - - if shell_lint { - eprintln!("linting shell files"); - - let mut file_args_shc = file_args.clone(); - let files; - if file_args_shc.is_empty() { - files = find_with_extension(root_path, None, &[OsStr::new("sh")])?; - file_args_shc.extend(files.iter().map(|p| p.as_os_str())); - } - - shellcheck_runner(&merge_args(&cfg_args, &file_args_shc))?; - } - - if spellcheck { - let config_path = root_path.join("typos.toml"); - let mut args = vec!["-c", config_path.as_os_str().to_str().unwrap()]; - - args.extend_from_slice(SPELLCHECK_DIRS); - - if bless { - eprintln!("spellcheck files and fix"); - args.push("--write-changes"); - } else { - eprintln!("spellcheck files"); - } - spellcheck_runner(&args)?; - } - - if js_lint || js_typecheck { - rustdoc_js::npm_install(root_path, outdir, npm)?; - } - - if js_lint { - rustdoc_js::lint(outdir, librustdoc_path, tools_path)?; - rustdoc_js::es_check(outdir, librustdoc_path)?; - } - - if js_typecheck { - rustdoc_js::typecheck(outdir, librustdoc_path)?; - } - - Ok(()) -} - -fn run_ruff( - root_path: &Path, - outdir: &Path, - py_path: &Path, - cfg_args: &[&OsStr], - file_args: &[&OsStr], - ruff_args: &[&OsStr], -) -> Result<(), Error> { - let mut cfg_args_ruff = cfg_args.to_vec(); - let mut file_args_ruff = file_args.to_vec(); - - let mut cfg_path = root_path.to_owned(); - cfg_path.extend(RUFF_CONFIG_PATH); - let mut cache_dir = outdir.to_owned(); - cache_dir.extend(RUFF_CACHE_PATH); - - cfg_args_ruff.extend([ - "--config".as_ref(), - cfg_path.as_os_str(), - "--cache-dir".as_ref(), - cache_dir.as_os_str(), - ]); - - if file_args_ruff.is_empty() { - file_args_ruff.push(root_path.as_os_str()); - } - - let mut args: Vec<&OsStr> = ruff_args.to_vec(); - args.extend(merge_args(&cfg_args_ruff, &file_args_ruff)); - py_runner(py_path, true, None, "ruff", &args) -} - -/// Helper to create `cfg1 cfg2 -- file1 file2` output -fn merge_args<'a>(cfg_args: &[&'a OsStr], file_args: &[&'a OsStr]) -> Vec<&'a OsStr> { - let mut args = cfg_args.to_owned(); - args.push("--".as_ref()); - args.extend(file_args); - args -} - -/// Run a python command with given arguments. `py_path` should be a virtualenv. -/// -/// Captures `stdout` to a string if provided, otherwise prints the output. -fn py_runner( - py_path: &Path, - as_module: bool, - stdout: Option<&mut String>, - bin: &'static str, - args: &[&OsStr], -) -> Result<(), Error> { - let mut cmd = Command::new(py_path); - if as_module { - cmd.arg("-m").arg(bin).args(args); - } else { - let bin_path = py_path.with_file_name(bin); - cmd.arg(bin_path).args(args); - } - let status = if let Some(stdout) = stdout { - let output = cmd.output()?; - if let Ok(s) = std::str::from_utf8(&output.stdout) { - stdout.push_str(s); - } - output.status - } else { - cmd.status()? - }; - if status.success() { Ok(()) } else { Err(Error::FailedCheck(bin)) } -} - -/// Create a virtuaenv at a given path if it doesn't already exist, or validate -/// the install if it does. Returns the path to that venv's python executable. -fn get_or_create_venv(venv_path: &Path, src_reqs_path: &Path) -> Result { - let mut should_create = true; - let dst_reqs_path = venv_path.join("requirements.txt"); - let mut py_path = venv_path.to_owned(); - py_path.extend(REL_PY_PATH); - - if let Ok(req) = fs::read_to_string(&dst_reqs_path) { - if req == fs::read_to_string(src_reqs_path)? { - // found existing environment - should_create = false; - } else { - eprintln!("requirements.txt file mismatch, recreating environment"); - } - } - - if should_create { - eprintln!("removing old virtual environment"); - if venv_path.is_dir() { - fs::remove_dir_all(venv_path).unwrap_or_else(|_| { - panic!("failed to remove directory at {}", venv_path.display()) - }); - } - create_venv_at_path(venv_path)?; - install_requirements(&py_path, src_reqs_path, &dst_reqs_path)?; - } - - verify_py_version(&py_path)?; - Ok(py_path) -} - -/// Attempt to create a virtualenv at this path. Cycles through all expected -/// valid python versions to find one that is installed. -fn create_venv_at_path(path: &Path) -> Result<(), Error> { - /// Preferred python versions in order. Newest to oldest then current - /// development versions - const TRY_PY: &[&str] = &[ - "python3.13", - "python3.12", - "python3.11", - "python3.10", - "python3.9", - "python3", - "python", - "python3.14", - ]; - - let mut sys_py = None; - let mut found = Vec::new(); - - for py in TRY_PY { - match verify_py_version(Path::new(py)) { - Ok(_) => { - sys_py = Some(*py); - break; - } - // Skip not found errors - Err(Error::Io(e)) if e.kind() == io::ErrorKind::NotFound => (), - // Skip insufficient version errors - Err(Error::Version { installed, .. }) => found.push(installed), - // just log and skip unrecognized errors - Err(e) => eprintln!("note: error running '{py}': {e}"), - } - } - - let Some(sys_py) = sys_py else { - let ret = if found.is_empty() { - Error::MissingReq("python3", "python file checks", None) - } else { - found.sort(); - found.dedup(); - Error::Version { - program: "python3", - required: MIN_PY_REV_STR, - installed: found.join(", "), - } - }; - return Err(ret); - }; - - // First try venv, which should be packaged in the Python3 standard library. - // If it is not available, try to create the virtual environment using the - // virtualenv package. - if try_create_venv(sys_py, path, "venv").is_ok() { - return Ok(()); - } - try_create_venv(sys_py, path, "virtualenv") -} - -fn try_create_venv(python: &str, path: &Path, module: &str) -> Result<(), Error> { - eprintln!( - "creating virtual environment at '{}' using '{python}' and '{module}'", - path.display() - ); - let out = Command::new(python).args(["-m", module]).arg(path).output().unwrap(); - - if out.status.success() { - return Ok(()); - } - - let stderr = String::from_utf8_lossy(&out.stderr); - let err = if stderr.contains(&format!("No module named {module}")) { - Error::Generic(format!( - r#"{module} not found: you may need to install it: -`{python} -m pip install {module}` -If you see an error about "externally managed environment" when running the above command, -either install `{module}` using your system package manager -(e.g. `sudo apt-get install {python}-{module}`) or create a virtual environment manually, install -`{module}` in it and then activate it before running tidy. -"# - )) - } else { - Error::Generic(format!( - "failed to create venv at '{}' using {python} -m {module}: {stderr}", - path.display() - )) - }; - Err(err) -} - -/// Parse python's version output (`Python x.y.z`) and ensure we have a -/// suitable version. -fn verify_py_version(py_path: &Path) -> Result<(), Error> { - let out = Command::new(py_path).arg("--version").output()?; - let outstr = String::from_utf8_lossy(&out.stdout); - let vers = outstr.trim().split_ascii_whitespace().nth(1).unwrap().trim(); - let mut vers_comps = vers.split('.'); - let major: u32 = vers_comps.next().unwrap().parse().unwrap(); - let minor: u32 = vers_comps.next().unwrap().parse().unwrap(); - - if (major, minor) < MIN_PY_REV { - Err(Error::Version { - program: "python", - required: MIN_PY_REV_STR, - installed: vers.to_owned(), - }) - } else { - Ok(()) - } -} - -fn install_requirements( - py_path: &Path, - src_reqs_path: &Path, - dst_reqs_path: &Path, -) -> Result<(), Error> { - let stat = Command::new(py_path) - .args(["-m", "pip", "install", "--upgrade", "pip"]) - .status() - .expect("failed to launch pip"); - if !stat.success() { - return Err(Error::Generic(format!("pip install failed with status {stat}"))); - } - - let stat = Command::new(py_path) - .args(["-m", "pip", "install", "--quiet", "--require-hashes", "-r"]) - .arg(src_reqs_path) - .status()?; - if !stat.success() { - return Err(Error::Generic(format!( - "failed to install requirements at {}", - src_reqs_path.display() - ))); - } - fs::copy(src_reqs_path, dst_reqs_path)?; - assert_eq!( - fs::read_to_string(src_reqs_path).unwrap(), - fs::read_to_string(dst_reqs_path).unwrap() - ); - Ok(()) -} - -/// Check that shellcheck is installed then run it at the given path -fn shellcheck_runner(args: &[&OsStr]) -> Result<(), Error> { - match Command::new("shellcheck").arg("--version").status() { - Ok(_) => (), - Err(e) if e.kind() == io::ErrorKind::NotFound => { - return Err(Error::MissingReq( - "shellcheck", - "shell file checks", - Some( - "see \ - for installation instructions" - .to_owned(), - ), - )); - } - Err(e) => return Err(e.into()), - } - - let status = Command::new("shellcheck").args(args).status()?; - if status.success() { Ok(()) } else { Err(Error::FailedCheck("shellcheck")) } -} - -/// Check that spellchecker is installed then run it at the given path -fn spellcheck_runner(args: &[&str]) -> Result<(), Error> { - // sync version with .github/workflows/spellcheck.yml - let expected_version = "typos-cli 1.34.0"; - match Command::new("typos").arg("--version").output() { - Ok(o) => { - let stdout = String::from_utf8_lossy(&o.stdout); - if stdout.trim() != expected_version { - return Err(Error::Version { - program: "typos", - required: expected_version, - installed: stdout.trim().to_string(), - }); - } - } - Err(e) if e.kind() == io::ErrorKind::NotFound => { - return Err(Error::MissingReq( - "typos", - "spellcheck file checks", - // sync version with .github/workflows/spellcheck.yml - Some("install tool via `cargo install typos-cli@1.34.0`".to_owned()), - )); - } - Err(e) => return Err(e.into()), - } - - let status = Command::new("typos").args(args).status()?; - if status.success() { Ok(()) } else { Err(Error::FailedCheck("typos")) } -} - -/// Check git for tracked files matching an extension -fn find_with_extension( - root_path: &Path, - find_dir: Option<&Path>, - extensions: &[&OsStr], -) -> Result, Error> { - // Untracked files show up for short status and are indicated with a leading `?` - // -C changes git to be as if run from that directory - let stat_output = - Command::new("git").arg("-C").arg(root_path).args(["status", "--short"]).output()?.stdout; - - if String::from_utf8_lossy(&stat_output).lines().filter(|ln| ln.starts_with('?')).count() > 0 { - eprintln!("found untracked files, ignoring"); - } - - let mut output = Vec::new(); - let binding = { - let mut command = Command::new("git"); - command.arg("-C").arg(root_path).args(["ls-files"]); - if let Some(find_dir) = find_dir { - command.arg(find_dir); - } - command.output()? - }; - let tracked = String::from_utf8_lossy(&binding.stdout); - - for line in tracked.lines() { - let line = line.trim(); - let path = Path::new(line); - - let Some(ref extension) = path.extension() else { - continue; - }; - if extensions.contains(extension) { - output.push(root_path.join(path)); - } - } - - Ok(output) -} - -#[derive(Debug)] -enum Error { - Io(io::Error), - /// a is required to run b. c is extra info - MissingReq(&'static str, &'static str, Option), - /// Tool x failed the check - FailedCheck(&'static str), - /// Any message, just print it - Generic(String), - /// Installed but wrong version - Version { - program: &'static str, - required: &'static str, - installed: String, - }, -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::MissingReq(a, b, ex) => { - write!( - f, - "{a} is required to run {b} but it could not be located. Is it installed?" - )?; - if let Some(s) = ex { - write!(f, "\n{s}")?; - }; - Ok(()) - } - Self::Version { program, required, installed } => write!( - f, - "insufficient version of '{program}' to run external tools: \ - {required} required but found {installed}", - ), - Self::Generic(s) => f.write_str(s), - Self::Io(e) => write!(f, "IO error: {e}"), - Self::FailedCheck(s) => write!(f, "checks with external tool '{s}' failed"), - } - } -} - -impl From for Error { - fn from(value: io::Error) -> Self { - Self::Io(value) - } -} - -#[derive(Debug)] -enum ExtraCheckParseError { - #[allow(dead_code, reason = "shown through Debug")] - UnknownKind(String), - #[allow(dead_code)] - UnknownLang(String), - UnsupportedKindForLang, - /// Too many `:` - TooManyParts, - /// Tried to parse the empty string - Empty, - /// `auto` specified without lang part. - AutoRequiresLang, -} - -struct ExtraCheckArg { - auto: bool, - lang: ExtraCheckLang, - /// None = run all extra checks for the given lang - kind: Option, -} - -impl ExtraCheckArg { - fn matches(&self, lang: ExtraCheckLang, kind: ExtraCheckKind) -> bool { - self.lang == lang && self.kind.map(|k| k == kind).unwrap_or(true) - } - - /// Returns `true` if this is an auto arg and the relevant files are not modified. - fn is_inactive_auto(&self, ci_info: &CiInfo) -> bool { - if !self.auto { - return false; - } - let ext = match self.lang { - ExtraCheckLang::Py => ".py", - ExtraCheckLang::Cpp => ".cpp", - ExtraCheckLang::Shell => ".sh", - ExtraCheckLang::Js => ".js", - ExtraCheckLang::Spellcheck => { - return !crate::files_modified(ci_info, |s| { - SPELLCHECK_DIRS.iter().any(|dir| Path::new(s).starts_with(dir)) - }); - } - }; - !crate::files_modified(ci_info, |s| s.ends_with(ext)) - } - - fn has_supported_kind(&self) -> bool { - let Some(kind) = self.kind else { - // "run all extra checks" mode is supported for all languages. - return true; - }; - use ExtraCheckKind::*; - let supported_kinds: &[_] = match self.lang { - ExtraCheckLang::Py => &[Fmt, Lint], - ExtraCheckLang::Cpp => &[Fmt], - ExtraCheckLang::Shell => &[Lint], - ExtraCheckLang::Spellcheck => &[], - ExtraCheckLang::Js => &[Lint, Typecheck], - }; - supported_kinds.contains(&kind) - } -} - -impl FromStr for ExtraCheckArg { - type Err = ExtraCheckParseError; - - fn from_str(s: &str) -> Result { - let mut auto = false; - let mut parts = s.split(':'); - let Some(mut first) = parts.next() else { - return Err(ExtraCheckParseError::Empty); - }; - if first == "auto" { - let Some(part) = parts.next() else { - return Err(ExtraCheckParseError::AutoRequiresLang); - }; - auto = true; - first = part; - } - let second = parts.next(); - if parts.next().is_some() { - return Err(ExtraCheckParseError::TooManyParts); - } - let arg = Self { auto, lang: first.parse()?, kind: second.map(|s| s.parse()).transpose()? }; - if !arg.has_supported_kind() { - return Err(ExtraCheckParseError::UnsupportedKindForLang); - } - - Ok(arg) - } -} - -#[derive(PartialEq, Copy, Clone)] -enum ExtraCheckLang { - Py, - Shell, - Cpp, - Spellcheck, - Js, -} - -impl FromStr for ExtraCheckLang { - type Err = ExtraCheckParseError; - - fn from_str(s: &str) -> Result { - Ok(match s { - "py" => Self::Py, - "shell" => Self::Shell, - "cpp" => Self::Cpp, - "spellcheck" => Self::Spellcheck, - "js" => Self::Js, - _ => return Err(ExtraCheckParseError::UnknownLang(s.to_string())), - }) - } -} - -#[derive(PartialEq, Copy, Clone)] -enum ExtraCheckKind { - Lint, - Fmt, - Typecheck, - /// Never parsed, but used as a placeholder for - /// langs that never have a specific kind. - None, -} - -impl FromStr for ExtraCheckKind { - type Err = ExtraCheckParseError; - - fn from_str(s: &str) -> Result { - Ok(match s { - "lint" => Self::Lint, - "fmt" => Self::Fmt, - "typecheck" => Self::Typecheck, - _ => return Err(ExtraCheckParseError::UnknownKind(s.to_string())), - }) - } -} diff --git a/src/tools/tidy/src/ext_tool_checks/rustdoc_js.rs b/src/tools/tidy/src/ext_tool_checks/rustdoc_js.rs deleted file mode 100644 index 7708b128e23..00000000000 --- a/src/tools/tidy/src/ext_tool_checks/rustdoc_js.rs +++ /dev/null @@ -1,113 +0,0 @@ -//! Tidy check to ensure that rustdoc templates didn't forget a `{# #}` to strip extra whitespace -//! characters. - -use std::ffi::OsStr; -use std::io; -use std::path::{Path, PathBuf}; -use std::process::{Child, Command}; - -use build_helper::npm; -use ignore::DirEntry; - -use crate::walk::walk_no_read; - -fn node_module_bin(outdir: &Path, name: &str) -> PathBuf { - outdir.join("node_modules/.bin").join(name) -} - -fn spawn_cmd(cmd: &mut Command) -> Result { - cmd.spawn().map_err(|err| { - eprintln!("unable to run {cmd:?} due to {err:?}"); - err - }) -} - -/// install all js dependencies from package.json. -pub(super) fn npm_install(root_path: &Path, outdir: &Path, npm: &Path) -> Result<(), super::Error> { - npm::install(root_path, outdir, npm)?; - Ok(()) -} - -fn rustdoc_js_files(librustdoc_path: &Path) -> Vec { - let mut files = Vec::new(); - walk_no_read( - &[&librustdoc_path.join("html/static/js")], - |path, is_dir| is_dir || path.extension().is_none_or(|ext| ext != OsStr::new("js")), - &mut |path: &DirEntry| { - files.push(path.path().into()); - }, - ); - return files; -} - -fn run_eslint(outdir: &Path, args: &[PathBuf], config_folder: PathBuf) -> Result<(), super::Error> { - let mut child = spawn_cmd( - Command::new(node_module_bin(outdir, "eslint")) - .arg("-c") - .arg(config_folder.join(".eslintrc.js")) - .args(args), - )?; - match child.wait() { - Ok(exit_status) => { - if exit_status.success() { - return Ok(()); - } - Err(super::Error::FailedCheck("eslint command failed")) - } - Err(error) => Err(super::Error::Generic(format!("eslint command failed: {error:?}"))), - } -} - -pub(super) fn lint( - outdir: &Path, - librustdoc_path: &Path, - tools_path: &Path, -) -> Result<(), super::Error> { - let files_to_check = rustdoc_js_files(librustdoc_path); - println!("Running eslint on rustdoc JS files"); - run_eslint(outdir, &files_to_check, librustdoc_path.join("html/static"))?; - - run_eslint(outdir, &[tools_path.join("rustdoc-js/tester.js")], tools_path.join("rustdoc-js"))?; - run_eslint( - outdir, - &[tools_path.join("rustdoc-gui/tester.js")], - tools_path.join("rustdoc-gui"), - )?; - Ok(()) -} - -pub(super) fn typecheck(outdir: &Path, librustdoc_path: &Path) -> Result<(), super::Error> { - // use npx to ensure correct version - let mut child = spawn_cmd( - Command::new(node_module_bin(outdir, "tsc")) - .arg("-p") - .arg(librustdoc_path.join("html/static/js/tsconfig.json")), - )?; - match child.wait() { - Ok(exit_status) => { - if exit_status.success() { - return Ok(()); - } - Err(super::Error::FailedCheck("tsc command failed")) - } - Err(error) => Err(super::Error::Generic(format!("tsc command failed: {error:?}"))), - } -} - -pub(super) fn es_check(outdir: &Path, librustdoc_path: &Path) -> Result<(), super::Error> { - let files_to_check = rustdoc_js_files(librustdoc_path); - let mut cmd = Command::new(node_module_bin(outdir, "es-check")); - cmd.arg("es2019"); - for f in files_to_check { - cmd.arg(f); - } - match spawn_cmd(&mut cmd)?.wait() { - Ok(exit_status) => { - if exit_status.success() { - return Ok(()); - } - Err(super::Error::FailedCheck("es-check command failed")) - } - Err(error) => Err(super::Error::Generic(format!("es-check command failed: {error:?}"))), - } -} diff --git a/src/tools/tidy/src/extra_checks/mod.rs b/src/tools/tidy/src/extra_checks/mod.rs new file mode 100644 index 00000000000..8121eb057db --- /dev/null +++ b/src/tools/tidy/src/extra_checks/mod.rs @@ -0,0 +1,834 @@ +//! Optional checks for file types other than Rust source +//! +//! Handles python tool version management via a virtual environment in +//! `build/venv`. +//! +//! # Functional outline +//! +//! 1. Run tidy with an extra option: `--extra-checks=py,shell`, +//! `--extra-checks=py:lint`, or similar. Optionally provide specific +//! configuration after a double dash (`--extra-checks=py -- foo.py`) +//! 2. Build configuration based on args/environment: +//! - Formatters by default are in check only mode +//! - If in CI (TIDY_PRINT_DIFF=1 is set), check and print the diff +//! - If `--bless` is provided, formatters may run +//! - Pass any additional config after the `--`. If no files are specified, +//! use a default. +//! 3. Print the output of the given command. If it fails and `TIDY_PRINT_DIFF` +//! is set, rerun the tool to print a suggestion diff (for e.g. CI) + +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::str::FromStr; +use std::{fmt, fs, io}; + +use crate::CiInfo; + +mod rustdoc_js; + +const MIN_PY_REV: (u32, u32) = (3, 9); +const MIN_PY_REV_STR: &str = "≥3.9"; + +/// Path to find the python executable within a virtual environment +#[cfg(target_os = "windows")] +const REL_PY_PATH: &[&str] = &["Scripts", "python3.exe"]; +#[cfg(not(target_os = "windows"))] +const REL_PY_PATH: &[&str] = &["bin", "python3"]; + +const RUFF_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "ruff.toml"]; +/// Location within build directory +const RUFF_CACHE_PATH: &[&str] = &["cache", "ruff_cache"]; +const PIP_REQ_PATH: &[&str] = &["src", "tools", "tidy", "config", "requirements.txt"]; + +// this must be kept in sync with with .github/workflows/spellcheck.yml +const SPELLCHECK_DIRS: &[&str] = &["compiler", "library", "src/bootstrap", "src/librustdoc"]; + +pub fn check( + root_path: &Path, + outdir: &Path, + ci_info: &CiInfo, + librustdoc_path: &Path, + tools_path: &Path, + npm: &Path, + bless: bool, + extra_checks: Option<&str>, + pos_args: &[String], + bad: &mut bool, +) { + if let Err(e) = check_impl( + root_path, + outdir, + ci_info, + librustdoc_path, + tools_path, + npm, + bless, + extra_checks, + pos_args, + ) { + tidy_error!(bad, "{e}"); + } +} + +fn check_impl( + root_path: &Path, + outdir: &Path, + ci_info: &CiInfo, + librustdoc_path: &Path, + tools_path: &Path, + npm: &Path, + bless: bool, + extra_checks: Option<&str>, + pos_args: &[String], +) -> Result<(), Error> { + let show_diff = + std::env::var("TIDY_PRINT_DIFF").is_ok_and(|v| v.eq_ignore_ascii_case("true") || v == "1"); + + // Split comma-separated args up + let lint_args = match extra_checks { + Some(s) => s + .strip_prefix("--extra-checks=") + .unwrap() + .split(',') + .map(|s| { + if s == "spellcheck:fix" { + eprintln!("warning: `spellcheck:fix` is no longer valid, use `--extra-checks=spellcheck --bless`"); + } + (ExtraCheckArg::from_str(s), s) + }) + .filter_map(|(res, src)| match res { + Ok(arg) => { + if arg.is_inactive_auto(ci_info) { + None + } else { + Some(arg) + } + } + Err(err) => { + // only warn because before bad extra checks would be silently ignored. + eprintln!("warning: bad extra check argument {src:?}: {err:?}"); + None + } + }) + .collect(), + None => vec![], + }; + + macro_rules! extra_check { + ($lang:ident, $kind:ident) => { + lint_args.iter().any(|arg| arg.matches(ExtraCheckLang::$lang, ExtraCheckKind::$kind)) + }; + } + + let python_lint = extra_check!(Py, Lint); + let python_fmt = extra_check!(Py, Fmt); + let shell_lint = extra_check!(Shell, Lint); + let cpp_fmt = extra_check!(Cpp, Fmt); + let spellcheck = extra_check!(Spellcheck, None); + let js_lint = extra_check!(Js, Lint); + let js_typecheck = extra_check!(Js, Typecheck); + + let mut py_path = None; + + let (cfg_args, file_args): (Vec<_>, Vec<_>) = pos_args + .iter() + .map(OsStr::new) + .partition(|arg| arg.to_str().is_some_and(|s| s.starts_with('-'))); + + if python_lint || python_fmt || cpp_fmt { + let venv_path = outdir.join("venv"); + let mut reqs_path = root_path.to_owned(); + reqs_path.extend(PIP_REQ_PATH); + py_path = Some(get_or_create_venv(&venv_path, &reqs_path)?); + } + + if python_lint { + eprintln!("linting python files"); + let py_path = py_path.as_ref().unwrap(); + let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, &["check".as_ref()]); + + if res.is_err() && show_diff { + eprintln!("\npython linting failed! Printing diff suggestions:"); + + let _ = run_ruff( + root_path, + outdir, + py_path, + &cfg_args, + &file_args, + &["check".as_ref(), "--diff".as_ref()], + ); + } + // Rethrow error + res?; + } + + if python_fmt { + let mut args: Vec<&OsStr> = vec!["format".as_ref()]; + if bless { + eprintln!("formatting python files"); + } else { + eprintln!("checking python file formatting"); + args.push("--check".as_ref()); + } + + let py_path = py_path.as_ref().unwrap(); + let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, &args); + + if res.is_err() && !bless { + if show_diff { + eprintln!("\npython formatting does not match! Printing diff:"); + + let _ = run_ruff( + root_path, + outdir, + py_path, + &cfg_args, + &file_args, + &["format".as_ref(), "--diff".as_ref()], + ); + } + eprintln!("rerun tidy with `--extra-checks=py:fmt --bless` to reformat Python code"); + } + + // Rethrow error + res?; + } + + if cpp_fmt { + let mut cfg_args_clang_format = cfg_args.clone(); + let mut file_args_clang_format = file_args.clone(); + let config_path = root_path.join(".clang-format"); + let config_file_arg = format!("file:{}", config_path.display()); + cfg_args_clang_format.extend(&["--style".as_ref(), config_file_arg.as_ref()]); + if bless { + eprintln!("formatting C++ files"); + cfg_args_clang_format.push("-i".as_ref()); + } else { + eprintln!("checking C++ file formatting"); + cfg_args_clang_format.extend(&["--dry-run".as_ref(), "--Werror".as_ref()]); + } + let files; + if file_args_clang_format.is_empty() { + let llvm_wrapper = root_path.join("compiler/rustc_llvm/llvm-wrapper"); + files = find_with_extension( + root_path, + Some(llvm_wrapper.as_path()), + &[OsStr::new("h"), OsStr::new("cpp")], + )?; + file_args_clang_format.extend(files.iter().map(|p| p.as_os_str())); + } + let args = merge_args(&cfg_args_clang_format, &file_args_clang_format); + let res = py_runner(py_path.as_ref().unwrap(), false, None, "clang-format", &args); + + if res.is_err() && show_diff { + eprintln!("\nclang-format linting failed! Printing diff suggestions:"); + + let mut cfg_args_clang_format_diff = cfg_args.clone(); + cfg_args_clang_format_diff.extend(&["--style".as_ref(), config_file_arg.as_ref()]); + for file in file_args_clang_format { + let mut formatted = String::new(); + let mut diff_args = cfg_args_clang_format_diff.clone(); + diff_args.push(file); + let _ = py_runner( + py_path.as_ref().unwrap(), + false, + Some(&mut formatted), + "clang-format", + &diff_args, + ); + if formatted.is_empty() { + eprintln!( + "failed to obtain the formatted content for '{}'", + file.to_string_lossy() + ); + continue; + } + let actual = std::fs::read_to_string(file).unwrap_or_else(|e| { + panic!( + "failed to read the C++ file at '{}' due to '{e}'", + file.to_string_lossy() + ) + }); + if formatted != actual { + let diff = similar::TextDiff::from_lines(&actual, &formatted); + eprintln!( + "{}", + diff.unified_diff().context_radius(4).header( + &format!("{} (actual)", file.to_string_lossy()), + &format!("{} (formatted)", file.to_string_lossy()) + ) + ); + } + } + } + // Rethrow error + res?; + } + + if shell_lint { + eprintln!("linting shell files"); + + let mut file_args_shc = file_args.clone(); + let files; + if file_args_shc.is_empty() { + files = find_with_extension(root_path, None, &[OsStr::new("sh")])?; + file_args_shc.extend(files.iter().map(|p| p.as_os_str())); + } + + shellcheck_runner(&merge_args(&cfg_args, &file_args_shc))?; + } + + if spellcheck { + let config_path = root_path.join("typos.toml"); + let mut args = vec!["-c", config_path.as_os_str().to_str().unwrap()]; + + args.extend_from_slice(SPELLCHECK_DIRS); + + if bless { + eprintln!("spellcheck files and fix"); + args.push("--write-changes"); + } else { + eprintln!("spellcheck files"); + } + spellcheck_runner(&args)?; + } + + if js_lint || js_typecheck { + rustdoc_js::npm_install(root_path, outdir, npm)?; + } + + if js_lint { + rustdoc_js::lint(outdir, librustdoc_path, tools_path)?; + rustdoc_js::es_check(outdir, librustdoc_path)?; + } + + if js_typecheck { + rustdoc_js::typecheck(outdir, librustdoc_path)?; + } + + Ok(()) +} + +fn run_ruff( + root_path: &Path, + outdir: &Path, + py_path: &Path, + cfg_args: &[&OsStr], + file_args: &[&OsStr], + ruff_args: &[&OsStr], +) -> Result<(), Error> { + let mut cfg_args_ruff = cfg_args.to_vec(); + let mut file_args_ruff = file_args.to_vec(); + + let mut cfg_path = root_path.to_owned(); + cfg_path.extend(RUFF_CONFIG_PATH); + let mut cache_dir = outdir.to_owned(); + cache_dir.extend(RUFF_CACHE_PATH); + + cfg_args_ruff.extend([ + "--config".as_ref(), + cfg_path.as_os_str(), + "--cache-dir".as_ref(), + cache_dir.as_os_str(), + ]); + + if file_args_ruff.is_empty() { + file_args_ruff.push(root_path.as_os_str()); + } + + let mut args: Vec<&OsStr> = ruff_args.to_vec(); + args.extend(merge_args(&cfg_args_ruff, &file_args_ruff)); + py_runner(py_path, true, None, "ruff", &args) +} + +/// Helper to create `cfg1 cfg2 -- file1 file2` output +fn merge_args<'a>(cfg_args: &[&'a OsStr], file_args: &[&'a OsStr]) -> Vec<&'a OsStr> { + let mut args = cfg_args.to_owned(); + args.push("--".as_ref()); + args.extend(file_args); + args +} + +/// Run a python command with given arguments. `py_path` should be a virtualenv. +/// +/// Captures `stdout` to a string if provided, otherwise prints the output. +fn py_runner( + py_path: &Path, + as_module: bool, + stdout: Option<&mut String>, + bin: &'static str, + args: &[&OsStr], +) -> Result<(), Error> { + let mut cmd = Command::new(py_path); + if as_module { + cmd.arg("-m").arg(bin).args(args); + } else { + let bin_path = py_path.with_file_name(bin); + cmd.arg(bin_path).args(args); + } + let status = if let Some(stdout) = stdout { + let output = cmd.output()?; + if let Ok(s) = std::str::from_utf8(&output.stdout) { + stdout.push_str(s); + } + output.status + } else { + cmd.status()? + }; + if status.success() { Ok(()) } else { Err(Error::FailedCheck(bin)) } +} + +/// Create a virtuaenv at a given path if it doesn't already exist, or validate +/// the install if it does. Returns the path to that venv's python executable. +fn get_or_create_venv(venv_path: &Path, src_reqs_path: &Path) -> Result { + let mut should_create = true; + let dst_reqs_path = venv_path.join("requirements.txt"); + let mut py_path = venv_path.to_owned(); + py_path.extend(REL_PY_PATH); + + if let Ok(req) = fs::read_to_string(&dst_reqs_path) { + if req == fs::read_to_string(src_reqs_path)? { + // found existing environment + should_create = false; + } else { + eprintln!("requirements.txt file mismatch, recreating environment"); + } + } + + if should_create { + eprintln!("removing old virtual environment"); + if venv_path.is_dir() { + fs::remove_dir_all(venv_path).unwrap_or_else(|_| { + panic!("failed to remove directory at {}", venv_path.display()) + }); + } + create_venv_at_path(venv_path)?; + install_requirements(&py_path, src_reqs_path, &dst_reqs_path)?; + } + + verify_py_version(&py_path)?; + Ok(py_path) +} + +/// Attempt to create a virtualenv at this path. Cycles through all expected +/// valid python versions to find one that is installed. +fn create_venv_at_path(path: &Path) -> Result<(), Error> { + /// Preferred python versions in order. Newest to oldest then current + /// development versions + const TRY_PY: &[&str] = &[ + "python3.13", + "python3.12", + "python3.11", + "python3.10", + "python3.9", + "python3", + "python", + "python3.14", + ]; + + let mut sys_py = None; + let mut found = Vec::new(); + + for py in TRY_PY { + match verify_py_version(Path::new(py)) { + Ok(_) => { + sys_py = Some(*py); + break; + } + // Skip not found errors + Err(Error::Io(e)) if e.kind() == io::ErrorKind::NotFound => (), + // Skip insufficient version errors + Err(Error::Version { installed, .. }) => found.push(installed), + // just log and skip unrecognized errors + Err(e) => eprintln!("note: error running '{py}': {e}"), + } + } + + let Some(sys_py) = sys_py else { + let ret = if found.is_empty() { + Error::MissingReq("python3", "python file checks", None) + } else { + found.sort(); + found.dedup(); + Error::Version { + program: "python3", + required: MIN_PY_REV_STR, + installed: found.join(", "), + } + }; + return Err(ret); + }; + + // First try venv, which should be packaged in the Python3 standard library. + // If it is not available, try to create the virtual environment using the + // virtualenv package. + if try_create_venv(sys_py, path, "venv").is_ok() { + return Ok(()); + } + try_create_venv(sys_py, path, "virtualenv") +} + +fn try_create_venv(python: &str, path: &Path, module: &str) -> Result<(), Error> { + eprintln!( + "creating virtual environment at '{}' using '{python}' and '{module}'", + path.display() + ); + let out = Command::new(python).args(["-m", module]).arg(path).output().unwrap(); + + if out.status.success() { + return Ok(()); + } + + let stderr = String::from_utf8_lossy(&out.stderr); + let err = if stderr.contains(&format!("No module named {module}")) { + Error::Generic(format!( + r#"{module} not found: you may need to install it: +`{python} -m pip install {module}` +If you see an error about "externally managed environment" when running the above command, +either install `{module}` using your system package manager +(e.g. `sudo apt-get install {python}-{module}`) or create a virtual environment manually, install +`{module}` in it and then activate it before running tidy. +"# + )) + } else { + Error::Generic(format!( + "failed to create venv at '{}' using {python} -m {module}: {stderr}", + path.display() + )) + }; + Err(err) +} + +/// Parse python's version output (`Python x.y.z`) and ensure we have a +/// suitable version. +fn verify_py_version(py_path: &Path) -> Result<(), Error> { + let out = Command::new(py_path).arg("--version").output()?; + let outstr = String::from_utf8_lossy(&out.stdout); + let vers = outstr.trim().split_ascii_whitespace().nth(1).unwrap().trim(); + let mut vers_comps = vers.split('.'); + let major: u32 = vers_comps.next().unwrap().parse().unwrap(); + let minor: u32 = vers_comps.next().unwrap().parse().unwrap(); + + if (major, minor) < MIN_PY_REV { + Err(Error::Version { + program: "python", + required: MIN_PY_REV_STR, + installed: vers.to_owned(), + }) + } else { + Ok(()) + } +} + +fn install_requirements( + py_path: &Path, + src_reqs_path: &Path, + dst_reqs_path: &Path, +) -> Result<(), Error> { + let stat = Command::new(py_path) + .args(["-m", "pip", "install", "--upgrade", "pip"]) + .status() + .expect("failed to launch pip"); + if !stat.success() { + return Err(Error::Generic(format!("pip install failed with status {stat}"))); + } + + let stat = Command::new(py_path) + .args(["-m", "pip", "install", "--quiet", "--require-hashes", "-r"]) + .arg(src_reqs_path) + .status()?; + if !stat.success() { + return Err(Error::Generic(format!( + "failed to install requirements at {}", + src_reqs_path.display() + ))); + } + fs::copy(src_reqs_path, dst_reqs_path)?; + assert_eq!( + fs::read_to_string(src_reqs_path).unwrap(), + fs::read_to_string(dst_reqs_path).unwrap() + ); + Ok(()) +} + +/// Check that shellcheck is installed then run it at the given path +fn shellcheck_runner(args: &[&OsStr]) -> Result<(), Error> { + match Command::new("shellcheck").arg("--version").status() { + Ok(_) => (), + Err(e) if e.kind() == io::ErrorKind::NotFound => { + return Err(Error::MissingReq( + "shellcheck", + "shell file checks", + Some( + "see \ + for installation instructions" + .to_owned(), + ), + )); + } + Err(e) => return Err(e.into()), + } + + let status = Command::new("shellcheck").args(args).status()?; + if status.success() { Ok(()) } else { Err(Error::FailedCheck("shellcheck")) } +} + +/// Check that spellchecker is installed then run it at the given path +fn spellcheck_runner(args: &[&str]) -> Result<(), Error> { + // sync version with .github/workflows/spellcheck.yml + let expected_version = "typos-cli 1.34.0"; + match Command::new("typos").arg("--version").output() { + Ok(o) => { + let stdout = String::from_utf8_lossy(&o.stdout); + if stdout.trim() != expected_version { + return Err(Error::Version { + program: "typos", + required: expected_version, + installed: stdout.trim().to_string(), + }); + } + } + Err(e) if e.kind() == io::ErrorKind::NotFound => { + return Err(Error::MissingReq( + "typos", + "spellcheck file checks", + // sync version with .github/workflows/spellcheck.yml + Some("install tool via `cargo install typos-cli@1.34.0`".to_owned()), + )); + } + Err(e) => return Err(e.into()), + } + + let status = Command::new("typos").args(args).status()?; + if status.success() { Ok(()) } else { Err(Error::FailedCheck("typos")) } +} + +/// Check git for tracked files matching an extension +fn find_with_extension( + root_path: &Path, + find_dir: Option<&Path>, + extensions: &[&OsStr], +) -> Result, Error> { + // Untracked files show up for short status and are indicated with a leading `?` + // -C changes git to be as if run from that directory + let stat_output = + Command::new("git").arg("-C").arg(root_path).args(["status", "--short"]).output()?.stdout; + + if String::from_utf8_lossy(&stat_output).lines().filter(|ln| ln.starts_with('?')).count() > 0 { + eprintln!("found untracked files, ignoring"); + } + + let mut output = Vec::new(); + let binding = { + let mut command = Command::new("git"); + command.arg("-C").arg(root_path).args(["ls-files"]); + if let Some(find_dir) = find_dir { + command.arg(find_dir); + } + command.output()? + }; + let tracked = String::from_utf8_lossy(&binding.stdout); + + for line in tracked.lines() { + let line = line.trim(); + let path = Path::new(line); + + let Some(ref extension) = path.extension() else { + continue; + }; + if extensions.contains(extension) { + output.push(root_path.join(path)); + } + } + + Ok(output) +} + +#[derive(Debug)] +enum Error { + Io(io::Error), + /// a is required to run b. c is extra info + MissingReq(&'static str, &'static str, Option), + /// Tool x failed the check + FailedCheck(&'static str), + /// Any message, just print it + Generic(String), + /// Installed but wrong version + Version { + program: &'static str, + required: &'static str, + installed: String, + }, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MissingReq(a, b, ex) => { + write!( + f, + "{a} is required to run {b} but it could not be located. Is it installed?" + )?; + if let Some(s) = ex { + write!(f, "\n{s}")?; + }; + Ok(()) + } + Self::Version { program, required, installed } => write!( + f, + "insufficient version of '{program}' to run external tools: \ + {required} required but found {installed}", + ), + Self::Generic(s) => f.write_str(s), + Self::Io(e) => write!(f, "IO error: {e}"), + Self::FailedCheck(s) => write!(f, "checks with external tool '{s}' failed"), + } + } +} + +impl From for Error { + fn from(value: io::Error) -> Self { + Self::Io(value) + } +} + +#[derive(Debug)] +enum ExtraCheckParseError { + #[allow(dead_code, reason = "shown through Debug")] + UnknownKind(String), + #[allow(dead_code)] + UnknownLang(String), + UnsupportedKindForLang, + /// Too many `:` + TooManyParts, + /// Tried to parse the empty string + Empty, + /// `auto` specified without lang part. + AutoRequiresLang, +} + +struct ExtraCheckArg { + auto: bool, + lang: ExtraCheckLang, + /// None = run all extra checks for the given lang + kind: Option, +} + +impl ExtraCheckArg { + fn matches(&self, lang: ExtraCheckLang, kind: ExtraCheckKind) -> bool { + self.lang == lang && self.kind.map(|k| k == kind).unwrap_or(true) + } + + /// Returns `true` if this is an auto arg and the relevant files are not modified. + fn is_inactive_auto(&self, ci_info: &CiInfo) -> bool { + if !self.auto { + return false; + } + let ext = match self.lang { + ExtraCheckLang::Py => ".py", + ExtraCheckLang::Cpp => ".cpp", + ExtraCheckLang::Shell => ".sh", + ExtraCheckLang::Js => ".js", + ExtraCheckLang::Spellcheck => { + return !crate::files_modified(ci_info, |s| { + SPELLCHECK_DIRS.iter().any(|dir| Path::new(s).starts_with(dir)) + }); + } + }; + !crate::files_modified(ci_info, |s| s.ends_with(ext)) + } + + fn has_supported_kind(&self) -> bool { + let Some(kind) = self.kind else { + // "run all extra checks" mode is supported for all languages. + return true; + }; + use ExtraCheckKind::*; + let supported_kinds: &[_] = match self.lang { + ExtraCheckLang::Py => &[Fmt, Lint], + ExtraCheckLang::Cpp => &[Fmt], + ExtraCheckLang::Shell => &[Lint], + ExtraCheckLang::Spellcheck => &[], + ExtraCheckLang::Js => &[Lint, Typecheck], + }; + supported_kinds.contains(&kind) + } +} + +impl FromStr for ExtraCheckArg { + type Err = ExtraCheckParseError; + + fn from_str(s: &str) -> Result { + let mut auto = false; + let mut parts = s.split(':'); + let Some(mut first) = parts.next() else { + return Err(ExtraCheckParseError::Empty); + }; + if first == "auto" { + let Some(part) = parts.next() else { + return Err(ExtraCheckParseError::AutoRequiresLang); + }; + auto = true; + first = part; + } + let second = parts.next(); + if parts.next().is_some() { + return Err(ExtraCheckParseError::TooManyParts); + } + let arg = Self { auto, lang: first.parse()?, kind: second.map(|s| s.parse()).transpose()? }; + if !arg.has_supported_kind() { + return Err(ExtraCheckParseError::UnsupportedKindForLang); + } + + Ok(arg) + } +} + +#[derive(PartialEq, Copy, Clone)] +enum ExtraCheckLang { + Py, + Shell, + Cpp, + Spellcheck, + Js, +} + +impl FromStr for ExtraCheckLang { + type Err = ExtraCheckParseError; + + fn from_str(s: &str) -> Result { + Ok(match s { + "py" => Self::Py, + "shell" => Self::Shell, + "cpp" => Self::Cpp, + "spellcheck" => Self::Spellcheck, + "js" => Self::Js, + _ => return Err(ExtraCheckParseError::UnknownLang(s.to_string())), + }) + } +} + +#[derive(PartialEq, Copy, Clone)] +enum ExtraCheckKind { + Lint, + Fmt, + Typecheck, + /// Never parsed, but used as a placeholder for + /// langs that never have a specific kind. + None, +} + +impl FromStr for ExtraCheckKind { + type Err = ExtraCheckParseError; + + fn from_str(s: &str) -> Result { + Ok(match s { + "lint" => Self::Lint, + "fmt" => Self::Fmt, + "typecheck" => Self::Typecheck, + _ => return Err(ExtraCheckParseError::UnknownKind(s.to_string())), + }) + } +} diff --git a/src/tools/tidy/src/extra_checks/rustdoc_js.rs b/src/tools/tidy/src/extra_checks/rustdoc_js.rs new file mode 100644 index 00000000000..7708b128e23 --- /dev/null +++ b/src/tools/tidy/src/extra_checks/rustdoc_js.rs @@ -0,0 +1,113 @@ +//! Tidy check to ensure that rustdoc templates didn't forget a `{# #}` to strip extra whitespace +//! characters. + +use std::ffi::OsStr; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; + +use build_helper::npm; +use ignore::DirEntry; + +use crate::walk::walk_no_read; + +fn node_module_bin(outdir: &Path, name: &str) -> PathBuf { + outdir.join("node_modules/.bin").join(name) +} + +fn spawn_cmd(cmd: &mut Command) -> Result { + cmd.spawn().map_err(|err| { + eprintln!("unable to run {cmd:?} due to {err:?}"); + err + }) +} + +/// install all js dependencies from package.json. +pub(super) fn npm_install(root_path: &Path, outdir: &Path, npm: &Path) -> Result<(), super::Error> { + npm::install(root_path, outdir, npm)?; + Ok(()) +} + +fn rustdoc_js_files(librustdoc_path: &Path) -> Vec { + let mut files = Vec::new(); + walk_no_read( + &[&librustdoc_path.join("html/static/js")], + |path, is_dir| is_dir || path.extension().is_none_or(|ext| ext != OsStr::new("js")), + &mut |path: &DirEntry| { + files.push(path.path().into()); + }, + ); + return files; +} + +fn run_eslint(outdir: &Path, args: &[PathBuf], config_folder: PathBuf) -> Result<(), super::Error> { + let mut child = spawn_cmd( + Command::new(node_module_bin(outdir, "eslint")) + .arg("-c") + .arg(config_folder.join(".eslintrc.js")) + .args(args), + )?; + match child.wait() { + Ok(exit_status) => { + if exit_status.success() { + return Ok(()); + } + Err(super::Error::FailedCheck("eslint command failed")) + } + Err(error) => Err(super::Error::Generic(format!("eslint command failed: {error:?}"))), + } +} + +pub(super) fn lint( + outdir: &Path, + librustdoc_path: &Path, + tools_path: &Path, +) -> Result<(), super::Error> { + let files_to_check = rustdoc_js_files(librustdoc_path); + println!("Running eslint on rustdoc JS files"); + run_eslint(outdir, &files_to_check, librustdoc_path.join("html/static"))?; + + run_eslint(outdir, &[tools_path.join("rustdoc-js/tester.js")], tools_path.join("rustdoc-js"))?; + run_eslint( + outdir, + &[tools_path.join("rustdoc-gui/tester.js")], + tools_path.join("rustdoc-gui"), + )?; + Ok(()) +} + +pub(super) fn typecheck(outdir: &Path, librustdoc_path: &Path) -> Result<(), super::Error> { + // use npx to ensure correct version + let mut child = spawn_cmd( + Command::new(node_module_bin(outdir, "tsc")) + .arg("-p") + .arg(librustdoc_path.join("html/static/js/tsconfig.json")), + )?; + match child.wait() { + Ok(exit_status) => { + if exit_status.success() { + return Ok(()); + } + Err(super::Error::FailedCheck("tsc command failed")) + } + Err(error) => Err(super::Error::Generic(format!("tsc command failed: {error:?}"))), + } +} + +pub(super) fn es_check(outdir: &Path, librustdoc_path: &Path) -> Result<(), super::Error> { + let files_to_check = rustdoc_js_files(librustdoc_path); + let mut cmd = Command::new(node_module_bin(outdir, "es-check")); + cmd.arg("es2019"); + for f in files_to_check { + cmd.arg(f); + } + match spawn_cmd(&mut cmd)?.wait() { + Ok(exit_status) => { + if exit_status.success() { + return Ok(()); + } + Err(super::Error::FailedCheck("es-check command failed")) + } + Err(error) => Err(super::Error::Generic(format!("es-check command failed: {error:?}"))), + } +} diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs index 5f6796a9150..4f9fb308a86 100644 --- a/src/tools/tidy/src/lib.rs +++ b/src/tools/tidy/src/lib.rs @@ -165,8 +165,8 @@ pub mod debug_artifacts; pub mod deps; pub mod edition; pub mod error_codes; -pub mod ext_tool_checks; pub mod extdeps; +pub mod extra_checks; pub mod features; pub mod filenames; pub mod fluent_alphabetical; diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs index 11ee2ae688d..794b0addee3 100644 --- a/src/tools/tidy/src/main.rs +++ b/src/tools/tidy/src/main.rs @@ -177,7 +177,7 @@ fn main() { check!(unstable_book, &src_path, collected); check!( - ext_tool_checks, + extra_checks, &root_path, &output_directory, &ci_info, -- cgit 1.4.1-3-g733a5