From efc49e4dfaeeda3c0ee3d2eca90bb532dc7f7610 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 9 Jun 2023 02:41:34 -0400 Subject: Add support for tidy linting via external tools for non-rust files This change adds the flag `--check-extras` to `tidy`. It accepts a comma separated list of any of the options: - py (test everything applicable for python files) - py:lint (lint python files using `ruff`) - py:fmt (check formatting for python files using `black`) - shell or shell:lint (lint shell files using `shellcheck`) Specific files to check can also be specified via positional args. Examples: - `./x test tidy --check-extras=shell,py` - `./x test tidy --check-extras=py:fmt -- src/bootstrap/bootstrap.py` - `./x test tidy --check-extras=shell -- src/ci/*.sh` - Python formatting can be applied with bless: `./x test tidy --ckeck-extras=py:fmt --bless` `ruff` and `black` need to be installed via pip; this tool manages these within a virtual environment at `build/venv`. `shellcheck` needs to be installed on the system already. --- src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/ci/docker') diff --git a/src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile b/src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile index 34b93be412e..0a49eab4d50 100644 --- a/src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile +++ b/src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile @@ -26,11 +26,13 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh COPY host-x86_64/mingw-check/reuse-requirements.txt /tmp/ -RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-requirements.txt +RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-requirements.txt \ + && pip3 install virtualenv COPY host-x86_64/mingw-check/validate-toolstate.sh /scripts/ COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/ # NOTE: intentionally uses python2 for x.py so we can test it still works. # validate-toolstate only runs in our CI, so it's ok for it to only support python3. -ENV SCRIPT python2.7 ../x.py test --stage 0 src/tools/tidy tidyselftest +ENV SCRIPT TIDY_PRINT_DIFF=1 python2.7 ../x.py test \ + --stage 0 src/tools/tidy tidyselftest --extra-checks=py:lint -- cgit 1.4.1-3-g733a5 From 9df0f5d43341a67741b55020def1474ae3748128 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Thu, 22 Jun 2023 01:59:24 -0400 Subject: Fix recent python linting errors - Remove unneeded imports in 'fuscia-test-runner.py' - Add explicit stacklevel to 'x.py' - Fix mutable types as default args in `bootstrap.py` and `bootstrap_test.py` --- src/bootstrap/bootstrap.py | 4 +++- src/bootstrap/bootstrap_test.py | 19 ++++++++++++++----- src/ci/docker/scripts/fuchsia-test-runner.py | 6 ++---- x.py | 2 +- 4 files changed, 20 insertions(+), 11 deletions(-) (limited to 'src/ci/docker') diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index fe66b1d41bf..70079106689 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -472,7 +472,9 @@ class FakeArgs: class RustBuild(object): """Provide all the methods required to build Rust""" - def __init__(self, config_toml="", args=FakeArgs()): + def __init__(self, config_toml="", args=None): + if args is None: + args = FakeArgs() self.git_version = None self.nix_deps_dir = None self._should_fix_bins_and_dylibs = None diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py index 3c91e403df3..dc06a4c9734 100644 --- a/src/bootstrap/bootstrap_test.py +++ b/src/bootstrap/bootstrap_test.py @@ -4,7 +4,6 @@ Run these with `x test bootstrap`, or `python -m unittest src/bootstrap/bootstra from __future__ import absolute_import, division, print_function import os -import doctest import unittest import tempfile import hashlib @@ -16,12 +15,15 @@ from shutil import rmtree bootstrap_dir = os.path.dirname(os.path.abspath(__file__)) # For the import below, have Python search in src/bootstrap first. sys.path.insert(0, bootstrap_dir) -import bootstrap -import configure +import bootstrap # noqa: E402 +import configure # noqa: E402 -def serialize_and_parse(configure_args, bootstrap_args=bootstrap.FakeArgs()): +def serialize_and_parse(configure_args, bootstrap_args=None): from io import StringIO + if bootstrap_args is None: + bootstrap_args = bootstrap.FakeArgs() + section_order, sections, targets = configure.parse_args(configure_args) buffer = StringIO() configure.write_config_toml(buffer, section_order, targets, sections) @@ -129,7 +131,14 @@ class GenerateAndParseConfig(unittest.TestCase): class BuildBootstrap(unittest.TestCase): """Test that we generate the appropriate arguments when building bootstrap""" - def build_args(self, configure_args=[], args=[], env={}): + def build_args(self, configure_args=None, args=None, env=None): + if configure_args is None: + configure_args = [] + if args is None: + args = [] + if env is None: + env = {} + env = env.copy() env["PATH"] = os.environ["PATH"] diff --git a/src/ci/docker/scripts/fuchsia-test-runner.py b/src/ci/docker/scripts/fuchsia-test-runner.py index af01f9ccbbc..f78d446c8b7 100755 --- a/src/ci/docker/scripts/fuchsia-test-runner.py +++ b/src/ci/docker/scripts/fuchsia-test-runner.py @@ -15,12 +15,10 @@ import hashlib import json import os import platform -import re import shutil -import signal import subprocess import sys -from typing import ClassVar, List, Optional +from typing import ClassVar, List @dataclass @@ -523,7 +521,7 @@ class TestEnvironment: env_vars += '\n "RUST_BACKTRACE=0",' # Use /tmp as the test temporary directory - env_vars += f'\n "RUST_TEST_TMPDIR=/tmp",' + env_vars += '\n "RUST_TEST_TMPDIR=/tmp",' cml.write( self.CML_TEMPLATE.format(env_vars=env_vars, exe_name=exe_name) diff --git a/x.py b/x.py index ba959a3047e..6c4c106817e 100755 --- a/x.py +++ b/x.py @@ -40,7 +40,7 @@ if __name__ == '__main__': This message can be suppressed by setting `RUST_IGNORE_OLD_PYTHON=1` """.format(major, minor)) - warnings.warn(msg) + warnings.warn(msg, stacklevel=1) rust_dir = os.path.dirname(os.path.abspath(__file__)) # For the import below, have Python search in src/bootstrap first. -- cgit 1.4.1-3-g733a5