diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a16ab4a..6d9a76b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,26 +15,14 @@ jobs: matrix: include: - PYVER: current_arch - SKIP_SHELLCHECK: 0 - SKIP_MYPY: 0 - SKIP_VULTURE: 0 - SKIP_RUFF: 0 - SKIP_PYPROJECT: 0 + CI_MAKE_TARGET: lint - PYVER: python310_ubuntu_2204 - SKIP_SHELLCHECK: 1 - SKIP_MYPY: 1 - SKIP_VULTURE: 1 - SKIP_RUFF: 1 - SKIP_PYPROJECT: 1 + CI_MAKE_TARGET: lint_ubuntu_310 env: PYVER: ${{ matrix.PYVER }} - SKIP_SHELLCHECK: ${{ matrix.SKIP_SHELLCHECK }} - SKIP_MYPY: ${{ matrix.SKIP_MYPY }} - SKIP_VULTURE: ${{ matrix.SKIP_VULTURE }} - SKIP_RUFF: ${{ matrix.SKIP_RUFF }} - SKIP_PYPROJECT: ${{ matrix.SKIP_PYPROJECT }} + CI_MAKE_TARGET: ${{ matrix.CI_MAKE_TARGET }} steps: - uses: actions/checkout@v2 @@ -50,10 +38,5 @@ jobs: - name: run ci in docker run: docker run - -e SKIP_SHELLCHECK=${SKIP_SHELLCHECK} - -e SKIP_MYPY=${SKIP_MYPY} - -e SKIP_VULTURE=${SKIP_VULTURE} - -e SKIP_RUFF=${SKIP_RUFF} - -e SKIP_PYPROJECT=${SKIP_PYPROJECT} oomox:latest - ./maintenance_scripts/lint.sh + make ${CI_MAKE_TARGET} diff --git a/Makefile b/Makefile index ff5afd52..f77b745f 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,41 @@ DEST_APPDIR = $(DESTDIR)$(APPDIR) DEST_PLUGIN_DIR = $(DESTDIR)$(APPDIR)/plugins DEST_PREFIX = $(DESTDIR)$(PREFIX) +SHELL := bash +PYTHON := $(shell which python) +ifeq (,$(PYTHON)) +$(error Can't find Python) +endif + +# lint: +RUFF := ruff +script_dir := $(shell readlink -e .) +APP_DIR := $(shell readlink -e "$(script_dir)") +TARGET_MODULE := oomox_gui +TARGETS := $(APP_DIR)/oomox_gui/ $(shell ls $(APP_DIR)/plugins/*/oomox_plugin.py) $(shell ls $(APP_DIR)/maintenance_scripts/*.py) +GLOBALS_IGNORES := \ + -e ': Final' \ + -e ' \# nonfinal-ignore' \ + -e ' \# checkglobals-ignore' \ + \ + -e TypeVar \ + -e namedtuple \ + -e Generic \ + -e Sequence \ + \ + -e 'BaseClass' \ + -e 'HexColor' \ + -e 'ColorScheme' \ + \ + -e './maintenance_scripts/find_.*.py.*:.*:' \ + -e '.SRCINFO' + +################################################################################ + +.PHONY: all +all: install + +################################################################################ install_gui: install_import_random $(eval PACKAGING_TMP_DIR := $(shell mktemp -d)) @@ -220,5 +255,140 @@ install_icons_suruplus_aspromauros: .PHONY: install_gui install_import_random install_theme_arc install_theme_oomox install_theme_materia install_export_oomoxify install_import_images install_plugin_base16 install_icons_archdroid install_icons_gnomecolors install_icons_numix install_icons_papirus install_icons_suruplus install_icons_suruplus_aspromauros install_import_xresources install_export_xresources install: install_gui install_theme_oomox install_theme_materia install_export_oomoxify install_import_images install_plugin_base16 install_icons_archdroid install_icons_gnomecolors install_icons_numix install_icons_papirus install_icons_suruplus install_icons_suruplus_aspromauros install_import_xresources install_export_xresources -.PHONY: all -all: install +################################################################################ + +lint_fix: + $(RUFF) check --fix $(TARGETS) + +compile_all: + export PYTHONWARNINGS='ignore,error:::$(TARGET_MODULE)[.*],error:::pikaur_test[.*]' + # Running python compile: + $(PYTHON) -O -m compileall $(TARGETS) \ + | (\ + grep -v -e '^Listing' -e '^Compiling' || true \ + ) + # :: python compile passed :: + +python_import: + # Running python import: + $(PYTHON) -c "import $(TARGET_MODULE).main" + # :: python import passed :: + +non_final_globals: + # Checking for non-Final globals: + result=$$( \ + grep -REn "^[a-zA-Z_]+ = " $(TARGETS) --color=always \ + | grep -Ev \ + \ + -e '=.*\|' \ + -e '=.*(dict|list|Callable)\[' \ + \ + $(GLOBALS_IGNORES) \ + | sort \ + ) ; \ + echo -n "$$result" ; \ + exit "$$(test "$$result" = "" && echo 0 || echo 1)" + # :: non-final globals check passed :: + +unreasonable_globals: + # Checking for unreasonable global vars: + result=$$( \ + grep -REn "^[a-zA-Z_]+ = [^'\"].*" $(TARGETS) --color=always \ + | grep -Ev \ + \ + -e ' =.*\|' \ + -e ' = [a-zA-Z_]+\[' \ + -e ' = str[^(]' \ + \ + $(GLOBALS_IGNORES) \ + | sort \ + ) ; \ + echo -n "$$result" ; \ + exit "$$(test "$$result" = "" && echo 0 || echo 1)" + # :: global vars check passed :: + +ruff: + # Checking Ruff rules up-to-date: + diff --color -u \ + <(awk '/select = \[/,/]/' pyproject.toml \ + | sed -e 's|", "|/|g' \ + | head -n -1 \ + | tail -n +2 \ + | tr -d '",\#' \ + | awk '{print $$1;}' \ + | sort) \ + <($(RUFF) linter \ + | awk '{print $$1;}' \ + | sort) + # Running ruff... + $(RUFF) check $(TARGETS) + # :: ruff passed :: + +flake8: + # Running flake8: + $(PYTHON) -m flake8 $(TARGETS) + # :: flake8 passed :: + +pylint: + # Running pylint: + $(PYTHON) -m pylint $(TARGETS) --score no + # :: pylint passed :: + +mypy: + # Running mypy: + #$(PYTHON) -m mypy $(TARGETS) --no-error-summary + $(PYTHON) -m mypy $(TARGET_MODULE) --no-error-summary + # :: mypy passed :: + +vulture: + # Running vulture: + $(PYTHON) -m vulture $(TARGETS) \ + --min-confidence=1 \ + --sort-by-size + # :: vulture passed :: + +shellcheck: + # shellcheck disable=SC2046 + # Running shellcheck: + ( \ + cd $(APP_DIR) || exit ; \ + shellcheck $$(find . \ + -name '*.sh' \ + -not -path './plugins/icons_archdroid/archdroid-icon-theme/*' \ + -not -path './plugins/icons_gnomecolors/gnome-colors-icon-theme/*' \ + -not -path './plugins/icons_papirus/papirus-icon-theme/*' \ + -not -path './plugins/icons_suruplus/suru-plus/*' \ + -not -path './plugins/icons_suruplus_aspromauros/suru-plus-aspromauros/*' \ + -not -path './plugins/base16/*.tmp/*' \ + -not -path './plugins/oomoxify/*' \ + -not -path './plugins/theme_arc/arc-theme/*' \ + -not -path './plugins/theme_materia/materia-theme/*' \ + -not -path './plugins/theme_oomox/gtk-theme/*' \ + -or -path './packaging/bin/*' \ + ) \ + ) + # :: shellcheck passed :: + +shellcheck_makefile: + # Running shellcheck on Makefile... + ( \ + cd $(APP_DIR) || exit ; \ + $(PYTHON) ./maintenance_scripts/makefile_shellcheck.py --skip lint ; \ + ) + # :: shellcheck makefile passed :: + +validate_pyproject: + # Validate pyproject file... + ( \ + exit_code=0 ; \ + result=$$(validate-pyproject pyproject.toml 2>&1) || exit_code=$$? ; \ + if [[ $$exit_code -gt 0 ]] ; then \ + echo "$$result" ; \ + exit "$$exit_code" ; \ + fi \ + ) + # :: pyproject validation passed :: + +.PHONY: lint compile_all python_import non_final_globals unreasonable_globals ruff flake8 pylint mypy vulture shellcheck shellcheck_makefile validate_pyproject +lint: compile_all python_import non_final_globals unreasonable_globals ruff flake8 pylint mypy vulture shellcheck shellcheck_makefile validate_pyproject +lint_ubuntu_310: compile_all python_import non_final_globals unreasonable_globals flake8 pylint diff --git a/dockerfiles/Dockerfile_current_arch b/dockerfiles/Dockerfile_current_arch index f1495a8e..e4fa5dc1 100644 --- a/dockerfiles/Dockerfile_current_arch +++ b/dockerfiles/Dockerfile_current_arch @@ -3,7 +3,7 @@ WORKDIR /opt/oomox-build/ # App and test (xvfb, pylint) deps RUN pacman -Syu --noconfirm && \ - pacman -S --needed --noconfirm gtk3 python-gobject python-yaml flake8 python-pylint xorg-server-xvfb mypy python-typing_extensions shellcheck && \ + pacman -S --needed --noconfirm gtk3 python-gobject python-yaml ruff flake8 python-pylint xorg-server-xvfb mypy python-typing_extensions shellcheck && \ pacman -S --needed --noconfirm git base-devel && \ (useradd -m user && echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers || true) && \ (which pikaur || sudo -u user bash -c "\ diff --git a/maintenance_scripts/get_global_expressions.sh b/maintenance_scripts/get_global_expressions.sh deleted file mode 100755 index 24d8bda0..00000000 --- a/maintenance_scripts/get_global_expressions.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -ue - -result=$( - grep -REn "^[a-zA-Z_]+ = [^'\"].*" "$@" --color=always \ - | grep -Ev \ - -e ': Final' \ - -e ' # nonfinal-ignore' \ - -e ' # checkglobals-ignore' \ - \ - -e ' =.*\|' \ - -e ' = [a-zA-Z_]+\[' \ - -e ' = str[^(]' \ - -e TypeVar \ - -e namedtuple \ - \ - -e 'create_logger\(|running_as_root|sudo' \ - | sort -) -echo -n "$result" -exit "$(test "$result" = "" && echo 0 || echo 1)" diff --git a/maintenance_scripts/get_non_final_expressions.sh b/maintenance_scripts/get_non_final_expressions.sh deleted file mode 100755 index 8ebb793f..00000000 --- a/maintenance_scripts/get_non_final_expressions.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -ue - -result=$( - grep -REn "^[a-zA-Z_]+ = " "$@" --color=always \ - | grep -Ev \ - -e ': Final' \ - \ - -e '\|' \ - -e '(dict|list)\[' \ - -e TypeVar \ - -e Sequence \ - -e Generic \ - -e namedtuple \ - \ - -e 'BaseClass' \ - -e 'HexColor' \ - -e 'ColorScheme' \ - | sort -) -echo -n "$result" -exit "$(test "$result" = "" && echo 0 || echo 1)" diff --git a/maintenance_scripts/lint.sh b/maintenance_scripts/lint.sh deleted file mode 100755 index 1404fca7..00000000 --- a/maintenance_scripts/lint.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env bash -set -ueo pipefail - -script_dir=$(readlink -e "$(dirname "${0}")") -APP_DIR="$(readlink -e "${script_dir}"/..)" - - -if [[ -z "${DISPLAY:-}" ]] ; then - # we need it as we're a GTK app: - Xvfb :99 -ac -screen 0 1920x1080x16 -nolisten tcp 2>&1 & - xvfb_pid="$!" - - clean_up() { - echo -e "\n== Killing Xvfb..." - kill "$xvfb_pid" - echo "== Done." - } - trap clean_up EXIT SIGHUP SIGINT SIGTERM - - echo '== Started Xvfb' - export DISPLAY=:99 - sleep 3 -fi - - -FIX_MODE=0 -CHECK_RUFF_RULES=0 -while getopts fr name -do - case $name in - f) FIX_MODE=1;; - r) CHECK_RUFF_RULES=1;; - ?) printf "Usage: %s: [-f] [TARGETS]\n" "$0" - echo "Arguments:" - echo " -f run in fix mode" - echo " -r check if ruff rules config up-to-date" - exit 2;; - esac -done -shift $((OPTIND - 1)) -printf "Remaining arguments are: %s\n$*" - - -PYTHON=python3 -TARGET_MODULE='oomox_gui' -TARGETS=( - './oomox_gui/' - ./plugins/*/oomox_plugin.py - ./maintenance_scripts/*.py -) -if [[ -n "${1:-}" ]] ; then - TARGETS=("$@") -fi - - -install_ruff() { - if [[ ! -f "${APP_DIR}/env/bin/activate" ]] ; then - "$PYTHON" -m venv "${APP_DIR}/env" --system-site-packages - # shellcheck disable=SC1091 - . "${APP_DIR}/env/bin/activate" - "$PYTHON" -m pip install ruff --upgrade - deactivate - fi -} -RUFF="${APP_DIR}/env/bin/ruff" - - -if [[ "$CHECK_RUFF_RULES" -eq 1 ]] ; then - echo -e "\n== Checking Ruff rules up-to-date:" - install_ruff - "$APP_DIR"/env/bin/pip install -U ruff - diff --color -u \ - <(awk '/select = \[/,/]/' pyproject.toml \ - | sed -e 's|", "|/|g' \ - | head -n -1 \ - | tail -n +2 \ - | tr -d '",#' \ - | awk '{print $1;}' \ - | sort) \ - <("$RUFF" linter \ - | awk '{print $1;}' \ - | sort) -elif [[ "$FIX_MODE" -eq 1 ]] ; then - for file in $(flake8 "${TARGETS[@]}" 2>&1 | cut -d: -f1 | uniq) ; do - autopep8 --in-place "$file" - done - "$RUFF" check --fix "${TARGETS[@]}" -else - export PYTHONWARNINGS='default,error:::'"$TARGET_MODULE"'[.*],error:::plugins[.*]' - - echo '== Running on system python' - "$PYTHON" --version - - echo -e "\n== Running python compile:" - "$PYTHON" -O -m compileall "${TARGETS[@]}" \ - | (\ - grep -v -e '^Listing' -e '^Compiling' || true \ - ) - echo ':: python compile passed ::' - - echo -e "\n== Running python import:" - "$PYTHON" -c "import ${TARGET_MODULE}.main" - echo ':: python import passed ::' - - echo -e "\n== Checking for non-Final globals:" - ./maintenance_scripts/get_non_final_expressions.sh "${TARGETS[@]}" - echo ':: check passed ::' - - echo -e "\n== Checking for unreasonable global vars:" - ./maintenance_scripts/get_global_expressions.sh "${TARGETS[@]}" - echo ':: check passed ::' - - if [[ "${SKIP_RUFF:-}" = "1" ]] ; then - echo -e "\n!! WARNING !! skipping Ruff" - else - echo -e "\n== Ruff..." - install_ruff - "$RUFF" check "${TARGETS[@]}" - echo ':: ruff passed ::' - fi - - echo -e "\n== Running flake8:" - flake8 "${TARGETS[@]}" 2>&1 - echo ':: flake8 passed ::' - - echo -e "\n== Running pylint:" - pylint "${TARGETS[@]}" --score no 2>&1 - #| ( - # grep -v \ - # -e "^ warnings.warn($" \ - # -e "^/usr/lib/python3.10/site-packages/" \ - # || true \ - #) - echo ':: pylint passed ::' - - - if [[ "${SKIP_MYPY:-}" = "1" ]] ; then - echo -e "\n!! WARNING !! skipping mypy" - else - #python -m venv mypy_venv --system-site-packages - #( - # # shellcheck disable=SC1091 - # . ./mypy_venv/bin/activate - # python -m pip install types-Pillow - echo -e "\n== Running mypy:" - python -m mypy "$TARGET_MODULE" - echo ':: mypy passed ::' - #) - fi - - - if [[ "${SKIP_VULTURE:-}" = "1" ]] ; then - echo -e "\n!! WARNING !! skipping vulture" - else - echo -e "\n== Running vulture:" - vulture "${TARGETS[@]}" \ - ./maintenance_scripts/vulture_whitelist.py \ - --min-confidence=1 \ - --sort-by-size - echo ':: vulture passed ::' - fi - - - if [[ "${SKIP_SHELLCHECK:-}" = "1" ]] ; then - echo -e "\n!! WARNING !! skipping shellcheck" - else - echo -e "\n== Running shellcheck:" - ./maintenance_scripts/shellcheck.sh - echo ':: shellcheck passed ::' - echo -e "\n== Running shellcheck on Makefile..." - ./maintenance_scripts/makefile_shellcheck.py - echo ':: shellcheck makefile passed ::' - fi - - if [[ "${SKIP_PYPROJECT:-}" = "1" ]] ; then - echo -e "\n!! WARNING !! skipping pyproject validation" - else - echo -e "\n== Validate pyproject file..." - ( - exit_code=0 - result=$(validate-pyproject pyproject.toml 2>&1) || exit_code=$? - if [[ $exit_code -gt 0 ]] ; then - echo "$result" - exit $exit_code - fi - ) - echo ':: pyproject validation passed ::' - fi -fi - - -echo -e "\n"'$$ All checks have been passed successfully $$' diff --git a/maintenance_scripts/makefile_shellcheck.py b/maintenance_scripts/makefile_shellcheck.py index 194a53b4..20fde8e3 100755 --- a/maintenance_scripts/makefile_shellcheck.py +++ b/maintenance_scripts/makefile_shellcheck.py @@ -1,31 +1,52 @@ -#!/usr/bin/python +#!/usr/bin/env python -import os +import argparse import subprocess # nosec B404 import sys import tempfile from typing import Final -MAKEFILE: str = "./Makefile" -if len(sys.argv) > 1: - MAKEFILE = sys.argv[1] - DEFAULT_ENCODING: Final = "utf-8" -MAKE_SHELL: Final = os.environ.get("MAKE_SHELL", "sh") # SKIP_TARGETS_WITH_CHARS = ("%", ) SKIP_TARGETS_WITH_CHARS: Final = ("%", "/") -SKIP_TARGETS: Final = (".PHONY", ".PRECIOUS") +SKIP_TARGETS: Final = [".PHONY", ".PRECIOUS"] _ALL: Final = "all" -def get_targets() -> list[str]: +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Makefile shellcheck", + ) + parser.add_argument( + "makefile", + nargs="?", + default="./Makefile", + help="path to Makefile to check", + ) + parser.add_argument( + "-s", "--skip", + action="append", + default=[], + help="make target to skip (arg could be used multiple times)", + ) + parser.add_argument( + "--shell", + nargs="?", + const="sh", + default="sh", + help="make shell", + ) + return parser.parse_args() + + +def get_targets(args: argparse.Namespace) -> tuple[list[str], str | None]: lines = subprocess.check_output( # nosec B603 args=[ "make", "--dry-run", - f"--makefile={MAKEFILE}", + f"--makefile={args.makefile}", "--print-data-base", "--no-builtin-rules", "--no-builtin-variables", @@ -34,8 +55,15 @@ def get_targets() -> list[str]: ).splitlines() not_a_target_comment = "# Not a target:" + make_shell: str | None = None + targets = [] for idx, line in enumerate(lines): + if not make_shell: + words = line.split(" ") + if (len(words) == 3) and (words[0] == "SHELL") and (words[1] == ":="): # noqa: PLR2004: + make_shell = words[2] + if lines[idx - 1] == not_a_target_comment: continue @@ -51,7 +79,7 @@ def get_targets() -> list[str]: continue target = word.rstrip(":") - if target in SKIP_TARGETS: + if target in SKIP_TARGETS + args.skip: continue targets.append(target) @@ -60,7 +88,7 @@ def get_targets() -> list[str]: # check it last: targets.remove(_ALL) targets.append(_ALL) - return targets + return targets, make_shell def print_by_lines(text: str) -> None: @@ -77,8 +105,9 @@ def print_error_in_target(target: str) -> None: def main() -> None: + args = parse_args() print("Starting the check...") - targets = get_targets() + targets, make_shell = get_targets(args) if _ALL not in targets: print(f"ERROR: `{_ALL}` target is not defined.") sys.exit(1) @@ -91,7 +120,7 @@ def main() -> None: args=[ "make", "--dry-run", - f"--makefile={MAKEFILE}", + f"--makefile={args.makefile}", target, ], encoding=DEFAULT_ENCODING, @@ -101,6 +130,11 @@ def main() -> None: print_error_in_target(target) print_by_lines(err.output) sys.exit(1) + make_result = "\n".join( + line + for line in make_result.splitlines() + if not line.startswith("make[1]") + ) with tempfile.NamedTemporaryFile("w", encoding=DEFAULT_ENCODING) as fobj: fobj.write(make_result) fobj.seek(0) @@ -109,7 +143,7 @@ def main() -> None: args=[ "shellcheck", fobj.name, - f"--shell={MAKE_SHELL}", + f"--shell={make_shell or args.shell}", "--color=always", ], encoding=DEFAULT_ENCODING, @@ -117,7 +151,7 @@ def main() -> None: except subprocess.CalledProcessError as err: print_error_in_target(target) print_by_lines(make_result) - print(err.output.replace(fobj.name, f"{MAKEFILE}:{target}")) + print(err.output.replace(fobj.name, f"{args.makefile}:{target}")) sys.exit(1) print("\n:: OK ::") diff --git a/maintenance_scripts/shellcheck.sh b/maintenance_scripts/shellcheck.sh deleted file mode 100755 index 7b69e0ee..00000000 --- a/maintenance_scripts/shellcheck.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -IFS=$'\n\t' - -TEST_DIR=$(readlink -e "$(dirname "${0}")") -SCRIPT_DIR="$(readlink -e "${TEST_DIR}"/..)" - -( - cd "${SCRIPT_DIR}" - # shellcheck disable=SC2046 - shellcheck $(find . \ - -name '*.sh' \ - -not -path './plugins/icons_archdroid/archdroid-icon-theme/*' \ - -not -path './plugins/icons_gnomecolors/gnome-colors-icon-theme/*' \ - -not -path './plugins/icons_papirus/papirus-icon-theme/*' \ - -not -path './plugins/icons_suruplus/suru-plus/*' \ - -not -path './plugins/icons_suruplus_aspromauros/suru-plus-aspromauros/*' \ - -not -path './plugins/base16/*.tmp/*' \ - -not -path './plugins/oomoxify/*' \ - -not -path './plugins/theme_arc/arc-theme/*' \ - -not -path './plugins/theme_materia/materia-theme/*' \ - -not -path './plugins/theme_oomox/gtk-theme/*' \ - -or -path './packaging/bin/*' \ - ) -) - -echo ':: shellcheck passed ::' -exit 0 diff --git a/pyproject.toml b/pyproject.toml index efccf513..c444169d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ select = [ "ASYNC", #"CPY", # @TODO: add copyrights #"DJ", # django + #"FAST", # fastapi "FA", "FIX", "FLY", @@ -98,6 +99,7 @@ select = [ "SLF", "SLOT", "TD", + #"DOC", pydoclint ] ignore = [ # enable back later: