From 18291bd1e3105046592d221426704c2418c34b7c Mon Sep 17 00:00:00 2001 From: Eric L Frederich Date: Fri, 22 Mar 2024 12:24:03 -0400 Subject: [PATCH 1/5] feat: new terraform_fmt_v2 with better Windows support --- .gitignore | 2 ++ .pre-commit-hooks.yaml | 8 +++++ hooks/__init__.py | 4 --- hooks/common.py | 61 +++++++++++++++++++++++++++++++++ hooks/terraform_docs_replace.py | 5 +++ hooks/terraform_fmt.py | 32 +++++++++++++++++ setup.py | 1 + 7 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 hooks/common.py create mode 100644 hooks/terraform_fmt.py diff --git a/.gitignore b/.gitignore index 0bbeada90..1de9c2b4a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ tests/results/* +__pycache__/ +*.py[cod] diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index cbe506dd3..f08dbb6c0 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -15,6 +15,14 @@ files: (\.tf|\.tfvars)$ exclude: \.terraform/.*$ +- id: terraform_fmt_py + name: Terraform fmt + description: Rewrites all Terraform configuration files to a canonical format. + entry: terraform_fmt + language: python + files: \.tf(vars)?$ + exclude: \.terraform/.*$ + - id: terraform_docs name: Terraform docs description: Inserts input and output documentation into README.md (using terraform-docs). diff --git a/hooks/__init__.py b/hooks/__init__.py index aeb6f9b27..e69de29bb 100644 --- a/hooks/__init__.py +++ b/hooks/__init__.py @@ -1,4 +0,0 @@ -print( - '`terraform_docs_replace` hook is DEPRECATED.' - 'For migration instructions see https://github.com/antonbabenko/pre-commit-terraform/issues/248#issuecomment-1290829226' -) diff --git a/hooks/common.py b/hooks/common.py new file mode 100644 index 000000000..28bef23ef --- /dev/null +++ b/hooks/common.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import argparse +import logging +import os +from collections.abc import Sequence + +logger = logging.getLogger(__name__) + + +def setup_logging(): + logging.basicConfig( + level={ + "error": logging.ERROR, + "warn": logging.WARNING, + "warning": logging.WARNING, + "info": logging.INFO, + "debug": logging.DEBUG, + }[os.environ.get("PRE_COMMIT_TERRAFORM_LOG_LEVEL", "warning").lower()] + ) + + +def parse_env_vars(env_var_strs: list[str]) -> dict[str, str]: + ret = {} + for env_var_str in env_var_strs: + name, val = env_var_str.split("=", 1) + if val.startswith('"') and val.endswith('"'): + val = val[1:-1] + ret[name] = val + return ret + + +def parse_cmdline( + argv: Sequence[str] | None = None, +) -> tuple[list[str], list[str], list[str], list[str], dict[str, str]]: + parser = argparse.ArgumentParser( + add_help=False, # Allow the use of `-h` for compatiblity with Bash version of the hook + ) + parser.add_argument("-a", "--args", action="append", help="Arguments") + parser.add_argument("-h", "--hook-config", action="append", help="Hook Config") + parser.add_argument("-i", "--init-args", "--tf-init-args", action="append", help="Init Args") + parser.add_argument("-e", "--envs", "--env-vars", action="append", help="Environment Variables") + parser.add_argument("FILES", nargs="*", help="Files") + + parsed_args = parser.parse_args(argv) + + args = parsed_args.args or [] + hook_config = parsed_args.hook_config or [] + files = parsed_args.FILES or [] + tf_init_args = parsed_args.init_args or [] + env_vars = parsed_args.envs or [] + + env_var_dict = parse_env_vars(env_vars) + + if hook_config: + raise NotImplementedError("TODO: implement: hook_config") + + if tf_init_args: + raise NotImplementedError("TODO: implement: tf_init_args") + + return args, hook_config, files, tf_init_args, env_var_dict diff --git a/hooks/terraform_docs_replace.py b/hooks/terraform_docs_replace.py index a9cf6c9bc..600cb08bf 100644 --- a/hooks/terraform_docs_replace.py +++ b/hooks/terraform_docs_replace.py @@ -3,6 +3,11 @@ import subprocess import sys +print( + '`terraform_docs_replace` hook is DEPRECATED.' + 'For migration instructions see https://github.com/antonbabenko/pre-commit-terraform/issues/248#issuecomment-1290829226' +) + def main(argv=None): parser = argparse.ArgumentParser( diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py new file mode 100644 index 000000000..bd07aba41 --- /dev/null +++ b/hooks/terraform_fmt.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import logging +import os +import shlex +import sys +from subprocess import PIPE, run +from typing import Sequence + +from .common import parse_cmdline, setup_logging + +logger = logging.getLogger(__name__) + + +def main(argv: Sequence[str] | None = None) -> int: + setup_logging() + logger.debug(sys.version_info) + args, hook_config, files, tf_init_args, env_vars = parse_cmdline(argv) + if os.environ.get("PRE_COMMIT_COLOR") == "never": + args.append("-no-color") + cmd = ["terraform", "fmt", *args, *files] + logger.info("calling %s", shlex.join(cmd)) + logger.debug("env_vars: %r", env_vars) + logger.debug("args: %r", args) + completed_process = run(cmd, env={**os.environ, **env_vars}, text=True, stdout=PIPE) + if completed_process.stdout: + print(completed_process.stdout) + return completed_process.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.py b/setup.py index 2d88425b9..ce1ee2623 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ entry_points={ 'console_scripts': [ 'terraform_docs_replace = hooks.terraform_docs_replace:main', + 'terraform_fmt = hooks.terraform_fmt:main', ], }, ) From 806a87a3943a29984a07b786a3ff399474c6b90f Mon Sep 17 00:00:00 2001 From: MaxymVlasov Date: Fri, 26 Apr 2024 20:32:51 +0300 Subject: [PATCH 2/5] Add linters andd apply part which not requiers tests --- .gitignore | 1 + .pre-commit-config.yaml | 68 ++++++++++++++++++++++++- hooks/common.py | 89 +++++++++++++++++++++++++-------- hooks/terraform_docs_replace.py | 43 ++++++++++------ hooks/terraform_fmt.py | 32 ++++++++---- 5 files changed, 186 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index 1de9c2b4a..820940260 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ tests/results/* __pycache__/ *.py[cod] +node_modules/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6944a0adb..a44d55046 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,7 +38,7 @@ repos: # Dockerfile linter - repo: https://github.com/hadolint/hadolint - rev: v2.12.1-beta + rev: v2.13.0-beta hooks: - id: hadolint args: [ @@ -54,8 +54,72 @@ repos: # JSON5 Linter - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.1.0 + rev: v4.0.0-alpha.8 hooks: - id: prettier # https://prettier.io/docs/en/options.html#parser files: '.json5$' + + +########## +# PYTHON # +########## + +- repo: https://github.com/asottile/reorder_python_imports + rev: v3.12.0 + hooks: + - id: reorder-python-imports + +- repo: https://github.com/asottile/add-trailing-comma + rev: v3.1.0 + hooks: + - id: add-trailing-comma + +- repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v2.0.4 + hooks: + - id: autopep8 + args: + - -i + - --max-line-length=100 + +# Usage: http://pylint.pycqa.org/en/latest/user_guide/message-control.html +- repo: https://github.com/PyCQA/pylint + rev: v3.1.0 + hooks: + - id: pylint + args: + - --disable=import-error # E0401. Locally you could not have all imports. + - --disable=fixme # W0511. 'TODO' notations. + - --disable=logging-fstring-interpolation # Conflict with "use a single formatting" WPS323 + - --disable=ungrouped-imports # ignore `if TYPE_CHECKING` case. Other do reorder-python-imports + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.10.0 + hooks: + - id: mypy + args: [ + --ignore-missing-imports, + --disallow-untyped-calls, + --warn-redundant-casts, + ] + +- repo: https://github.com/pycqa/flake8.git + rev: 7.0.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-2020 + - flake8-docstrings + - flake8-pytest-style + - wemake-python-styleguide + args: + - --max-returns=2 # Default settings + - --max-arguments=4 # Default settings + # https://www.flake8rules.com/ + # https://wemake-python-stylegui.de/en/latest/pages/usage/violations/index.html + - --extend-ignore= + WPS305, + E501, + I, + # RST, diff --git a/hooks/common.py b/hooks/common.py index 28bef23ef..f981f89bd 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -1,3 +1,8 @@ +""" +Here located common functions for hooks. + +It not executed directly, but imported by other hooks. +""" from __future__ import annotations import argparse @@ -9,38 +14,80 @@ def setup_logging(): - logging.basicConfig( - level={ - "error": logging.ERROR, - "warn": logging.WARNING, - "warning": logging.WARNING, - "info": logging.INFO, - "debug": logging.DEBUG, - }[os.environ.get("PRE_COMMIT_TERRAFORM_LOG_LEVEL", "warning").lower()] - ) + """ + Set up the logging configuration based on the value of the 'PCT_LOG' environment variable. + + The 'PCT_LOG' environment variable determines the logging level to be used. + The available levels are: + - 'error': Only log error messages. + - 'warn' or 'warning': Log warning messages and above. + - 'info': Log informational messages and above. + - 'debug': Log debug messages and above. + + If the 'PCT_LOG' environment variable is not set or has an invalid value, + the default logging level is 'warning'. + + Returns: + None + """ + log_level = { + 'error': logging.ERROR, + 'warn': logging.WARNING, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG, + }[os.environ.get('PCT_LOG', 'warning').lower()] + + logging.basicConfig(level=log_level) def parse_env_vars(env_var_strs: list[str]) -> dict[str, str]: + """ + Expand environment variables definition into their values in '--args'. + + Args: + env_var_strs (list[str]): A list of environment variable strings in the format "name=value". + + Returns: + dict[str, str]: A dictionary mapping variable names to their corresponding values. + """ ret = {} for env_var_str in env_var_strs: - name, val = env_var_str.split("=", 1) - if val.startswith('"') and val.endswith('"'): - val = val[1:-1] - ret[name] = val + name, env_var_value = env_var_str.split('=', 1) + if env_var_value.startswith('"') and env_var_value.endswith('"'): + env_var_value = env_var_value[1:-1] + ret[name] = env_var_value return ret def parse_cmdline( argv: Sequence[str] | None = None, ) -> tuple[list[str], list[str], list[str], list[str], dict[str, str]]: + """ + Parse the command line arguments and return a tuple containing the parsed values. + + Args: + argv (Sequence[str] | None): The command line arguments to parse. + If None, the arguments from sys.argv will be used. + + Returns: + tuple[list[str], list[str], list[str], list[str], dict[str, str]]: + A tuple containing the parsed values: + - args (list[str]): The parsed arguments. + - hook_config (list[str]): The parsed hook configurations. + - files (list[str]): The parsed files. + - tf_init_args (list[str]): The parsed Terraform initialization arguments. + - env_var_dict (dict[str, str]): The parsed environment variables as a dictionary. + """ + parser = argparse.ArgumentParser( - add_help=False, # Allow the use of `-h` for compatiblity with Bash version of the hook + add_help=False, # Allow the use of `-h` for compatibility with the Bash version of the hook ) - parser.add_argument("-a", "--args", action="append", help="Arguments") - parser.add_argument("-h", "--hook-config", action="append", help="Hook Config") - parser.add_argument("-i", "--init-args", "--tf-init-args", action="append", help="Init Args") - parser.add_argument("-e", "--envs", "--env-vars", action="append", help="Environment Variables") - parser.add_argument("FILES", nargs="*", help="Files") + parser.add_argument('-a', '--args', action='append', help='Arguments') + parser.add_argument('-h', '--hook-config', action='append', help='Hook Config') + parser.add_argument('-i', '--init-args', '--tf-init-args', action='append', help='Init Args') + parser.add_argument('-e', '--envs', '--env-vars', action='append', help='Environment Variables') + parser.add_argument('FILES', nargs='*', help='Files') parsed_args = parser.parse_args(argv) @@ -53,9 +100,9 @@ def parse_cmdline( env_var_dict = parse_env_vars(env_vars) if hook_config: - raise NotImplementedError("TODO: implement: hook_config") + raise NotImplementedError('TODO: implement: hook_config') if tf_init_args: - raise NotImplementedError("TODO: implement: tf_init_args") + raise NotImplementedError('TODO: implement: tf_init_args') return args, hook_config, files, tf_init_args, env_var_dict diff --git a/hooks/terraform_docs_replace.py b/hooks/terraform_docs_replace.py index 600cb08bf..204034e35 100644 --- a/hooks/terraform_docs_replace.py +++ b/hooks/terraform_docs_replace.py @@ -1,3 +1,4 @@ +"""Deprecated hook to replace README.md with the output of terraform-docs.""" import argparse import os import subprocess @@ -5,15 +6,25 @@ print( '`terraform_docs_replace` hook is DEPRECATED.' - 'For migration instructions see https://github.com/antonbabenko/pre-commit-terraform/issues/248#issuecomment-1290829226' + 'For migration instructions see ' + + 'https://github.com/antonbabenko/pre-commit-terraform/issues/248#issuecomment-1290829226', ) def main(argv=None): + """ + TODO: Add docstring. + + Args: + argv (list): List of command-line arguments (default: None) + + Returns: + int: The return value indicating the success or failure of the function + """ parser = argparse.ArgumentParser( description="""Run terraform-docs on a set of files. Follows the standard convention of pulling the documentation from main.tf in order to replace the entire - README.md file each time.""" + README.md file each time.""", ) parser.add_argument( '--dest', dest='dest', default='README.md', @@ -34,25 +45,27 @@ def main(argv=None): dirs = [] for filename in args.filenames: - if (os.path.realpath(filename) not in dirs and - (filename.endswith(".tf") or filename.endswith(".tfvars"))): + if ( + os.path.realpath(filename) not in dirs and + (filename.endswith('.tf') or filename.endswith('.tfvars')) + ): dirs.append(os.path.dirname(filename)) retval = 0 - for dir in dirs: + for directory in dirs: try: - procArgs = [] - procArgs.append('terraform-docs') + proc_args = [] + proc_args.append('terraform-docs') if args.sort: - procArgs.append('--sort-by-required') - procArgs.append('md') - procArgs.append("./{dir}".format(dir=dir)) - procArgs.append('>') - procArgs.append("./{dir}/{dest}".format(dir=dir, dest=args.dest)) - subprocess.check_call(" ".join(procArgs), shell=True) - except subprocess.CalledProcessError as e: - print(e) + proc_args.append('--sort-by-required') + proc_args.append('md') + proc_args.append(f'./{directory}') + proc_args.append('>') + proc_args.append(f'./{directory}/{args.dest}') + subprocess.check_call(' '.join(proc_args), shell=True) + except subprocess.CalledProcessError as exeption: + print(exeption) retval = 1 return retval diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index bd07aba41..7a6dbe62e 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -1,32 +1,46 @@ +""" +Pre-commit hook for terraform fmt. +""" from __future__ import annotations import logging import os import shlex import sys -from subprocess import PIPE, run +from subprocess import PIPE +from subprocess import run from typing import Sequence -from .common import parse_cmdline, setup_logging +from .common import parse_cmdline +from .common import setup_logging logger = logging.getLogger(__name__) def main(argv: Sequence[str] | None = None) -> int: + setup_logging() + logger.debug(sys.version_info) + args, hook_config, files, tf_init_args, env_vars = parse_cmdline(argv) - if os.environ.get("PRE_COMMIT_COLOR") == "never": - args.append("-no-color") - cmd = ["terraform", "fmt", *args, *files] - logger.info("calling %s", shlex.join(cmd)) - logger.debug("env_vars: %r", env_vars) - logger.debug("args: %r", args) + + if os.environ.get('PRE_COMMIT_COLOR') == 'never': + args.append('-no-color') + + cmd = ['terraform', 'fmt', *args, *files] + + logger.info('calling %s', shlex.join(cmd)) + logger.debug('env_vars: %r', env_vars) + logger.debug('args: %r', args) + completed_process = run(cmd, env={**os.environ, **env_vars}, text=True, stdout=PIPE) + if completed_process.stdout: print(completed_process.stdout) + return completed_process.returncode -if __name__ == "__main__": +if __name__ == '__main__': raise SystemExit(main()) From 3f3fdd97d74634a46179f9dbe004681c9221edab Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Fri, 26 Apr 2024 20:35:11 +0300 Subject: [PATCH 3/5] Apply suggestions from code review --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a44d55046..27f92d4c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,7 +54,7 @@ repos: # JSON5 Linter - repo: https://github.com/pre-commit/mirrors-prettier - rev: v4.0.0-alpha.8 + rev: v3.1.0 hooks: - id: prettier # https://prettier.io/docs/en/options.html#parser From a37971b9f179ab4c539b455b075068446d8d0b59 Mon Sep 17 00:00:00 2001 From: Maksym Vlasov Date: Fri, 26 Apr 2024 20:35:29 +0300 Subject: [PATCH 4/5] Apply suggestions from code review --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27f92d4c5..f6894f8f9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,7 +54,7 @@ repos: # JSON5 Linter - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.1.0 + rev: v3.1.0 hooks: - id: prettier # https://prettier.io/docs/en/options.html#parser From 85269df0250ab9ce85f2fa1bd29217bb32cbdeb2 Mon Sep 17 00:00:00 2001 From: Eric L Frederich Date: Wed, 1 May 2024 09:41:22 -0400 Subject: [PATCH 5/5] fix mypy andy pylint issues --- hooks/common.py | 2 +- hooks/terraform_docs_replace.py | 2 +- hooks/terraform_fmt.py | 11 +++++++++-- setup.py | 1 + 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/hooks/common.py b/hooks/common.py index f981f89bd..9ea175d02 100644 --- a/hooks/common.py +++ b/hooks/common.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -def setup_logging(): +def setup_logging() -> None: """ Set up the logging configuration based on the value of the 'PCT_LOG' environment variable. diff --git a/hooks/terraform_docs_replace.py b/hooks/terraform_docs_replace.py index 204034e35..2561fdc3f 100644 --- a/hooks/terraform_docs_replace.py +++ b/hooks/terraform_docs_replace.py @@ -11,7 +11,7 @@ ) -def main(argv=None): +def main(argv=None) -> int: """ TODO: Add docstring. diff --git a/hooks/terraform_fmt.py b/hooks/terraform_fmt.py index 7a6dbe62e..44fb7c956 100644 --- a/hooks/terraform_fmt.py +++ b/hooks/terraform_fmt.py @@ -18,12 +18,16 @@ def main(argv: Sequence[str] | None = None) -> int: + """ + Main entry point for terraform_fmt_py pre-commit hook. + Parses args and calls `terraform fmt` on list of files provided by pre-commit. + """ setup_logging() logger.debug(sys.version_info) - args, hook_config, files, tf_init_args, env_vars = parse_cmdline(argv) + args, _hook_config, files, _tf_init_args, env_vars = parse_cmdline(argv) if os.environ.get('PRE_COMMIT_COLOR') == 'never': args.append('-no-color') @@ -34,7 +38,10 @@ def main(argv: Sequence[str] | None = None) -> int: logger.debug('env_vars: %r', env_vars) logger.debug('args: %r', args) - completed_process = run(cmd, env={**os.environ, **env_vars}, text=True, stdout=PIPE) + completed_process = run( + cmd, env={**os.environ, **env_vars}, + text=True, stdout=PIPE, check=False, + ) if completed_process.stdout: print(completed_process.stdout) diff --git a/setup.py b/setup.py index ce1ee2623..58535a0fb 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +# pylint: skip-file from setuptools import find_packages from setuptools import setup