From 2910aec616e844c481067df159c5116b8c35150b Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 23 Nov 2023 21:43:54 -0600 Subject: [PATCH] Switch to ruff for linting and autoformatting --- .pre-commit-config.yaml | 20 ++++------------ doc/source/toctree_filter.py | 2 +- polar2grid/_glue_argparser.py | 14 ++++++----- polar2grid/add_coastlines.py | 6 ++--- polar2grid/compare.py | 6 +++++ polar2grid/core/dtype.py | 8 +++---- polar2grid/filters/_utils.py | 6 +---- polar2grid/glue.py | 12 ++++------ polar2grid/resample/resample_decisions.py | 6 +++-- .../tests/etc/enhancements/generic.yaml | 4 ++-- polar2grid/tests/test_enhancements.py | 1 + .../test_utils/test_convert_grids_conf.py | 2 +- polar2grid/utils/config.py | 4 ++-- polar2grid/utils/legacy_compat.py | 6 ++--- polar2grid/writers/binary.py | 8 +------ polar2grid/writers/hdf5.py | 1 + pyproject.toml | 24 ++++++------------- setup.cfg | 6 ----- 18 files changed, 54 insertions(+), 82 deletions(-) delete mode 100644 setup.cfg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a62210f..4f98b350 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,12 @@ exclude: '^$' fail_fast: false repos: - - repo: https://github.com/psf/black - rev: 23.11.0 # Replace by any tag/version: https://github.com/psf/black/tags + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.1.6' hooks: - - id: black - language_version: python3 # Should be a command that runs python3.6+ - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - language_version: python3 - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - additional_dependencies: [flake8-docstrings, flake8-debugger, flake8-bugbear, mccabe] - args: [--max-complexity, "10"] + - id: ruff + args: ["--fix"] + - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: diff --git a/doc/source/toctree_filter.py b/doc/source/toctree_filter.py index 76741efb..d8a6906d 100644 --- a/doc/source/toctree_filter.py +++ b/doc/source/toctree_filter.py @@ -49,7 +49,7 @@ def filter_entries(self, entries): for e in entries: m = self.hasPat.match(e) if m is not None: - if not m.groups()[0] in excl: + if m.groups()[0] not in excl: filtered.append(m.groups()[1]) else: filtered.append(e) diff --git a/polar2grid/_glue_argparser.py b/polar2grid/_glue_argparser.py index a01f1161..856f07f3 100644 --- a/polar2grid/_glue_argparser.py +++ b/polar2grid/_glue_argparser.py @@ -163,7 +163,7 @@ def _separate_scene_init_load_args(self, reader_subgroups) -> None: def _parse_reader_args(self, reader_subgroups: list) -> tuple[dict, dict]: reader_args = {} load_args = {} - for reader_name, (sgrp1, sgrp2) in zip(self._reader_names, reader_subgroups): + for reader_name, (sgrp1, sgrp2) in zip(self._reader_names, reader_subgroups, strict=True): if sgrp1 is None: continue rargs = _args_to_dict(self._args, sgrp1._group_actions) @@ -175,7 +175,7 @@ def _parse_reader_args(self, reader_subgroups: list) -> tuple[dict, dict]: def _parse_one_writer_args(self, writer_subgroups: list) -> dict: writer_names: list[str] = self._writer_args["writers"] writer_specific_args = {} - for writer_name, (sgrp1, sgrp2) in zip(writer_names, writer_subgroups): + for writer_name, (sgrp1, sgrp2) in zip(writer_names, writer_subgroups, strict=True): wargs = _args_to_dict(self._args, sgrp1._group_actions) if sgrp2 is not None: wargs.update(_args_to_dict(self._args, sgrp2._group_actions)) @@ -283,8 +283,9 @@ def _validate_reader_writer_args(parser, args, use_polar2grid_defaults): parser.print_usage() parser.exit( 1, - "\nERROR: Reader must be provided (-r flag).\n" - "Supported readers:\n\t{}\n".format("\n\t".join(_supported_readers(use_polar2grid_defaults))), + "\nERROR: Reader must be provided (-r flag).\n" "Supported readers:\n\t{}\n".format( + "\n\t".join(_supported_readers(use_polar2grid_defaults)) + ), ) elif len(args.readers) > 1: parser.print_usage() @@ -297,8 +298,9 @@ def _validate_reader_writer_args(parser, args, use_polar2grid_defaults): parser.print_usage() parser.exit( 1, - "\nERROR: Writer must be provided (-w flag) with one or more writer.\n" - "Supported writers:\n\t{}\n".format("\n\t".join(_supported_writers(use_polar2grid_defaults))), + "\nERROR: Writer must be provided (-w flag) with one or more writer.\n" "Supported writers:\n\t{}\n".format( + "\n\t".join(_supported_writers(use_polar2grid_defaults)) + ), ) diff --git a/polar2grid/add_coastlines.py b/polar2grid/add_coastlines.py index 61f2a00f..562893a0 100644 --- a/polar2grid/add_coastlines.py +++ b/polar2grid/add_coastlines.py @@ -351,7 +351,7 @@ def main(argv=sys.argv[1:]): # gather all options into a single dictionary that we can pass to pycoast pycoast_options = _args_to_pycoast_dict(args) colorbar_kwargs = _args_to_colorbar_kwargs(args) if args.add_colorbar else {} - for input_tiff, output_filename in zip(args.input_tiff, args.output_filename): + for input_tiff, output_filename in zip(args.input_tiff, args.output_filename, strict=True): _process_one_image(input_tiff, output_filename, pycoast_options, args.shapes_dir, colorbar_kwargs) return 0 @@ -484,10 +484,10 @@ def find_font(font_name, size): try: font = ImageFont.truetype(font_name, size) return font.path - except IOError: + except IOError as err: font_path = get_resource_filename("polar2grid.fonts", font_name) if not os.path.exists(font_path): - raise ValueError("Font path does not exist: {}".format(font_path)) + raise ValueError("Font path does not exist: {}".format(font_path)) from err return font_path diff --git a/polar2grid/compare.py b/polar2grid/compare.py index bc2b66e0..0cc50bd3 100644 --- a/polar2grid/compare.py +++ b/polar2grid/compare.py @@ -105,6 +105,12 @@ def isclose_array(array1, array2, atol=0.0, rtol=0.0, margin_of_error=0.0, **kwa Args: array1: numpy array for comparison array2: numpy array for comparison + atol: absolute tolerance (see numpy ``isclose``) + rtol: relative tolerance (see numpy ``isclose``) + margin_of_error: percentage of pixels that can be different and still + be considered a passing amount. + kwargs: Unused. + Returns: 1 if more than margin_of_error pixels are different, 0 otherwise. diff --git a/polar2grid/core/dtype.py b/polar2grid/core/dtype.py index 0c022e97..d2782cb7 100644 --- a/polar2grid/core/dtype.py +++ b/polar2grid/core/dtype.py @@ -103,8 +103,8 @@ def str_to_dtype(dtype_str): try: return str2dtype[dtype_str] - except KeyError: - raise ValueError("Not a valid data type string: %s" % (dtype_str,)) + except KeyError as err: + raise ValueError("Not a valid data type string: %s" % (dtype_str,)) from err def dtype_to_str(numpy_dtype): @@ -114,8 +114,8 @@ def dtype_to_str(numpy_dtype): try: return dtype2str[np.dtype(numpy_dtype).type] - except KeyError: - raise ValueError("Unsupported np data type: %r" % (numpy_dtype,)) + except KeyError as err: + raise ValueError("Unsupported np data type: %r" % (numpy_dtype,)) from err def clip_to_data_type(data, data_type): diff --git a/polar2grid/filters/_utils.py b/polar2grid/filters/_utils.py index 4da11317..b317fc29 100644 --- a/polar2grid/filters/_utils.py +++ b/polar2grid/filters/_utils.py @@ -36,11 +36,7 @@ from typing import Union from pyresample.boundary import AreaBoundary, AreaDefBoundary, Boundary -from pyresample.geometry import ( - AreaDefinition, - SwathDefinition, - get_geostationary_bounding_box, -) +from pyresample.geometry import AreaDefinition, SwathDefinition, get_geostationary_bounding_box from pyresample.spherical import SphPolygon logger = logging.getLogger(__name__) diff --git a/polar2grid/glue.py b/polar2grid/glue.py index ae89b641..8ce6bd78 100644 --- a/polar2grid/glue.py +++ b/polar2grid/glue.py @@ -51,11 +51,7 @@ from satpy.writers import compute_writer_results from polar2grid._glue_argparser import GlueArgumentParser, get_p2g_defaults_env_var -from polar2grid.core.script_utils import ( - create_exc_handler, - rename_log_file, - setup_logging, -) +from polar2grid.core.script_utils import create_exc_handler, rename_log_file, setup_logging from polar2grid.filters import filter_scene from polar2grid.readers._base import ReaderProxyBase from polar2grid.resample import resample_scene @@ -150,7 +146,7 @@ def _write_scene_with_writer(scn: Scene, writer_name: str, data_ids: list[DataID res = scn.save_datasets(writer=writer_name, compute=False, datasets=data_ids, **wargs) if res and isinstance(res[0], (tuple, list)): # list of (dask-array, file-obj) tuples - to_save.extend(zip(*res)) + to_save.extend(zip(*res, strict=True)) else: # list of delayed objects to_save.extend(res) @@ -472,12 +468,12 @@ def _persist_swath_definition_in_scene(scn: Scene) -> None: if not to_persist_swath_defs: return scn - to_update_data_arrays, to_persist_lonlats = zip(*to_persist_swath_defs.values()) + to_update_data_arrays, to_persist_lonlats = zip(*to_persist_swath_defs.values(), strict=True) LOG.info("Loading swath geolocation into memory...") persisted_lonlats = dask.persist(*to_persist_lonlats) persisted_swath_defs = [SwathDefinition(plons, plats) for plons, plats in persisted_lonlats] new_scn = scn.copy() - for arrays_to_update, persisted_swath_def in zip(to_update_data_arrays, persisted_swath_defs): + for arrays_to_update, persisted_swath_def in zip(to_update_data_arrays, persisted_swath_defs, strict=True): for array_to_update in arrays_to_update: array_to_update.attrs["area"] = persisted_swath_def new_scn._datasets[array_to_update.attrs["_satpy_id"]] = array_to_update diff --git a/polar2grid/resample/resample_decisions.py b/polar2grid/resample/resample_decisions.py index a36e2930..d83b3382 100644 --- a/polar2grid/resample/resample_decisions.py +++ b/polar2grid/resample/resample_decisions.py @@ -94,6 +94,8 @@ def find_match(self, **query_dict): try: return super().find_match(**query_dict) - except KeyError: + except KeyError as err: # give a more understandable error message - raise KeyError(f"No resampling configuration found for {query_dict['area_type']=} | {query_dict['name']=}") + raise KeyError( + f"No resampling configuration found for {query_dict['area_type']=} | {query_dict['name']=}" + ) from err diff --git a/polar2grid/tests/etc/enhancements/generic.yaml b/polar2grid/tests/etc/enhancements/generic.yaml index 3f9aad40..fb31313b 100644 --- a/polar2grid/tests/etc/enhancements/generic.yaml +++ b/polar2grid/tests/etc/enhancements/generic.yaml @@ -20,7 +20,7 @@ enhancements: method: !!python/name:polar2grid.enhancements.palettize kwargs: palettes: - - filename: $POLAR2GRID_HOME/../etc/colormaps/amsr2_36h.cmap + - filename: $POLAR2GRID_HOME/../polar2grid/etc/colormaps/amsr2_36h.cmap min_value: 180 max_value: 280 test_p2g_palettize2: @@ -62,6 +62,6 @@ enhancements: method: !!python/name:polar2grid.enhancements.colorize kwargs: palettes: - - filename: $POLAR2GRID_HOME/../etc/colormaps/amsr2_36h.cmap + - filename: $POLAR2GRID_HOME/../polar2grid/etc/colormaps/amsr2_36h.cmap min_value: 180 max_value: 280 diff --git a/polar2grid/tests/test_enhancements.py b/polar2grid/tests/test_enhancements.py index 3e0db521..0f8ffde7 100644 --- a/polar2grid/tests/test_enhancements.py +++ b/polar2grid/tests/test_enhancements.py @@ -58,6 +58,7 @@ def setup_method(self): add_polar2grid_config_paths() # add test specific configs curr_path = satpy.config.get("config_path") + print(curr_path, TEST_ETC_DIR) satpy.config.set(config_path=[TEST_ETC_DIR] + curr_path) def teardown_method(self): diff --git a/polar2grid/tests/test_utils/test_convert_grids_conf.py b/polar2grid/tests/test_utils/test_convert_grids_conf.py index 73de6ef7..af317f7c 100644 --- a/polar2grid/tests/test_utils/test_convert_grids_conf.py +++ b/polar2grid/tests/test_utils/test_convert_grids_conf.py @@ -64,5 +64,5 @@ def test_conf_conversion(tmpdir, capsys, conf_content, num_areas, area_types): s.seek(0) areas = parse_area_file([s]) assert len(areas) == num_areas - for area_obj, area_type in zip(areas, area_types): + for area_obj, area_type in zip(areas, area_types, strict=True): assert isinstance(area_obj, area_type) diff --git a/polar2grid/utils/config.py b/polar2grid/utils/config.py index 63ff2261..4fdfe24f 100644 --- a/polar2grid/utils/config.py +++ b/polar2grid/utils/config.py @@ -32,8 +32,8 @@ def get_polar2grid_etc(): p2g_pkg_location = impr.files("polar2grid") if _is_editable_installation(): - return str(p2g_pkg_location.parent / "etc") - return os.path.join(sys.prefix, "etc", "polar2grid") + return str(p2g_pkg_location / "etc") + return p2g_pkg_location / "etc" / "polar2grid" def _is_editable_installation(): diff --git a/polar2grid/utils/legacy_compat.py b/polar2grid/utils/legacy_compat.py index 87c3015e..2cd920bb 100644 --- a/polar2grid/utils/legacy_compat.py +++ b/polar2grid/utils/legacy_compat.py @@ -129,7 +129,7 @@ def remove_unknown_user_products( """ satpy_names = self.convert_p2g_name_to_satpy(self._user_products) new_user_products = [] - for user_name, satpy_name in zip(self._user_products, satpy_names): + for user_name, satpy_name in zip(self._user_products, satpy_names, strict=True): # convert DataID/DataQuery to string satpy_name = satpy_name if isinstance(satpy_name, str) else satpy_name["name"] if satpy_name not in known_dataset_names: @@ -212,7 +212,7 @@ def apply_p2g_name_to_scene( """ all_ids = list(scn.keys()) all_p2g_names = list(self.convert_satpy_to_p2g_name(all_ids)) - for data_id, p2g_name in zip(all_ids, all_p2g_names): + for data_id, p2g_name in zip(all_ids, all_p2g_names, strict=True): if p2g_name is None: # the Satpy ID doesn't have a Polar2Grid compatible name logger.debug("Satpy DataID %s does not have a compatible polar2grid name.", data_id) @@ -225,7 +225,7 @@ def available_product_names( ) -> tuple[list[str], list[str], list[str]]: """Get separate lists of available Satpy products and Polar2Grid products.""" available_ids_as_p2g_names = list(self.convert_satpy_to_p2g_name(available_satpy_ids, all_p2g_products)) - satpy_id_to_p2g_name = dict(zip(available_satpy_ids, available_ids_as_p2g_names)) + satpy_id_to_p2g_name = dict(zip(available_satpy_ids, available_ids_as_p2g_names, strict=True)) available_p2g_names = [] available_custom_names = [] available_satpy_names = [] diff --git a/polar2grid/writers/binary.py b/polar2grid/writers/binary.py index 4b8a77a2..c4e2778b 100644 --- a/polar2grid/writers/binary.py +++ b/polar2grid/writers/binary.py @@ -42,13 +42,7 @@ import xarray as xr from satpy.writers import ImageWriter, get_enhanced_image -from polar2grid.core.dtype import ( - NUMPY_DTYPE_STRS, - clip_to_data_type, - dtype_to_str, - int_or_float, - str_to_dtype, -) +from polar2grid.core.dtype import NUMPY_DTYPE_STRS, clip_to_data_type, dtype_to_str, int_or_float, str_to_dtype from polar2grid.core.script_utils import NumpyDtypeList from polar2grid.utils.legacy_compat import convert_p2g_pattern_to_satpy diff --git a/polar2grid/writers/hdf5.py b/polar2grid/writers/hdf5.py index 10fda59f..9109915c 100644 --- a/polar2grid/writers/hdf5.py +++ b/polar2grid/writers/hdf5.py @@ -121,6 +121,7 @@ def iter_by_area(self, datasets: list[xr.DataArray]): Args: datasets (list[xr.DataArray]): A list of dataArray objects stored in Scene. + Returns: dictionary: a dictionary of {AreaDef: list[xr.DataArray]} """ diff --git a/pyproject.toml b/pyproject.toml index 5dff3a63..4dacc5af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,20 +56,6 @@ include-package-data = true [tool.setuptools.packages] find = {} -[tool.isort] -sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"] -profile = "black" -skip_gitignore = true -force_to_top = true -default_section = "THIRDPARTY" -known_first_party = "polar2grid" -line_length = 120 - -[tool.black] -line-length = 120 -target-version = ['py310'] -include = '\.pyi?$' - [tool.coverage.run] relative_files = true @@ -77,15 +63,19 @@ relative_files = true # See https://docs.astral.sh/ruff/rules/ select = ["E", "W", "B", "D", "T10", "C90"] # Remove D416 when all docstrings have been converted to google-style -ignore = ["D107", "D416"] +ignore = ["D101", "D102", "D103", "D104", "D105", "D106", "D107", "E203", "D416"] line-length = 120 -exclude = ["versioneer.py", "pyresample/version.py"] [tool.ruff.per-file-ignores] -"pyresample/test/*.py" = ["D102", "S101"] # assert allowed in tests +"doc/source/conf.py" = ["E501"] +"polar2grid/readers/*.py" = ["D205", "D400", "D415", "S101"] # assert allowed in tests [tool.ruff.pydocstyle] convention = "google" [tool.ruff.mccabe] max-complexity = 10 + +[tool.ruff.lint.isort] +known-first-party = ["polar2grid"] +known-third-party = ["src"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 69402b55..00000000 --- a/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[flake8] -max-line-length = 120 -ignore = D101,D102,D103,D105,D106,D107,E203 -per-file-ignores = - doc/source/conf.py:E501 - polar2grid/readers/*.py:D205,D400