diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..7da1f96 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 100 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9b4cda7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +settings.yaml +**/__pycache__/ +*.json +*.csv +.idea +trust diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..34d5395 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/asottile/reorder_python_imports + rev: v3.12.0 + hooks: + - id: reorder-python-imports + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + - id: debug-statements + - repo: https://github.com/psf/black + rev: 23.11.0 + hooks: + - id: black + - repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 diff --git a/apix/diff.py b/apix/diff.py index 7499bfd..7e58211 100644 --- a/apix/diff.py +++ b/apix/diff.py @@ -27,13 +27,10 @@ def __attrs_post_init__(self): self.api_name = get_latest(data_dir=self.data_dir, mock=self.mock) if not self.ver1: # get the latest saved version - self.ver1 = get_latest( - api_name=self.api_name, data_dir=self.data_dir, mock=self.mock - ) + self.ver1 = get_latest(api_name=self.api_name, data_dir=self.data_dir, mock=self.mock) if not self.ver2: # get the version before ver1 - self.ver2 = get_previous( - self.api_name, self.ver1, self.data_dir, self.mock) + self.ver2 = get_previous(self.api_name, self.ver1, self.data_dir, self.mock) @staticmethod def _truncate(diff_dict): @@ -161,8 +158,7 @@ def save_diff(self, return_path=False): else: ftype = "comp-diff" if self.compact else "diff" fpath = Path( - f"{self.data_dir}APIs/{self.api_name}/" - f"{self.ver2}-to-{self.ver1}-{ftype}.yaml" + f"{self.data_dir}APIs/{self.api_name}/" f"{self.ver2}-to-{self.ver1}-{ftype}.yaml" ) if fpath.exists(): fpath.unlink() diff --git a/apix/explore.py b/apix/explore.py index beaa61b..c0038b8 100644 --- a/apix/explore.py +++ b/apix/explore.py @@ -64,11 +64,9 @@ def _visit_links(self, links, retries=3): try: loop = asyncio.get_event_loop() loop.run_until_complete(self._async_loop(links)) - except aiohttp.client_exceptions.ServerDisconnectedError as err: + except aiohttp.client_exceptions.ServerDisconnectedError: logger.warning( - "Lost connection to host.{}".join( - "Retrying in 10 seconds" if retries else "" - ) + "Lost connection to host.".join("Retrying in 10 seconds" if retries else "") ) if retries: time.sleep(10) @@ -120,8 +118,7 @@ def explore(self): result = requests.get(self.host_url + self.base_path, verify=False) if not result: logger.warning( - f"I couldn't find anything useful at " - f"{self.host_url}{self.base_path}." + f"I couldn't find anything useful at " f"{self.host_url}{self.base_path}." ) return self.base_path = self.base_path.replace(".html", "") # for next step diff --git a/apix/helpers.py b/apix/helpers.py index da68140..4a6d2a0 100644 --- a/apix/helpers.py +++ b/apix/helpers.py @@ -14,9 +14,7 @@ def get_api_list(data_dir=None, mock=False): if not api_dir.exists(): return None # get all versions in directory, that aren't diffs - apis = [ - (api.name, api.stat().st_mtime) for api in api_dir.iterdir() if api.is_dir() - ] or [] + apis = [(api.name, api.stat().st_mtime) for api in api_dir.iterdir() if api.is_dir()] or [] apis = [api for api, _ in sorted(apis, key=lambda x: x[1], reverse=True)] return apis @@ -34,9 +32,7 @@ def get_ver_list(api_name, data_dir=None, mock=False): versions = [ v_file.name.replace(".yaml", "") for v_file in save_path.iterdir() - if "-diff." not in v_file.name - and "-comp." not in v_file.name - and ".yaml" in v_file.name + if "-diff." not in v_file.name and "-comp." not in v_file.name and ".yaml" in v_file.name ] or [] return sorted(versions, reverse=True) @@ -87,13 +83,10 @@ def save_api(api_name, version, api_dict, data_dir=None, compact=False, mock=Fal """Save the dict to yaml, if the file doesn't exist""" if mock: a_path = Path( - f"{data_dir}tests/APIs/{api_name}/{version}" - f"{'-comp' if compact else ''}.yaml" + f"{data_dir}tests/APIs/{api_name}/{version}" f"{'-comp' if compact else ''}.yaml" ) else: - a_path = Path( - f"{data_dir}APIs/{api_name}/{version}" f"{'-comp' if compact else ''}.yaml" - ) + a_path = Path(f"{data_dir}APIs/{api_name}/{version}" f"{'-comp' if compact else ''}.yaml") a_path.parent.mkdir(parents=True, exist_ok=True) logger.info(f"Saving {api_name} v{version} to {a_path}") with a_path.open("w") as f: diff --git a/apix/parsers/apipie.py b/apix/parsers/apipie.py index 2f936a3..12a1f0d 100644 --- a/apix/parsers/apipie.py +++ b/apix/parsers/apipie.py @@ -29,10 +29,7 @@ def _compile_method(method_dict): ) for param in method_dict["params"] ] - paths = [ - f'{path["http_method"].upper()} {path["api_url"]}' - for path in method_dict["apis"] - ] + paths = [f'{path["http_method"].upper()} {path["api_url"]}' for path in method_dict["apis"]] return {"paths": paths, "params": params} def scrape_content(self, result): @@ -42,9 +39,7 @@ def scrape_content(self, result): logger.debug(f"Compiling {name} with {len(data['methods'])} methods") self._data[name] = {"methods": []} for method in data["methods"]: - self._data[name]["methods"].append( - {method["name"]: self._compile_method(method)} - ) + self._data[name]["methods"].append({method["name"]: self._compile_method(method)}) self.params.update({param["name"]: param for param in method["params"]}) def yaml_format(self, ingore=None): diff --git a/apix/parsers/test.py b/apix/parsers/test.py index de4e650..fe822b6 100644 --- a/apix/parsers/test.py +++ b/apix/parsers/test.py @@ -46,12 +46,7 @@ def pull_links(result, base_path): links, last = [], None for link in g_links: url = link[2].replace("../", "") - if ( - "JacobCallahan" in url - and "sparkline" not in url - and link[0].text - and url != last - ): + if "JacobCallahan" in url and "sparkline" not in url and link[0].text and url != last: links.append((link[0].text, url)) last = url return links diff --git a/candore/__init__.py b/candore/__init__.py index 0a6badc..d8ac851 100644 --- a/candore/__init__.py +++ b/candore/__init__.py @@ -1,4 +1,4 @@ -import asyncio +import asyncio # noqa: F401 import json from pathlib import Path @@ -31,9 +31,7 @@ async def save_all_entities(self, mode, output_file, full): if mode not in ["pre", "post"]: raise ModeError("Extracting mode must be 'pre' or 'post'") - async with Extractor( - settings=self.settings, apilister=self.api_lister - ) as extractor: + async with Extractor(settings=self.settings, apilister=self.api_lister) as extractor: if full: extractor.full = True data = await extractor.extract_all_entities() diff --git a/candore/cli.py b/candore/cli.py index 3fb8795..8898a6a 100644 --- a/candore/cli.py +++ b/candore/cli.py @@ -36,9 +36,6 @@ def candore(ctx, version, settings_file, components_file): def apis(ctx): """List API lister endpoints from Product""" print("List of API lister endpoints from Product") - import ipdb - - ipdb.set_trace() candore_obj = ctx.parent.candore pprint(candore_obj.list_endpoints()) @@ -46,16 +43,12 @@ def apis(ctx): @candore.command(help="Extract and save data using API lister endpoints") @click.option("--mode", type=str, help="The mode must be 'pre' or 'post'") @click.option("-o", "--output", type=str, help="The output file name") -@click.option( - "--full", is_flag=True, help="Extract data from all the pages of a component" -) +@click.option("--full", is_flag=True, help="Extract data from all the pages of a component") @click.pass_context def extract(ctx, mode, output, full): loop = asyncio.get_event_loop() candore_obj = ctx.parent.candore - loop.run_until_complete( - candore_obj.save_all_entities(mode=mode, output_file=output, full=full) - ) + loop.run_until_complete(candore_obj.save_all_entities(mode=mode, output_file=output, full=full)) @candore.command(help="Compare pre and post upgrade data") @@ -69,9 +62,7 @@ def extract(ctx, mode, output, full): default="json", help="The type of report GSheet, JSON, or webpage", ) -@click.option( - "--record-evs", is_flag=True, help="Record Expected Variations in reporting" -) +@click.option("--record-evs", is_flag=True, help="Record Expected Variations in reporting") @click.pass_context def compare(ctx, pre, post, output, report_type, record_evs): candore_obj = ctx.parent.candore diff --git a/candore/modules/comparator.py b/candore/modules/comparator.py index 1759517..ebafb22 100644 --- a/candore/modules/comparator.py +++ b/candore/modules/comparator.py @@ -15,7 +15,8 @@ def __init__(self, settings): def remove_non_variant_key(self, key): reversed_bk = self.big_key[::-1] - reversed_bk.remove(key) + if key in reversed_bk: + reversed_bk.remove(key) self.big_key = reversed_bk[::-1] def remove_path(self, identy): @@ -27,13 +28,8 @@ def remove_path(self, identy): def record_variation(self, pre, post, var_details=None): big_key = [str(itm) for itm in self.big_key] full_path = "/".join(big_key) - var_full_path = "/".join( - [itm for itm in self.big_key if not isinstance(itm, int)] - ) - if ( - var_full_path in self.expected_variations - or var_full_path in self.skipped_variations - ): + var_full_path = "/".join([itm for itm in self.big_key if not isinstance(itm, int)]) + if var_full_path in self.expected_variations or var_full_path in self.skipped_variations: if self.record_evs: variation = { "pre": pre, @@ -89,9 +85,7 @@ def _is_data_type_list(self, pre, post, unique_key=""): self.record_variation(pre, post) self.remove_path(unique_key) - def compare_all_pres_with_posts( - self, pre_data, post_data, unique_key="", var_details=None - ): + def compare_all_pres_with_posts(self, pre_data, post_data, unique_key="", var_details=None): if unique_key: self.big_key.append(unique_key) if type(pre_data) is dict: @@ -104,7 +98,7 @@ def compare_all_pres_with_posts( self.remove_non_variant_key(unique_key) def compare_json(self, pre_file, post_file): - pre_data = post_Data = None + pre_data = post_data = None with open(pre_file, "r") as fpre: pre_data = json.load(fpre) diff --git a/candore/modules/extractor.py b/candore/modules/extractor.py index a1a7c79..b7d1a48 100644 --- a/candore/modules/extractor.py +++ b/candore/modules/extractor.py @@ -1,4 +1,4 @@ -import asyncio +import asyncio # noqa: F401 from functools import cached_property import aiohttp @@ -37,9 +37,7 @@ def api_endpoints(self): async def _start_session(self): if not self.client: - self.client = aiohttp.ClientSession( - auth=self.auth, connector=self.connector - ) + self.client = aiohttp.ClientSession(auth=self.auth, connector=self.connector) return self.client async def _end_session(self): diff --git a/candore/modules/report.py b/candore/modules/report.py index 2f9221b..00c3d75 100644 --- a/candore/modules/report.py +++ b/candore/modules/report.py @@ -78,12 +78,8 @@ def _generate_csv_report(self, output_file): output_file = Path(output_file) # Convert json to csv and write to output file csv_writer = csv.writer(output_file.open("w")) - csv_writer.writerow( - ["Variation Path", "Pre-Upgrade", "Post-Upgrade", "Variation"] - ) + csv_writer.writerow(["Variation Path", "Pre-Upgrade", "Post-Upgrade", "Variation"]) for var_path, vals in self.results.items(): - csv_writer.writerow( - [var_path, vals["pre"], vals["post"], vals["variation"]] - ) + csv_writer.writerow([var_path, vals["pre"], vals["post"], vals["variation"]]) print("Wrote CSV report to {}".format(output_file)) print("CSV report contains {} results".format(len(self.results))) diff --git a/candore/modules/variatons.py b/candore/modules/variatons.py index f53dcfd..d42d312 100644 --- a/candore/modules/variatons.py +++ b/candore/modules/variatons.py @@ -9,7 +9,7 @@ class Variations: - def __int__(self, settings): + def __init__(self, settings): self.settings = settings def get_paths(self, variations, prefix="", separator="/"): diff --git a/pyproject.toml b/pyproject.toml index 21a2d0c..9fc0f43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,3 +61,20 @@ test = [ [project.scripts] candore = "candore.cli:candore" + +[tool.black] +line-length = 100 +skip-string-normalization = true +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.hg + | \.mypy_cache + | \.venv + | _build + | buck-out + | build + | dist +)/ +'''