Pre-commit and gitignore

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 100

.gitignore

@@ -0,0 +1,6 @@
+settings.yaml
+**/__pycache__/
+*.json
+*.csv
+.idea
+trust

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+  - repo: https://github.com/asottile/reorder_python_imports
+    rev: v3.12.0
+    hooks:
+      - id: reorder-python-imports
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: debug-statements
+  - repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8

apix/diff.py

@@ -27,13 +27,10 @@ def __attrs_post_init__(self):
             self.api_name = get_latest(data_dir=self.data_dir, mock=self.mock)
         if not self.ver1:
             # get the latest saved version
-            self.ver1 = get_latest(
-                api_name=self.api_name, data_dir=self.data_dir, mock=self.mock
-            )
+            self.ver1 = get_latest(api_name=self.api_name, data_dir=self.data_dir, mock=self.mock)
         if not self.ver2:
             # get the version before ver1
-            self.ver2 = get_previous(
-                self.api_name, self.ver1, self.data_dir, self.mock)
+            self.ver2 = get_previous(self.api_name, self.ver1, self.data_dir, self.mock)
 
     @staticmethod
     def _truncate(diff_dict):
@@ -161,8 +158,7 @@ def save_diff(self, return_path=False):
         else:
             ftype = "comp-diff" if self.compact else "diff"
             fpath = Path(
-                f"{self.data_dir}APIs/{self.api_name}/"
-                f"{self.ver2}-to-{self.ver1}-{ftype}.yaml"
+                f"{self.data_dir}APIs/{self.api_name}/" f"{self.ver2}-to-{self.ver1}-{ftype}.yaml"
             )
         if fpath.exists():
             fpath.unlink()

apix/explore.py

@@ -64,11 +64,9 @@ def _visit_links(self, links, retries=3):
         try:
             loop = asyncio.get_event_loop()
             loop.run_until_complete(self._async_loop(links))
-        except aiohttp.client_exceptions.ServerDisconnectedError as err:
+        except aiohttp.client_exceptions.ServerDisconnectedError:
             logger.warning(
-                "Lost connection to host.{}".join(
-                    "Retrying in 10 seconds" if retries else ""
-                )
+                "Lost connection to host.".join("Retrying in 10 seconds" if retries else "")
             )
             if retries:
                 time.sleep(10)
@@ -120,8 +118,7 @@ def explore(self):
         result = requests.get(self.host_url + self.base_path, verify=False)
         if not result:
             logger.warning(
-                f"I couldn't find anything useful at "
-                f"{self.host_url}{self.base_path}."
+                f"I couldn't find anything useful at " f"{self.host_url}{self.base_path}."
             )
             return
         self.base_path = self.base_path.replace(".html", "")  # for next step

apix/helpers.py

@@ -14,9 +14,7 @@ def get_api_list(data_dir=None, mock=False):
     if not api_dir.exists():
         return None
     # get all versions in directory, that aren't diffs
-    apis = [
-        (api.name, api.stat().st_mtime) for api in api_dir.iterdir() if api.is_dir()
-    ] or []
+    apis = [(api.name, api.stat().st_mtime) for api in api_dir.iterdir() if api.is_dir()] or []
     apis = [api for api, _ in sorted(apis, key=lambda x: x[1], reverse=True)]
     return apis
 
@@ -34,9 +32,7 @@ def get_ver_list(api_name, data_dir=None, mock=False):
     versions = [
         v_file.name.replace(".yaml", "")
         for v_file in save_path.iterdir()
-        if "-diff." not in v_file.name
-        and "-comp." not in v_file.name
-        and ".yaml" in v_file.name
+        if "-diff." not in v_file.name and "-comp." not in v_file.name and ".yaml" in v_file.name
     ] or []
     return sorted(versions, reverse=True)
 
@@ -87,13 +83,10 @@ def save_api(api_name, version, api_dict, data_dir=None, compact=False, mock=Fal
     """Save the dict to yaml, if the file doesn't exist"""
     if mock:
         a_path = Path(
-            f"{data_dir}tests/APIs/{api_name}/{version}"
-            f"{'-comp' if compact else ''}.yaml"
+            f"{data_dir}tests/APIs/{api_name}/{version}" f"{'-comp' if compact else ''}.yaml"
         )
     else:
-        a_path = Path(
-            f"{data_dir}APIs/{api_name}/{version}" f"{'-comp' if compact else ''}.yaml"
-        )
+        a_path = Path(f"{data_dir}APIs/{api_name}/{version}" f"{'-comp' if compact else ''}.yaml")
     a_path.parent.mkdir(parents=True, exist_ok=True)
     logger.info(f"Saving {api_name} v{version} to {a_path}")
     with a_path.open("w") as f:

apix/parsers/apipie.py

@@ -29,10 +29,7 @@ def _compile_method(method_dict):
             )
             for param in method_dict["params"]
         ]
-        paths = [
-            f'{path["http_method"].upper()} {path["api_url"]}'
-            for path in method_dict["apis"]
-        ]
+        paths = [f'{path["http_method"].upper()} {path["api_url"]}' for path in method_dict["apis"]]
         return {"paths": paths, "params": params}
 
     def scrape_content(self, result):
@@ -42,9 +39,7 @@ def scrape_content(self, result):
             logger.debug(f"Compiling {name} with {len(data['methods'])} methods")
             self._data[name] = {"methods": []}
             for method in data["methods"]:
-                self._data[name]["methods"].append(
-                    {method["name"]: self._compile_method(method)}
-                )
+                self._data[name]["methods"].append({method["name"]: self._compile_method(method)})
                 self.params.update({param["name"]: param for param in method["params"]})
 
     def yaml_format(self, ingore=None):

apix/parsers/test.py

@@ -46,12 +46,7 @@ def pull_links(result, base_path):
         links, last = [], None
         for link in g_links:
             url = link[2].replace("../", "")
-            if (
-                "JacobCallahan" in url
-                and "sparkline" not in url
-                and link[0].text
-                and url != last
-            ):
+            if "JacobCallahan" in url and "sparkline" not in url and link[0].text and url != last:
                 links.append((link[0].text, url))
                 last = url
         return links

candore/__init__.py

@@ -1,4 +1,4 @@
-import asyncio
+import asyncio  # noqa: F401
 import json
 from pathlib import Path
 
@@ -31,9 +31,7 @@ async def save_all_entities(self, mode, output_file, full):
         if mode not in ["pre", "post"]:
             raise ModeError("Extracting mode must be 'pre' or 'post'")
 
-        async with Extractor(
-            settings=self.settings, apilister=self.api_lister
-        ) as extractor:
+        async with Extractor(settings=self.settings, apilister=self.api_lister) as extractor:
             if full:
                 extractor.full = True
             data = await extractor.extract_all_entities()

candore/cli.py

@@ -36,26 +36,19 @@ def candore(ctx, version, settings_file, components_file):
 def apis(ctx):
     """List API lister endpoints from Product"""
     print("List of API lister endpoints from Product")
-    import ipdb
-
-    ipdb.set_trace()
     candore_obj = ctx.parent.candore
     pprint(candore_obj.list_endpoints())
 
 
 @candore.command(help="Extract and save data using API lister endpoints")
 @click.option("--mode", type=str, help="The mode must be 'pre' or 'post'")
 @click.option("-o", "--output", type=str, help="The output file name")
-@click.option(
-    "--full", is_flag=True, help="Extract data from all the pages of a component"
-)
+@click.option("--full", is_flag=True, help="Extract data from all the pages of a component")
 @click.pass_context
 def extract(ctx, mode, output, full):
     loop = asyncio.get_event_loop()
     candore_obj = ctx.parent.candore
-    loop.run_until_complete(
-        candore_obj.save_all_entities(mode=mode, output_file=output, full=full)
-    )
+    loop.run_until_complete(candore_obj.save_all_entities(mode=mode, output_file=output, full=full))
 
 
 @candore.command(help="Compare pre and post upgrade data")
@@ -69,9 +62,7 @@ def extract(ctx, mode, output, full):
     default="json",
     help="The type of report GSheet, JSON, or webpage",
 )
-@click.option(
-    "--record-evs", is_flag=True, help="Record Expected Variations in reporting"
-)
+@click.option("--record-evs", is_flag=True, help="Record Expected Variations in reporting")
 @click.pass_context
 def compare(ctx, pre, post, output, report_type, record_evs):
     candore_obj = ctx.parent.candore

candore/modules/comparator.py

@@ -15,7 +15,8 @@ def __init__(self, settings):
 
     def remove_non_variant_key(self, key):
         reversed_bk = self.big_key[::-1]
-        reversed_bk.remove(key)
+        if key in reversed_bk:
+            reversed_bk.remove(key)
         self.big_key = reversed_bk[::-1]
 
     def remove_path(self, identy):
@@ -27,13 +28,8 @@ def remove_path(self, identy):
     def record_variation(self, pre, post, var_details=None):
         big_key = [str(itm) for itm in self.big_key]
         full_path = "/".join(big_key)
-        var_full_path = "/".join(
-            [itm for itm in self.big_key if not isinstance(itm, int)]
-        )
-        if (
-            var_full_path in self.expected_variations
-            or var_full_path in self.skipped_variations
-        ):
+        var_full_path = "/".join([itm for itm in self.big_key if not isinstance(itm, int)])
+        if var_full_path in self.expected_variations or var_full_path in self.skipped_variations:
             if self.record_evs:
                 variation = {
                     "pre": pre,
@@ -89,9 +85,7 @@ def _is_data_type_list(self, pre, post, unique_key=""):
                     self.record_variation(pre, post)
         self.remove_path(unique_key)
 
-    def compare_all_pres_with_posts(
-        self, pre_data, post_data, unique_key="", var_details=None
-    ):
+    def compare_all_pres_with_posts(self, pre_data, post_data, unique_key="", var_details=None):
         if unique_key:
             self.big_key.append(unique_key)
         if type(pre_data) is dict:
@@ -104,7 +98,7 @@ def compare_all_pres_with_posts(
             self.remove_non_variant_key(unique_key)
 
     def compare_json(self, pre_file, post_file):
-        pre_data = post_Data = None
+        pre_data = post_data = None
 
         with open(pre_file, "r") as fpre:
             pre_data = json.load(fpre)

candore/modules/extractor.py

@@ -1,4 +1,4 @@
-import asyncio
+import asyncio  # noqa: F401
 from functools import cached_property
 
 import aiohttp
@@ -37,9 +37,7 @@ def api_endpoints(self):
 
     async def _start_session(self):
         if not self.client:
-            self.client = aiohttp.ClientSession(
-                auth=self.auth, connector=self.connector
-            )
+            self.client = aiohttp.ClientSession(auth=self.auth, connector=self.connector)
         return self.client
 
     async def _end_session(self):

candore/modules/report.py

@@ -78,12 +78,8 @@ def _generate_csv_report(self, output_file):
         output_file = Path(output_file)
         # Convert json to csv and write to output file
         csv_writer = csv.writer(output_file.open("w"))
-        csv_writer.writerow(
-            ["Variation Path", "Pre-Upgrade", "Post-Upgrade", "Variation"]
-        )
+        csv_writer.writerow(["Variation Path", "Pre-Upgrade", "Post-Upgrade", "Variation"])
         for var_path, vals in self.results.items():
-            csv_writer.writerow(
-                [var_path, vals["pre"], vals["post"], vals["variation"]]
-            )
+            csv_writer.writerow([var_path, vals["pre"], vals["post"], vals["variation"]])
         print("Wrote CSV report to {}".format(output_file))
         print("CSV report contains {} results".format(len(self.results)))

candore/modules/variatons.py

@@ -9,7 +9,7 @@
 
 
 class Variations:
-    def __int__(self, settings):
+    def __init__(self, settings):
         self.settings = settings
 
     def get_paths(self, variations, prefix="", separator="/"):

pyproject.toml

@@ -61,3 +61,20 @@ test = [
 
 [project.scripts]
 candore = "candore.cli:candore"
+
+[tool.black]
+line-length = 100
+skip-string-normalization = true
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''