Add debug level logging statements

Onboard pre-commit

Author: dormant-user

.github/workflows/python-publish.yaml

@@ -0,0 +1,14 @@
+name: pypi-publish
+
+on:
+  release:
+    types: [ published ]
+  workflow_dispatch:
+
+jobs:
+  pypi-publisher:
+    runs-on: thevickypedia-lite
+    steps:
+      - uses: thevickypedia/pypi-publisher@v3
+        env:
+          token: ${{ secrets.PYPI_TOKEN }}

.pre-commit-config.yaml

@@ -0,0 +1,58 @@
+fail_fast: true
+exclude: ^(notebooks/|scripts/|.github/|docs/)
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-ast
+      - id: check-byte-order-marker
+      - id: check-builtin-literals
+      - id: check-case-conflict
+      - id: check-docstring-first
+      - id: check-executables-have-shebangs
+      - id: check-shebang-scripts-are-executable
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: check-vcs-permalinks
+      - id: check-xml
+      - id: check-yaml
+      - id: debug-statements
+      - id: detect-aws-credentials
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: fix-byte-order-marker
+      - id: mixed-line-ending
+      - id: name-tests-test
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+        - id: black
+          exclude: docs_gen/
+
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-docstrings
+          - flake8-sfs
+        args: [ --max-line-length=120, --extend-ignore=SFS3 D107 SFS301 D100 D104 D401 SFS101 SFS201 D412 ]
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: [ --profile, black ]
+
+  - repo: local
+    hooks:
+      - id: pytest_docs
+        name: run pytest and generate runbook
+        entry: /bin/bash pre_commit.sh
+        language: system
+        pass_filenames: false
+        always_run: true

git2s3/__init__.py

@@ -1 +1,3 @@
-"""Placeholder for package"""
+"""Placeholder for packaging."""
+
+version = "0.0.0-a"

git2s3/config.py

@@ -2,20 +2,20 @@
 import os
 from datetime import datetime
 
-from git2s3.models import LogOptions
+from git2s3.models import LogOptions, EnvConfig
 
 
-def default_logger(log: LogOptions) -> logging.Logger:
+def default_logger(env: EnvConfig) -> logging.Logger:
     """Generates a default console logger.
 
     Args:
-        log: Log option chosen by the user.
+        env: Environment configuration.
 
     Returns:
         logging.Logger:
         Logger object.
     """
-    if log == LogOptions.file:
+    if env.log == LogOptions.file:
         if not os.path.isdir("logs"):
             os.mkdir("logs")
         logfile: str = datetime.now().strftime(
@@ -25,7 +25,10 @@ def default_logger(log: LogOptions) -> logging.Logger:
     else:
         handler = logging.StreamHandler()
     logger = logging.getLogger(__name__)
-    logger.setLevel(level=logging.INFO)
+    if env.debug:
+        logger.setLevel(level=logging.DEBUG)
+    else:
+        logger.setLevel(level=logging.INFO)
     handler.setFormatter(
         fmt=logging.Formatter(
             fmt="%(asctime)s - %(levelname)-8s - [%(funcName)s:%(lineno)d] - %(message)s"

git2s3/models.py

@@ -9,7 +9,6 @@
 else:
     from enum import Enum
 
-
     class StrEnum(str, Enum):
         """Override for python 3.10 due to lack of StrEnum."""
 
@@ -54,9 +53,7 @@ def parse_clone_dir(cls, value: str) -> DirectoryPath:
         if os.path.isdir(os.path.dirname(value)):
             os.makedirs(value)
             return value
-        raise ValueError(
-            f"{value!r} is neither a valid path, nor a potential path"
-        )
+        raise ValueError(f"{value!r} is neither a valid path, nor a potential path")
 
     class Config:
         """Environment variables configuration."""

main.py

@@ -13,14 +13,14 @@
 class Git2S3:
     def __init__(self, **kwargs):
         self.env = models.EnvConfig(**kwargs)
-        self.logger = kwargs.get("logger", config.default_logger(self.env.log))
+        self.logger = kwargs.get("logger", config.default_logger(self.env))
         self.repo = git.Repo()
         self.session = requests.Session()
         self.session.headers = {
-            'Accept': 'application/vnd.github+json',
-            'Authorization': f'Bearer {self.env.git_token}',
-            'X-GitHub-Api-Version': '2022-11-28',
-            'Content-Type': 'application/x-www-form-urlencoded',
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {self.env.git_token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+            "Content-Type": "application/x-www-form-urlencoded",
         }
 
     def get_all_repos(self) -> Generator[Dict[str, str]]:
@@ -30,10 +30,9 @@ def get_all_repos(self) -> Generator[Dict[str, str]]:
             Generator[Dict[str, str]]:
             Yields a dictionary of each repo's information.
         """
-        # todo: Add debug level logging for each API call, along with number of repos fetched and in break statements
-        #   Add .pre-commit config and pyproject.toml
         idx = 1
         while True:
+            self.logger.debug("Fetching repos from page %d", idx)
             try:
                 response = self.session.get(
                     url=f"{self.env.git_api_url}orgs/{self.env.git_owner}/repos?per_page=1&page={idx}"
@@ -42,9 +41,13 @@ def get_all_repos(self) -> Generator[Dict[str, str]]:
                 break
             json_response = response.json()
             if json_response:
+                self.logger.debug(
+                    "Repositories in page %d: %d", idx, len(json_response)
+                )
                 yield from json_response
                 idx += 1
             else:
+                self.logger.debug("No repos found in page: %d, ending loop.", idx)
                 break
 
     def worker(self, repo: Dict[str, str]):
@@ -53,15 +56,15 @@ def worker(self, repo: Dict[str, str]):
         Args:
             repo: Repository information as JSON payload.
         """
-        self.logger.info("Cloning %s", repo.get('name'))
-        repo_dest = os.path.join(self.env.clone_dir, repo.get('name'))
+        self.logger.info("Cloning %s", repo.get("name"))
+        repo_dest = os.path.join(self.env.clone_dir, repo.get("name"))
         if not os.path.isdir(repo_dest):
             os.makedirs(repo_dest)
         try:
-            self.repo.clone_from(repo.get('clone_url'), str(repo_dest))
+            self.repo.clone_from(repo.get("clone_url"), str(repo_dest))
         except GitCommandError as error:
             msg = error.stderr or error.stdout or ""
-            msg = msg.strip().replace('\n', '').replace("'", "").replace('"', '')
+            msg = msg.strip().replace("\n", "").replace("'", "").replace('"', "")
             self.logger.error(msg)
             # Raise an exception to indicate that the thread failed
             raise Exception(msg)
@@ -76,15 +79,15 @@ def cloner(self):
         with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
             for repo in self.get_all_repos():
                 future = executor.submit(self.worker, repo)
-                futures[future] = repo.get('name')
+                futures[future] = repo.get("name")
         for future in as_completed(futures):
             if future.exception():
                 self.logger.error(
                     "Thread processing for '%s' received an exception: %s",
                     futures[future],
-                    future.exception()
+                    future.exception(),
                 )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     Git2S3().cloner()

pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "git2s3"
+dynamic = ["version"]
+description = "Backup GitHub projects to AWS S3"
+readme = "README.md"
+authors = [{ name = "Vignesh Rao", email = "[email protected]" }]
+license = { file = "LICENSE" }
+classifiers = [
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Development Status :: 2 - Pre-Alpha",  # todo: change dev status to "Development Status :: 5 - Production/Stable"
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX :: Linux",
+    "Topic :: System :: Archiving :: Backup"
+]
+keywords = ["git2s3", "backup"]
+requires-python = ">=3.10"
+
+[tool.setuptools]
+packages = ["git2s3"]
+
+[tool.setuptools.dynamic]
+version = {attr = "git2s3.version"}
+
+[build-system]
+requires      = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project.urls]
+Homepage = "https://github.com/thevickypedia/git2s3"
+Docs = "https://thevickypedia.github.io/git2s3"
+Source = "https://github.com/thevickypedia/git2s3"
+"Bug Tracker" = "https://github.com/thevickypedia/git2s3/issues"
+"Release Notes" = "https://github.com/thevickypedia/git2s3/blob/main/release_notes.rst"

requirements.txt

@@ -1,4 +1,4 @@
-requests==2.32.3
+GitPython==3.1.43
 pydantic==2.7.4
 pydantic-settings==2.3.4
-GitPython==3.1.43
+requests==2.32.3