Merge pull request #558 from openlawlibrary/renatav/mirrors-fix

Release 0.32.0

CHANGELOG.md

@@ -13,6 +13,21 @@ and this project adheres to [Semantic Versioning][semver].
 
 ### Fixed
 
+## [0.32.0] - 10/23/2024
+
+### Added
+
+
+### Changed
+
+
+### Fixed
+
+- Fix specification of pygit2 version depending on the Python version [(558)]
+- Fix validation and listing targets of an auth repo that does not contain `mirrors.json` [(558)]
+
+[558]: https://github.com/openlawlibrary/taf/pull/558
+
 
 ## [0.31.2] - 10/16/2024
 
@@ -1305,6 +1320,7 @@ and this project adheres to [Semantic Versioning][semver].
 [keepachangelog]: https://keepachangelog.com/en/1.0.0/
 [semver]: https://semver.org/spec/v2.0.0.html
 [unreleased]: https://github.com/openlawlibrary/taf/compare/v0.31.2...HEAD
+[0.32.0]: https://github.com/openlawlibrary/taf/compare/v0.31.2...v0.32.0
 [0.31.2]: https://github.com/openlawlibrary/taf/compare/v0.31.1...v0.31.2
 [0.31.1]: https://github.com/openlawlibrary/taf/compare/v0.31.0...v0.31.1
 [0.31.0]: https://github.com/openlawlibrary/taf/compare/v0.30.2...0.31.0

setup.py

@@ -1,8 +1,7 @@
 from setuptools import find_packages, setup
-import sys
 
 PACKAGE_NAME = "taf"
-VERSION = "0.31.2"
+VERSION = "0.32.0"
 AUTHOR = "Open Law Library"
 AUTHOR_EMAIL = "[email protected]"
 DESCRIPTION = "Implementation of archival authentication"
@@ -36,11 +35,6 @@
 
 yubikey_require = ["yubikey-manager==5.1.*"]
 
-# Determine the appropriate version of pygit2 based on the Python version
-if sys.version_info >= (3, 11):
-    pygit2_version = "pygit2==1.14.1"
-elif sys.version_info >= (3, 7) and sys.version_info < (3, 11):
-    pygit2_version = "pygit2==1.9.*"
 
 kwargs = {
     "name": PACKAGE_NAME,
@@ -64,7 +58,8 @@
         "cryptography==38.0.*",
         "securesystemslib==0.25.*",
         "loguru==0.7.*",
-        pygit2_version,
+        'pygit2==1.9.*; python_version < "3.11"',
+        'pygit2==1.14.*; python_version >= "3.11"',
         "pyOpenSSL==22.1.*",
         "logdecorator==2.*",
     ],

taf/api/dependencies.py

@@ -8,7 +8,7 @@
     update_snapshot_and_timestamp,
     update_target_metadata,
 )
-from taf.api.utils._git import check_if_clean, commit_and_push
+from taf.api.utils._git import check_if_clean
 from taf.messages import git_commit_message
 from pathlib import Path
 
@@ -176,7 +176,7 @@ def add_dependency(
         commit_msg = git_commit_message(
             "add-dependency", dependency_name=dependency_name
         )
-        commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+        auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
     else:
         print("\nPlease commit manually.\n")
 
@@ -266,7 +266,7 @@ def remove_dependency(
         commit_msg = git_commit_message(
             "remove-dependency", dependency_name=dependency_name
         )
-        commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+        auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
     else:
         print("\nPlease commit manually.\n")
 

taf/api/metadata.py

@@ -2,7 +2,7 @@
 from logging import INFO, ERROR
 from typing import Dict, List, Optional, Tuple
 from logdecorator import log_on_end, log_on_error
-from taf.api.utils._git import check_if_clean, commit_and_push
+from taf.api.utils._git import check_if_clean
 from taf.exceptions import TAFError
 from taf.keys import load_signing_keys
 from taf.constants import DEFAULT_RSA_SIGNATURE_SCHEME
@@ -156,7 +156,7 @@ def update_metadata_expiration_date(
         commit_msg = git_commit_message(
             "update-expiration-dates", roles=",".join(roles)
         )
-        commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+        auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
 
 
 @log_on_end(INFO, "Updated expiration date of {role:s}", logger=taf_logger)

taf/api/repository.py

@@ -6,7 +6,6 @@
 from taf.messages import git_commit_message
 from taf.models.types import RolesIterator
 from taf.models.types import RolesKeysData
-from taf.api.utils._git import commit_and_push
 from taf.models.converter import from_dict
 
 from pathlib import Path
@@ -125,7 +124,7 @@ def create_repository(
     if commit:
         auth_repo.init_repo()
         commit_msg = git_commit_message("create-repo")
-        commit_and_push(auth_repo, push=False, commit_msg=commit_msg)
+        auth_repo.commit_and_push(push=False, commit_msg=commit_msg)
     else:
         print("\nPlease commit manually.\n")
 
@@ -194,7 +193,7 @@ def taf_status(path: str, library_dir: Optional[str] = None, indent: int = 0) ->
     print(f"{indent_str}Something to commit: {auth_repo.something_to_commit()}")
     print(f"{indent_str}Target Repositories Status:")
     # Call the list_targets function
-    list_targets(path=path, library_dir=library_dir)
+    list_targets(path=path)
 
     # Load dependencies using repositoriesdb.get_auth_repositories
     repositoriesdb.load_dependencies(auth_repo, library_dir=library_dir)

taf/api/roles.py

@@ -10,7 +10,7 @@
 from taf.api.utils._roles import _role_obj, create_delegations
 from taf.messages import git_commit_message
 from tuf.repository_tool import Targets
-from taf.api.utils._git import check_if_clean, commit_and_push
+from taf.api.utils._git import check_if_clean
 from taf.exceptions import KeystoreError, TAFError
 from taf.models.converter import from_dict
 from taf.models.types import RolesIterator, TargetsRole, compare_roles_data
@@ -140,7 +140,7 @@ def add_role(
             auth_repo, keystore, scheme=scheme, prompt_for_keys=prompt_for_keys
         )
         commit_msg = git_commit_message("add-role", role=role)
-        commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+        auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
     else:
         taf_logger.log("NOTICE", "\nPlease commit manually\n")
 
@@ -206,7 +206,7 @@ def add_role_paths(
             commit_msg = git_commit_message(
                 "add-role-paths", paths=", ".join(paths), role=delegated_role
             )
-            commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+            auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
         else:
             taf_logger.log("NOTICE", "\nPlease commit manually\n")
     else:
@@ -303,7 +303,7 @@ def add_multiple_roles(
     if commit:
         roles_names = [role.name for role in new_roles]
         commit_msg = git_commit_message("add-roles", roles=", ".join(roles_names))
-        commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+        auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
     else:
         taf_logger.log("NOTICE", "\nPlease commit manually\n")
 
@@ -414,7 +414,7 @@ def add_signing_key(
         # commit_msg = git_commit_message(
         #     "add-signing-key", role={role}
         # )
-        commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+        auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
     else:
         taf_logger.log("NOTICE", "\nPlease commit manually\n")
 
@@ -842,7 +842,7 @@ def remove_role(
     )
     if commit:
         commit_msg = git_commit_message("remove-role", role=role)
-        commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+        auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
     else:
         taf_logger.log("NOTICE", "Please commit manually")
 
@@ -903,7 +903,7 @@ def remove_paths(
         commit_msg = git_commit_message(
             "remove-role-paths", paths=", ".join(paths), role=delegated_role
         )
-        commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+        auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
     elif delegation_existed:
         taf_logger.log("NOTICE", "\nPlease commit manually\n")
     return delegation_existed

taf/api/targets.py

@@ -16,7 +16,7 @@
     add_role_paths,
     remove_paths,
 )
-from taf.api.utils._git import check_if_clean, commit_and_push
+from taf.api.utils._git import check_if_clean
 from taf.constants import DEFAULT_RSA_SIGNATURE_SCHEME
 from taf.exceptions import TAFError
 from taf.git import GitRepository
@@ -185,7 +185,7 @@ def add_target_repo(
     )
     if commit:
         commit_msg = git_commit_message("add-target", target_name=target_name)
-        commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+        auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
     else:
         taf_logger.log("NOTICE", "\nPlease commit manually\n")
 
@@ -246,10 +246,9 @@ def export_targets_history(
 
 def list_targets(
     path: str,
-    library_dir: Optional[str] = None,
-) -> None:
+) -> Dict:
     """
-    Prints a list of target repositories of an authentication repository and their states (are the work directories clean, are there
+    Returns a dictionary containing target repositories of an authentication repository and their states (are the work directories clean, are there
     remove changes that have not yed been pulled, are there commits that have not yet been signed).
 
     Arguments:
@@ -266,7 +265,7 @@ def list_targets(
     head_commit = auth_repo.head_commit_sha()
     if head_commit is None:
         taf_logger.log("NOTICE", "Repository is empty")
-        return
+        return {}
     top_commit = [head_commit]
     repositoriesdb.load_repositories(auth_repo)
     target_repositories = repositoriesdb.get_deduplicated_repositories(auth_repo)
@@ -285,28 +284,33 @@ def list_targets(
         repo_output["cloned"] = local_repo_exists
         if local_repo_exists:
             repo_output["bare"] = repo.is_bare_repository
+            repo_output["unsigned"] = []
             # there will only be one branch since only data corresponding to the top auth commit was loaded
             for branch, branch_data in repo_data.items():
-                repo_output["unsigned"] = False
+                has_remote = repo.has_remote()
+                repo_output["has-remote"] = has_remote
+
                 if not repo.branch_exists(branch, include_remotes=False):
                     repo_output["up-to-date"] = False
                 else:
-                    is_synced_with_remote = repo.synced_with_remote(branch=branch)
-                    repo_output["up-to-date"] = is_synced_with_remote
-                    if not is_synced_with_remote:
-                        last_signed_commit = branch_data[0]["commit"]
-                        if branch in repo.branches_containing_commit(
-                            last_signed_commit
+                    if has_remote:
+                        is_synced_with_remote = repo.synced_with_remote(branch=branch)
+                        repo_output["up-to-date"] = is_synced_with_remote
+
+                    last_signed_commit = branch_data[0]["commit"]
+                    if branch in repo.branches_containing_commit(last_signed_commit):
+                        branch_top_commit = repo.top_commit_of_branch(branch)
+                        unsigned_commits = repo.all_commits_since_commit(
+                            last_signed_commit, branch
+                        )
+                        if (
+                            len(unsigned_commits)
+                            and branch_top_commit in unsigned_commits
                         ):
-                            branch_top_commit = repo.top_commit_of_branch(branch)
-                            repo_output[
-                                "unsigned"
-                            ] = branch_top_commit in repo.all_commits_since_commit(
-                                last_signed_commit, branch
-                            )
+                            repo_output["unsigned"].append(branch)
             repo_output["something-to-commit"] = repo.something_to_commit()
 
-    taf_logger.log("NOTICE", json.dumps(output, indent=4))
+    return output
 
 
 @log_on_start(INFO, "Signing target files", logger=taf_logger)
@@ -373,7 +377,7 @@ def register_target_files(
         if commit:
             auth_repo = AuthenticationRepository(path=taf_repo.path)
             commit_msg = git_commit_message("update-targets")
-            commit_and_push(auth_repo, commit_msg=commit_msg, push=push)
+            auth_repo.commit_and_push(commit_msg=commit_msg, push=push)
         elif not no_commit_warning:
             taf_logger.log("NOTICE", "\nPlease commit manually\n")
 
@@ -439,7 +443,7 @@ def remove_target_repo(
         os.unlink(str(target_file_path))
         removed_targets_data[target_name] = {}
     else:
-        taf_logger.log("NOTICE", f"{target_file_path} target file does not exist")
+        taf_logger.info(f"{target_file_path} target file does not exist")
 
     changes_committed = False
     if len(added_targets_data) or len(removed_targets_data):
@@ -476,7 +480,7 @@ def remove_target_repo(
         )
         changes_committed = True
     else:
-        taf_logger.log("NOTICE", f"{target_name} not among delegated paths")
+        taf_logger.info(f"{target_name} not among delegated paths")
     # update snapshot and timestamp calls write_all, so targets updates will be saved too
     if changes_committed and push:
         auth_repo.push()

taf/api/utils/_git.py

@@ -1,9 +1,6 @@
 import functools
-from typing import Optional
-from taf.auth_repo import AuthenticationRepository
-from taf.exceptions import GitError, RepositoryNotCleanError, TAFError
+from taf.exceptions import RepositoryNotCleanError
 from taf.git import GitRepository
-from taf.log import taf_logger
 
 
 def check_if_clean(func):
@@ -20,34 +17,3 @@ def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
 
     return wrapper
-
-
-def commit_and_push(
-    auth_repo: AuthenticationRepository,
-    commit_msg: Optional[str] = None,
-    push: Optional[bool] = True,
-) -> None:
-    if commit_msg is None:
-        commit_msg = input("\nEnter commit message and press ENTER\n\n")
-    auth_repo.commit(commit_msg)
-
-    if push and auth_repo.has_remote():
-        try:
-            auth_repo.push()
-            taf_logger.log("NOTICE", "Successfully pushed to remote")
-
-            new_commit_branch = auth_repo.default_branch
-            if new_commit_branch:
-                new_commit = auth_repo.top_commit_of_branch(new_commit_branch)
-                if new_commit:
-                    auth_repo.set_last_validated_commit(new_commit)
-                    taf_logger.info(f"Updated last_validated_commit to {new_commit}")
-            else:
-                taf_logger.warning(
-                    "Default branch is None, skipping last_validated_commit update."
-                )
-        except GitError as e:
-            taf_logger.error(
-                f"Push failed: {str(e)}. Please check if there are upstream changes."
-            )
-            raise TAFError("Push operation failed") from e

taf/auth_repo.py

@@ -150,6 +150,34 @@ def log_prefix(self) -> str:
             return f"{self.alias}: "
         return f"Auth repo {self.name}: "
 
+    def commit_and_push(
+        self,
+        commit_msg: Optional[str] = None,
+        push: Optional[bool] = True,
+        commit: Optional[bool] = True,
+    ) -> None:
+
+        if commit:
+            if commit_msg is None:
+                commit_msg = input("\nEnter commit message and press ENTER\n\n")
+            self.commit(commit_msg)
+
+        if push:
+            push_successful = self.push()
+            if push_successful:
+                new_commit_branch = self.default_branch
+                if new_commit_branch:
+                    new_commit = self.top_commit_of_branch(new_commit_branch)
+                    if new_commit:
+                        self.set_last_validated_commit(new_commit)
+                        self._log_notice(
+                            f"Updated last_validated_commit to {new_commit}"
+                        )
+                else:
+                    self._log_warning(
+                        "Default branch is None, skipping last_validated_commit update."
+                    )
+
     def get_target(self, target_name, commit=None, safely=True) -> Optional[Dict]:
         if commit is None:
             commit = self.head_commit_sha()