From 166401e3e9a649f7f62ffd1f0715e7c7cbdcd799 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 18:02:56 +0300 Subject: [PATCH 01/18] :truck: move helper method to SseListener class --- app/event_handling/sse_listener.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/event_handling/sse_listener.py b/app/event_handling/sse_listener.py index 456e9bbe4..048b38683 100644 --- a/app/event_handling/sse_listener.py +++ b/app/event_handling/sse_listener.py @@ -69,3 +69,8 @@ async def wait_for_event(self, field, field_id, desired_state, timeout: int = 12 class SseListenerTimeout(Exception): """Exception raised when the Listener times out waiting for a matching event.""" + + +def create_sse_listener(wallet_id: str, topic: str) -> SseListener: + # Helper method for passing a MockListener to a class + return SseListener(topic=topic, wallet_id=wallet_id) From cc9ce39b3fa4899912174c2dcc30a7afbda94d87 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 18:03:05 +0300 Subject: [PATCH 02/18] empty init --- app/services/onboarding/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 app/services/onboarding/__init__.py diff --git a/app/services/onboarding/__init__.py b/app/services/onboarding/__init__.py new file mode 100644 index 000000000..e69de29bb From 4f45e0be9cfee633af09ef9285f311ca20e5d8c7 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 18:05:53 +0300 Subject: [PATCH 03/18] :truck::art: split onboarding service into multi modules --- app/routes/admin/tenants.py | 2 +- .../{onboarding.py => onboarding/issuer.py} | 192 +----------------- app/services/onboarding/tenants.py | 111 ++++++++++ app/services/onboarding/verifier.py | 88 ++++++++ app/tests/admin/test_onboarding.py | 3 +- 5 files changed, 204 insertions(+), 192 deletions(-) rename app/services/{onboarding.py => onboarding/issuer.py} (55%) create mode 100644 app/services/onboarding/tenants.py create mode 100644 app/services/onboarding/verifier.py diff --git a/app/routes/admin/tenants.py b/app/routes/admin/tenants.py index 9d9c372b7..617fe7ef4 100644 --- a/app/routes/admin/tenants.py +++ b/app/routes/admin/tenants.py @@ -25,7 +25,7 @@ UpdateTenantRequest, WalletListWithGroups, ) -from app.services.onboarding import handle_tenant_update, onboard_tenant +from app.services.onboarding.tenants import handle_tenant_update, onboard_tenant from app.services.trust_registry import ( Actor, TrustRegistryException, diff --git a/app/services/onboarding.py b/app/services/onboarding/issuer.py similarity index 55% rename from app/services/onboarding.py rename to app/services/onboarding/issuer.py index e801ecd44..cf663daf3 100644 --- a/app/services/onboarding.py +++ b/app/services/onboarding/issuer.py @@ -1,20 +1,9 @@ -from typing import List +from aries_cloudcontroller import AcaPyClient, InvitationCreateRequest -from aries_cloudcontroller import AcaPyClient, InvitationCreateRequest, InvitationRecord -from aries_cloudcontroller.model.create_wallet_token_request import ( - CreateWalletTokenRequest, -) -from fastapi.exceptions import HTTPException - -from app.dependencies.acapy_clients import ( - get_governance_controller, - get_tenant_controller, -) -from app.event_handling.sse_listener import SseListener +from app.event_handling.sse_listener import create_sse_listener from app.exceptions.cloud_api_error import CloudApiException -from app.models.tenants import OnboardResult, UpdateTenantRequest +from app.models.tenants import OnboardResult from app.services import acapy_ledger, acapy_wallet -from app.services.trust_registry import TrustRegistryRole, actor_by_id, update_actor from app.util.assert_connection_metadata import ( assert_author_role_set, assert_endorser_info_set, @@ -27,102 +16,6 @@ logger = get_logger(__name__) -def create_sse_listener(wallet_id: str, topic: str) -> SseListener: - # Helper method for passing MockListener to class - return SseListener(topic=topic, wallet_id=wallet_id) - - -async def handle_tenant_update( - admin_controller: AcaPyClient, - tenant_id: str, - update: UpdateTenantRequest, -): - bound_logger = logger.bind(body={"tenant_id": tenant_id}) - bound_logger.bind(body=update).info("Handling tenant update") - - bound_logger.debug("Retrieving the wallet") - wallet = await admin_controller.multitenancy.get_wallet(wallet_id=tenant_id) - if not wallet: - bound_logger.error("Bad request: Wallet not found.") - raise HTTPException(404, f"Wallet for tenant id `{tenant_id}` not found.") - - bound_logger.debug("Retrieving tenant from trust registry") - actor = await actor_by_id(wallet.wallet_id) - if not actor: - bound_logger.error( - "Tenant not found in trust registry. " - "Holder tenants cannot be updated with new roles." - ) - raise HTTPException(409, "Holder tenants cannot be updated with new roles.") - - updated_actor = actor.copy() - if update.name: - updated_actor["name"] = update.name - - if update.roles: - bound_logger.info("Updating tenant roles") - # We only care about the added roles, as that's what needs the setup. - # Teardown is not required at the moment, besides from removing it from - # the trust registry - added_roles = list(set(update.roles) - set(actor["roles"])) - - # We need to pose as the tenant to onboard for the specified role - token_response = await admin_controller.multitenancy.get_auth_token( - wallet_id=tenant_id, body=CreateWalletTokenRequest() - ) - - onboard_result = await onboard_tenant( - name=updated_actor["name"], - roles=added_roles, - tenant_auth_token=token_response.token, - tenant_id=tenant_id, - ) - - # Remove duplicates from the role list - updated_actor["roles"] = list(set(update.roles)) - updated_actor["did"] = onboard_result.did - updated_actor["didcomm_invitation"] = onboard_result.didcomm_invitation - - await update_actor(updated_actor) - bound_logger.info("Tenant update handled successfully.") - - -async def onboard_tenant( - *, name: str, roles: List[TrustRegistryRole], tenant_auth_token: str, tenant_id: str -) -> OnboardResult: - bound_logger = logger.bind( - body={"name": name, "roles": roles, "tenant_id": tenant_id} - ) - bound_logger.bind(body=roles).info("Start onboarding tenant") - - if "issuer" in roles: - bound_logger.debug("Tenant has 'issuer' role, onboarding as issuer") - # Get governance and tenant controllers, onboard issuer - async with get_governance_controller() as governance_controller, get_tenant_controller( - tenant_auth_token - ) as tenant_controller: - onboard_result = await onboard_issuer( - name=name, - endorser_controller=governance_controller, - issuer_controller=tenant_controller, - issuer_wallet_id=tenant_id, - ) - bound_logger.info("Onboarding as issuer completed successfully.") - return onboard_result - - elif "verifier" in roles: - bound_logger.debug("Tenant has 'verifier' role, onboarding as verifier") - async with get_tenant_controller(tenant_auth_token) as tenant_controller: - onboard_result = await onboard_verifier( - name=name, verifier_controller=tenant_controller - ) - bound_logger.info("Onboarding as verifier completed successfully.") - return onboard_result - - bound_logger.error("Tenant request does not have valid role(s) for onboarding.") - raise CloudApiException("Unable to onboard tenant without role(s).") - - async def onboard_issuer( *, name: str = None, @@ -355,82 +248,3 @@ async def create_connection_with_endorser(endorser_did): bound_logger.info("Successfully registered DID for issuer.") return issuer_did - - -async def onboard_verifier(*, name: str, verifier_controller: AcaPyClient): - """Onboard the controller as verifier. - - The onboarding will take care of the following: - - create a multi_use invitation to use in the - - Args: - verifier_controller (AcaPyClient): authenticated ACA-Py client for verifier - """ - bound_logger = logger.bind(body={"name": name}) - bound_logger.info("Onboarding verifier") - - onboarding_result = {} - - # If the verifier already has a public did it doesn't need an invitation. The invitation - # is just to bypass having to pay for a public did for every verifier - try: - bound_logger.debug("Getting public DID for to-be verifier") - public_did = await acapy_wallet.get_public_did(controller=verifier_controller) - - onboarding_result["did"] = qualified_did_sov(public_did.did) - except CloudApiException: - bound_logger.info( - "No public DID found for to-be verifier. " - "Creating OOB invitation on their behalf." - ) - # create a multi_use invitation from the did - invitation: InvitationRecord = ( - await verifier_controller.out_of_band.create_invitation( - auto_accept=True, - multi_use=True, - body=InvitationCreateRequest( - use_public_did=False, - alias=f"Trust Registry {name}", - handshake_protocols=["https://didcomm.org/didexchange/1.0"], - ), - ) - ) - - # check if invitation and necessary attributes exist - if invitation and invitation.invitation and invitation.invitation.services: - try: - # Because we're not creating an invitation with a public did the invitation will always - # contain a did:key as the first recipientKey in the first service - bound_logger.debug("Getting DID from verifier's invitation") - service = invitation.invitation.services[0] - if ( - service - and "recipientKeys" in service - and len(service["recipientKeys"]) > 0 - ): - onboarding_result["did"] = service["recipientKeys"][0] - else: - raise KeyError( - f"RecipientKeys not present in the invitation service: `{service}`." - ) - onboarding_result["didcomm_invitation"] = invitation.invitation_url - except (KeyError, IndexError) as e: - bound_logger.error( - "Created invitation does not contain expected keys: {}", e - ) - raise CloudApiException( - "Error onboarding verifier: No public DID found. " - "Tried to create invitation, but found no service/recipientKeys." - ) from e - else: - bound_logger.error( - "Created invitation does not have necessary attributes. Got: `{}`.", - invitation, - ) - raise CloudApiException( - "Error onboarding verifier: No public DID found. " - "Tried and failed to create invitation on their behalf." - ) - - bound_logger.info("Returning verifier onboard result.") - return OnboardResult(**onboarding_result) diff --git a/app/services/onboarding/tenants.py b/app/services/onboarding/tenants.py new file mode 100644 index 000000000..e711314be --- /dev/null +++ b/app/services/onboarding/tenants.py @@ -0,0 +1,111 @@ +from typing import List + +from aries_cloudcontroller import AcaPyClient +from aries_cloudcontroller.model.create_wallet_token_request import ( + CreateWalletTokenRequest, +) +from fastapi.exceptions import HTTPException + +from app.dependencies.acapy_clients import ( + get_governance_controller, + get_tenant_controller, +) +from app.exceptions.cloud_api_error import CloudApiException +from app.models.tenants import OnboardResult, UpdateTenantRequest +from app.services.onboarding.issuer import onboard_issuer +from app.services.onboarding.verifier import onboard_verifier +from app.services.trust_registry import TrustRegistryRole, actor_by_id, update_actor +from shared.log_config import get_logger + +logger = get_logger(__name__) + + +async def handle_tenant_update( + admin_controller: AcaPyClient, + tenant_id: str, + update: UpdateTenantRequest, +): + bound_logger = logger.bind(body={"tenant_id": tenant_id}) + bound_logger.bind(body=update).info("Handling tenant update") + + bound_logger.debug("Retrieving the wallet") + wallet = await admin_controller.multitenancy.get_wallet(wallet_id=tenant_id) + if not wallet: + bound_logger.error("Bad request: Wallet not found.") + raise HTTPException(404, f"Wallet for tenant id `{tenant_id}` not found.") + + bound_logger.debug("Retrieving tenant from trust registry") + actor = await actor_by_id(wallet.wallet_id) + if not actor: + bound_logger.error( + "Tenant not found in trust registry. " + "Holder tenants cannot be updated with new roles." + ) + raise HTTPException(409, "Holder tenants cannot be updated with new roles.") + + updated_actor = actor.copy() + if update.name: + updated_actor["name"] = update.name + + if update.roles: + bound_logger.info("Updating tenant roles") + # We only care about the added roles, as that's what needs the setup. + # Teardown is not required at the moment, besides from removing it from + # the trust registry + added_roles = list(set(update.roles) - set(actor["roles"])) + + # We need to pose as the tenant to onboard for the specified role + token_response = await admin_controller.multitenancy.get_auth_token( + wallet_id=tenant_id, body=CreateWalletTokenRequest() + ) + + onboard_result = await onboard_tenant( + name=updated_actor["name"], + roles=added_roles, + tenant_auth_token=token_response.token, + tenant_id=tenant_id, + ) + + # Remove duplicates from the role list + updated_actor["roles"] = list(set(update.roles)) + updated_actor["did"] = onboard_result.did + updated_actor["didcomm_invitation"] = onboard_result.didcomm_invitation + + await update_actor(updated_actor) + bound_logger.info("Tenant update handled successfully.") + + +async def onboard_tenant( + *, name: str, roles: List[TrustRegistryRole], tenant_auth_token: str, tenant_id: str +) -> OnboardResult: + bound_logger = logger.bind( + body={"name": name, "roles": roles, "tenant_id": tenant_id} + ) + bound_logger.bind(body=roles).info("Start onboarding tenant") + + if "issuer" in roles: + bound_logger.debug("Tenant has 'issuer' role, onboarding as issuer") + # Get governance and tenant controllers, onboard issuer + async with get_governance_controller() as governance_controller, get_tenant_controller( + tenant_auth_token + ) as tenant_controller: + onboard_result = await onboard_issuer( + name=name, + endorser_controller=governance_controller, + issuer_controller=tenant_controller, + issuer_wallet_id=tenant_id, + ) + bound_logger.info("Onboarding as issuer completed successfully.") + return onboard_result + + elif "verifier" in roles: + bound_logger.debug("Tenant has 'verifier' role, onboarding as verifier") + async with get_tenant_controller(tenant_auth_token) as tenant_controller: + onboard_result = await onboard_verifier( + name=name, verifier_controller=tenant_controller + ) + bound_logger.info("Onboarding as verifier completed successfully.") + return onboard_result + + bound_logger.error("Tenant request does not have valid role(s) for onboarding.") + raise CloudApiException("Unable to onboard tenant without role(s).") diff --git a/app/services/onboarding/verifier.py b/app/services/onboarding/verifier.py new file mode 100644 index 000000000..75b79efc9 --- /dev/null +++ b/app/services/onboarding/verifier.py @@ -0,0 +1,88 @@ +from aries_cloudcontroller import AcaPyClient, InvitationCreateRequest, InvitationRecord + +from app.exceptions.cloud_api_error import CloudApiException +from app.models.tenants import OnboardResult +from app.services import acapy_wallet +from app.util.did import qualified_did_sov +from shared.log_config import get_logger + +logger = get_logger(__name__) + + +async def onboard_verifier(*, name: str, verifier_controller: AcaPyClient): + """Onboard the controller as verifier. + + The onboarding will take care of the following: + - create a multi_use invitation to use in the + + Args: + verifier_controller (AcaPyClient): authenticated ACA-Py client for verifier + """ + bound_logger = logger.bind(body={"name": name}) + bound_logger.info("Onboarding verifier") + + onboarding_result = {} + + # If the verifier already has a public did it doesn't need an invitation. The invitation + # is just to bypass having to pay for a public did for every verifier + try: + bound_logger.debug("Getting public DID for to-be verifier") + public_did = await acapy_wallet.get_public_did(controller=verifier_controller) + + onboarding_result["did"] = qualified_did_sov(public_did.did) + except CloudApiException: + bound_logger.info( + "No public DID found for to-be verifier. " + "Creating OOB invitation on their behalf." + ) + # create a multi_use invitation from the did + invitation: InvitationRecord = ( + await verifier_controller.out_of_band.create_invitation( + auto_accept=True, + multi_use=True, + body=InvitationCreateRequest( + use_public_did=False, + alias=f"Trust Registry {name}", + handshake_protocols=["https://didcomm.org/didexchange/1.0"], + ), + ) + ) + + # check if invitation and necessary attributes exist + if invitation and invitation.invitation and invitation.invitation.services: + try: + # Because we're not creating an invitation with a public did the invitation will always + # contain a did:key as the first recipientKey in the first service + bound_logger.debug("Getting DID from verifier's invitation") + service = invitation.invitation.services[0] + if ( + service + and "recipientKeys" in service + and len(service["recipientKeys"]) > 0 + ): + onboarding_result["did"] = service["recipientKeys"][0] + else: + raise KeyError( + f"RecipientKeys not present in the invitation service: `{service}`." + ) + onboarding_result["didcomm_invitation"] = invitation.invitation_url + except (KeyError, IndexError) as e: + bound_logger.error( + "Created invitation does not contain expected keys: {}", e + ) + raise CloudApiException( + "Error onboarding verifier: No public DID found. " + "Tried to create invitation, but found no service/recipientKeys." + ) from e + else: + bound_logger.error( + "Created invitation does not have necessary attributes. Got: `{}`.", + invitation, + ) + raise CloudApiException( + "Error onboarding verifier: No public DID found. " + "Tried and failed to create invitation on their behalf." + ) + + bound_logger.info("Returning verifier onboard result.") + return OnboardResult(**onboarding_result) diff --git a/app/tests/admin/test_onboarding.py b/app/tests/admin/test_onboarding.py index ad0bb0b55..e998571cd 100644 --- a/app/tests/admin/test_onboarding.py +++ b/app/tests/admin/test_onboarding.py @@ -13,9 +13,8 @@ from app.event_handling.sse_listener import SseListener from app.exceptions.cloud_api_error import CloudApiException -from app.services import onboarding +from app.services import acapy_ledger, acapy_wallet, onboarding from app.services.acapy_wallet import Did -from app.services.onboarding import acapy_ledger, acapy_wallet from app.tests.util.mock import to_async from shared.util.mock_agent_controller import get_mock_agent_controller From 66c1d9bdaa96ce960ebfcc7cfa6f3ff9b8340f98 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 18:06:16 +0300 Subject: [PATCH 04/18] :see_no_evil: add junit.xml to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c3198ddee..7c7bc7757 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ coverage.xml .hypothesis/ .pytest_cache/ cover/ +junit.xml # Translations *.mo From 8a9caffe3a4304c4ebb55850ab934f1f49022e93 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 20:39:59 +0300 Subject: [PATCH 05/18] :truck::art: move and rename util module. refactor with logging --- .../onboarding/util.py} | 52 ++++++++++++++----- 1 file changed, 38 insertions(+), 14 deletions(-) rename app/{util/assert_connection_metadata.py => services/onboarding/util.py} (69%) diff --git a/app/util/assert_connection_metadata.py b/app/services/onboarding/util.py similarity index 69% rename from app/util/assert_connection_metadata.py rename to app/services/onboarding/util.py index 237ce3420..05bcb75a3 100644 --- a/app/util/assert_connection_metadata.py +++ b/app/services/onboarding/util.py @@ -1,4 +1,5 @@ import asyncio +from logging import Logger from typing import Callable from aiohttp import ClientResponseError @@ -6,7 +7,7 @@ from app.exceptions.cloud_api_error import CloudApiException -DEFAULT_RETRIES = 10 +DEFAULT_NUM_TRIES = 10 DEFAULT_DELAY = 0.2 @@ -14,7 +15,8 @@ async def assert_metadata_set( controller: AcaPyClient, conn_id: str, check_fn: Callable, - retries=DEFAULT_RETRIES, + logger: Logger, + num_tries=DEFAULT_NUM_TRIES, delay=DEFAULT_DELAY, ): """Checks if connection record metadata has been set according to a custom check function. @@ -23,42 +25,51 @@ async def assert_metadata_set( controller: The AcaPyClient instance for the respective agent conn_id: Connection id of the connection you're interested in check_fn: A function that takes the metadata and returns True if it meets the desired condition - retries: Number of retries before failing + logger: A logger instance + num_tries: Number of num_tries before failing delay: Delay in seconds between each retry Returns: True if condition is met, raises an exception otherwise. """ - for _ in range(retries): + for _ in range(num_tries): # Delay is placed at the start to avoid race condition in ACA-Py, where reading metadata causes duplicate # record error if metadata is still due to be updated + logger.debug(f"Sleep {delay}s before trying to fetch metadata") await asyncio.sleep(delay) try: + logger.debug("Fetching connection metadata") connection_metadata = await controller.connection.get_metadata( conn_id=conn_id ) + logger.debug("Successfully fetched metadata") metadata_dict = connection_metadata.results if check_fn(metadata_dict): return True - except ClientResponseError: - # A duplicate record error (aries_cloudagent.storage.error.StorageDuplicateError) may occur in ACA-Py - # if we fetch metadata while it's being updated + except ClientResponseError as e: + logger.error("Exception occurred when getting metadata: {}", e) pass raise CloudApiException( - f"Failed to assert that metadata meets the desired condition after {retries} attempts." + f"Failed to assert that metadata meets the desired condition after {num_tries} attempts." ) async def assert_endorser_role_set( - controller, conn_id, retries=DEFAULT_RETRIES, delay=DEFAULT_DELAY + controller: AcaPyClient, + conn_id: str, + logger: Logger, + num_tries=1, + delay=DEFAULT_DELAY, ): check_fn = ( lambda metadata: metadata.get("transaction_jobs", {}).get("transaction_my_job") == "TRANSACTION_ENDORSER" ) try: - await assert_metadata_set(controller, conn_id, check_fn, retries, delay) + await assert_metadata_set( + controller, conn_id, check_fn, logger, num_tries, delay + ) except Exception as e: raise CloudApiException( "Failed to assert that the endorser role has been set in the connection metadata." @@ -66,7 +77,11 @@ async def assert_endorser_role_set( async def assert_author_role_set( - controller, conn_id, retries=DEFAULT_RETRIES, delay=DEFAULT_DELAY + controller: AcaPyClient, + conn_id: str, + logger: Logger, + num_tries=1, + delay=DEFAULT_DELAY, ): check_fn = ( lambda metadata: metadata.get("transaction_jobs", {}).get("transaction_my_job") @@ -75,7 +90,9 @@ async def assert_author_role_set( == "TRANSACTION_ENDORSER" ) try: - await assert_metadata_set(controller, conn_id, check_fn, retries, delay) + await assert_metadata_set( + controller, conn_id, check_fn, logger, num_tries, delay + ) except Exception as e: raise CloudApiException( "Failed to assert that the author role has been set in the connection metadata." @@ -83,7 +100,12 @@ async def assert_author_role_set( async def assert_endorser_info_set( - controller, conn_id, endorser_did, retries=DEFAULT_RETRIES, delay=DEFAULT_DELAY + controller: AcaPyClient, + conn_id: str, + endorser_did: str, + logger: Logger, + num_tries=1, + delay=DEFAULT_DELAY, ): check_fn = ( lambda metadata: metadata.get("transaction_jobs", {}).get("transaction_my_job") @@ -93,7 +115,9 @@ async def assert_endorser_info_set( and metadata.get("endorser_info", {}).get("endorser_did") == endorser_did ) try: - await assert_metadata_set(controller, conn_id, check_fn, retries, delay) + await assert_metadata_set( + controller, conn_id, check_fn, logger, num_tries, delay + ) except Exception as e: raise CloudApiException( "Failed to assert that the endorser info has been set in the connection metadata." From f03532264925a0f11f43c24aab3014419669e44a Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 20:40:44 +0300 Subject: [PATCH 06/18] :art: add custom exception --- app/services/onboarding/util.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/services/onboarding/util.py b/app/services/onboarding/util.py index 05bcb75a3..9c9f6afc7 100644 --- a/app/services/onboarding/util.py +++ b/app/services/onboarding/util.py @@ -50,7 +50,7 @@ async def assert_metadata_set( logger.error("Exception occurred when getting metadata: {}", e) pass - raise CloudApiException( + raise SettingMetadataException( f"Failed to assert that metadata meets the desired condition after {num_tries} attempts." ) @@ -71,7 +71,7 @@ async def assert_endorser_role_set( controller, conn_id, check_fn, logger, num_tries, delay ) except Exception as e: - raise CloudApiException( + raise SettingMetadataException( "Failed to assert that the endorser role has been set in the connection metadata." ) from e @@ -94,7 +94,7 @@ async def assert_author_role_set( controller, conn_id, check_fn, logger, num_tries, delay ) except Exception as e: - raise CloudApiException( + raise SettingMetadataException( "Failed to assert that the author role has been set in the connection metadata." ) from e @@ -119,6 +119,10 @@ async def assert_endorser_info_set( controller, conn_id, check_fn, logger, num_tries, delay ) except Exception as e: - raise CloudApiException( + raise SettingMetadataException( "Failed to assert that the endorser info has been set in the connection metadata." ) from e + + +class SettingMetadataException(CloudApiException): + pass From 568cc3e7faf061fe85a882e196d974d943de9040 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 20:41:53 +0300 Subject: [PATCH 07/18] :sparkles: define set metadata methods that assert success, with backing-off delayed retry if exception occurs --- app/services/onboarding/util.py | 136 ++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/app/services/onboarding/util.py b/app/services/onboarding/util.py index 9c9f6afc7..83012ad6e 100644 --- a/app/services/onboarding/util.py +++ b/app/services/onboarding/util.py @@ -11,6 +11,142 @@ DEFAULT_DELAY = 0.2 +async def set_endorser_role( + endorser_controller: AcaPyClient, endorser_connection_id: str, logger: Logger +): + delay = DEFAULT_DELAY + for n in range(DEFAULT_NUM_TRIES): + try: + logger.debug( + f"Setting roles for endorser on endorser-issuer connection. Try: {n}" + ) + await endorser_controller.endorse_transaction.set_endorser_role( + conn_id=endorser_connection_id, + transaction_my_job="TRANSACTION_ENDORSER", + ) + + # Try assert that it's done. Checking too soon may raise ACA-Py error (bug). + # So if it fails, retry with backing-off delay + logger.debug("Assert that the endorser role is set before continuing") + await assert_endorser_role_set( + endorser_controller, + endorser_connection_id, + logger, + num_tries=1, + delay=delay, + ) + logger.debug(f"Successfully set endorser role on try: {n}") + return True # success, exit retries + except (SettingMetadataException, ClientResponseError) as e: + if n == DEFAULT_NUM_TRIES: + logger.error( + "Failed to set endorser role after {} retries.", DEFAULT_NUM_TRIES + ) + raise CloudApiException( + "Failed to set the endorser role in the endorser-issuer connection, " + f"with connection id {endorser_connection_id}." + ) from e + logger.warning( + f"Setting endorser role has failed on try {n} with delay {delay}s" + ) + await asyncio.sleep(delay) # Secondary delay. Primary occurs in assert_ method + + delay *= 2 + logger.info(f"Retry setting of endorser role with increased delay: {delay}s") + + +async def set_author_role( + issuer_controller: AcaPyClient, issuer_connection_id: str, logger: Logger +): + delay = DEFAULT_DELAY + for n in range(DEFAULT_NUM_TRIES): + try: + logger.debug( + f"Setting roles for author on issuer-endorser connection. Try: {n}" + ) + await issuer_controller.endorse_transaction.set_endorser_role( + conn_id=issuer_connection_id, + transaction_my_job="TRANSACTION_AUTHOR", + ) + + # Try assert that it's done. Checking too soon may raise ACA-Py error (bug). + # So if it fails, retry with backing-off delay + logger.debug("Assert that the author role is set before continuing") + + await assert_author_role_set( + issuer_controller, + issuer_connection_id, + logger, + num_tries=1, + delay=delay, + ) + logger.debug(f"Successfully set author role on try: {n}") + return True # success, exit retries + except (SettingMetadataException, ClientResponseError) as e: + if n == DEFAULT_NUM_TRIES: + logger.error( + "Failed to set author role after {} retries.", DEFAULT_NUM_TRIES + ) + raise CloudApiException( + "Failed to set the author role in the issuer-endorser connection, " + f"with connection id {issuer_connection_id}." + ) from e + logger.warning(f"Setting author role has failed on try {n} with delay {delay}s") + await asyncio.sleep(delay) # Secondary delay. Primary occurs in assert_ method + + delay *= 2 + logger.info(f"Retry setting of author role with increased delay: {delay}s") + + +async def set_endorser_info( + issuer_controller: AcaPyClient, + issuer_connection_id: str, + endorser_did: str, + logger: Logger, +): + delay = DEFAULT_DELAY + for n in range(DEFAULT_NUM_TRIES): + try: + logger.debug( + f"Setting endorser info on issuer-endorser connection. Try: {n}" + ) + await issuer_controller.endorse_transaction.set_endorser_info( + conn_id=issuer_connection_id, + endorser_did=endorser_did, + ) + + # Try assert that it's done. Checking too soon may raise ACA-Py error (bug). + # So if it fails, retry with backing-off delay + logger.debug("Assert that the endorser info is set before continuing") + + await assert_endorser_info_set( + issuer_controller, + issuer_connection_id, + endorser_did, + logger, + num_tries=1, + delay=delay, + ) + logger.debug(f"Successfully set author role on try: {n}") + return True # success, exit retries + except (SettingMetadataException, ClientResponseError) as e: + if n == DEFAULT_NUM_TRIES: + logger.error( + "Failed to set endorser info after {} retries.", DEFAULT_NUM_TRIES + ) + raise CloudApiException( + "Failed to set the endorser info in the issuer-endorser connection, " + f"with connection id {issuer_connection_id}." + ) from e + logger.warning( + f"Setting endorser info has failed on try {n} with delay {delay}s" + ) + await asyncio.sleep(delay) # Secondary delay. Primary occurs in assert_ method + + delay *= 2 + logger.info(f"Retry setting of endorser info with increased delay: {delay}s") + + async def assert_metadata_set( controller: AcaPyClient, conn_id: str, From 51618a37d4518e00bb34d09e739ae986c84727a2 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 20:42:29 +0300 Subject: [PATCH 08/18] :art: move helper method --- app/event_handling/sse_listener.py | 5 ----- app/services/onboarding/issuer.py | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/event_handling/sse_listener.py b/app/event_handling/sse_listener.py index 048b38683..456e9bbe4 100644 --- a/app/event_handling/sse_listener.py +++ b/app/event_handling/sse_listener.py @@ -69,8 +69,3 @@ async def wait_for_event(self, field, field_id, desired_state, timeout: int = 12 class SseListenerTimeout(Exception): """Exception raised when the Listener times out waiting for a matching event.""" - - -def create_sse_listener(wallet_id: str, topic: str) -> SseListener: - # Helper method for passing a MockListener to a class - return SseListener(topic=topic, wallet_id=wallet_id) diff --git a/app/services/onboarding/issuer.py b/app/services/onboarding/issuer.py index cf663daf3..b3ac7dd2b 100644 --- a/app/services/onboarding/issuer.py +++ b/app/services/onboarding/issuer.py @@ -248,3 +248,8 @@ async def create_connection_with_endorser(endorser_did): bound_logger.info("Successfully registered DID for issuer.") return issuer_did + + +def create_sse_listener(wallet_id: str, topic: str) -> SseListener: + # Helper method for passing a MockListener to a class + return SseListener(topic=topic, wallet_id=wallet_id) From c64951196cfed5c7e2d09fc149b0722d647cd0a5 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 20:43:05 +0300 Subject: [PATCH 09/18] :white_check_mark: fix test after modules renamed --- app/tests/admin/test_onboarding.py | 31 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/app/tests/admin/test_onboarding.py b/app/tests/admin/test_onboarding.py index e998571cd..eedcb670e 100644 --- a/app/tests/admin/test_onboarding.py +++ b/app/tests/admin/test_onboarding.py @@ -13,8 +13,9 @@ from app.event_handling.sse_listener import SseListener from app.exceptions.cloud_api_error import CloudApiException -from app.services import acapy_ledger, acapy_wallet, onboarding +from app.services import acapy_ledger, acapy_wallet from app.services.acapy_wallet import Did +from app.services.onboarding import issuer, verifier from app.tests.util.mock import to_async from shared.util.mock_agent_controller import get_mock_agent_controller @@ -56,10 +57,10 @@ async def test_onboard_issuer_public_did_exists( ) # Mock event listeners - when(onboarding).create_sse_listener( - topic="connections", wallet_id="admin" - ).thenReturn(MockSseListener(topic="connections", wallet_id="admin")) - when(onboarding).create_sse_listener( + when(issuer).create_sse_listener(topic="connections", wallet_id="admin").thenReturn( + MockSseListener(topic="connections", wallet_id="admin") + ) + when(issuer).create_sse_listener( topic="endorsements", wallet_id="admin" ).thenReturn( MockListenerEndorserConnectionId(topic="endorsements", wallet_id="admin") @@ -75,7 +76,7 @@ async def test_onboard_issuer_public_did_exists( ) ) - onboard_result = await onboarding.onboard_issuer( + onboard_result = await issuer.onboard_issuer( name="issuer_name", endorser_controller=endorser_controller, issuer_controller=mock_agent_controller, @@ -104,12 +105,10 @@ async def test_onboard_issuer_no_public_did( ) # Mock event listeners - when(onboarding).create_sse_listener( - topic="connections", wallet_id="admin" - ).thenReturn( + when(issuer).create_sse_listener(topic="connections", wallet_id="admin").thenReturn( MockListenerEndorserConnectionId(topic="connections", wallet_id="admin") ) - when(onboarding).create_sse_listener( + when(issuer).create_sse_listener( topic="endorsements", wallet_id="admin" ).thenReturn(MockListenerRequestReceived(topic="endorsements", wallet_id="admin")) @@ -136,8 +135,8 @@ async def test_onboard_issuer_no_public_did( when(endorser_controller.endorse_transaction).set_endorser_role(...).thenReturn( to_async() ) - when(mock_agent_controller.endorse_transaction).set_endorser_info(...).thenReturn( - to_async() + when(mock_agent_controller.endorse_transaction).set_endorser_info(...).thenAnswer( + lambda conn_id, endorser_did: to_async() ) # Expanding the test scenario: we want to ensure that the coroutine is successfully retried in the @@ -211,7 +210,7 @@ def __init__(self): ) ) - onboard_result = await onboarding.onboard_issuer( + onboard_result = await issuer.onboard_issuer( name="issuer_name", endorser_controller=endorser_controller, issuer_controller=mock_agent_controller, @@ -245,7 +244,7 @@ async def test_onboard_verifier_public_did_exists(mock_agent_controller: AcaPyCl ) ) - onboard_result = await onboarding.onboard_verifier( + onboard_result = await verifier.onboard_verifier( name="verifier_name", verifier_controller=mock_agent_controller ) @@ -271,7 +270,7 @@ async def test_onboard_verifier_no_public_did(mock_agent_controller: AcaPyClient ) ) - onboard_result = await onboarding.onboard_verifier( + onboard_result = await verifier.onboard_verifier( name="verifier_name", verifier_controller=mock_agent_controller ) @@ -302,7 +301,7 @@ async def test_onboard_verifier_no_recipient_keys(mock_agent_controller: AcaPyCl ) with pytest.raises(CloudApiException): - await onboarding.onboard_verifier( + await verifier.onboard_verifier( name="verifier_name", verifier_controller=mock_agent_controller ) From 89a9fe8fac64cf6765ee080c185b43bbde5e6f5d Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 20:49:58 +0300 Subject: [PATCH 10/18] :art: add method signature types --- app/event_handling/sse_listener.py | 7 ++++-- app/services/onboarding/issuer.py | 39 +++++++++++++----------------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/app/event_handling/sse_listener.py b/app/event_handling/sse_listener.py index 456e9bbe4..72513727a 100644 --- a/app/event_handling/sse_listener.py +++ b/app/event_handling/sse_listener.py @@ -1,4 +1,5 @@ import json +from typing import Any, Dict import httpx @@ -26,7 +27,7 @@ def __init__( self.wallet_id = wallet_id self.topic = topic - async def wait_for_state(self, desired_state, timeout: int = 120): + async def wait_for_state(self, desired_state, timeout: int = 120) -> Dict[str, Any]: """ Start listening for SSE events. When an event is received that matches the specified parameters. """ @@ -46,7 +47,9 @@ async def wait_for_state(self, desired_state, timeout: int = 120): raise SseListenerTimeout("Event with request state was not returned by server.") - async def wait_for_event(self, field, field_id, desired_state, timeout: int = 120): + async def wait_for_event( + self, field, field_id, desired_state, timeout: int = 120 + ) -> Dict[str, Any]: """ Start listening for SSE events. When an event is received that matches the specified parameters. """ diff --git a/app/services/onboarding/issuer.py b/app/services/onboarding/issuer.py index b3ac7dd2b..f53c7a1de 100644 --- a/app/services/onboarding/issuer.py +++ b/app/services/onboarding/issuer.py @@ -1,9 +1,15 @@ -from aries_cloudcontroller import AcaPyClient, InvitationCreateRequest +from aries_cloudcontroller import ( + AcaPyClient, + InvitationCreateRequest, + InvitationRecord, + OobRecord, +) from app.event_handling.sse_listener import create_sse_listener from app.exceptions.cloud_api_error import CloudApiException from app.models.tenants import OnboardResult from app.services import acapy_ledger, acapy_wallet +from app.services.acapy_wallet import Did from app.util.assert_connection_metadata import ( assert_author_role_set, assert_endorser_info_set, @@ -105,7 +111,7 @@ async def create_endorser_invitation(): bound_logger.debug("Created OOB invitation") return invitation - async def wait_for_connection_completion(invitation): + async def wait_for_connection_completion(invitation: InvitationRecord): connections_listener = create_sse_listener( topic="connections", wallet_id="admin" ) @@ -135,22 +141,9 @@ async def wait_for_connection_completion(invitation): bound_logger.info("Connection complete between issuer and endorser.") return endorser_connection, connection_record - async def set_endorser_roles(endorser_connection, connection_record): - endorser_connection_id = endorser_connection["connection_id"] - issuer_connection_id = connection_record.connection_id - - bound_logger.debug("Setting roles for endorser") - await endorser_controller.endorse_transaction.set_endorser_role( - conn_id=endorser_connection_id, - transaction_my_job="TRANSACTION_ENDORSER", - ) - - bound_logger.debug("Assert that the endorser role is set") - await assert_endorser_role_set(endorser_controller, endorser_connection_id) - - await issuer_controller.endorse_transaction.set_endorser_role( - conn_id=issuer_connection_id, - transaction_my_job="TRANSACTION_AUTHOR", + async def set_endorser_roles( + endorser_connection_id: str, issuer_connection_id: str + ): ) bound_logger.debug("Assert that the author role is set") @@ -158,7 +151,7 @@ async def set_endorser_roles(endorser_connection, connection_record): bound_logger.debug("Successfully set roles for connection.") - async def configure_endorsement(connection_record, endorser_did): + async def configure_endorsement(connection_record: OobRecord, endorser_did: str): # Make sure endorsement has been configured # There is currently no way to retrieve endorser info. We'll just set it # to make sure the endorser info is set. @@ -221,13 +214,15 @@ async def register_issuer_did(): bound_logger.debug("Issuer DID registered and endorsed successfully.") return issuer_did - async def create_connection_with_endorser(endorser_did): + async def create_connection_with_endorser(endorser_did: Did): invitation = await create_endorser_invitation() endorser_connection, connection_record = await wait_for_connection_completion( invitation ) - await set_endorser_roles(endorser_connection, connection_record) - await configure_endorsement(connection_record, endorser_did) + await set_endorser_roles( + endorser_connection["connection_id"], connection_record.connection_id + ) + await configure_endorsement(connection_record, endorser_did.did) try: logger.debug("Getting public DID for endorser") From 64c213322f1b4d8248adc0a4e070be7f5bc6a678 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 20:51:39 +0300 Subject: [PATCH 11/18] :art: use new set and assert methods from util module --- app/services/onboarding/issuer.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/app/services/onboarding/issuer.py b/app/services/onboarding/issuer.py index f53c7a1de..ff91ee99a 100644 --- a/app/services/onboarding/issuer.py +++ b/app/services/onboarding/issuer.py @@ -5,15 +5,15 @@ OobRecord, ) -from app.event_handling.sse_listener import create_sse_listener +from app.event_handling.sse_listener import SseListener from app.exceptions.cloud_api_error import CloudApiException from app.models.tenants import OnboardResult from app.services import acapy_ledger, acapy_wallet from app.services.acapy_wallet import Did -from app.util.assert_connection_metadata import ( - assert_author_role_set, - assert_endorser_info_set, - assert_endorser_role_set, +from app.services.onboarding.util import ( + set_author_role, + set_endorser_info, + set_endorser_role, ) from app.util.did import qualified_did_sov from shared import ACAPY_ENDORSER_ALIAS @@ -144,10 +144,13 @@ async def wait_for_connection_completion(invitation: InvitationRecord): async def set_endorser_roles( endorser_connection_id: str, issuer_connection_id: str ): + bound_logger.debug("Setting roles for endorser") + await set_endorser_role( + endorser_controller, endorser_connection_id, bound_logger ) - bound_logger.debug("Assert that the author role is set") - await assert_author_role_set(issuer_controller, issuer_connection_id) + bound_logger.debug("Setting roles for author") + await set_author_role(issuer_controller, issuer_connection_id, bound_logger) bound_logger.debug("Successfully set roles for connection.") @@ -156,14 +159,11 @@ async def configure_endorsement(connection_record: OobRecord, endorser_did: str) # There is currently no way to retrieve endorser info. We'll just set it # to make sure the endorser info is set. bound_logger.debug("Setting endorser info") - await issuer_controller.endorse_transaction.set_endorser_info( - conn_id=connection_record.connection_id, - endorser_did=endorser_did.did, - ) - - bound_logger.debug("Assert that the endorser info is set") - await assert_endorser_info_set( - issuer_controller, connection_record.connection_id, endorser_did.did + await set_endorser_info( + issuer_controller, + connection_record.connection_id, + endorser_did, + bound_logger, ) bound_logger.debug("Successfully set endorser info.") From 39200345901573f15f3e0966b38a9f94b0402312 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 21:26:32 +0300 Subject: [PATCH 12/18] :art: --- app/services/onboarding/issuer.py | 2 +- app/services/onboarding/util/__init__.py | 0 .../onboarding/{util.py => util/set_endorser_metadata.py} | 1 - app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py | 2 +- app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py | 4 ++-- 5 files changed, 4 insertions(+), 5 deletions(-) create mode 100644 app/services/onboarding/util/__init__.py rename app/services/onboarding/{util.py => util/set_endorser_metadata.py} (99%) diff --git a/app/services/onboarding/issuer.py b/app/services/onboarding/issuer.py index ff91ee99a..1a9fda91d 100644 --- a/app/services/onboarding/issuer.py +++ b/app/services/onboarding/issuer.py @@ -246,5 +246,5 @@ async def create_connection_with_endorser(endorser_did: Did): def create_sse_listener(wallet_id: str, topic: str) -> SseListener: - # Helper method for passing a MockListener to a class + # Helper method for passing a MockListener to this module in tests return SseListener(topic=topic, wallet_id=wallet_id) diff --git a/app/services/onboarding/util/__init__.py b/app/services/onboarding/util/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/app/services/onboarding/util.py b/app/services/onboarding/util/set_endorser_metadata.py similarity index 99% rename from app/services/onboarding/util.py rename to app/services/onboarding/util/set_endorser_metadata.py index 83012ad6e..8aefee775 100644 --- a/app/services/onboarding/util.py +++ b/app/services/onboarding/util/set_endorser_metadata.py @@ -184,7 +184,6 @@ async def assert_metadata_set( return True except ClientResponseError as e: logger.error("Exception occurred when getting metadata: {}", e) - pass raise SettingMetadataException( f"Failed to assert that metadata meets the desired condition after {num_tries} attempts." diff --git a/app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py b/app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py index 268902678..5ae311a5b 100644 --- a/app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py +++ b/app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py @@ -15,7 +15,7 @@ CREDENTIALS_BASE_PATH = issuer_router.prefix OOB_BASE_PATH = oob_router.prefix -CON = con_router.prefix +CONNECTIONS_BASE_PATH = con_router.prefix credential_ = SendCredential( diff --git a/app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py b/app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py index 8f6a89e1d..92d6168ca 100644 --- a/app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py +++ b/app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py @@ -16,8 +16,8 @@ CREDENTIALS_BASE_PATH = issuer_router.prefix OOB_BASE_PATH = oob_router.prefix -WALLET = wallet_router.prefix -CON = con_router.prefix +WALLET_BASE_PATH = wallet_router.prefix +CONNECTIONS_BASE_PATH = con_router.prefix credential_ = SendCredential( type="ld_proof", From c405c21f693f7404f7cefcdbaf27cd77664ce96d Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 21:29:03 +0300 Subject: [PATCH 13/18] :truck: move endorser configuration and registering issuer did methods to own util module --- app/services/onboarding/issuer.py | 162 ++-------------- .../onboarding/util/register_issuer_did.py | 173 ++++++++++++++++++ app/tests/admin/test_onboarding.py | 15 +- 3 files changed, 195 insertions(+), 155 deletions(-) create mode 100644 app/services/onboarding/util/register_issuer_did.py diff --git a/app/services/onboarding/issuer.py b/app/services/onboarding/issuer.py index 1a9fda91d..103305c4a 100644 --- a/app/services/onboarding/issuer.py +++ b/app/services/onboarding/issuer.py @@ -1,22 +1,13 @@ -from aries_cloudcontroller import ( - AcaPyClient, - InvitationCreateRequest, - InvitationRecord, - OobRecord, -) +from aries_cloudcontroller import AcaPyClient, InvitationCreateRequest -from app.event_handling.sse_listener import SseListener from app.exceptions.cloud_api_error import CloudApiException from app.models.tenants import OnboardResult -from app.services import acapy_ledger, acapy_wallet -from app.services.acapy_wallet import Did -from app.services.onboarding.util import ( - set_author_role, - set_endorser_info, - set_endorser_role, +from app.services import acapy_wallet +from app.services.onboarding.util.register_issuer_did import ( + create_connection_with_endorser, + register_issuer_did, ) from app.util.did import qualified_did_sov -from shared import ACAPY_ENDORSER_ALIAS from shared.log_config import get_logger logger = get_logger(__name__) @@ -98,143 +89,21 @@ async def onboard_issuer_no_public_did( bound_logger = logger.bind(body={"issuer_wallet_id": issuer_wallet_id}) bound_logger.info("Onboarding issuer that has no public DID") - async def create_endorser_invitation(): - bound_logger.debug("Create OOB invitation on behalf of endorser") - invitation = await endorser_controller.out_of_band.create_invitation( - auto_accept=True, - body=InvitationCreateRequest( - alias=name, - handshake_protocols=["https://didcomm.org/didexchange/1.0"], - use_public_did=True, - ), - ) - bound_logger.debug("Created OOB invitation") - return invitation - - async def wait_for_connection_completion(invitation: InvitationRecord): - connections_listener = create_sse_listener( - topic="connections", wallet_id="admin" - ) - - bound_logger.debug("Receive invitation from endorser on behalf of issuer") - connection_record = await issuer_controller.out_of_band.receive_invitation( - auto_accept=True, - use_existing_connection=False, - body=invitation.invitation, - alias=ACAPY_ENDORSER_ALIAS, - ) - - try: - bound_logger.debug("Waiting for event signalling invitation complete") - endorser_connection = await connections_listener.wait_for_event( - field="invitation_msg_id", - field_id=invitation.invi_msg_id, - desired_state="completed", - ) - except TimeoutError as e: - bound_logger.error("Waiting for invitation complete event has timed out.") - raise CloudApiException( - "Timeout occurred while waiting for connection with endorser to complete.", - 504, - ) from e - - bound_logger.info("Connection complete between issuer and endorser.") - return endorser_connection, connection_record - - async def set_endorser_roles( - endorser_connection_id: str, issuer_connection_id: str - ): - bound_logger.debug("Setting roles for endorser") - await set_endorser_role( - endorser_controller, endorser_connection_id, bound_logger - ) - - bound_logger.debug("Setting roles for author") - await set_author_role(issuer_controller, issuer_connection_id, bound_logger) - - bound_logger.debug("Successfully set roles for connection.") - - async def configure_endorsement(connection_record: OobRecord, endorser_did: str): - # Make sure endorsement has been configured - # There is currently no way to retrieve endorser info. We'll just set it - # to make sure the endorser info is set. - bound_logger.debug("Setting endorser info") - await set_endorser_info( - issuer_controller, - connection_record.connection_id, - endorser_did, - bound_logger, - ) - bound_logger.debug("Successfully set endorser info.") - - async def register_issuer_did(): - bound_logger.info("Creating DID for issuer") - issuer_did = await acapy_wallet.create_did(issuer_controller) - - await acapy_ledger.register_nym_on_ledger( - endorser_controller, - did=issuer_did.did, - verkey=issuer_did.verkey, - alias=name, - ) - - bound_logger.debug("Accepting TAA on behalf of issuer") - await acapy_ledger.accept_taa_if_required(issuer_controller) - # NOTE: Still needs endorsement in 0.7.5 release - # Otherwise did has no associated services. - bound_logger.debug("Setting public DID for issuer") - await acapy_wallet.set_public_did( - issuer_controller, - did=issuer_did.did, - create_transaction_for_endorser=True, - ) - - endorsements_listener = create_sse_listener( - topic="endorsements", wallet_id="admin" - ) - - try: - bound_logger.debug("Waiting for endorsement request received") - txn_record = await endorsements_listener.wait_for_state( - desired_state="request-received" - ) - except TimeoutError as e: - bound_logger.error("Waiting for endorsement request has timed out.") - raise CloudApiException( - "Timeout occurred while waiting for endorsement request.", 504 - ) from e - - bound_logger.bind(body=txn_record["transaction_id"]).debug( - "Endorsing transaction" - ) - await endorser_controller.endorse_transaction.endorse_transaction( - tran_id=txn_record["transaction_id"] - ) - - bound_logger.debug("Issuer DID registered and endorsed successfully.") - return issuer_did - - async def create_connection_with_endorser(endorser_did: Did): - invitation = await create_endorser_invitation() - endorser_connection, connection_record = await wait_for_connection_completion( - invitation - ) - await set_endorser_roles( - endorser_connection["connection_id"], connection_record.connection_id - ) - await configure_endorsement(connection_record, endorser_did.did) - try: - logger.debug("Getting public DID for endorser") + bound_logger.debug("Getting public DID for endorser") endorser_did = await acapy_wallet.get_public_did(controller=endorser_controller) except Exception as e: - logger.critical("Endorser has no public DID.") + bound_logger.critical("Endorser has no public DID.") raise CloudApiException("Unable to get endorser public DID.") from e try: bound_logger.info("Creating connection with endorser") - await create_connection_with_endorser(endorser_did) - issuer_did = await register_issuer_did() + await create_connection_with_endorser( + endorser_controller, issuer_controller, endorser_did, name, bound_logger + ) + issuer_did = await register_issuer_did( + endorser_controller, issuer_controller, name, bound_logger + ) except Exception as e: bound_logger.exception("Could not create connection with endorser.") raise CloudApiException( @@ -243,8 +112,3 @@ async def create_connection_with_endorser(endorser_did: Did): bound_logger.info("Successfully registered DID for issuer.") return issuer_did - - -def create_sse_listener(wallet_id: str, topic: str) -> SseListener: - # Helper method for passing a MockListener to this module in tests - return SseListener(topic=topic, wallet_id=wallet_id) diff --git a/app/services/onboarding/util/register_issuer_did.py b/app/services/onboarding/util/register_issuer_did.py new file mode 100644 index 000000000..9ff4b5f28 --- /dev/null +++ b/app/services/onboarding/util/register_issuer_did.py @@ -0,0 +1,173 @@ +from logging import Logger + +from aries_cloudcontroller import AcaPyClient, InvitationCreateRequest, InvitationRecord + +from app.event_handling.sse_listener import SseListener +from app.exceptions.cloud_api_error import CloudApiException +from app.services import acapy_ledger, acapy_wallet +from app.services.acapy_wallet import Did +from app.services.onboarding.util.set_endorser_metadata import ( + set_author_role, + set_endorser_info, + set_endorser_role, +) +from shared import ACAPY_ENDORSER_ALIAS + + +async def create_connection_with_endorser( + endorser_controller: AcaPyClient, + issuer_controller: AcaPyClient, + endorser_did: Did, + name: str, + logger: Logger, +): + invitation = await create_endorser_invitation(endorser_controller, name, logger) + endorser_connection_id, issuer_connection_id = await wait_for_connection_completion( + issuer_controller, invitation, logger + ) + await set_endorser_roles( + endorser_controller, + issuer_controller, + endorser_connection_id, + issuer_connection_id, + logger, + ) + await configure_endorsement( + issuer_controller, issuer_connection_id, endorser_did.did, logger + ) + + +async def create_endorser_invitation( + endorser_controller: AcaPyClient, name: str, logger: Logger +): + logger.debug("Create OOB invitation on behalf of endorser") + invitation = await endorser_controller.out_of_band.create_invitation( + auto_accept=True, + body=InvitationCreateRequest( + alias=name, + handshake_protocols=["https://didcomm.org/didexchange/1.0"], + use_public_did=True, + ), + ) + logger.debug("Created OOB invitation") + return invitation + + +async def wait_for_connection_completion( + issuer_controller: AcaPyClient, invitation: InvitationRecord, logger: Logger +) -> tuple[str, str]: + connections_listener = create_sse_listener(topic="connections", wallet_id="admin") + + logger.debug("Receive invitation from endorser on behalf of issuer") + connection_record = await issuer_controller.out_of_band.receive_invitation( + auto_accept=True, + use_existing_connection=False, + body=invitation.invitation, + alias=ACAPY_ENDORSER_ALIAS, + ) + + try: + logger.debug("Waiting for event signalling invitation complete") + endorser_connection = await connections_listener.wait_for_event( + field="invitation_msg_id", + field_id=invitation.invi_msg_id, + desired_state="completed", + ) + except TimeoutError as e: + logger.error("Waiting for invitation complete event has timed out.") + raise CloudApiException( + "Timeout occurred while waiting for connection with endorser to complete.", + 504, + ) from e + + logger.info("Connection complete between issuer and endorser.") + return endorser_connection["connection_id"], connection_record.connection_id + + +async def set_endorser_roles( + endorser_controller: AcaPyClient, + issuer_controller: AcaPyClient, + endorser_connection_id: str, + issuer_connection_id: str, + logger: Logger, +): + logger.debug("Setting roles for endorser") + await set_endorser_role(endorser_controller, endorser_connection_id, logger) + + logger.debug("Setting roles for author") + await set_author_role(issuer_controller, issuer_connection_id, logger) + + logger.debug("Successfully set roles for connection.") + + +async def configure_endorsement( + issuer_controller: AcaPyClient, + issuer_connection_id: str, + endorser_did: str, + logger: Logger, +): + # Make sure endorsement has been configured + # There is currently no way to retrieve endorser info. We'll just set it + # to make sure the endorser info is set. + logger.debug("Setting endorser info") + await set_endorser_info( + issuer_controller, + issuer_connection_id, + endorser_did, + logger, + ) + logger.debug("Successfully set endorser info.") + + +async def register_issuer_did( + endorser_controller: AcaPyClient, + issuer_controller: AcaPyClient, + name: str, + logger: Logger, +): + logger.info("Creating DID for issuer") + issuer_did = await acapy_wallet.create_did(issuer_controller) + + await acapy_ledger.register_nym_on_ledger( + endorser_controller, + did=issuer_did.did, + verkey=issuer_did.verkey, + alias=name, + ) + + logger.debug("Accepting TAA on behalf of issuer") + await acapy_ledger.accept_taa_if_required(issuer_controller) + # NOTE: Still needs endorsement in 0.7.5 release + # Otherwise did has no associated services. + logger.debug("Setting public DID for issuer") + await acapy_wallet.set_public_did( + issuer_controller, + did=issuer_did.did, + create_transaction_for_endorser=True, + ) + + endorsements_listener = create_sse_listener(topic="endorsements", wallet_id="admin") + + try: + logger.debug("Waiting for endorsement request received") + txn_record = await endorsements_listener.wait_for_state( + desired_state="request-received" + ) + except TimeoutError as e: + logger.error("Waiting for endorsement request has timed out.") + raise CloudApiException( + "Timeout occurred while waiting for endorsement request.", 504 + ) from e + + logger.bind(body=txn_record["transaction_id"]).debug("Endorsing transaction") + await endorser_controller.endorse_transaction.endorse_transaction( + tran_id=txn_record["transaction_id"] + ) + + logger.debug("Issuer DID registered and endorsed successfully.") + return issuer_did + + +def create_sse_listener(wallet_id: str, topic: str) -> SseListener: + # Helper method for passing a MockListener to this module in tests + return SseListener(topic=topic, wallet_id=wallet_id) diff --git a/app/tests/admin/test_onboarding.py b/app/tests/admin/test_onboarding.py index eedcb670e..30650dc07 100644 --- a/app/tests/admin/test_onboarding.py +++ b/app/tests/admin/test_onboarding.py @@ -16,6 +16,7 @@ from app.services import acapy_ledger, acapy_wallet from app.services.acapy_wallet import Did from app.services.onboarding import issuer, verifier +from app.services.onboarding.util import register_issuer_did from app.tests.util.mock import to_async from shared.util.mock_agent_controller import get_mock_agent_controller @@ -57,10 +58,10 @@ async def test_onboard_issuer_public_did_exists( ) # Mock event listeners - when(issuer).create_sse_listener(topic="connections", wallet_id="admin").thenReturn( - MockSseListener(topic="connections", wallet_id="admin") - ) - when(issuer).create_sse_listener( + when(register_issuer_did).create_sse_listener( + topic="connections", wallet_id="admin" + ).thenReturn(MockSseListener(topic="connections", wallet_id="admin")) + when(register_issuer_did).create_sse_listener( topic="endorsements", wallet_id="admin" ).thenReturn( MockListenerEndorserConnectionId(topic="endorsements", wallet_id="admin") @@ -105,10 +106,12 @@ async def test_onboard_issuer_no_public_did( ) # Mock event listeners - when(issuer).create_sse_listener(topic="connections", wallet_id="admin").thenReturn( + when(register_issuer_did).create_sse_listener( + topic="connections", wallet_id="admin" + ).thenReturn( MockListenerEndorserConnectionId(topic="connections", wallet_id="admin") ) - when(issuer).create_sse_listener( + when(register_issuer_did).create_sse_listener( topic="endorsements", wallet_id="admin" ).thenReturn(MockListenerRequestReceived(topic="endorsements", wallet_id="admin")) From a83e44781d51d0a54691c5f5e66fbcf4d1867ad2 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 21:42:58 +0300 Subject: [PATCH 14/18] :art: set DEFAULT_NUM_TRIES to 5 with 0.2 seconds, this will try with second delays as follows: 0.2 -> 0.4 -> 0.8 -> 1.6 -> 3.2 so, we can be as fast as 0.2 seconds, or as slow as ~6 seconds. good compromise to workaround the aca-py bug --- app/services/onboarding/util/set_endorser_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/onboarding/util/set_endorser_metadata.py b/app/services/onboarding/util/set_endorser_metadata.py index 8aefee775..8f1958e29 100644 --- a/app/services/onboarding/util/set_endorser_metadata.py +++ b/app/services/onboarding/util/set_endorser_metadata.py @@ -7,7 +7,7 @@ from app.exceptions.cloud_api_error import CloudApiException -DEFAULT_NUM_TRIES = 10 +DEFAULT_NUM_TRIES = 5 DEFAULT_DELAY = 0.2 From 411668ab995d75ab81cec97dad6f503f715021d3 Mon Sep 17 00:00:00 2001 From: ff137 Date: Thu, 5 Oct 2023 21:55:36 +0300 Subject: [PATCH 15/18] :art: --- app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py | 2 +- app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py b/app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py index 5ae311a5b..421987967 100644 --- a/app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py +++ b/app/tests/e2e/issuer/did_key_bbs/test_ld_bbs.py @@ -162,7 +162,7 @@ async def test_send_jsonld_bbs_oob( assert_that(accept_response.status_code).is_equal_to(200) assert_that(oob_record).contains("created_at", "oob_id", "invitation") - faber_con = await faber_client.get(CON) + faber_con = await faber_client.get(CONNECTIONS_BASE_PATH) faber_connections = faber_con.json() for con in faber_connections: diff --git a/app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py b/app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py index 92d6168ca..615b179ec 100644 --- a/app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py +++ b/app/tests/e2e/issuer/did_key_ed/test_ld_ed25519.py @@ -8,7 +8,6 @@ from app.routes.connections import router as con_router from app.routes.issuer import router as issuer_router from app.routes.oob import router as oob_router -from app.routes.wallet import router as wallet_router from app.tests.util.ecosystem_connections import FaberAliceConnect from app.tests.util.trust_registry import DidKey from app.tests.util.webhooks import check_webhook_state @@ -16,7 +15,6 @@ CREDENTIALS_BASE_PATH = issuer_router.prefix OOB_BASE_PATH = oob_router.prefix -WALLET_BASE_PATH = wallet_router.prefix CONNECTIONS_BASE_PATH = con_router.prefix credential_ = SendCredential( @@ -165,7 +163,7 @@ async def test_send_jsonld_oob( assert_that(accept_response.status_code).is_equal_to(200) assert_that(oob_record).contains("created_at", "oob_id", "invitation") - faber_con = await faber_client.get(CON) + faber_con = await faber_client.get(CONNECTIONS_BASE_PATH) faber_connections = faber_con.json() for con in faber_connections: From 0c35ddcde304a7e1fd07d2faaaa4b8f79624407c Mon Sep 17 00:00:00 2001 From: ff137 Date: Fri, 6 Oct 2023 12:21:25 +0300 Subject: [PATCH 16/18] :bug::art: remove usage of retry + assert logic and opt for simpler solution: 1 try with longer delays. Mention "known bug" in exception messages. --- .../onboarding/util/set_endorser_metadata.py | 162 +++++------------- 1 file changed, 46 insertions(+), 116 deletions(-) diff --git a/app/services/onboarding/util/set_endorser_metadata.py b/app/services/onboarding/util/set_endorser_metadata.py index 8f1958e29..8187eee5b 100644 --- a/app/services/onboarding/util/set_endorser_metadata.py +++ b/app/services/onboarding/util/set_endorser_metadata.py @@ -7,95 +7,48 @@ from app.exceptions.cloud_api_error import CloudApiException -DEFAULT_NUM_TRIES = 5 +DEFAULT_NUM_TRIES = 1 DEFAULT_DELAY = 0.2 async def set_endorser_role( endorser_controller: AcaPyClient, endorser_connection_id: str, logger: Logger ): - delay = DEFAULT_DELAY - for n in range(DEFAULT_NUM_TRIES): - try: - logger.debug( - f"Setting roles for endorser on endorser-issuer connection. Try: {n}" - ) - await endorser_controller.endorse_transaction.set_endorser_role( - conn_id=endorser_connection_id, - transaction_my_job="TRANSACTION_ENDORSER", - ) - - # Try assert that it's done. Checking too soon may raise ACA-Py error (bug). - # So if it fails, retry with backing-off delay - logger.debug("Assert that the endorser role is set before continuing") - await assert_endorser_role_set( - endorser_controller, - endorser_connection_id, - logger, - num_tries=1, - delay=delay, - ) - logger.debug(f"Successfully set endorser role on try: {n}") - return True # success, exit retries - except (SettingMetadataException, ClientResponseError) as e: - if n == DEFAULT_NUM_TRIES: - logger.error( - "Failed to set endorser role after {} retries.", DEFAULT_NUM_TRIES - ) - raise CloudApiException( - "Failed to set the endorser role in the endorser-issuer connection, " - f"with connection id {endorser_connection_id}." - ) from e - logger.warning( - f"Setting endorser role has failed on try {n} with delay {delay}s" + try: + logger.debug("Setting roles for endorser on endorser-issuer connection.") + await endorser_controller.endorse_transaction.set_endorser_role( + conn_id=endorser_connection_id, + transaction_my_job="TRANSACTION_ENDORSER", ) - await asyncio.sleep(delay) # Secondary delay. Primary occurs in assert_ method - - delay *= 2 - logger.info(f"Retry setting of endorser role with increased delay: {delay}s") + logger.debug("Successfully set endorser role.") + await asyncio.sleep(DEFAULT_DELAY) # Allow ACA-Py records to update + except ClientResponseError as e: + logger.error("Failed to set endorser role: {}.", e) + raise CloudApiException( + "Failed to set the endorser role in the endorser-issuer connection, " + f"with connection id {endorser_connection_id}. " + "This is a known bug in ACA-Py. Please retry." + ) from e async def set_author_role( issuer_controller: AcaPyClient, issuer_connection_id: str, logger: Logger ): - delay = DEFAULT_DELAY - for n in range(DEFAULT_NUM_TRIES): - try: - logger.debug( - f"Setting roles for author on issuer-endorser connection. Try: {n}" - ) - await issuer_controller.endorse_transaction.set_endorser_role( - conn_id=issuer_connection_id, - transaction_my_job="TRANSACTION_AUTHOR", - ) - - # Try assert that it's done. Checking too soon may raise ACA-Py error (bug). - # So if it fails, retry with backing-off delay - logger.debug("Assert that the author role is set before continuing") - - await assert_author_role_set( - issuer_controller, - issuer_connection_id, - logger, - num_tries=1, - delay=delay, - ) - logger.debug(f"Successfully set author role on try: {n}") - return True # success, exit retries - except (SettingMetadataException, ClientResponseError) as e: - if n == DEFAULT_NUM_TRIES: - logger.error( - "Failed to set author role after {} retries.", DEFAULT_NUM_TRIES - ) - raise CloudApiException( - "Failed to set the author role in the issuer-endorser connection, " - f"with connection id {issuer_connection_id}." - ) from e - logger.warning(f"Setting author role has failed on try {n} with delay {delay}s") - await asyncio.sleep(delay) # Secondary delay. Primary occurs in assert_ method - - delay *= 2 - logger.info(f"Retry setting of author role with increased delay: {delay}s") + try: + logger.debug("Setting roles for author on issuer-endorser connection") + await issuer_controller.endorse_transaction.set_endorser_role( + conn_id=issuer_connection_id, + transaction_my_job="TRANSACTION_AUTHOR", + ) + logger.debug("Successfully set author role.") + await asyncio.sleep(DEFAULT_DELAY) # Allow ACA-Py records to update + except ClientResponseError as e: + logger.error("Failed to set author role: {}.", e) + raise CloudApiException( + "Failed to set the author role in the issuer-endorser connection, " + f"with connection id {issuer_connection_id}. " + "This is a known bug in ACA-Py. Please retry." + ) from e async def set_endorser_info( @@ -104,47 +57,24 @@ async def set_endorser_info( endorser_did: str, logger: Logger, ): - delay = DEFAULT_DELAY - for n in range(DEFAULT_NUM_TRIES): - try: - logger.debug( - f"Setting endorser info on issuer-endorser connection. Try: {n}" - ) - await issuer_controller.endorse_transaction.set_endorser_info( - conn_id=issuer_connection_id, - endorser_did=endorser_did, - ) - - # Try assert that it's done. Checking too soon may raise ACA-Py error (bug). - # So if it fails, retry with backing-off delay - logger.debug("Assert that the endorser info is set before continuing") - - await assert_endorser_info_set( - issuer_controller, - issuer_connection_id, - endorser_did, - logger, - num_tries=1, - delay=delay, - ) - logger.debug(f"Successfully set author role on try: {n}") - return True # success, exit retries - except (SettingMetadataException, ClientResponseError) as e: - if n == DEFAULT_NUM_TRIES: - logger.error( - "Failed to set endorser info after {} retries.", DEFAULT_NUM_TRIES - ) - raise CloudApiException( - "Failed to set the endorser info in the issuer-endorser connection, " - f"with connection id {issuer_connection_id}." - ) from e - logger.warning( - f"Setting endorser info has failed on try {n} with delay {delay}s" + try: + logger.debug(f"Setting endorser info on issuer-endorser connection") + await issuer_controller.endorse_transaction.set_endorser_info( + conn_id=issuer_connection_id, + endorser_did=endorser_did, ) - await asyncio.sleep(delay) # Secondary delay. Primary occurs in assert_ method + logger.debug(f"Successfully set endorser info.") + await asyncio.sleep(DEFAULT_DELAY) # Allow ACA-Py records to update + except ClientResponseError as e: + logger.error("Failed to set endorser info: {}.", e) + raise CloudApiException( + "Failed to set the endorser info in the issuer-endorser connection, " + f"with connection id {issuer_connection_id}. " + "This is a known bug in ACA-Py. Please retry." + ) from e + - delay *= 2 - logger.info(f"Retry setting of endorser info with increased delay: {delay}s") +# Unused code at the moment: may be useful in avoiding ACA-Py delays resulting in duplicate record bug async def assert_metadata_set( From 0e71f5f90840f1ee46b219b19eb7345acffe7a20 Mon Sep 17 00:00:00 2001 From: ff137 Date: Fri, 6 Oct 2023 12:21:47 +0300 Subject: [PATCH 17/18] :art: set environ variable for DEFAULT_DELAY between endorser configurations --- app/services/onboarding/util/set_endorser_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/services/onboarding/util/set_endorser_metadata.py b/app/services/onboarding/util/set_endorser_metadata.py index 8187eee5b..e14e0b37c 100644 --- a/app/services/onboarding/util/set_endorser_metadata.py +++ b/app/services/onboarding/util/set_endorser_metadata.py @@ -1,4 +1,5 @@ import asyncio +import os from logging import Logger from typing import Callable @@ -8,7 +9,7 @@ from app.exceptions.cloud_api_error import CloudApiException DEFAULT_NUM_TRIES = 1 -DEFAULT_DELAY = 0.2 +DEFAULT_DELAY = float(os.environ.get("SET_ENDORSER_INFO_DELAY", "1.5")) async def set_endorser_role( From 6df22f141abde798cba583f68e8bf21b2778f40b Mon Sep 17 00:00:00 2001 From: ff137 Date: Fri, 6 Oct 2023 12:38:53 +0300 Subject: [PATCH 18/18] :art: --- app/services/onboarding/util/set_endorser_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/onboarding/util/set_endorser_metadata.py b/app/services/onboarding/util/set_endorser_metadata.py index e14e0b37c..ba7b2c757 100644 --- a/app/services/onboarding/util/set_endorser_metadata.py +++ b/app/services/onboarding/util/set_endorser_metadata.py @@ -59,12 +59,12 @@ async def set_endorser_info( logger: Logger, ): try: - logger.debug(f"Setting endorser info on issuer-endorser connection") + logger.debug("Setting endorser info on issuer-endorser connection") await issuer_controller.endorse_transaction.set_endorser_info( conn_id=issuer_connection_id, endorser_did=endorser_did, ) - logger.debug(f"Successfully set endorser info.") + logger.debug("Successfully set endorser info.") await asyncio.sleep(DEFAULT_DELAY) # Allow ACA-Py records to update except ClientResponseError as e: logger.error("Failed to set endorser info: {}.", e)