[DPE-3654] Peer cluster data stored in a secret

lib/charms/opensearch/v0/opensearch_peer_clusters.py

@@ -39,6 +39,7 @@
 )
 from charms.opensearch.v0.opensearch_exceptions import OpenSearchError
 from charms.opensearch.v0.opensearch_internal_data import Scope
+from charms.opensearch.v0.opensearch_relation_peer_cluster import rel_data_from_secret
 from ops import BlockedStatus
 from shortuuid import ShortUUID
 
@@ -522,10 +523,10 @@ def rel_data(self) -> Optional[PeerClusterRelData]:
         rel = self._charm.model.get_relation(
             PeerClusterOrchestratorRelationName, orchestrators.main_rel_id
         )
-        if not (data := rel.data[rel.app].get("data")):
+        if not (secret_id := rel.data[rel.app].get("data")):
             return None
 
-        return PeerClusterRelData.from_str(data)
+        return rel_data_from_secret(self._charm, secret_id)
 
     def _pre_validate_roles_change(self, new_roles: List[str], prev_roles: List[str]):
         """Validate that the config changes of roles are allowed to happen."""

lib/charms/opensearch/v0/opensearch_relation_peer_cluster.py

@@ -4,6 +4,7 @@
 """Peer clusters relation related classes for OpenSearch."""
 import json
 import logging
+from hashlib import sha1
 from typing import TYPE_CHECKING, Any, Dict, List, MutableMapping, Optional, Union
 
 from charms.opensearch.v0.constants_charm import (
@@ -45,6 +46,7 @@
     RelationDepartedEvent,
     RelationEvent,
     RelationJoinedEvent,
+    Secret,
     WaitingStatus,
 )
 from tenacity import RetryError, Retrying, stop_after_attempt, wait_fixed
@@ -64,7 +66,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 1
+LIBPATCH = 2
 
 
 class OpenSearchPeerClusterRelation(Object):
@@ -260,6 +262,19 @@ def refresh_relation_data(
         # compute the data that needs to be broadcast to all related clusters (success or error)
         rel_data = self._rel_data(deployment_desc, orchestrators)
 
+        # if rel_data is an error, prepare to broadcast it to all related clusters
+        if isinstance(rel_data, PeerClusterRelData):
+            rel_data_secret_content = self._rel_data_secret_content(rel_data)
+
+            # check if a secret already exists for this orchestrator's relations
+            # and update it if so
+            rel_data_secret = self._update_or_create_rel_data_secret(
+                rel_data_secret_content, all_relation_ids
+            )
+
+            # grant the secrets inside the rel_data to all the related clusters
+            self._grant_rel_data_secrets(rel_data_secret_content, all_relation_ids)
+
         # exit if current cluster should not have been considered a provider
         if self._notify_if_wrong_integration(rel_data, all_relation_ids) and event_rel_id:
             self.delete_from_rel("trigger", rel_id=event_rel_id)
@@ -281,9 +296,9 @@ def refresh_relation_data(
         orchestrators[f"{cluster_type}_app"] = deployment_desc.app.to_dict()
         self.charm.peers_data.put_object(Scope.APP, "orchestrators", orchestrators)
 
-        peer_rel_data_key, should_defer = "data", False
+        should_defer = False
         if isinstance(rel_data, PeerClusterRelErrorData):
-            peer_rel_data_key, should_defer = "error_data", rel_data.should_wait
+            should_defer = rel_data.should_wait
 
         # save the orchestrators of this fleet
         for rel_id in all_relation_ids:
@@ -299,9 +314,21 @@ def refresh_relation_data(
             # there is no error to broadcast - we clear any previously broadcasted error
             if isinstance(rel_data, PeerClusterRelData):
                 self.delete_from_rel("error_data", rel_id=rel_id)
+                # we add the hash of the rel_data to only emit a change event
+                # if the data has actually changed
+                self.put_in_rel(
+                    data={
+                        "data": rel_data_secret.id,
+                        "rel_data_hash": sha1(
+                            json.dumps(rel_data.to_dict(), sort_keys=True).encode()
+                        ).hexdigest(),
+                    },
+                    rel_id=rel_id,
+                )
+                rel_data_secret.grant(self.get_rel(rel_id=rel_id))
 
-            # are we potentially overriding stuff here?
-            self.put_in_rel(data={peer_rel_data_key: rel_data.to_str()}, rel_id=rel_id)
+            else:
+                self.put_in_rel(data={"error_data": rel_data.to_str()}, rel_id=rel_id)
 
         if can_defer and should_defer:
             event.defer()
@@ -519,6 +546,73 @@ def _fetch_local_cm_nodes(self, deployment_desc: DeploymentDescription) -> List[
             if node.is_cm_eligible() and node.app.id == deployment_desc.app.id
         ]
 
+    def _rel_data_secret_content(self, rel_data: PeerClusterRelData) -> dict[str, str]:
+        """Convert the secret data to a dict for storage in a secret."""
+        # If we use the values of the secrets then the size of the secret
+        # would be too large for juju we store the secret ids instead and
+        # fetch the secrets when needed in the requirer side
+        secrets = self.charm.secrets
+
+        credentials = {
+            "admin-username": AdminUser,
+            "admin-password": secrets.get_secret_id(Scope.APP, secrets.password_key(AdminUser)),
+            "admin-password-hash": secrets.get_secret_id(Scope.APP, secrets.hash_key(AdminUser)),
+            "kibana-password": secrets.get_secret_id(
+                Scope.APP, secrets.password_key(KibanaserverUser)
+            ),
+            "kibana-password-hash": secrets.get_secret_id(
+                Scope.APP, secrets.hash_key(KibanaserverUser)
+            ),
+        }
+
+        if monitor_password := secrets.get_secret_id(Scope.APP, secrets.password_key(COSUser)):
+            credentials["monitor-password"] = monitor_password
+        if admin_tls := secrets.get_secret_id(Scope.APP, CertType.APP_ADMIN.val):
+            credentials["admin-tls"] = admin_tls
+
+        if s3_creds := rel_data.credentials.s3:
+            credentials["s3"] = {
+                "access-key": s3_creds.access_key,
+                "secret-key": s3_creds.secret_key,
+            }
+
+        return {
+            "cluster-name": rel_data.cluster_name,
+            "cm-nodes": json.dumps([node.to_dict() for node in rel_data.cm_nodes]),
+            "credentials": json.dumps(credentials),
+            "deployment-desc": json.dumps(rel_data.deployment_desc.to_dict()),
+        }
+
+    def _update_or_create_rel_data_secret(
+        self, rel_data_secret_content: dict[str, str], all_rel_ids: list[int]
+    ) -> Secret:
+        """Update or create the secret for the peer cluster relation data."""
+        for rel_id in all_rel_ids:
+            rel_data_secret_id = self.get_from_rel("data", rel_id=rel_id)
+            if rel_data_secret_id:
+                rel_data_secret = self.model.get_secret(id=rel_data_secret_id)
+                if rel_data_secret_content != rel_data_secret.get_content():
+                    rel_data_secret.set_content(rel_data_secret_content)
+
+                return rel_data_secret
+
+        return self.model.app.add_secret(rel_data_secret_content)
+
+    def _grant_rel_data_secrets(
+        self, rel_data_secret_content: dict[str, str], all_rel_ids: list[int]
+    ):
+        """Grant the secrets to all the related apps."""
+        credentials = json.loads(rel_data_secret_content["credentials"])
+        for key, secret_id in credentials.items():
+            # s3 and admin-username are not secrets
+            if key == "s3" or key == "admin-username":
+                continue
+
+            secret = self.model.get_secret(id=secret_id)
+            for rel_id in all_rel_ids:
+                if relation := self.get_rel(rel_id=rel_id):
+                    secret.grant(relation)
+
 
 class OpenSearchPeerClusterRequirer(OpenSearchPeerClusterRelation):
     """Peer cluster relation requirer class."""
@@ -573,8 +667,7 @@ def _on_peer_cluster_relation_changed(self, event: RelationChangedEvent):  # noq
             return
 
         # fetch the success data
-        data = PeerClusterRelData.from_str(data["data"])
-
+        data = rel_data_from_secret(self.charm, data["data"])
         # check errors that can only be figured out from the requirer side
         if self._error_set_from_requirer(orchestrators, deployment_desc, data, event.relation.id):
             return
@@ -806,11 +899,11 @@ def _cm_nodes(self, orchestrators: PeerClusterOrchestrators) -> List[Node]:
             if rel_id == -1:
                 continue
 
-            data = self.get_obj_from_rel(key="data", rel_id=rel_id)
-            if not data:  # not ready yet
+            secret_id = self.get_from_rel(key="data", rel_id=rel_id, remote_app=True)
+            if not secret_id:  # not ready yet
                 continue
 
-            data = PeerClusterRelData.from_dict(data)
+            data = rel_data_from_secret(self.charm, secret_id)
             cm_nodes = {**cm_nodes, **{node.name: node for node in data.cm_nodes}}
 
         # attempt to have an opensearch reported list of CMs - the response
@@ -847,7 +940,7 @@ def _error_set_from_providers(
 
         error = None
         for rel_id in orchestrator_rel_ids:
-            data = self.get_obj_from_rel("data", rel_id=rel_id)
+            data = self.get_from_rel("data", rel_id=rel_id, remote_app=True)
             error_data = self.get_obj_from_rel("error_data", rel_id=rel_id)
             if not data and not error_data:  # relation data still incomplete
                 return True
@@ -960,3 +1053,76 @@ def _clear_errors(self, *error_labels: str):
             error = self.charm.peers_data.get(Scope.APP, error_label, "")
             self.charm.status.clear(error, app=True)
             self.charm.peers_data.delete(Scope.APP, error_label)
+
+
+def rel_data_from_secret(
+    charm: "OpenSearchBaseCharm", secret_id: str
+) -> PeerClusterRelData | None:
+    """Construct the peer cluster rel data from the secret data."""
+    secret = charm.model.get_secret(id=secret_id)
+    if not secret:
+        return None
+
+    content = secret.get_content()
+    credentials = json.loads(content["credentials"])
+
+    secrets = charm.secrets
+
+    admin_password = (
+        charm.model.get_secret(id=credentials["admin-password"])
+        .get_content()
+        .get(secrets.password_key(AdminUser))
+    )
+
+    admin_password_hash = (
+        charm.model.get_secret(id=credentials["admin-password-hash"])
+        .get_content()
+        .get(secrets.hash_key(AdminUser))
+    )
+
+    kibana_password = (
+        charm.model.get_secret(id=credentials["kibana-password"])
+        .get_content()
+        .get(secrets.password_key(KibanaserverUser))
+    )
+
+    kibana_password_hash = (
+        charm.model.get_secret(id=credentials["kibana-password-hash"])
+        .get_content()
+        .get(secrets.hash_key(KibanaserverUser))
+    )
+
+    monitor_password = None
+    if "monitor-password" in credentials:
+        monitor_password = (
+            charm.model.get_secret(id=credentials["monitor-password"])
+            .get_content()
+            .get(secrets.password_key(COSUser))
+        )
+
+    admin_tls = None
+    if "admin-tls" in credentials:
+        admin_tls = charm.model.get_secret(id=credentials["admin-tls"]).get_content()
+
+    s3 = None
+    if "s3" in credentials:
+        s3 = S3RelDataCredentials(
+            access_key=credentials["s3"]["access-key"],
+            secret_key=credentials["s3"]["secret-key"],
+        )
+
+    return PeerClusterRelData(
+        cluster_name=content["cluster-name"],
+        cm_nodes=[Node.from_dict(node) for node in json.loads(content["cm-nodes"])],
+        credentials=PeerClusterRelDataCredentials(
+            admin_username=AdminUser,
+            admin_password=admin_password,
+            admin_password_hash=admin_password_hash,
+            kibana_password=kibana_password,
+            kibana_password_hash=kibana_password_hash,
+            monitor_password=monitor_password,
+            admin_tls=admin_tls,
+            s3=s3,
+        ),
+        deployment_desc=DeploymentDescription.from_dict(json.loads(content["deployment-desc"])),
+    )

lib/charms/opensearch/v0/opensearch_secrets.py

@@ -40,7 +40,7 @@
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 1
+LIBPATCH = 2
 
 
 if TYPE_CHECKING:
@@ -368,3 +368,8 @@ def delete(self, scope: Scope, key: str) -> None:
         self._remove_juju_secret(scope, key)
 
         logging.debug(f"Deleted secret {scope}:{key}")
+
+    def get_secret_id(self, scope: Scope, key: str) -> Optional[str]:
+        """Get the secret ID from the cache."""
+        label = self.label(scope, key)
+        return self._charm.peers_data.get(scope, label)

tests/unit/lib/test_opensearch_secrets.py

@@ -185,3 +185,15 @@ def test_bad_label(self):
     @parameterized.expand([Scope.APP, Scope.UNIT])
     def test_put_and_get_complex_obj(self, scope):
         return
+
+    def test_get_secret_id(self):
+        # add a secret to the store
+        content = {"secret": "value"}
+        self.store.put(Scope.APP, "super-secret-key", content)
+        # get the secret id
+        secret_id = self.store.get_secret_id(Scope.APP, "super-secret-key")
+        self.assertIsNotNone(secret_id)
+        # check the secret content
+        secret = self.charm.model.get_secret(id=secret_id)
+        secret_content = secret.get_content()
+        self.assertDictEqual(secret_content, {"super-secret-key": str(content)})