[DPE-3724] Update labels to support spark-integrator-hub (#86)

pyproject.toml

@@ -42,7 +42,7 @@ ignore_missing_imports = true
 
 [tool.poetry]
 name = "spark8t"
-version = "0.0.6"
+version = "0.0.7"
 description = "This project provides some utilities function and CLI commands to run Spark on K8s."
 authors = [
     "Canonical Data Platform <[email protected]>"

requirements.txt

@@ -1,9 +1,9 @@
-anyio==4.3.0; python_full_version > "3.8.0" and python_version < "4.0" \
-    --hash=sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8 \
-    --hash=sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6
-certifi==2024.2.2; python_full_version > "3.8.0" and python_version < "4.0" \
-    --hash=sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f \
-    --hash=sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1
+anyio==3.7.0 ; python_full_version > "3.8.0" and python_version < "4.0" \
+    --hash=sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce \
+    --hash=sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0
+certifi==2023.5.7 ; python_full_version > "3.8.0" and python_version < "4.0" \
+    --hash=sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7 \
+    --hash=sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716
 envyaml==1.10.211231 ; python_full_version > "3.8.0" and python_version < "4.0" \
     --hash=sha256:88f8a076159e3c317d3450a5f404132b6ac91aecee4934ea72eac65f911f1244 \
     --hash=sha256:8d7a7a6be12587cc5da32a587067506b47b849f4643981099ad148015a72de52
@@ -13,15 +13,15 @@ exceptiongroup==1.1.1 ; python_full_version > "3.8.0" and python_version < "3.11
 h11==0.14.0 ; python_full_version > "3.8.0" and python_version < "4.0" \
     --hash=sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d \
     --hash=sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761
-httpcore==1.0.5; python_full_version > "3.8.0" and python_version < "4.0" \
-    --hash=sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61 \
-    --hash=sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5
+httpcore==0.17.2 ; python_full_version > "3.8.0" and python_version < "4.0" \
+    --hash=sha256:125f8375ab60036db632f34f4b627a9ad085048eef7cb7d2616fea0f739f98af \
+    --hash=sha256:5581b9c12379c4288fe70f43c710d16060c10080617001e6b22a3b6dbcbefd36
 httpx==0.24.1 ; python_full_version > "3.8.0" and python_version < "4.0" \
     --hash=sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd \
     --hash=sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd
-idna==3.7; python_full_version > "3.8.0" and python_version < "4.0" \
-    --hash=sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc \
-    --hash=sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0
+idna==3.4 ; python_full_version > "3.8.0" and python_version < "4.0" \
+    --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \
+    --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2
 jinja2==3.1.3 ; python_full_version > "3.8.0" and python_version < "4.0" \
     --hash=sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa \
     --hash=sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90

spark8t/cli/params.py

@@ -44,16 +44,16 @@ def add_logging_arguments(parser: ArgumentParser) -> ArgumentParser:
     return parser
 
 
-def add_ignore_configuration_hub(parser: ArgumentParser) -> ArgumentParser:
+def add_ignore_integrator_hub(parser: ArgumentParser) -> ArgumentParser:
     """
-    Add option to exclude the configuration provided by the Spark Configuration Hub
+    Add option to exclude the configuration provided by the Spark Integrator Hub
 
     :param parser: Input parser to decorate with parsing support for logging args.
     """
     parser.add_argument(
-        "--ignore-configuration-hub",
+        "--ignore-integrator-hub",
         action="store_true",
-        help="Ignore the configuration provided by Spark Configuration Hub Charm.",
+        help="Ignore the configuration provided by Spark Integrator Hub Charm.",
     )
 
     return parser

spark8t/cli/pyspark.py

@@ -7,7 +7,7 @@
 
 from spark8t.cli.params import (
     add_config_arguments,
-    add_ignore_configuration_hub,
+    add_ignore_integrator_hub,
     add_logging_arguments,
     defaults,
     get_kube_interface,
@@ -43,8 +43,8 @@ def main(args: Namespace, logger: Logger):
             else PrimaryAccountNotFound()
         )
 
-    if args.ignore_configuration_hub:
-        service_account.configuration_hub_confs = PropertyFile.empty()
+    if args.ignore_integrator_hub:
+        service_account.integrator_hub_confs = PropertyFile.empty()
 
     SparkInterface(
         service_account=service_account,
@@ -60,7 +60,7 @@ def main(args: Namespace, logger: Logger):
             k8s_parser,
             spark_user_parser,
             add_config_arguments,
-            add_ignore_configuration_hub,
+            add_ignore_integrator_hub,
         ]
     ).parse_known_args()
 

spark8t/cli/service_account_registry.py

@@ -84,9 +84,9 @@ def create_service_account_registry_parser(parser: ArgumentParser):
         [spark_user_parser],
         subparsers.add_parser(Actions.GET_CONFIG.value, parents=[base_parser]),
     ).add_argument(
-        "--ignore-configuration-hub",
+        "--ignore-integrator-hub",
         action="store_true",
-        help="Boolean to ignore configuration hub generated options.",
+        help="Boolean to ignore Spark Integrator Hub generated options.",
     )
     #  subparser for sa-conf-del
     parse_arguments_with(
@@ -169,8 +169,8 @@ def main(args: Namespace, logger: Logger):
         if maybe_service_account is None:
             raise AccountNotFound(input_service_account.id)
 
-        if args.ignore_configuration_hub:
-            maybe_service_account.configuration_hub_confs = PropertyFile.empty()
+        if args.ignore_integrator_hub:
+            maybe_service_account.integrator_hub_confs = PropertyFile.empty()
         maybe_service_account.configurations.log(print)
 
     elif args.action == Actions.CLEAR_CONFIG:

spark8t/cli/spark_shell.py

@@ -7,7 +7,7 @@
 
 from spark8t.cli.params import (
     add_config_arguments,
-    add_ignore_configuration_hub,
+    add_ignore_integrator_hub,
     add_logging_arguments,
     defaults,
     get_kube_interface,
@@ -43,8 +43,8 @@ def main(args: Namespace, logger: Logger):
             else PrimaryAccountNotFound()
         )
 
-    if args.ignore_configuration_hub:
-        service_account.configuration_hub_confs = PropertyFile.empty()
+    if args.ignore_integrator_hub:
+        service_account.integrator_hub_confs = PropertyFile.empty()
 
     SparkInterface(
         service_account=service_account,
@@ -60,7 +60,7 @@ def main(args: Namespace, logger: Logger):
             k8s_parser,
             spark_user_parser,
             add_config_arguments,
-            add_ignore_configuration_hub,
+            add_ignore_integrator_hub,
         ]
     ).parse_known_args()
 

spark8t/cli/spark_sql.py

@@ -7,7 +7,7 @@
 
 from spark8t.cli.params import (
     add_config_arguments,
-    add_ignore_configuration_hub,
+    add_ignore_integrator_hub,
     add_logging_arguments,
     defaults,
     get_kube_interface,
@@ -43,8 +43,8 @@ def main(args: Namespace, logger: Logger):
             else PrimaryAccountNotFound()
         )
 
-    if args.ignore_configuration_hub:
-        service_account.configuration_hub_confs = PropertyFile.empty()
+    if args.ignore_integrator_hub:
+        service_account.integrator_hub_confs = PropertyFile.empty()
 
     SparkInterface(
         service_account=service_account,
@@ -60,7 +60,7 @@ def main(args: Namespace, logger: Logger):
             k8s_parser,
             spark_user_parser,
             add_config_arguments,
-            add_ignore_configuration_hub,
+            add_ignore_integrator_hub,
         ]
     ).parse_known_args()
 

spark8t/cli/spark_submit.py

@@ -8,7 +8,7 @@
 from spark8t.cli.params import (
     add_config_arguments,
     add_deploy_arguments,
-    add_ignore_configuration_hub,
+    add_ignore_integrator_hub,
     add_logging_arguments,
     defaults,
     get_kube_interface,
@@ -44,8 +44,8 @@ def main(args: Namespace, logger: Logger):
             else PrimaryAccountNotFound()
         )
 
-    if args.ignore_configuration_hub:
-        service_account.configuration_hub_confs = PropertyFile.empty()
+    if args.ignore_integrator_hub:
+        service_account.integrator_hub_confs = PropertyFile.empty()
 
     SparkInterface(
         service_account=service_account,
@@ -67,7 +67,7 @@ def main(args: Namespace, logger: Logger):
             spark_user_parser,
             add_deploy_arguments,
             add_config_arguments,
-            add_ignore_configuration_hub,
+            add_ignore_integrator_hub,
         ]
     ).parse_known_args()
 

spark8t/domain.py

@@ -300,7 +300,7 @@ class ServiceAccount:
     api_server: str
     primary: bool = False
     extra_confs: PropertyFile = PropertyFile.empty()
-    configuration_hub_confs: PropertyFile = PropertyFile.empty()
+    integrator_hub_confs: PropertyFile = PropertyFile.empty()
 
     @property
     def id(self):
@@ -319,9 +319,7 @@ def _k8s_configurations(self):
     @property
     def configurations(self) -> PropertyFile:
         """Return the service account configuration, associated to a given spark service account."""
-        return (
-            self.extra_confs + self.configuration_hub_confs + self._k8s_configurations
-        )
+        return self.extra_confs + self.integrator_hub_confs + self._k8s_configurations
 
 
 class KubernetesResourceType(str, Enum):

spark8t/literals.py

@@ -1,4 +1,4 @@
 MANAGED_BY_LABELNAME = "app.kubernetes.io/managed-by"
 PRIMARY_LABELNAME = "app.kubernetes.io/spark8t-primary"
 SPARK8S_LABEL = "spark8t"
-CONFIGURATION_HUB_LABEL = "configuration-hub-conf"
+HUB_LABEL = "integrator-hub-conf"

spark8t/resources/templates/role_yaml.tmpl

@@ -37,6 +37,6 @@ rules:
   resources:
   - secrets
   resourceNames:
-  - configuration-hub-conf-{{username}}
+  - integrator-hub-conf-{{username}}
   verbs:
   - get

spark8t/services.py

@@ -35,7 +35,7 @@
     ResourceAlreadyExists,
 )
 from spark8t.literals import (
-    CONFIGURATION_HUB_LABEL,
+    HUB_LABEL,
     MANAGED_BY_LABELNAME,
     PRIMARY_LABELNAME,
     SPARK8S_LABEL,
@@ -1157,8 +1157,8 @@ def _get_secret_name(name):
         return f"{SPARK8S_LABEL}-sa-conf-{name}"
 
     @staticmethod
-    def _get_configuration_hub_secret_name(name):
-        return f"{CONFIGURATION_HUB_LABEL}-{name}"
+    def _get_integrator_hub_secret_name(name):
+        return f"{HUB_LABEL}-{name}"
 
     def _retrieve_secret_configurations(
         self, name: str, namespace: str, secret_name: str
@@ -1183,7 +1183,7 @@ def _build_service_account_from_raw(self, metadata: Dict[str, Any]):
         primary = PRIMARY_LABELNAME in metadata["labels"]
 
         account_secret_name = self._get_secret_name(name)
-        configuration_hub_secret_name = self._get_configuration_hub_secret_name(name)
+        integrator_hub_secret_name = self._get_integrator_hub_secret_name(name)
 
         return ServiceAccount(
             name=name,
@@ -1193,8 +1193,8 @@ def _build_service_account_from_raw(self, metadata: Dict[str, Any]):
             extra_confs=self._retrieve_secret_configurations(
                 name, namespace, account_secret_name
             ),
-            configuration_hub_confs=self._retrieve_secret_configurations(
-                name, namespace, configuration_hub_secret_name
+            integrator_hub_confs=self._retrieve_secret_configurations(
+                name, namespace, integrator_hub_secret_name
             ),
         )
 

tests/integration/test_cli_service_accounts.py

@@ -9,7 +9,7 @@
 
 from spark8t.domain import KubernetesResourceType, PropertyFile
 from spark8t.literals import (
-    CONFIGURATION_HUB_LABEL,
+    HUB_LABEL,
     MANAGED_BY_LABELNAME,
     PRIMARY_LABELNAME,
     SPARK8S_LABEL,
@@ -132,7 +132,7 @@ def test_create_service_account(namespace, backend, primary):
     role_name = f"{username}-role"
     role_binding_name = f"{username}-role-binding"
     secret_name = f"{SPARK8S_LABEL}-sa-conf-{username}"
-    hub_secret_name = f"configuration-hub-conf-{username}"
+    hub_secret_name = f"{HUB_LABEL}-{username}"
 
     create_args = [
         "create",
@@ -561,8 +561,8 @@ def test_service_account_get_config(service_account, backend, request):
     }
     assert actual_configs == expected_configs
 
-    # add configuration hub secret for the test service account
-    secret_name = f"{CONFIGURATION_HUB_LABEL}-{username}"
+    # add integrator hub secret for the test service account
+    secret_name = f"{HUB_LABEL}-{username}"
 
     property_file = PropertyFile({"key": "value"})
 
@@ -589,7 +589,7 @@ def test_service_account_get_config(service_account, backend, request):
         KubernetesResourceType.SECRET_GENERIC, secret_name, namespace
     )
 
-    # check that configuration hub config is there
+    # check that integrator hub config is there
     # Get the default configs created with a service account
     stdout, stderr, ret_code = run_service_account_registry(
         "get-config",
@@ -617,7 +617,7 @@ def test_service_account_get_config(service_account, backend, request):
         namespace,
         "--backend",
         backend,
-        "--ignore-configuration-hub",
+        "--ignore-integrator-hub",
     )
     actual_configs = set(stdout.splitlines())
     assert actual_configs == expected_configs
@@ -669,7 +669,7 @@ def test_service_account_add_config(service_account, backend):
         namespace,
         "--backend",
         backend,
-        "--ignore-configuration-hub",
+        "--ignore-integrator-hub",
     )
     updated_configs = set(stdout.splitlines())