Update workers to match FM release 7.0.0

inventory/python/CHANGELOG.md

@@ -34,3 +34,7 @@
 # 1.5.0
 - Bumped version of frinx-inventory-api to 2.3.0.
 - Implemented workers for reading, adding, updating and removing inventory locations.
+
+# 2.0.0
+- Bumped version of frinx-inventory-api to 3.0.0.
+- Bumped version of frinx-python-sdk to 2.2.1.

inventory/python/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 python = "^3.10"
 pydantic = "^2"
 frinx-python-sdk = "^2"
-frinx-inventory-api = { git = "https://github.com/FRINXio/frinx-services-python-api.git", tag = "frinx-inventory-api_v2.3.0", subdirectory = "frinx-inventory-server/python" }
+frinx-inventory-api = { git = "https://github.com/FRINXio/frinx-services-python-api.git", tag = "frinx-inventory-api_v3.0.0", subdirectory = "frinx-inventory-server/python" }
 
 [tool.poetry.group.dev.dependencies]
 ruff = "^0"
@@ -19,7 +19,7 @@ packages = [{ include = "frinx_worker" }]
 name = "frinx-inventory-worker"
 description = "Conductor worker for Frinx Device Inventory"
 authors = ["Jozef Volak <[email protected]>"]
-version = "1.5.0"
+version = "2.0.0"
 readme = ["README.md", "CHANGELOG.md", "RELEASE.md"]
 keywords = ["frinx-machine", "device inventory", "worker"]
 license = "Apache 2.0"

misc/python/kafka/CHANGELOG.md

@@ -9,3 +9,6 @@
 
 # 1.1.0
 - Support Kafka headers on the input of the Kafka_publish task.
+
+# 1.2.0
+- Updated frinx-python-sdk to version 2.2.1 from 2.1.3.

misc/python/kafka/pyproject.toml

@@ -19,7 +19,7 @@ packages = [{ include = "frinx_worker" }]
 name = "frinx-kafka-worker"
 description = "Conductor Kafka worker for Frinx Machine"
 authors = ["Jozef Volak <[email protected]>"]
-version = "1.1.1"
+version = "1.2.0"
 readme = ["README.md", "CHANGELOG.md"]
 keywords = ["frinx-machine", "kafka", "worker"]
 license = "Apache 2.0"

misc/python/python-lambda/CHANGELOG.md

@@ -2,4 +2,8 @@
 - Updated frinx-python-sdk to version ^1.1
 
 # 1.0.1
-- Fix timeout decorator signal thread issue
+- Fix timeout decorator signal thread issue
+
+# 1.1.0
+- Bump dependencies to match frinx-python-sdk 2.2.1.
+

misc/python/python-lambda/pyproject.toml

@@ -18,7 +18,7 @@ packages = [{ include = "frinx_worker" }]
 name = "frinx-python-lambda"
 description = "Conductor Python Lambda worker for Frinx Machine"
 authors = ["Jozef Volak <[email protected]>"]
-version = "1.0.2"
+version = "1.1.0"
 readme = ["README.md", "CHANGELOG.md"]
 keywords = ["frinx-machine", "python-lambda", "worker"]
 license = "Apache 2.0"

uniconfig/python/CHANGELOG.md

@@ -54,3 +54,8 @@
 
 # 2.3.5
 - Fix handling of null or empty input in the 'close_transactions' worker.
+
+# 3.0.0
+- Upgrade UniConfig API to v2.0.0 (server version 7.0.0).
+- Removed temporary workaround related to unwrapping of choice nodes in the Device Discovery and Install Node RPC.
+- Integrated new key-value escaping mechanism into workers.

uniconfig/python/frinx_worker/uniconfig/__init__.py

@@ -98,8 +98,8 @@ def class_to_json(dataclass: Any) -> Any:
         Any: A JSON string representation of the dataclass with keys in kebab-case.
     """
     if dataclasses.is_dataclass(dataclass):
-        return json.dumps(snake_to_kebab_case(remove_empty_elements_from_dict(dataclasses.asdict(dataclass))))
+        return json.dumps(snake_to_kebab_case(remove_empty_elements_from_dict(dataclasses.asdict(dataclass))))  # type: ignore[call-overload]
     elif isinstance(dataclass, BaseModel):
-        return dataclass.json(exclude_none=True, by_alias=True, exclude_defaults=True)
+        return dataclass.model_dump_json(exclude_none=True, by_alias=True, exclude_defaults=True)
     else:
         return json.dumps(snake_to_kebab_case(remove_empty_elements_from_dict(dataclass)))

uniconfig/python/frinx_worker/uniconfig/cli_network_topology.py

@@ -5,6 +5,7 @@
 from frinx.common.frinx_rest import UNICONFIG_REQUEST_PARAMS
 from frinx.common.frinx_rest import UNICONFIG_URL_BASE
 from frinx.common.type_aliases import ListAny
+from frinx.common.util import escape_uniconfig_uri_key
 from frinx.common.worker.service import ServiceWorkersImpl
 from frinx.common.worker.task_def import TaskDefinition
 from frinx.common.worker.task_def import TaskInput
@@ -43,9 +44,10 @@ def execute(self, worker_input: WorkerInput) -> TaskResult[WorkerOutput]:
             if self.UniconfigApi.request is None:
                 raise Exception(f"Failed to create request {self.UniconfigApi.request}")
 
+            escaped_node_id = escape_uniconfig_uri_key(worker_input.node_id)
             response = requests.request(
                 url=worker_input.uniconfig_url_base
-                + self.UniconfigApi.uri.format(topology_id=worker_input.topology_id, node_id=worker_input.node_id),
+                + self.UniconfigApi.uri.format(topology_id=worker_input.topology_id, node_id=escaped_node_id),
                 method=self.UniconfigApi.method,
                 data=class_to_json(
                     self.UniconfigApi.request(
@@ -91,9 +93,10 @@ def execute(self, worker_input: WorkerInput) -> TaskResult[WorkerOutput]:
             if self.UniconfigApi.request is None:
                 raise Exception(f"Failed to create request {self.UniconfigApi.request}")
 
+            escaped_node_id = escape_uniconfig_uri_key(worker_input.node_id)
             response = requests.request(
                 url=worker_input.uniconfig_url_base
-                + self.UniconfigApi.uri.format(topology_id=worker_input.topology_id, node_id=worker_input.node_id),
+                + self.UniconfigApi.uri.format(topology_id=worker_input.topology_id, node_id=escaped_node_id),
                 method=self.UniconfigApi.method,
                 data=class_to_json(
                     self.UniconfigApi.request(

uniconfig/python/frinx_worker/uniconfig/connection_manager.py

@@ -1,11 +1,11 @@
-import urllib.parse
 from typing import Literal
 
 import requests
 from frinx.common.frinx_rest import UNICONFIG_HEADERS
 from frinx.common.frinx_rest import UNICONFIG_REQUEST_PARAMS
 from frinx.common.frinx_rest import UNICONFIG_URL_BASE
 from frinx.common.type_aliases import DictAny
+from frinx.common.util import escape_uniconfig_uri_key
 from frinx.common.worker.service import ServiceWorkersImpl
 from frinx.common.worker.task_def import TaskDefinition
 from frinx.common.worker.task_def import TaskExecutionProperties
@@ -14,18 +14,18 @@
 from frinx.common.worker.task_result import TaskResult
 from frinx.common.worker.worker import WorkerImpl
 from frinx_api.uniconfig.connection.manager import MountType
+from frinx_api.uniconfig.connection.manager import installnode
 
 from . import class_to_json
 from . import handle_response
-from . import snake_to_kebab_case
 
 
 class ConnectionManager(ServiceWorkersImpl):
     class InstallNode(WorkerImpl):
+        from frinx_api.uniconfig.connection.manager.installnode import Cli
+        from frinx_api.uniconfig.connection.manager.installnode import Gnmi
+        from frinx_api.uniconfig.connection.manager.installnode import Netconf
         from frinx_api.uniconfig.rest_api import InstallNode as UniconfigApi
-        # from frinx_api.uniconfig.connection.manager.installnode import Cli
-        # from frinx_api.uniconfig.connection.manager.installnode import Gnmi
-        # from frinx_api.uniconfig.connection.manager.installnode import Netconf
 
         class ExecutionProperties(TaskExecutionProperties):
             exclude_empty_inputs: bool = True
@@ -48,66 +48,31 @@ def execute(self, worker_input: WorkerInput) -> TaskResult[WorkerOutput]:
             if self.UniconfigApi.request is None:
                 raise Exception(f"Failed to create request {self.UniconfigApi.request}")
 
-            import json
             response = requests.request(
                 url=worker_input.uniconfig_url_base + self.UniconfigApi.uri,
                 method=self.UniconfigApi.method,
-                # FIXME prepare input with credentials (TEMPORARY SOLUTION)
-                data=json.dumps(snake_to_kebab_case(self._prepare_input(worker_input))),
-                # data=class_to_json(
-                #     self.UniconfigApi.request(
-                #         input=self.Input(
-                #             node_id=worker_input.node_id,
-                #             cli=self.Cli(**worker_input.install_params)
-                #             if worker_input.connection_type == "cli"
-                #             else None,
-                #             netconf=self.Netconf(**worker_input.install_params)
-                #             if worker_input.connection_type == "netconf"
-                #             else None,
-                #             gnmi=self.Gnmi(**worker_input.install_params)
-                #             if worker_input.connection_type == "gnmi"
-                #             else None,
-                #         ),
-                #     ),
-                # ),
+                data=class_to_json(
+                    self.UniconfigApi.request(
+                        input=installnode.Input(
+                            node_id=worker_input.node_id,
+                            cli=self.Cli(**worker_input.install_params)
+                            if worker_input.connection_type == "cli"
+                            else None,
+                            netconf=self.Netconf(**worker_input.install_params)
+                            if worker_input.connection_type == "netconf"
+                            else None,
+                            gnmi=self.Gnmi(**worker_input.install_params)
+                            if worker_input.connection_type == "gnmi"
+                            else None,
+                        ),
+                    ),
+                ),
                 headers=dict(UNICONFIG_HEADERS),
                 params=UNICONFIG_REQUEST_PARAMS,
             )
 
             return handle_response(response, self.WorkerOutput)
 
-        def _prepare_input(self, worker_input: WorkerInput) -> DictAny:
-            """
-            The input now contains multiple choice nodes (keepalive, credentials and other). Until UniConfig can parse
-            choice nodes, this is a workaround to prepare the input for the installation request correctly.
-            https://frinxhelpdesk.atlassian.net/browse/UNIC-1764
-            :param worker_input: Worker input.
-            :return: Request input data as dict.
-            """
-            if worker_input.connection_type == "cli":
-                return {
-                    "input": {
-                        "node_id": worker_input.node_id,
-                        "cli": worker_input.install_params
-                    }
-                }
-            elif worker_input.connection_type == "netconf":
-                return {
-                    "input": {
-                        "node_id": worker_input.node_id,
-                        "netconf": worker_input.install_params
-                    }
-                }
-            elif worker_input.connection_type == "gnmi":
-                return {
-                    "input": {
-                        "node_id": worker_input.node_id,
-                        "gnmi": worker_input.install_params
-                    }
-                }
-            else:
-                raise ValueError(f"Unknown connection type '{worker_input.connection_type}'")
-
 
     class UninstallNode(WorkerImpl):
         from frinx_api.uniconfig.connection.manager import ConnectionType
@@ -352,7 +317,7 @@ class WorkerOutput(TaskOutput):
 
         def execute(self, worker_input: WorkerInput) -> TaskResult[WorkerOutput]:
             topology_id = self._derive_topology_id(worker_input.connection_type)
-            escaped_node_id = urllib.parse.quote(worker_input.node_id)
+            escaped_node_id = escape_uniconfig_uri_key(worker_input.node_id)
             response = requests.request(
                 url=f"{worker_input.uniconfig_url_base}/data/network-topology"
                     f"/topology={topology_id}/node={escaped_node_id}",

uniconfig/python/frinx_worker/uniconfig/device_discovery.py

@@ -1,6 +1,5 @@
 from ipaddress import IPv4Address
 from ipaddress import IPv6Address
-from typing import Any
 
 import pydantic
 import requests
@@ -17,14 +16,14 @@
 from frinx.common.worker.task_result import TaskResult
 from frinx_api.uniconfig import OperationsDiscoverPostResponse
 from frinx_api.uniconfig.device.discovery.discover import Address
+from frinx_api.uniconfig.device.discovery.discover import DeviceDiscoveryTypeOfAddressModel
+from frinx_api.uniconfig.device.discovery.discover import DeviceDiscoveryTypeOfAddressModel1
+from frinx_api.uniconfig.device.discovery.discover import DeviceDiscoveryTypeOfAddressModel2
+from frinx_api.uniconfig.device.discovery.discover import DeviceDiscoveryTypeOfAddressModel3
+from frinx_api.uniconfig.device.discovery.discover import DeviceDiscoveryTypeOfPortModel
+from frinx_api.uniconfig.device.discovery.discover import DeviceDiscoveryTypeOfPortModel2
 from frinx_api.uniconfig.device.discovery.discover import Input
 from frinx_api.uniconfig.device.discovery.discover import TcpPortItem
-from frinx_api.uniconfig.device.discovery.discover import TypeOfAddressModel
-from frinx_api.uniconfig.device.discovery.discover import TypeOfAddressModel1
-from frinx_api.uniconfig.device.discovery.discover import TypeOfAddressModel2
-from frinx_api.uniconfig.device.discovery.discover import TypeOfAddressModel3
-from frinx_api.uniconfig.device.discovery.discover import TypeOfPortModel
-from frinx_api.uniconfig.device.discovery.discover import TypeOfPortModel2
 from frinx_api.uniconfig.device.discovery.discover import UdpPortItem
 from frinx_api.uniconfig.rest_api import Discover
 from pydantic import Field
@@ -37,37 +36,6 @@
 from .util import parse_ranges
 
 
-def _unwrap_data(discovery_input: Input) -> dict[str, Any]:
-    """
-    Unwrapping is necessary because of the input now containing choice wrapper nodes which cannot be parsed by
-    UniConfig yet. https://frinxhelpdesk.atlassian.net/browse/UNIC-1764
-    :param discovery_input: Device Discovery input.
-    :return: Unwrapped data as dict.
-    """
-    tcp_port: list[dict[str, Any]] = []
-    udp_port: list[dict[str, Any]] = []
-    address: list[dict[str, Any]] = []
-    if discovery_input.tcp_port is not None:
-        for tcp_port_item in discovery_input.tcp_port:
-            if tcp_port_item.type_of_port is not None:
-                tcp_port.append(tcp_port_item.type_of_port.model_dump())
-    if discovery_input.udp_port is not None:
-        for udp_port_item in discovery_input.udp_port:
-            if udp_port_item.type_of_port is not None:
-                udp_port.append(udp_port_item.type_of_port.model_dump())
-    if discovery_input.address is not None:
-        for address_item in discovery_input.address:
-            if address_item.type_of_address is not None:
-                address.append(address_item.type_of_address.model_dump())
-    return {
-        "input": {
-            "address": None if not address else address,
-            "tcp_port": None if not tcp_port else tcp_port,
-            "udp_port": None if not udp_port else udp_port
-        }
-    }
-
-
 class DeviceDiscoveryWorkers(ServiceWorkersImpl):
     class DeviceDiscoveryWorker(WorkerImpl):
         class ExecutionProperties(TaskExecutionProperties):
@@ -90,27 +58,27 @@ def validate_ip(cls, ip: str) -> list[Address]:
                 if len(ip_list) == 1:
                     if "/" in ip_list[0]:
                         address = Address(
-                            type_of_address=TypeOfAddressModel3(
+                            type_of_address=DeviceDiscoveryTypeOfAddressModel3(
                                 network=str(IPvAnyNetwork(ip_list[0]))
                             )
                         )
                     else:
                         address = Address(
-                            type_of_address=TypeOfAddressModel(
+                            type_of_address=DeviceDiscoveryTypeOfAddressModel(
                                 ip_address=str(IPvAnyAddress(ip_list[0]))
                             )
                         )
                 else:
                     try:
                         address = Address(
-                            type_of_address=TypeOfAddressModel1(
+                            type_of_address=DeviceDiscoveryTypeOfAddressModel1(
                                 start_ipv4_address=str(IPv4Address(ip_list[0])),
                                 end_ipv4_address=str(IPv4Address(ip_list[1]))
                             )
                         )
                     except ValueError:
                         address = Address(
-                            type_of_address=TypeOfAddressModel2(
+                            type_of_address=DeviceDiscoveryTypeOfAddressModel2(
                                 start_ipv6_address=str(IPv6Address(ip_list[0])),
                                 end_ipv6_address=str(IPv6Address(ip_list[1]))
                             )
@@ -122,7 +90,7 @@ def validate_ip(cls, ip: str) -> list[Address]:
             def validate_tcp(cls, tcp_port: str) -> list[TcpPortItem] | None:
                 if tcp_port:
                     return [TcpPortItem(
-                                type_of_port=TypeOfPortModel(
+                                type_of_port=DeviceDiscoveryTypeOfPortModel(
                                     port=p
                                 )
                         ) for p in parse_ranges(tcp_port.split(","))]
@@ -133,7 +101,7 @@ def validate_tcp(cls, tcp_port: str) -> list[TcpPortItem] | None:
             def validate_udp(cls, udp_port: str) -> list[UdpPortItem] | None:
                 if udp_port:
                     return [UdpPortItem(
-                        type_of_port=TypeOfPortModel2(
+                        type_of_port=DeviceDiscoveryTypeOfPortModel2(
                                     port=p
                                 )
                         ) for p in parse_ranges(udp_port.split(","))]
@@ -161,9 +129,8 @@ def execute(self, worker_input: WorkerInput) -> TaskResult[WorkerOutput]:
                 headers=dict(UNICONFIG_HEADERS),
                 params=UNICONFIG_REQUEST_PARAMS,
                 method=Discover.method,
-                # temporary workaround until UC adds possibility to accept choice nodes
                 data=class_to_json(
-                    _unwrap_data(template),
+                    template,
                 ),
             )