From 35101e77ff28916325f8085b58655a60fceee716 Mon Sep 17 00:00:00 2001 From: Ian Foster Date: Sat, 28 Sep 2024 21:04:14 -0700 Subject: [PATCH 1/5] docker updates --- .dockerignore | 4 +++- Dockerfile | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0cb38c89..b6ee78ef 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,4 +6,6 @@ k8s build dist mktxp.egg-info -tests \ No newline at end of file +tests/ +.github/ +deploy/ diff --git a/Dockerfile b/Dockerfile index e4a26a50..2840eaf6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,14 @@ FROM python:3-alpine LABEL org.opencontainers.image.source github.com/akpw/mktxp +RUN addgroup -S mktxp && adduser -S mktxp -G mktxp +RUN apk add nano + WORKDIR /mktxp COPY . . -RUN pip install ./ && apk add nano +RUN pip install ./ + EXPOSE 49090 -RUN addgroup -S mktxp && adduser -S mktxp -G mktxp + USER mktxp ENTRYPOINT ["/usr/local/bin/mktxp"] CMD ["export"] From 917b4fd54c96bab054dbfb35c2b34a70038cd2bf Mon Sep 17 00:00:00 2001 From: Ian Foster Date: Sat, 28 Sep 2024 21:04:29 -0700 Subject: [PATCH 2/5] added support for routing process stats --- mktxp/cli/config/config.py | 6 ++- mktxp/cli/config/mktxp.conf | 1 + mktxp/collector/routing_stats_collector.py | 46 ++++++++++++++++++++++ mktxp/datasource/routing_stats_ds.py | 27 +++++++++++++ mktxp/flow/collector_registry.py | 2 + mktxp/flow/router_entry.py | 1 + 6 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 mktxp/collector/routing_stats_collector.py create mode 100644 mktxp/datasource/routing_stats_ds.py diff --git a/mktxp/cli/config/config.py b/mktxp/cli/config/config.py index 3ccbf916..3b1f99a2 100755 --- a/mktxp/cli/config/config.py +++ b/mktxp/cli/config/config.py @@ -47,6 +47,7 @@ class CollectorKeys: KID_CONTROL_DEVICE_COLLECTOR = 'KidControlCollector' USER_COLLECTOR = 'UserCollector' BGP_COLLECTOR = 'BGPCollector' + ROUTING_STATS_COLLECTOR = 'RoutingStatsCollector' IPSEC_COLLECTOR = 'IPSecCollector' LTE_COLLECTOR = 'LTECollector' SWITCH_PORT_COLLECTOR = 'SwitchPortCollector' @@ -102,6 +103,7 @@ class MKTXPConfigKeys: FE_USER_KEY = 'user' FE_QUEUE_KEY = 'queue' FE_BGP_KEY = 'bgp' + FE_ROUTING_STATS_KEY = 'routing_stats' FE_REMOTE_DHCP_ENTRY = 'remote_dhcp_entry' FE_REMOTE_CAPSMAN_ENTRY = 'remote_capsman_entry' @@ -157,7 +159,7 @@ class MKTXPConfigKeys: BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, FE_KID_CONTROL_DEVICE, FE_KID_CONTROL_DYNAMIC, SSL_CERTIFICATE_VERIFY, FE_IPV6_ROUTE_KEY, FE_IPV6_DHCP_POOL_KEY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY, FE_BGP_KEY, - FE_IPSEC_KEY, FE_LTE_KEY, FE_SWITCH_PORT_KEY} + FE_IPSEC_KEY, FE_LTE_KEY, FE_SWITCH_PORT_KEY, FE_ROUTING_STATS_KEY} # Feature keys enabled by default BOOLEAN_KEYS_YES = {PLAINTEXT_LOGIN_KEY, FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, @@ -190,7 +192,7 @@ class ConfigEntry: MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_NEIGHBOR_KEY, MKTXPConfigKeys.FE_IPV6_ROUTE_KEY, MKTXPConfigKeys.FE_IPV6_DHCP_POOL_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY, MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_REMOTE_CAPSMAN_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_BGP_KEY, - MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_KID_CONTROL_DYNAMIC, MKTXPConfigKeys.FE_LTE_KEY, MKTXPConfigKeys.FE_IPSEC_KEY, MKTXPConfigKeys.FE_SWITCH_PORT_KEY + MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_KID_CONTROL_DYNAMIC, MKTXPConfigKeys.FE_LTE_KEY, MKTXPConfigKeys.FE_IPSEC_KEY, MKTXPConfigKeys.FE_SWITCH_PORT_KEY, MKTXPConfigKeys.FE_ROUTING_STATS_KEY ]) MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.LISTEN_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT, MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY, diff --git a/mktxp/cli/config/mktxp.conf b/mktxp/cli/config/mktxp.conf index 3d00ce35..f540b301 100644 --- a/mktxp/cli/config/mktxp.conf +++ b/mktxp/cli/config/mktxp.conf @@ -69,6 +69,7 @@ queue = True # Queues metrics bgp = False # BGP sessions metrics + routing_stats = False # routing process stats remote_dhcp_entry = None # An MKTXP entry to provide for remote DHCP info / resolution remote_capsman_entry = None # An MKTXP entry to provide for remote capsman info diff --git a/mktxp/collector/routing_stats_collector.py b/mktxp/collector/routing_stats_collector.py new file mode 100644 index 00000000..0862b4be --- /dev/null +++ b/mktxp/collector/routing_stats_collector.py @@ -0,0 +1,46 @@ +# coding=utf8 + +from mktxp.collector.base_collector import BaseCollector +from mktxp.datasource.routing_stats_ds import RoutingStatsMetricsDataSource +from mktxp.utils.utils import parse_mkt_uptime + + +class RoutingStatsCollector(BaseCollector): + '''Routing Stats collector''' + @staticmethod + def collect(router_entry): + if not router_entry.config_entry.routing_stats: + return + + routing_stats_labels = ['tasks', 'id', 'private_mem_blocks', 'shared_mem_blocks', 'kernel_time', 'process_time', 'max_busy', 'max_calc'] + translation_table = { + 'kernel_time': lambda value: parse_mkt_uptime(value), + 'process_time': lambda value: parse_mkt_uptime(value), + 'max_busy': lambda value: parse_mkt_uptime(value), + 'max_calc': lambda value: parse_mkt_uptime(value), + } + routing_stats_records = RoutingStatsMetricsDataSource.metric_records(router_entry, metric_labels=routing_stats_labels, translation_table = translation_table) + + if routing_stats_records: + session_info_labels = ['tasks', 'id', 'pid'] + routing_stats_processes_metrics = BaseCollector.info_collector('routing_stats_processes', 'Routing Process Stats', routing_stats_records, session_info_labels) + yield routing_stats_processes_metrics + + session_id_labels = ['tasks'] + routing_stats_private_mem_metrics = BaseCollector.gauge_collector('routing_stats_private_mem', 'Private Memory Blocks Used', routing_stats_records, 'private_mem_blocks', session_id_labels) + yield routing_stats_private_mem_metrics + + routing_stats_shared_mem_metrics = BaseCollector.gauge_collector('routing_stats_shared_mem', 'Shared Memory Blocks Used', routing_stats_records, 'shared_mem_blocks', session_id_labels) + yield routing_stats_shared_mem_metrics + + kernel_time_metrics = BaseCollector.counter_collector('routing_stats_kernel_time', 'Process Kernel Time', routing_stats_records, 'kernel_time', session_id_labels) + yield kernel_time_metrics + + process_time_metrics = BaseCollector.counter_collector('routing_stats_process_time', 'Process Time', routing_stats_records, 'process_time', session_id_labels) + yield process_time_metrics + + max_busy_metrics = BaseCollector.counter_collector('routing_stats_max_busy', 'Max Busy Time', routing_stats_records, 'max_busy', session_id_labels) + yield max_busy_metrics + + max_calc_metrics = BaseCollector.counter_collector('routing_stats_max_calc', 'Max Calc Time', routing_stats_records, 'max_calc', session_id_labels) + yield max_calc_metrics diff --git a/mktxp/datasource/routing_stats_ds.py b/mktxp/datasource/routing_stats_ds.py new file mode 100644 index 00000000..f79354ef --- /dev/null +++ b/mktxp/datasource/routing_stats_ds.py @@ -0,0 +1,27 @@ + +from mktxp.datasource.base_ds import BaseDSProcessor +from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource +from mktxp.utils.utils import routerOS7_version + + +class RoutingStatsMetricsDataSource: + ''' Wireless Metrics data provider + ''' + @staticmethod + def metric_records(router_entry, *, metric_labels = None, translation_table = None): + if metric_labels is None: + metric_labels = [] + try: + routing_stats = '/routing/stats/process' + + # legacy 6.x versions are untested + ver = SystemResourceMetricsDataSource.os_version(router_entry) + if not routerOS7_version(ver): + return + + routing_stats_records = router_entry.api_connection.router_api().get_resource(routing_stats).get() + return BaseDSProcessor.trimmed_records(router_entry, router_records = routing_stats_records, metric_labels = metric_labels, translation_table = translation_table) + except Exception as exc: + print(f'Error getting routing stats sessions info from router {router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}') + return None + diff --git a/mktxp/flow/collector_registry.py b/mktxp/flow/collector_registry.py index 771c7efe..56737e8d 100644 --- a/mktxp/flow/collector_registry.py +++ b/mktxp/flow/collector_registry.py @@ -39,6 +39,7 @@ from mktxp.collector.queue_collector import QueueSimpleCollector from mktxp.collector.kid_control_device_collector import KidDeviceCollector from mktxp.collector.bgp_collector import BGPCollector +from mktxp.collector.routing_stats_collector import RoutingStatsCollector from mktxp.collector.lte_collector import LTECollector from mktxp.collector.switch_collector import SwitchPortCollector @@ -80,6 +81,7 @@ def __init__(self): self.register(CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR, KidDeviceCollector.collect) self.register(CollectorKeys.BGP_COLLECTOR, BGPCollector.collect) + self.register(CollectorKeys.ROUTING_STATS_COLLECTOR, RoutingStatsCollector.collect) self.register(CollectorKeys.LTE_COLLECTOR, LTECollector.collect) self.register(CollectorKeys.SWITCH_PORT_COLLECTOR, SwitchPortCollector.collect) diff --git a/mktxp/flow/router_entry.py b/mktxp/flow/router_entry.py index d56ada19..f66f4f1b 100644 --- a/mktxp/flow/router_entry.py +++ b/mktxp/flow/router_entry.py @@ -73,6 +73,7 @@ def __init__(self, router_name): CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR: 0, CollectorKeys.USER_COLLECTOR: 0, CollectorKeys.BGP_COLLECTOR: 0, + CollectorKeys.ROUTING_STATS_COLLECTOR: 0, CollectorKeys.LTE_COLLECTOR: 0, CollectorKeys.SWITCH_PORT_COLLECTOR: 0, CollectorKeys.MKTXP_COLLECTOR: 0 From 4a9cee4f583835ae8f19efc5acab065c870a0f01 Mon Sep 17 00:00:00 2001 From: Ian Foster Date: Tue, 1 Oct 2024 11:36:59 -0700 Subject: [PATCH 3/5] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d36022d1..1963f788 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ The default configuration file comes with a sample configuration, making it easy queue = True # Queues metrics bgp = False # BGP sessions metrics + routing_stats = False # routing process stats remote_dhcp_entry = None # An MKTXP entry to provide for remote DHCP info / resolution remote_capsman_entry = None # An MKTXP entry to provide for remote capsman info From 9456e5fa090887bb5a5522acf082a08d83c0ebfd Mon Sep 17 00:00:00 2001 From: Ian Foster Date: Tue, 1 Oct 2024 11:41:54 -0700 Subject: [PATCH 4/5] Update README.md fix formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1963f788..74253a02 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ The default configuration file comes with a sample configuration, making it easy queue = True # Queues metrics bgp = False # BGP sessions metrics - routing_stats = False # routing process stats + routing_stats = False # routing process stats remote_dhcp_entry = None # An MKTXP entry to provide for remote DHCP info / resolution remote_capsman_entry = None # An MKTXP entry to provide for remote capsman info From 33d3bbee65b6e1a7198be2d8cc759b76ea271c61 Mon Sep 17 00:00:00 2001 From: Ian Foster Date: Tue, 1 Oct 2024 15:06:24 -0700 Subject: [PATCH 5/5] Update routing_stats_ds.py fix comment --- mktxp/datasource/routing_stats_ds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mktxp/datasource/routing_stats_ds.py b/mktxp/datasource/routing_stats_ds.py index f79354ef..92abd690 100644 --- a/mktxp/datasource/routing_stats_ds.py +++ b/mktxp/datasource/routing_stats_ds.py @@ -5,7 +5,7 @@ class RoutingStatsMetricsDataSource: - ''' Wireless Metrics data provider + ''' Routing Stats data provider ''' @staticmethod def metric_records(router_entry, *, metric_labels = None, translation_table = None):