Merge pull request #195 from lanrat/routing_stats

Add Routing Process Stats collector

.dockerignore

@@ -6,4 +6,6 @@ k8s
 build
 dist
 mktxp.egg-info
-tests
+tests/
+.github/
+deploy/

Dockerfile

@@ -1,10 +1,14 @@
 FROM python:3-alpine
 LABEL org.opencontainers.image.source github.com/akpw/mktxp
+RUN addgroup -S mktxp && adduser -S mktxp -G mktxp
+RUN apk add nano
+
 WORKDIR /mktxp
 COPY . .
-RUN pip install ./ && apk add nano
+RUN pip install ./
+
 EXPOSE 49090
-RUN addgroup -S mktxp && adduser -S mktxp -G mktxp
+
 USER mktxp
 ENTRYPOINT ["/usr/local/bin/mktxp"]
 CMD ["export"]

README.md

@@ -113,8 +113,9 @@ The default configuration file comes with a sample configuration, making it easy
     queue = True                    # Queues metrics
 
     bgp = False                     # BGP sessions metrics
+    routing_stats = False           # Routing process stats
     certificate = False             # Certificates metrics
-    
+   
     remote_dhcp_entry = None        # An MKTXP entry to provide for remote DHCP info / resolution
     remote_capsman_entry = None     # An MKTXP entry to provide for remote capsman info 
 

mktxp/cli/config/config.py

@@ -47,6 +47,7 @@ class CollectorKeys:
     KID_CONTROL_DEVICE_COLLECTOR = 'KidControlCollector'
     USER_COLLECTOR = 'UserCollector'
     BGP_COLLECTOR = 'BGPCollector'
+    ROUTING_STATS_COLLECTOR = 'RoutingStatsCollector'
     IPSEC_COLLECTOR = 'IPSecCollector'
     LTE_COLLECTOR = 'LTECollector'
     SWITCH_PORT_COLLECTOR = 'SwitchPortCollector'
@@ -103,6 +104,7 @@ class MKTXPConfigKeys:
     FE_USER_KEY = 'user'
     FE_QUEUE_KEY = 'queue'
     FE_BGP_KEY = 'bgp'
+    FE_ROUTING_STATS_KEY = 'routing_stats'
 
     FE_REMOTE_DHCP_ENTRY = 'remote_dhcp_entry'
     FE_REMOTE_CAPSMAN_ENTRY = 'remote_capsman_entry'
@@ -159,7 +161,7 @@ class MKTXPConfigKeys:
 
     BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, FE_KID_CONTROL_DEVICE, FE_KID_CONTROL_DYNAMIC,
                        SSL_CERTIFICATE_VERIFY, FE_IPV6_ROUTE_KEY, FE_IPV6_DHCP_POOL_KEY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY, FE_BGP_KEY,
-                       FE_IPSEC_KEY, FE_LTE_KEY, FE_SWITCH_PORT_KEY, FE_CERTIFICATE_KEY}
+                       FE_IPSEC_KEY, FE_LTE_KEY, FE_SWITCH_PORT_KEY, FE_ROUTING_STATS_KEY, FE_CERTIFICATE_KEY}
 
     # Feature keys enabled by default
     BOOLEAN_KEYS_YES = {PLAINTEXT_LOGIN_KEY, FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, 
@@ -192,7 +194,8 @@ class ConfigEntry:
                                                        MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_NEIGHBOR_KEY,
                                                        MKTXPConfigKeys.FE_IPV6_ROUTE_KEY, MKTXPConfigKeys.FE_IPV6_DHCP_POOL_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,                                               
                                                        MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_REMOTE_CAPSMAN_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_BGP_KEY,
-                                                       MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_KID_CONTROL_DYNAMIC, MKTXPConfigKeys.FE_LTE_KEY, MKTXPConfigKeys.FE_IPSEC_KEY, MKTXPConfigKeys.FE_SWITCH_PORT_KEY, MKTXPConfigKeys.FE_CERTIFICATE_KEY
+                                                       MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_KID_CONTROL_DYNAMIC, MKTXPConfigKeys.FE_LTE_KEY, MKTXPConfigKeys.FE_IPSEC_KEY, MKTXPConfigKeys.FE_SWITCH_PORT_KEY, 
+                                                       MKTXPConfigKeys.FE_ROUTING_STATS_KEY, MKTXPConfigKeys.FE_CERTIFICATE_KEY
                                                        ])
     MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.LISTEN_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
                                                        MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,

mktxp/cli/config/mktxp.conf

@@ -69,6 +69,7 @@
     queue = True                    # Queues metrics
 
     bgp = False                     # BGP sessions metrics
+    routing_stats = False           # routing process stats
     certificate = False             # Certificates metrics
 
     remote_dhcp_entry = None        # An MKTXP entry to provide for remote DHCP info / resolution

mktxp/collector/routing_stats_collector.py

@@ -0,0 +1,46 @@
+# coding=utf8
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.routing_stats_ds import RoutingStatsMetricsDataSource
+from mktxp.utils.utils import parse_mkt_uptime
+
+
+class RoutingStatsCollector(BaseCollector):
+    '''Routing Stats collector'''
+    @staticmethod
+    def collect(router_entry):
+        if not router_entry.config_entry.routing_stats:
+            return
+
+        routing_stats_labels = ['tasks', 'id', 'private_mem_blocks', 'shared_mem_blocks', 'kernel_time', 'process_time', 'max_busy', 'max_calc']
+        translation_table = {
+                'kernel_time': lambda value:  parse_mkt_uptime(value),
+                'process_time': lambda value:  parse_mkt_uptime(value),
+                'max_busy': lambda value:  parse_mkt_uptime(value),
+                'max_calc': lambda value:  parse_mkt_uptime(value),
+            }
+        routing_stats_records = RoutingStatsMetricsDataSource.metric_records(router_entry, metric_labels=routing_stats_labels, translation_table = translation_table)
+
+        if routing_stats_records:
+            session_info_labels = ['tasks', 'id', 'pid']
+            routing_stats_processes_metrics = BaseCollector.info_collector('routing_stats_processes', 'Routing Process Stats', routing_stats_records, session_info_labels)
+            yield routing_stats_processes_metrics
+
+            session_id_labels = ['tasks']
+            routing_stats_private_mem_metrics = BaseCollector.gauge_collector('routing_stats_private_mem', 'Private Memory Blocks Used', routing_stats_records, 'private_mem_blocks', session_id_labels)
+            yield routing_stats_private_mem_metrics
+
+            routing_stats_shared_mem_metrics = BaseCollector.gauge_collector('routing_stats_shared_mem', 'Shared Memory Blocks Used', routing_stats_records, 'shared_mem_blocks', session_id_labels)
+            yield routing_stats_shared_mem_metrics
+
+            kernel_time_metrics = BaseCollector.counter_collector('routing_stats_kernel_time', 'Process Kernel Time', routing_stats_records, 'kernel_time', session_id_labels)
+            yield kernel_time_metrics
+
+            process_time_metrics = BaseCollector.counter_collector('routing_stats_process_time', 'Process Time', routing_stats_records, 'process_time', session_id_labels)
+            yield process_time_metrics
+
+            max_busy_metrics = BaseCollector.counter_collector('routing_stats_max_busy', 'Max Busy Time', routing_stats_records, 'max_busy', session_id_labels)
+            yield max_busy_metrics
+
+            max_calc_metrics = BaseCollector.counter_collector('routing_stats_max_calc', 'Max Calc Time', routing_stats_records, 'max_calc', session_id_labels)
+            yield max_calc_metrics

mktxp/datasource/routing_stats_ds.py

@@ -0,0 +1,27 @@
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource
+from mktxp.utils.utils import routerOS7_version
+
+
+class RoutingStatsMetricsDataSource:
+    ''' Routing Stats data provider
+    '''             
+    @staticmethod
+    def metric_records(router_entry, *, metric_labels = None, translation_table = None):
+        if metric_labels is None:
+            metric_labels = []                
+        try:
+            routing_stats = '/routing/stats/process'
+
+            # legacy 6.x versions are untested
+            ver = SystemResourceMetricsDataSource.os_version(router_entry)
+            if not routerOS7_version(ver):
+                return
+
+            routing_stats_records = router_entry.api_connection.router_api().get_resource(routing_stats).get()
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = routing_stats_records, metric_labels = metric_labels, translation_table = translation_table)
+        except Exception as exc:
+            print(f'Error getting routing stats sessions info from router {router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None
+

mktxp/flow/collector_registry.py

@@ -39,6 +39,7 @@
 from mktxp.collector.queue_collector import QueueSimpleCollector
 from mktxp.collector.kid_control_device_collector import KidDeviceCollector
 from mktxp.collector.bgp_collector import BGPCollector
+from mktxp.collector.routing_stats_collector import RoutingStatsCollector
 from mktxp.collector.lte_collector import LTECollector
 from mktxp.collector.switch_collector import SwitchPortCollector
 from mktxp.collector.certificate_collector import CertificateCollector
@@ -81,6 +82,7 @@ def __init__(self):
 
         self.register(CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR, KidDeviceCollector.collect)
         self.register(CollectorKeys.BGP_COLLECTOR, BGPCollector.collect)
+        self.register(CollectorKeys.ROUTING_STATS_COLLECTOR, RoutingStatsCollector.collect)
         self.register(CollectorKeys.LTE_COLLECTOR, LTECollector.collect)
         self.register(CollectorKeys.SWITCH_PORT_COLLECTOR, SwitchPortCollector.collect)
 

mktxp/flow/router_entry.py

@@ -73,6 +73,7 @@ def __init__(self, router_name):
                             CollectorKeys.KID_CONTROL_DEVICE_COLLECTOR: 0,
                             CollectorKeys.USER_COLLECTOR: 0,
                             CollectorKeys.BGP_COLLECTOR: 0,
+                            CollectorKeys.ROUTING_STATS_COLLECTOR: 0,
                             CollectorKeys.LTE_COLLECTOR: 0,
                             CollectorKeys.SWITCH_PORT_COLLECTOR: 0,
                             CollectorKeys.MKTXP_COLLECTOR: 0,