Merge branch 'main' into routing_stats

README.md

@@ -113,8 +113,9 @@ The default configuration file comes with a sample configuration, making it easy
     queue = True                    # Queues metrics
 
     bgp = False                     # BGP sessions metrics
-    routing_stats = False           # routing process stats
-    
+    routing_stats = False           # Routing process stats
+    certificate = False             # Certificates metrics
+   
     remote_dhcp_entry = None        # An MKTXP entry to provide for remote DHCP info / resolution
     remote_capsman_entry = None     # An MKTXP entry to provide for remote capsman info 
 
@@ -472,7 +473,4 @@ mktxp is running as pid 36704
 
 **Running Tests**
 - TDB
-- Run via: `$ python setup.py test`
-
-
-
+- Run via: `$ python setup.py test`

deploy/kubernetes/secret.yaml

@@ -110,7 +110,8 @@ stringData:
             queue = True                    # Queues metrics
 
             bgp = False                     # BGP sessions metrics
-            
+            certificate = False             # Certificates metrics
+
             remote_dhcp_entry = None        # An MKTXP entry to provide for remote DHCP info / resolution
             remote_capsman_entry = None     # An MKTXP entry to provide for remote capsman info 
 

mktxp/cli/config/config.py

@@ -52,6 +52,7 @@ class CollectorKeys:
     LTE_COLLECTOR = 'LTECollector'
     SWITCH_PORT_COLLECTOR = 'SwitchPortCollector'
     MKTXP_COLLECTOR = 'MKTXPCollector'
+    CERTIFICATE_COLLECTOR = 'CertificateCollector'
 
 
 class MKTXPConfigKeys:
@@ -112,6 +113,7 @@ class MKTXPConfigKeys:
 
     FE_KID_CONTROL_DEVICE = 'kid_control_assigned'
     FE_KID_CONTROL_DYNAMIC = 'kid_control_dynamic'
+    FE_CERTIFICATE_KEY = 'certificate'
 
     MKTXP_SOCKET_TIMEOUT = 'socket_timeout'
     MKTXP_INITIAL_DELAY = 'initial_delay_on_failure'
@@ -159,7 +161,7 @@ class MKTXPConfigKeys:
 
     BOOLEAN_KEYS_NO = {ENABLED_KEY, SSL_KEY, NO_SSL_CERTIFICATE, FE_CHECK_FOR_UPDATES, FE_KID_CONTROL_DEVICE, FE_KID_CONTROL_DYNAMIC,
                        SSL_CERTIFICATE_VERIFY, FE_IPV6_ROUTE_KEY, FE_IPV6_DHCP_POOL_KEY, FE_IPV6_FIREWALL_KEY, FE_IPV6_NEIGHBOR_KEY, FE_CONNECTION_STATS_KEY, FE_BGP_KEY,
-                       FE_IPSEC_KEY, FE_LTE_KEY, FE_SWITCH_PORT_KEY, FE_ROUTING_STATS_KEY}
+                       FE_IPSEC_KEY, FE_LTE_KEY, FE_SWITCH_PORT_KEY, FE_ROUTING_STATS_KEY, FE_CERTIFICATE_KEY}
 
     # Feature keys enabled by default
     BOOLEAN_KEYS_YES = {PLAINTEXT_LOGIN_KEY, FE_DHCP_KEY, FE_PACKAGE_KEY, FE_DHCP_LEASE_KEY, FE_IP_CONNECTIONS_KEY, FE_INTERFACE_KEY, 
@@ -192,7 +194,8 @@ class ConfigEntry:
                                                        MKTXPConfigKeys.FE_ROUTE_KEY, MKTXPConfigKeys.FE_DHCP_POOL_KEY, MKTXPConfigKeys.FE_FIREWALL_KEY, MKTXPConfigKeys.FE_NEIGHBOR_KEY,
                                                        MKTXPConfigKeys.FE_IPV6_ROUTE_KEY, MKTXPConfigKeys.FE_IPV6_DHCP_POOL_KEY, MKTXPConfigKeys.FE_IPV6_FIREWALL_KEY, MKTXPConfigKeys.FE_IPV6_NEIGHBOR_KEY,                                               
                                                        MKTXPConfigKeys.FE_USER_KEY, MKTXPConfigKeys.FE_QUEUE_KEY, MKTXPConfigKeys.FE_REMOTE_DHCP_ENTRY, MKTXPConfigKeys.FE_REMOTE_CAPSMAN_ENTRY, MKTXPConfigKeys.FE_CHECK_FOR_UPDATES, MKTXPConfigKeys.FE_BGP_KEY,
-                                                       MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_KID_CONTROL_DYNAMIC, MKTXPConfigKeys.FE_LTE_KEY, MKTXPConfigKeys.FE_IPSEC_KEY, MKTXPConfigKeys.FE_SWITCH_PORT_KEY, MKTXPConfigKeys.FE_ROUTING_STATS_KEY
+                                                       MKTXPConfigKeys.FE_KID_CONTROL_DEVICE, MKTXPConfigKeys.FE_KID_CONTROL_DYNAMIC, MKTXPConfigKeys.FE_LTE_KEY, MKTXPConfigKeys.FE_IPSEC_KEY, MKTXPConfigKeys.FE_SWITCH_PORT_KEY, 
+                                                       MKTXPConfigKeys.FE_ROUTING_STATS_KEY, MKTXPConfigKeys.FE_CERTIFICATE_KEY
                                                        ])
     MKTXPSystemEntry = namedtuple('MKTXPSystemEntry', [MKTXPConfigKeys.PORT_KEY, MKTXPConfigKeys.LISTEN_KEY, MKTXPConfigKeys.MKTXP_SOCKET_TIMEOUT,
                                                        MKTXPConfigKeys.MKTXP_INITIAL_DELAY, MKTXPConfigKeys.MKTXP_MAX_DELAY,

mktxp/cli/config/mktxp.conf

@@ -70,10 +70,10 @@
 
     bgp = False                     # BGP sessions metrics
     routing_stats = False           # routing process stats
+    certificate = False             # Certificates metrics
 
     remote_dhcp_entry = None        # An MKTXP entry to provide for remote DHCP info / resolution
     remote_capsman_entry = None     # An MKTXP entry to provide for remote capsman info 
 
     use_comments_over_names = True  # when available, forces using comments over the interfaces names
     check_for_updates = False       # check for available ROS updates
-

mktxp/collector/certificate_collector.py

@@ -0,0 +1,39 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+from mktxp.collector.base_collector import BaseCollector
+from mktxp.datasource.certificate_ds import CertificateMetricsDataSource
+from datetime import datetime
+
+class CertificateCollector(BaseCollector):
+    '''Certificate collector'''
+    @staticmethod
+    def collect(router_entry):
+        if not router_entry.config_entry.certificate:
+            return
+
+        certificate_labels = ['name', 'digest_algorithm', 'key_type', 'country', 'state', 'locality', 'organization', 
+                                'common_name', 'key_size', 'subject_alt_name', 'days_valid', 'trusted', 'key_usage', 
+                                'ca', 'serial_number', 'key_usage', 'ca', 'serial_number', 'fingerprint', 'akid', 'skid', 
+                                'invalid_before', 'invalid_after']
+
+        translation_table = {
+        }
+
+        certificate_records = CertificateMetricsDataSource.metric_records(router_entry, translation_table=translation_table, metric_labels = certificate_labels)   
+        if isinstance(certificate_records, list): 
+            for record in certificate_records:
+                # Convert invalid_after time to epoch time
+                record['invalid_after_epoch'] = int(datetime.strptime(record['invalid_after'], "%Y-%m-%d %H:%M:%S").timestamp())
+        if certificate_records:
+            yield BaseCollector.gauge_collector('certificate_expiration_timestamp_seconds', 'The number of seconds before expiration time the certificate should renew.', certificate_records, 'invalid_after_epoch', certificate_labels)

mktxp/collector/public_ip_collector.py

@@ -23,14 +23,13 @@ def collect(router_entry):
         if not router_entry.config_entry.public_ip:
             return
 
-        address_labels = ['public_address', 'dns_name']
+        address_labels = ['public_address', 'public_address_ipv6', 'dns_name']
         address_records = PublicIPAddressDatasource.metric_records(router_entry, metric_labels=address_labels)
 
         if address_records:
             for address_record in address_records:
                 if not 'dns_name' in address_record:
                     address_record['dns_name'] = 'ddns disabled'
-                    
+
             address_metrics = BaseCollector.info_collector('public_ip_address', 'Public IP address', address_records, address_labels)
             yield address_metrics
-

mktxp/collector/switch_collector.py

@@ -27,8 +27,13 @@ def collect(router_entry):
                               'rx_bytes', 'rx_broadcast', 'rx_pause', 'rx_multicast', 'rx_fcs_error', 'rx_align_error', 'rx_fragment', 'rx_overflow', 
                               'tx_bytes', 'tx_broadcast', 'tx_pause', 'tx_multicast', 'tx_underrun', 'tx_collision', 'tx_deferred']
 
-        switch_port_records = SwitchPortMetricsDataSource.metric_records(router_entry, metric_labels = switch_port_labels)   
+        switch_port_records = SwitchPortMetricsDataSource.metric_records(router_entry, metric_labels = switch_port_labels)
         if switch_port_records:
+            for record in switch_port_records:
+                for field in switch_port_labels[1:]:
+                    if field in record and ',' in record[field]: # https://help.mikrotik.com/docs/display/ROS/Switch+Chip+Features#SwitchChipFeatures-Statistics
+                        # Sum each CPU lane for the total
+                        record[field] = str(sum([int(count) for count in record[field].split(",")]))
             yield BaseCollector.counter_collector('switch_driver_rx_byte', 'Total count of received bytes', switch_port_records, 'driver_rx_byte', ['name'])
             yield BaseCollector.counter_collector('switch_driver_rx_packet', 'Total count of received packets', switch_port_records, 'driver_rx_packet', ['name'])
             yield BaseCollector.counter_collector('switch_driver_tx_byte', 'Total count of transmitted bytes', switch_port_records, 'driver_tx_byte', ['name'])

mktxp/datasource/certificate_ds.py

@@ -0,0 +1,31 @@
+# coding=utf8
+## Copyright (c) 2020 Arseniy Kuznetsov
+##
+## This program is free software; you can redistribute it and/or
+## modify it under the terms of the GNU General Public License
+## as published by the Free Software Foundation; either version 2
+## of the License, or (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+
+
+from mktxp.datasource.base_ds import BaseDSProcessor
+
+
+class CertificateMetricsDataSource:
+    '''Certificates Metrics data provider
+    '''             
+    @staticmethod    
+    def metric_records(router_entry, *, metric_labels = None, translation_table=None):
+        if metric_labels is None:
+            metric_labels = []                
+        try:
+            certificates_records = router_entry.api_connection.router_api().get_resource('/certificate').call('print', {'detail':''})
+            return BaseDSProcessor.trimmed_records(router_entry, router_records = certificates_records, 
+                                                            metric_labels = metric_labels, translation_table=translation_table)
+        except Exception as exc:
+            print(f'Error getting certificates info from router {router_entry.router_name}@{router_entry.config_entry.hostname}: {exc}')
+            return None

mktxp/flow/collector_registry.py

@@ -42,6 +42,7 @@
 from mktxp.collector.routing_stats_collector import RoutingStatsCollector
 from mktxp.collector.lte_collector import LTECollector
 from mktxp.collector.switch_collector import SwitchPortCollector
+from mktxp.collector.certificate_collector import CertificateCollector
 
 class CollectorRegistry:
     ''' MKTXP Collectors Registry
@@ -85,7 +86,10 @@ def __init__(self):
         self.register(CollectorKeys.LTE_COLLECTOR, LTECollector.collect)
         self.register(CollectorKeys.SWITCH_PORT_COLLECTOR, SwitchPortCollector.collect)
 
+        self.register(CollectorKeys.CERTIFICATE_COLLECTOR, CertificateCollector.collect)
+
         self.register(CollectorKeys.MKTXP_COLLECTOR, MKTXPCollector.collect)
+
 
     def register(self, collector_ID, collect_func):
         self.registered_collectors[collector_ID] = collect_func

mktxp/flow/router_entry.py

@@ -76,7 +76,8 @@ def __init__(self, router_name):
                             CollectorKeys.ROUTING_STATS_COLLECTOR: 0,
                             CollectorKeys.LTE_COLLECTOR: 0,
                             CollectorKeys.SWITCH_PORT_COLLECTOR: 0,
-                            CollectorKeys.MKTXP_COLLECTOR: 0
+                            CollectorKeys.MKTXP_COLLECTOR: 0,
+                            CollectorKeys.CERTIFICATE_COLLECTOR: 0
                             }         
         self._dhcp_entry = None        
         self._dhcp_records = {}