DS refactor, fixes/optimizations

akpw · akpw · commit cb2ff3c1a560 · 2021-02-07T15:40:49.000+01:00
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@
 
 ## Description
 MKTXP is a Prometheus Exporter for Mikrotik RouterOS devices.\
-It gathers a rich set of supported metrics across multiple routers, all easily configurable via built-in CLI interface. 
+It gathers and exports a rich set of metrics across multiple routers, all easily configurable via built-in CLI interface. 
 
 Apart from exporting to Prometheus, MKTXP can also print some of the metrics directly on the command line (see an example below).
 
diff --git a/mktxp/cli/config/_mktxp.conf b/mktxp/cli/config/_mktxp.conf
@@ -12,10 +12,10 @@
 
 
 [MKTXP]
-    port = 49090		# default metrics HTTP server port
+    port = 49090					# default metrics HTTP server port
     bandwidth_test_interval = 420	# Interval between periodic bandwidth tests
 
-    socket_timeout = 2	# Socket connection timeout
+    socket_timeout = 2				# Socket connection timeout
     initial_delay_on_failure = 120	# Delay untill next connection attempt to a RouterOS Device
     max_delay_on_failure = 900		# Max delay untill next connection attempt to a RouterOS Device
     delay_inc_div = 5				# Delay increment factor
diff --git a/mktxp/cli/config/mktxp.conf b/mktxp/cli/config/mktxp.conf
@@ -36,4 +36,4 @@
     capsman = True                  # CAPsMAN general metrics
     capsman_clients = True          # CAPsMAN clients metrics
 
-    use_comments_over_names = False  # when available, forces using comments over the interfaces names 
+    use_comments_over_names = True  # when available, forces using comments over the interfaces names 
diff --git a/mktxp/cli/dispatch.py b/mktxp/cli/dispatch.py
@@ -13,13 +13,10 @@
 ## GNU General Public License for more details.
 
 
-import sys
 import subprocess
-import mktxp.cli.checks.chk_pv
-from mktxp.utils.utils import run_cmd
+from mktxp.cli.config.config import config_handler
 from mktxp.cli.options import MKTXPOptionsParser, MKTXPCommands
-from mktxp.cli.config.config import config_handler, ConfigEntry
-from mktxp.basep import MKTXPProcessor, MKTXPCLIProcessor
+from mktxp.processor.mktxp import MKTXPProcessor, MKTXPCLIProcessor
 
 
 class MKTXPDispatcher:
@@ -104,8 +101,7 @@ def print(self, args):
 
         if args['dhcp_clients']:
             MKTXPCLIProcessor.dhcp_clients(args['entry_name'])
-
-
+            
 
 def main():
     MKTXPDispatcher().dispatch()
diff --git a/mktxp/cli/output/capsman_out.py b/mktxp/cli/output/capsman_out.py
@@ -12,8 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from tabulate import tabulate   
-from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanRegistrationsMetricsDataSource
 
@@ -43,14 +42,23 @@ def clients_summary(router_entry):
             else:
                 dhcp_rt_by_interface[interface] = [registration_record]         
 
-        num_records = 0
-        output_table = []
+
+        output_entry = BaseOutputProcessor.OutputCapsmanEntry
+        output_table = BaseOutputProcessor.output_table(output_entry)
+
+        output_records = 0
+        registration_records = len(registration_records)                
         for key in dhcp_rt_by_interface.keys():
             for record in dhcp_rt_by_interface[key]:
-                output_table.append(BaseOutputProcessor.OutputCapsmanEntry(**record))
-                num_records += 1
-            output_table.append({})
-        print()
-        print(tabulate(output_table, headers = "keys",  tablefmt="github"))
-        print(tabulate([{0:'Connected Wifi Devices:', 1:num_records}], tablefmt="text"))
+                output_table.add_row(output_entry(**record))
+                output_records += 1
+            if output_records < registration_records:
+                output_table.add_row(output_entry())
+                
+        print (output_table.draw())
+
+        for server in dhcp_rt_by_interface.keys():
+            print(f'{server} clients: {len(dhcp_rt_by_interface[server])}')
+        print(f'Total connected CAPsMAN clients: {output_records}', '\n')
+
 
diff --git a/mktxp/cli/output/dhcp_out.py b/mktxp/cli/output/dhcp_out.py
@@ -12,8 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from tabulate import tabulate   
-from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 
 
@@ -35,17 +34,22 @@ def clients_summary(router_entry):
                 dhcp_by_server[server].append(dhcp_lease_record)
             else:
                 dhcp_by_server[server] = [dhcp_lease_record]         
+        
+        output_entry = BaseOutputProcessor.OutputDHCPEntry
+        output_table = BaseOutputProcessor.output_table(output_entry)
 
-        num_records = 0
-        output_table = []
+        output_records = 0
+        lease_records = len(dhcp_lease_records)        
         for key in dhcp_by_server.keys():
             for record in dhcp_by_server[key]:
                 record['host_name'] = BaseOutputProcessor.dhcp_name(router_entry, record, drop_comment = True)
-                output_table.append(BaseOutputProcessor.OutputDHCPEntry(**record))
-                num_records += 1
-            output_table.append({})
-        print()
-        print(tabulate(output_table, headers = "keys",  tablefmt="github"))
-        print(tabulate([{0:'DHCP Clients:', 1:num_records}], tablefmt="text"))
-
-
+                output_table.add_row(output_entry(**record))
+                output_records += 1
+            if output_records < lease_records:
+                output_table.add_row(output_entry())
+                
+        print (output_table.draw())
+
+        for server in dhcp_by_server.keys():
+            print(f'{server} clients: {len(dhcp_by_server[server])}')
+        print(f'Total DHCP clients: {output_records}', '\n')
diff --git a/mktxp/cli/output/wifi_out.py b/mktxp/cli/output/wifi_out.py
@@ -12,8 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from tabulate import tabulate   
-from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
 
@@ -43,13 +42,21 @@ def clients_summary(router_entry):
             else:
                 dhcp_rt_by_interface[interface] = [registration_record]         
 
-        num_records = 0
-        output_table = []
+        output_entry = BaseOutputProcessor.OutputWiFiEntry
+        output_table = BaseOutputProcessor.output_table(output_entry)
+
+        output_records = 0
+        registration_records = len(registration_records)                
         for key in dhcp_rt_by_interface.keys():
             for record in dhcp_rt_by_interface[key]:
-                output_table.append(BaseOutputProcessor.OutputWiFiEntry(**record))
-                num_records += 1
-            output_table.append({})
-        print()
-        print(tabulate(output_table, headers = "keys",  tablefmt="github"))
-        print(tabulate([{0:'Connected Wifi Devices:', 1:num_records}], tablefmt="text"))
+                output_table.add_row(output_entry(**record))
+                output_records += 1
+            if output_records < registration_records:
+                output_table.add_row(output_entry())
+                
+        print (output_table.draw())
+
+        for server in dhcp_rt_by_interface.keys():
+            print(f'{server} clients: {len(dhcp_rt_by_interface[server])}')
+        print(f'Total connected WiFi devices: {output_records}', '\n')
+
diff --git a/mktxp/collector/capsman_collector.py b/mktxp/collector/capsman_collector.py
@@ -12,8 +12,8 @@
 ## GNU General Public License for more details.
 
 
-from mktxp.cli.output.base_out import BaseOutputProcessor
 from mktxp.cli.config.config import MKTXPConfigKeys
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.collector.base_collector import BaseCollector
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.capsman_ds import CapsmanCapsMetricsDataSource, CapsmanRegistrationsMetricsDataSource
diff --git a/mktxp/collector/monitor_collector.py b/mktxp/collector/monitor_collector.py
@@ -13,7 +13,7 @@
 
 
 from mktxp.collector.base_collector import BaseCollector
-from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.datasource.interface_ds import InterfaceMonitorMetricsDataSource
 
 
diff --git a/mktxp/collector/resource_collector.py b/mktxp/collector/resource_collector.py
@@ -13,7 +13,7 @@
 
 
 from mktxp.collector.base_collector import BaseCollector
-from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.datasource.system_resource_ds import SystemResourceMetricsDataSource
 
 
diff --git a/mktxp/collector/wlan_collector.py b/mktxp/collector/wlan_collector.py
@@ -12,7 +12,7 @@
 ## GNU General Public License for more details.
 
 
-from mktxp.cli.output.base_out import BaseOutputProcessor
+from mktxp.processor.output import BaseOutputProcessor
 from mktxp.collector.base_collector import BaseCollector
 from mktxp.datasource.dhcp_ds import DHCPMetricsDataSource
 from mktxp.datasource.wireless_ds import WirelessMetricsDataSource
diff --git a/mktxp/flow/__init__.py b/mktxp/flow/__init__.py
diff --git a/mktxp/flow/collectors_handler.py b/mktxp/flow/collectors_handler.py
diff --git a/mktxp/flow/router_connection.py b/mktxp/flow/router_connection.py
diff --git a/mktxp/flow/router_entries_handler.py b/mktxp/flow/router_entries_handler.py
@@ -13,7 +13,7 @@
 
 
 from mktxp.cli.config.config import config_handler
-from mktxp.router_entry import RouterEntry
+from mktxp.flow.router_entry import RouterEntry
 
 
 class RouterEntriesHandler:
diff --git a/mktxp/flow/router_entry.py b/mktxp/flow/router_entry.py
@@ -13,7 +13,7 @@
 
 
 from mktxp.cli.config.config import config_handler, MKTXPConfigKeys
-from mktxp.router_connection import RouterAPIConnection
+from mktxp.flow.router_connection import RouterAPIConnection
 
 
 class RouterEntry:
diff --git a/mktxp/processor/__init__.py b/mktxp/processor/__init__.py
diff --git a/mktxp/processor/mktxp.py b/mktxp/processor/mktxp.py
@@ -15,10 +15,10 @@
 from http.server import HTTPServer
 from datetime import datetime
 from prometheus_client.core import REGISTRY
-from prometheus_client import MetricsHandler, start_http_server
+from prometheus_client import MetricsHandler
 from mktxp.cli.config.config import config_handler
-from mktxp.collectors_handler import CollectorsHandler
-from mktxp.router_entries_handler import RouterEntriesHandler
+from mktxp.flow.collectors_handler import CollectorsHandler
+from mktxp.flow.router_entries_handler import RouterEntriesHandler
 
 from mktxp.cli.output.capsman_out import CapsmanOutput
 from mktxp.cli.output.wifi_out import WirelessOutput
diff --git a/mktxp/processor/output.py b/mktxp/processor/output.py
@@ -11,17 +11,23 @@
 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ## GNU General Public License for more details.
 
-import re
+import re, os
 from datetime import timedelta
 from collections import namedtuple
+from texttable import Texttable
 from humanize import naturaldelta
 from mktxp.cli.config.config import config_handler
 
 
 class BaseOutputProcessor:
     OutputCapsmanEntry = namedtuple('OutputCapsmanEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'rx_signal', 'interface', 'ssid', 'tx_rate', 'rx_rate', 'uptime'])
+    OutputCapsmanEntry.__new__.__defaults__ = ('',) * len(OutputCapsmanEntry._fields)
+
     OutputWiFiEntry = namedtuple('OutputWiFiEntry', ['dhcp_name', 'dhcp_address', 'mac_address', 'signal_strength', 'signal_to_noise', 'interface', 'tx_rate', 'rx_rate', 'uptime'])
+    OutputWiFiEntry.__new__.__defaults__ = ('',) * len(OutputWiFiEntry._fields)
+
     OutputDHCPEntry = namedtuple('OutputDHCPEntry', ['host_name', 'server', 'mac_address', 'address', 'active_address', 'expires_after'])
+    OutputDHCPEntry.__new__.__defaults__ = ('',) * len(OutputDHCPEntry._fields)
 
     @staticmethod
     def augment_record(router_entry, registration_record, dhcp_lease_records):
@@ -110,3 +116,15 @@ def parse_interface_rate(interface_rate):
             config_handler.re_compiled['interface_rate_rgx'] = interface_rate_rgx
         rate = lambda interface_rate: 1000 if interface_rate.find('Mbps') < 0 else 1
         return(int(float(interface_rate_rgx.sub('', interface_rate)) * rate(interface_rate)))
+
+    @staticmethod
+    def output_table(outputEntry = None):
+        table = Texttable(max_width = os.get_terminal_size().columns)
+        table.set_deco(Texttable.HEADER | Texttable.BORDER | Texttable.VLINES )        
+        if outputEntry:
+            table.header(outputEntry._fields)
+            table.set_cols_align(['l']+ ['c']*(len(outputEntry._fields)-1))
+        return table
+
+
+
diff --git a/setup.py b/setup.py
@@ -20,7 +20,7 @@
 
 setup(
     name='mktxp',
-    version='0.24',
+    version='0.25',
 
     url='https://github.com/akpw/mktxp',
 
@@ -47,7 +47,7 @@
                         'RouterOS-api>=0.17.0', 
                         'configobj>=5.0.6',
                         'humanize>=3.2.0',
-                        'tabulate>=0.8.7',
+                        'texttable>=1.6.3',
                         'speedtest-cli>=2.1.2'
                         ],
 
