Add binaries and optional GUI

Move source files to ./Source/
Add binaries for Windows and Linux
Add an optional GUI for Windows

Author: ddbnl

Linux/AuditLogCollector

@@ -1,6 +1,6 @@
 # Standard libs
-import collections
 import os
+import sys
 import json
 import logging
 import datetime
@@ -16,7 +16,7 @@
 
 class AuditLogCollector(ApiConnection.ApiConnection):
 
-    def __init__(self, content_types, *args, resume=True, fallback_time=None,
+    def __init__(self, *args, content_types=None, resume=True, fallback_time=None,
                  file_output=False, output_path=None,
                  graylog_output=False, graylog_address=None, graylog_port=None,
                  azure_oms_output=False, azure_oms_workspace_id=None, azure_oms_shared_key=None,
@@ -41,38 +41,46 @@ def __init__(self, content_types, *args, resume=True, fallback_time=None,
         self.graylog_output = graylog_output
         self.azure_oms_output = azure_oms_output
         self.output_path = output_path
-        self.content_types = content_types
+        self.content_types = content_types or collections.deque()
         self._last_run_times = {}
+        self.resume = resume
         if resume:
             self.get_last_run_times()
         self._fallback_time = fallback_time or datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=1)
         self._known_content = {}
-        if self.azure_oms_output:
-            self._azure_oms_interface = AzureOMSInterface.AzureOMSInterface(workspace_id=azure_oms_workspace_id,
-                                                                            shared_key=azure_oms_shared_key)
-        if self.graylog_output:
-            self._graylog_interface = GraylogInterface.GraylogInterface(graylog_address=graylog_address,
-                                                                        graylog_port=graylog_port)
+        self._azure_oms_interface = AzureOMSInterface.AzureOMSInterface(workspace_id=azure_oms_workspace_id,
+                                                                        shared_key=azure_oms_shared_key)
+        self._graylog_interface = GraylogInterface.GraylogInterface(graylog_address=graylog_address,
+                                                                    graylog_port=graylog_port)
         self.blobs_to_collect = collections.defaultdict(collections.deque)
         self.monitor_thread = threading.Thread()
         self.retrieve_available_content_threads = collections.deque()
         self.retrieve_content_threads = collections.deque()
+        self.run_started = None
         self.logs_retrieved = 0
 
     def run_once(self, start_time=None):
         """
         Check available content and retrieve it, then exit.
         """
-        run_started = datetime.datetime.now()
+        self._known_content.clear()
+        self.logs_retrieved = 0
+        self._graylog_interface.successfully_sent = 0
+        self._graylog_interface.unsuccessfully_sent = 0
+        self._azure_oms_interface.successfully_sent = 0
+        self._azure_oms_interface.unsuccessfully_sent = 0
+        self.run_started = datetime.datetime.now()
         self._clean_known_content()
+        if self.resume:
+            self.get_last_run_times()
         self.start_monitoring()
         self.get_all_available_content(start_time=start_time)
         self.monitor_thread.join()
-        if self._last_run_times:
+        if self.resume and self._last_run_times:
             with open('last_run_times', 'w') as ofile:
                 json.dump(fp=ofile, obj=self._last_run_times)
         logging.info("Finished. Total logs retrieved: {}. Run time: {}.".format(
-            self.logs_retrieved, datetime.datetime.now() - run_started))
+            self.logs_retrieved, datetime.datetime.now() - self.run_started))
         if self.azure_oms_output:
             logging.info("Azure OMS output report: {} successfully sent, {} errors".format(
                 self._azure_oms_interface.successfully_sent, self._azure_oms_interface.unsuccessfully_sent))
@@ -127,7 +135,7 @@ def get_all_available_content(self, start_time=None):
         """
         for content_type in self.content_types.copy():
             if not start_time:
-                if content_type in self._last_run_times.keys():
+                if self.resume and content_type in self._last_run_times.keys():
                     start_time = self._last_run_times[content_type]
                 else:
                     start_time = self._fallback_time
@@ -336,8 +344,12 @@ def known_content(self):
     elif argsdict['time_hours']:
         fallback_time = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=argsdict['time_hours'])
 
-    logging.basicConfig(filemode='w', filename=argsdict['log_path'],
-                        level=logging.INFO if not argsdict['debug_logging'] else logging.DEBUG)
+    logger = logging.getLogger()
+    fileHandler = logging.FileHandler(argsdict['log_path'], mode='w')
+    streamHandler = logging.StreamHandler(sys.stdout)
+    logger.addHandler(streamHandler)
+    logger.addHandler(fileHandler)
+    logger.setLevel(logging.INFO if not argsdict['debug_logging'] else logging.DEBUG)
     logging.log(level=logging.INFO, msg='Starting run @ {0}'.format(datetime.datetime.now()))
 
     collector = AuditLogCollector(

Linux/AuditLogSubscriber

@@ -31,23 +31,33 @@ def get_sub_status(self):
         status = self.make_api_request(url='subscriptions/list', append_url=True)
         return status.json()
 
-    def set_sub_status(self, ctype_stat):
+    def set_sub_status(self, ctype_stat=None, content_type=None, action=None):
         """
         Args:
             ctype_stat (tuple): content type, status (enabled | disabled)
-
         Returns:
             dict
         """
-        if ctype_stat[1] == 'enabled':
-            action = 'stop'
-        elif ctype_stat[1] == 'disabled':
-            action = 'start'
-        else:
-            return
-        status = self.make_api_request(url='subscriptions/{0}?contentType={1}'.format(action, ctype_stat[0]),
+        content_type = content_type or ctype_stat[0]
+        if not action:
+            if ctype_stat[1] == 'enabled':
+                action = 'stop'
+            elif ctype_stat[1] == 'disabled':
+                action = 'start'
+            else:
+                return
+        status = self.make_api_request(url='subscriptions/{0}?contentType={1}'.format(action, content_type),
                                        append_url=True, get=False)
-        logging.info("Set sub status response: {}".format(status))
+        logging.debug("Set sub status response: {}".format(status))
+        try:
+            logging.debug("Set sub status json: {}".format(status.json()))
+        except Exception as e:
+            pass
+        if 200 <= status.status_code <= 299:
+            logging.info('Successfully set sub status: {} > {}'.format(content_type, action))
+        else:
+            raise RuntimeError("Unable to set sub status: {} > {}".format(content_type, action))
+        status.close()
 
     def interactive(self):
 

ApiConnection.py Source/ApiConnection.py

@@ -129,7 +129,12 @@ def post_data(self, body, log_type, time_generated):
             'time-generated-field': time_generated
         }
         response = self.session.post(uri, data=body, headers=headers)
-        status_code, json_output = response.status_code, response.json
+        status_code = response.status_code
+        try:
+            json_output = response.json()
+        except:
+            json_output = ''
+
         response.close()
         if 200 <= status_code <= 299:
             logging.info('Accepted payload:' + body)