diff --git a/ConfigExamples/azureLogAnalytics.yaml b/ConfigExamples/azureLogAnalytics.yaml new file mode 100644 index 0000000..7693cff --- /dev/null +++ b/ConfigExamples/azureLogAnalytics.yaml @@ -0,0 +1,14 @@ +collect: # Settings determining which audit logs to collect and how to do it + contentTypes: + Audit.General: True + Audit.AzureActiveDirectory: True + Audit.Exchange: True + Audit.SharePoint: True + DLP.All: True + skipKnownLogs: True + resume: True +output: + azureLogAnalytics: + enabled: True + workspaceId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + sharedKey: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx \ No newline at end of file diff --git a/ConfigExamples/fileOutput.yaml b/ConfigExamples/fileOutput.yaml index 614b8b5..c8ec0b2 100644 --- a/ConfigExamples/fileOutput.yaml +++ b/ConfigExamples/fileOutput.yaml @@ -1,5 +1,3 @@ -log: - path: 'collector.log' collect: contentTypes: Audit.General: True @@ -7,10 +5,8 @@ collect: Audit.Exchange: True Audit.SharePoint: True DLP.All: True - autoSubscribe: True skipKnownLogs: True resume: True - hoursToCollect: 24 output: file: enabled: True diff --git a/ConfigExamples/filteredFileOutput.yaml b/ConfigExamples/filteredFileOutput.yaml index 9bd824e..0dc68fe 100644 --- a/ConfigExamples/filteredFileOutput.yaml +++ b/ConfigExamples/filteredFileOutput.yaml @@ -1,25 +1,19 @@ -log: - path: 'collector.log' collect: contentTypes: Audit.General: True Audit.AzureActiveDirectory: True - Audit.Exchange: True Audit.SharePoint: True - DLP.All: True - autoSubscribe: True skipKnownLogs: True resume: True - hoursToCollect: 24 # Collect logs concerning spoofing prevention in Audit.General, deleted files from Audit.SharePoint # and login failures from Audit.AzureActiveDirectory filter: Audit.General: - - Policy: Spoof + Policy: Spoof Audit.AzureActiveDirectory: - - Operation: UserLoginFailed + Operation: UserLoginFailed Audit.SharePoint: - - Operation: FileDeleted + Operation: FileDeleted # Output only to file output: file: diff --git a/Source/config.yaml b/ConfigExamples/fullConfig.yaml similarity index 83% rename from Source/config.yaml rename to ConfigExamples/fullConfig.yaml index 2c4269a..1a9faf0 100644 --- a/Source/config.yaml +++ b/ConfigExamples/fullConfig.yaml @@ -8,8 +8,10 @@ collect: # Settings determining which audit logs to collect and how to do it Audit.Exchange: True Audit.SharePoint: True DLP.All: True - maxThreads: 20 - autoSubscribe: True # Automatically subscribe to collected content types + maxThreads: 50 + retries: 3 # Times to retry retrieving a content blob if it fails + retryCooldown: 3 # Seconds to wait before retrying retrieving a content blob + autoSubscribe: True # Automatically subscribe to collected content types. Never unsubscribes from anything. skipKnownLogs: True # Remember retrieved log ID's, don't collect them twice resume: True # Remember last run time, resume collecting from there next run hoursToCollect: 24 # Look back this many hours for audit logs (can be overwritten by resume) diff --git a/ConfigExamples/graylog.yaml b/ConfigExamples/graylog.yaml index 6826ffe..13da73b 100644 --- a/ConfigExamples/graylog.yaml +++ b/ConfigExamples/graylog.yaml @@ -1,5 +1,3 @@ -log: - path: 'collector.log' collect: contentTypes: Audit.General: True @@ -7,10 +5,8 @@ collect: Audit.Exchange: True Audit.SharePoint: True DLP.All: True - autoSubscribe: True skipKnownLogs: True resume: True - hoursToCollect: 24 output: graylog: enabled: False diff --git a/ConfigExamples/prtg.yaml b/ConfigExamples/prtg.yaml index 0ac483b..4f5c0ed 100644 --- a/ConfigExamples/prtg.yaml +++ b/ConfigExamples/prtg.yaml @@ -1,14 +1,9 @@ -log: - path: 'collector.log' collect: contentTypes: Audit.General: True Audit.AzureActiveDirectory: True - Audit.Exchange: True Audit.SharePoint: True - DLP.All: True - autoSubscribe: True - skipKnownLogs: True + skipKnownLogs: False # Take all logs each time to count the number of active filter hits each interval resume: False # Take all logs each time to count the number of active filter hits each interval hoursToCollect: 1 # Period over which to alert, e.g. failed AAD logins over the last hour # The PRTG output defines channels which have filters associated to them. The output of the channel will be @@ -20,12 +15,12 @@ output: - name: Deleted Sharepoint files filters: Audit.SharePoint: - - Operation: FileDeleted + Operation: FileDeleted - name: Failed Azure AD logins filters: Audit.AzureActiveDirectory: - - Operation: UserLoginFailed + Operation: UserLoginFailed - name: Spoof attempts prevented filters: Audit.General: - - Policy: Spoof \ No newline at end of file + Policy: Spoof \ No newline at end of file diff --git a/Linux/AuditLogCollector b/Linux/AuditLogCollector deleted file mode 100644 index a6b819f..0000000 Binary files a/Linux/AuditLogCollector and /dev/null differ diff --git a/Linux/AuditLogSubscriber b/Linux/OfficeAuditLogCollector-V1.1 similarity index 96% rename from Linux/AuditLogSubscriber rename to Linux/OfficeAuditLogCollector-V1.1 index 2371bee..8db8bdf 100644 Binary files a/Linux/AuditLogSubscriber and b/Linux/OfficeAuditLogCollector-V1.1 differ diff --git a/README.md b/README.md index 69463d6..9f26865 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,41 @@ -# Office365 API audit log collector - -Collect Office365 and Azure audit logs through their respective APIs. No prior knowledge of APIs is required, -onboarding and script usage is described below. There is a GUI for Windows. Currently supports the following outputs: -- Azure Analytics Workspace (OMS) -- Graylog (or any other source that accepts a simple socket connection) -- File - -Simply download the executable(s) you need from the Windows or Linux folder.: +# Office365 audit log collector + +Collect/retrieve Office365, Azure and DLP audit logs, optionally filter them, then send them to one or more outputs such as file, PRTG, Azure Log Analytics or Graylog. +Onboarding is easy and takes only a few minutes (steps described below). There are Windows and Linux executables, and an optional GUI for Windows only. +Easy configuration with a YAML config file (see the 'ConfigExamples' folder for reference). +If you have any issues or questions, feel free to create an issue in this repo. +- The following Audit logs can be extracted: + - Audit.General + - Audit.AzureActiveDirectory + - Audit.Exchange + - Audit.SharePoint + - DLP.All +- The following outputs are supported: + - Azure Analytics Workspace (OMS) + - PRTG Network Monitor + - Graylog (or any other source that accepts a simple socket connection) + - Local file + +Simply download the executable you need from the Windows or Linux folder and copy a config file from the ConfigExamples folder that suits your need: - Windows: - - GUI - Office Audit Log Collector.exe - - GUI for collecting audit logs AND subscribing to audit log feeds (see onboarding instructions below) - - Office Audit Log Collector.exe - - Command line tool for collecting audit logs (see syntax below) - - Office Audit Log Subscriber.exe - - Command line tool for subscribing to audit logs feeds (see onboarding instructions below) + - GUI-OfficeAuditLogCollector.exe + - GUI for collecting audit logs and subscribing to audit log feeds + - OfficeAuditLogCollector.exe + - Command line tool for collecting audit logs and (automatically) subscribing to audit log feeds - Linux: - OfficeAuditLogCollector - - Command line tool for collecting audit logs (see syntax below) - - OfficeAuditLogSubscriber - - Command line tool for subscribing to audit logs (see onboarding instructions below) + - Command line tool for collecting audit logs and (automatically) subscribing to audit log feeds + +Find onboarding instructions and more detailed instructions for using the executables below. + +For a full audit trail, schedule to run the collector on a regular basis (preferably at least once every day). Previously +retrieved logs can be remembered to prevent duplicates. Consider using the following parameters in the config file for a robust audit trail: +- skipKnownLogs: True (prevent duplicates) +- hoursToCollect: 24 (the maximum, or a number larger than the amount of hours between runs, for safety overlap) +- resume: False (don't resume where the last run stopped, have some overlap in case anything was missed for any reason) +See below for a more detailed instruction of the config file. -For a full audit trail schedule to run the script on a regular basis (preferably at least once every day). The last -run time is recorded automatically, so that when the script runs again it starts to retrieve audit logs from when it last ran. -Feel free to contribute other outputs if you happen to build any. +Lastly, feel free to contribute other outputs if you happen to build any. Also open to any other useful pull requests! See the following link for more info on the management APIs: https://msdn.microsoft.com/en-us/office-365/office-365-management-activity-api-reference. ## Roadmap: @@ -32,6 +45,9 @@ See the following link for more info on the management APIs: https://msdn.micros - Create a tutorial for automatic onboarding + docker container for the easiest way to run this ## Latest changes: +- Added PRTG output +- Added filters +- Added YAML config file - Added a GUI for Windows - Added executables for Windows and Linux - Added Azure Log Analytics Workspace OMS output @@ -48,94 +64,87 @@ See the following link for more info on the management APIs: https://msdn.micros - Ad-lib log retrieval; - Scheduling regular execution to retrieve the full audit trail. +- Output to PRTG for alerts on audit logs ## Features: -- Subscribe to the audit logs of your choice through the subscription script; +- Subscribe to the audit logs of your choice through the --interactive-subscriber switch, or automatically when collecting logs; - Collect General, Exchange, Sharepoint, Azure active directory and/or DLP audit logs through the collector script; -- Output to file or to a Graylog input (i.e. send the logs over a network socket) +- Output to file, PRTG, Azure Log Analytics or to a Graylog input (i.e. send the logs over a network socket). ## Requirements: - Office365 tenant; -- Azure app registration created for this script (see instructions) -- AzureAD tenant ID; -- Client key of the new Azure app registration; +- Azure app registration created for this script (see onboarding instructions) - Secret key (created in the new Azure app registration, see instructions); - App permissions to access the APIs for the new Azure application (see instructions); -- Subscription to the APIs of your choice (General/Sharepoint/Exchange/AzureAD/DLP, run AuditLogSubscription script and follow the instructions). +- Subscription to the APIs of your choice (use autoSubscribe option in the config file to automate this). ## Instructions: -### Onboarding: -- Create an app registration: - - Create the app registration itself under Azure AD (own tenant only works fine for single tenant) - - Create app secret (only shown once upon creation, store it somewhere safe) - - Grant your new app permissions to read the Office API's: - - Graph: AuditLog.Read.All - - Office 365 Management APIs: ActivityFeed.Read - - Office 365 Management APIs: ActivityFeed.ReadDlp +### Onboarding (one time only): - Make sure Auditing is turned on for your tenant! - - https://docs.microsoft.com/en-us/microsoft-365/compliance/turn-audit-log-search-on-or-off?view=o365-worldwide + - Use these instructions: https://docs.microsoft.com/en-us/microsoft-365/compliance/turn-audit-log-search-on-or-off?view=o365-worldwide - If you had to turn it on, it may take a few hours to process -- Use the 'AuditLogSubscriber' script to subscribe to the audit API's of your choice - - You will need tenant id, client key and secret key for this - - Simply follow the instructions -- You can now run the script and retrieve logs. - +- Create App registration: + - Azure AD > 'App registrations' > 'New registration': + - Choose any name for the registration + - Choose "Accounts in this organizational directory only (xyz only - Single tenant)" + - Hit 'register' + - Save 'Tenant ID' and 'Application (Client) ID' from the overview page of the new registration, you will need it to run the collector +- Create app secret: + - Azure AD > 'App registrations' > Click your new app registration > 'Certificates and secrets' > 'New client secret': + - Choose any name and expire date and hit 'add' + - Actual key is only shown once upon creation, store it somewhere safe. You will need it to run the collector. +- Grant your new app registration 'application' permissions to read the Office API's: + - Azure AD > 'App registrations' > Click your new app registration > 'API permissions' > 'Add permissions' > 'Office 365 Management APIs' > 'Application permissions': + - Check 'ActivityFeed.Read' + - Check 'ActivityFeed.ReadDlp' + - Hit 'Add permissions' +- Subscribe to audit log feeds of your choice + - Set 'autoSubscribe: True' in YAML config file to automate this. + - OR Use the '--interactive-subscriber' parameter when executing the collector to manually subscribe to the audit API's of your choice +- You can now run the collector and retrieve logs. + + +### Running the collector: + +Running from the GUI should be self-explanatory. It can run once or on a schedule. Usually you will want to use the +command-line executable with a config file, and schedule it for periodic execution (e.g. through CRON, windows task +scheduler, or a PRTG sensor). + +To run the command-line executable use the following syntax: + +OfficeAuditLogCollector(.exe) %tenant_id% %client_key% %secret_key% --config %path/to/config.yaml% + +To create a config file you can start with the 'fullConfig.yaml' from the ConfigExamples folder. This has all the +possible options and some explanatory comments. Cross-reference with a config example using the output(s) of your choice, and you +should be set. ### (optional) Creating an Azure Log Analytics Workspace (OMS): -If you are running this script to get audit events in an Azure Analytics Workspace you will a Workspace ID and a shared key. -Create a workspace from "Create resource" in Azure (no configuration required). Then get the ID and key from "Agent management". -You do not need to prepare any tables or other settings. - +If you are running this script to get audit events in an Azure Analytics Workspace you will need a Workspace ID and a shared key. +- Create a workspace from "Create resource" in Azure (no configuration required); +- Get the ID and key from "Agent management"; +- You do not need to prepare any tables or other settings. + +### (optional) Creating a PRTG sensor + +To run with PRTG you must create a sensor: +- Copy the OfficeAuditLogCollector.exe executable to the "\Custom Sensors\EXE" sub folder of your PRTG installation +- Create a device in PRTG with any host name (e.g. "Office Audit Logs") +- Create a 'EXE/Script Advanced Sensor' on that device and choose the executable you just copied +- Enter parameters, e.g.: "*tenant_id* *client_key* *secret_key* --config *full/path/to/config.yaml*" +(use full path, because PRTG will execute the script from a different working directory) +- Copy the prtg.config from ConfigExamples and modify at least the channel names and filters for your needs. +- Set the timeout of the script to something generous that suits the amount of logs you will retrieve. +Probably at least 300 seconds. Run the script manually first to check how long it takes. +- Match the interval of the sensor to the amount of hours of logs to retrieve. If your interval is 1 hour, hoursToCollect +in the config file should also be set to one hour. ### (optional) Creating a Graylog input -If you are running this script to get audit events in Graylog you will need to create a Graylog input. If not, just skip this. - +If you are running this script to get audit events in Graylog you will need to create a Graylog input. - Create a 'raw/plaintext TCP' input; - Enter the IP and port you want to receive the logs on (you can use these in the script); - All other settings can be left default. - -### Running the script: - -- Retrieve all logs and send to a network socket / Graylog server: -`python3 AuditLogCollector.py 'tenant_id' 'client_key' 'secret_key' --exchange --dlp --azure_ad --general --sharepoint -p 'random_publisher_id' -g -gA 10.10.10.1 -gP 6000` - -#### Script options: -``` -usage: AuditLogCollector.py [-h] [--general] [--exchange] [--azure_ad] - [--sharepoint] [--dlp] [-p publisher_id] - [-l log_path] [-f] [-fP file_output_path] [-g] - [-gA graylog_address] [-gP graylog_port] - tenant_id client_key secret_key` - -positional arguments: - tenant_id Tenant ID of Azure AD - client_key Client key of Azure application - secret_key Secret key generated by Azure application` - -optional arguments: - -h, --help show this help message and exit - --general Retrieve General content - --exchange Retrieve Exchange content - --azure_ad Retrieve Azure AD content - --sharepoint Retrieve SharePoint content - --dlp Retrieve DLP content - -r Resume looking for content from last run time for each content type (takes precedence over -tH and -tD) - -tH Number of hours to to go back and look for content - -tD Number of days to to go back and look for content - -p publisher_id Publisher GUID to avoid API throttling - -l log_path Path of log file - -f Output to file. - -fP file_output_path Path of directory of output files - -a Output to Azure Log Analytics workspace - -aC ID of log analytics workspace. - -aS Shared key of log analytics workspace. - -g Output to graylog. - -gA graylog_address Address of graylog server. - -gP graylog_port Port of graylog server. - -d Enable debug logging (large log files and lower performance) -``` \ No newline at end of file diff --git a/Source/AuditLogCollector.py b/Source/AuditLogCollector.py index fff6c8e..b3f269c 100644 --- a/Source/AuditLogCollector.py +++ b/Source/AuditLogCollector.py @@ -1,8 +1,10 @@ from Interfaces import AzureOMSInterface, GraylogInterface, PRTGInterface +import AuditLogSubscriber import ApiConnection import os import sys import yaml +import time import json import logging import datetime @@ -14,15 +16,17 @@ class AuditLogCollector(ApiConnection.ApiConnection): def __init__(self, content_types=None, resume=True, fallback_time=None, skip_known_logs=True, - log_path='collector.log', debug=False, auto_subscribe=False, max_threads=20, - file_output=False, output_path=None, graylog_output=False, azure_oms_output=False, prtg_output=False, - **kwargs): + log_path='collector.log', debug=False, auto_subscribe=False, max_threads=20, retries=3, + retry_cooldown=3, file_output=False, output_path=None, graylog_output=False, azure_oms_output=False, + prtg_output=False, **kwargs): """ Object that can retrieve all available content blobs for a list of content types and then retrieve those blobs and output them to a file or Graylog input (i.e. send over a socket). :param content_types: list of content types to retrieve (e.g. 'Audit.Exchange', 'Audit.Sharepoint') :param resume: Resume from last known run time for each content type (Bool) :param fallback_time: if no last run times are found to resume from, run from this start time (Datetime) + :param retries: Times to retry retrieving a content blob if it fails (int) + :param retry_cooldown: Seconds to wait before retrying retrieving a content blob (int) :param skip_known_logs: record retrieved content blobs and log ids, skip them next time (Bool) :param file_output: path of file to output audit logs to (str) :param log_path: path of file to log to (str) @@ -38,6 +42,8 @@ def __init__(self, content_types=None, resume=True, fallback_time=None, skip_kno self.resume = resume self._fallback_time = fallback_time or datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta( hours=23) + self.retries = retries + self.retry_cooldown = retry_cooldown self.skip_known_logs = skip_known_logs self.log_path = log_path self.debug = debug @@ -64,6 +70,7 @@ def __init__(self, content_types=None, resume=True, fallback_time=None, skip_kno self.retrieve_content_threads = collections.deque() self.run_started = None self.logs_retrieved = 0 + self.errors_retrieving = 0 @property def all_content_types(self): @@ -105,6 +112,10 @@ def _load_collect_config(self, config): config['collect']['contentTypes'][x] is True] if 'maxThreads' in config['collect']: self.max_threads = config['collect']['maxThreads'] + if 'retries' in config['collect']: + self.retries = config['collect']['retries'] + if 'retryCooldown' in config['collect']: + self.retry_cooldown = config['collect']['retryCooldown'] if 'autoSubscribe' in config['collect']: self.auto_subscribe = config['collect']['autoSubscribe'] if 'skipKnownLogs' in config['collect']: @@ -192,6 +203,8 @@ def _prepare_to_run(self): """ Make sure that self.run_once can be called multiple times by resetting to an initial state. """ + if self.auto_subscribe: + self._auto_subscribe() if self.resume: self._get_last_run_times() if self.skip_known_logs: @@ -237,8 +250,8 @@ def _log_statistics(self): """ Write run statistics to log file / console. """ - logging.info("Finished. Total logs retrieved: {}. Run time: {}.".format( - self.logs_retrieved, datetime.datetime.now() - self.run_started)) + logging.info("Finished. Total logs retrieved: {}. Total logs with errors: {}. Run time: {}.".format( + self.logs_retrieved, self.errors_retrieving, datetime.datetime.now() - self.run_started)) if self.azure_oms_output: logging.info("Azure OMS output report: {} successfully sent, {} errors".format( self.azure_oms_interface.successfully_sent, self.azure_oms_interface.unsuccessfully_sent)) @@ -287,9 +300,26 @@ def _start_monitoring(self): """ Start a thread monitoring the list containing blobs that need collecting. """ - self.monitor_thread = threading.Thread(target=self.monitor_blobs_to_collect, daemon=True) + self.monitor_thread = threading.Thread(target=self._monitor_blobs_to_collect, daemon=True) self.monitor_thread.start() + def _auto_subscribe(self): + """ + Subscribe to all content types that are set to be retrieved. + """ + subscriber = AuditLogSubscriber.AuditLogSubscriber(tenant_id=self.tenant_id, client_key=self.client_key, + secret_key=self.secret_key) + status = subscriber.get_sub_status() + if status == '': + raise RuntimeError("Auto subscribe enabled but could not get subscription status") + unsubscribed_content_types = self.content_types.copy() + for s in status: + if s['contentType'] in self.content_types and s['status'].lower() == 'enabled': + unsubscribed_content_types.remove(s['contentType']) + for content_type in unsubscribed_content_types: + logging.info("Auto subscribing to: {}".format(content_type)) + subscriber.set_sub_status(content_type=content_type, action='start') + def _get_all_available_content(self, start_time=None): """ Start a thread to retrieve available content blobs for each content type to be collected. @@ -316,7 +346,8 @@ def _get_available_content(self, content_type, start_time): current_time = datetime.datetime.now(datetime.timezone.utc) formatted_end_time = str(current_time).replace(' ', 'T').rsplit('.', maxsplit=1)[0] formatted_start_time = str(start_time).replace(' ', 'T').rsplit('.', maxsplit=1)[0] - logging.info("Start time: {}. End time: {}.".format(formatted_start_time, formatted_end_time)) + logging.info("Retrieving {}. Start time: {}. End time: {}.".format( + content_type, formatted_start_time, formatted_end_time)) response = self.make_api_request(url='subscriptions/content?contentType={0}&startTime={1}&endTime={2}'.format( content_type, formatted_start_time, formatted_end_time)) self.blobs_to_collect[content_type] += response.json() @@ -343,7 +374,7 @@ def _start_interfaces(self): if self.graylog_output: self.graylog_interface.start() - def stop_interfaces(self): + def _stop_interfaces(self): if self.azure_oms_output: self.azure_oms_interface.stop() @@ -352,7 +383,7 @@ def stop_interfaces(self): if self.graylog_output: self.graylog_interface.stop() - def monitor_blobs_to_collect(self): + def _monitor_blobs_to_collect(self): """ Wait for the 'retrieve_available_content' function to retrieve content URI's. Once they become available start retrieving in a background thread. @@ -363,15 +394,17 @@ def monitor_blobs_to_collect(self): threads = [thread for thread in threads if thread.is_alive()] if self.done_collecting_available_content and self.done_retrieving_content and not threads: break - if not self.blobs_to_collect or len(threads) >= self.max_threads: + if not self.blobs_to_collect: continue for content_type, blobs_to_collect in self.blobs_to_collect.copy().items(): + if len(threads) >= self.max_threads: + break if self.blobs_to_collect[content_type]: blob_json = self.blobs_to_collect[content_type].popleft() - self.collect_blob(blob_json=blob_json, content_type=content_type, threads=threads) - self.stop_interfaces() + self._collect_blob(blob_json=blob_json, content_type=content_type, threads=threads) + self._stop_interfaces() - def collect_blob(self, blob_json, content_type, threads): + def _collect_blob(self, blob_json, content_type, threads): """ Collect a single content blob in a thread. :param blob_json: JSON @@ -381,17 +414,17 @@ def collect_blob(self, blob_json, content_type, threads): if blob_json and 'contentUri' in blob_json: logging.log(level=logging.DEBUG, msg='Retrieving content blob: "{0}"'.format(blob_json)) threads.append(threading.Thread( - target=self.retrieve_content, daemon=True, - kwargs={'content_json': blob_json, 'content_type': content_type})) + target=self._retrieve_content, daemon=True, + kwargs={'content_json': blob_json, 'content_type': content_type, 'retries': self.retries})) threads[-1].start() - def retrieve_content(self, content_json, content_type): + def _retrieve_content(self, content_json, content_type, retries): """ Get an available content blob. If it exists in the list of known content blobs it is skipped to ensure idempotence. :param content_json: JSON dict of the content blob as retrieved from the API (dict) :param content_type: Type of API being retrieved for, e.g. 'Audit.Exchange' (str) - :return: + :param retries: Times to retry retrieving a content blob if it fails (int) """ if self.skip_known_logs and self.known_content and content_json['contentId'] in self.known_content: return @@ -400,8 +433,13 @@ def retrieve_content(self, content_json, content_type): if not results: return except Exception as e: - logging.error("Error retrieving content: {}".format(e)) - return + if retries: + time.sleep(self.retry_cooldown) + return self._retrieve_content(content_json=content_json, content_type=content_type, retries=retries - 1) + else: + self.errors_retrieving += 1 + logging.error("Error retrieving content: {}".format(e)) + return else: self._handle_retrieved_content(content_json=content_json, content_type=content_type, results=results) @@ -421,18 +459,18 @@ def _handle_retrieved_content(self, content_json, content_type, results): results.remove(log) continue self.known_logs[log['Id']] = log['CreationTime'] - if self.filters and not self.check_filters(log=log, content_type=content_type): + if self.filters and not self._check_filters(log=log, content_type=content_type): results.remove(log) self.logs_retrieved += len(results) - self.output_results(results=results, content_type=content_type) + self._output_results(results=results, content_type=content_type) - def output_results(self, results, content_type): + def _output_results(self, results, content_type): """ :param content_type: Type of API being retrieved for, e.g. 'Audit.Exchange' (str) :param results: list of JSON """ if self.file_output: - self.output_results_to_file(results=results) + self._output_results_to_file(*results) if self.prtg_output: self.prtg_interface.send_messages(*results, content_type=content_type) if self.graylog_output: @@ -440,7 +478,7 @@ def output_results(self, results, content_type): if self.azure_oms_output: self.azure_oms_interface.send_messages(*results, content_type=content_type) - def check_filters(self, log, content_type): + def _check_filters(self, log, content_type): """ :param log: JSON :param content_type: Type of API being retrieved for, e.g. 'Audit.Exchange' (str) @@ -452,13 +490,14 @@ def check_filters(self, log, content_type): return False return True - def output_results_to_file(self, results): + def _output_results_to_file(self, *results): """ Dump received JSON messages to a file. :param results: retrieved JSON (dict) """ - with open(self.output_path, 'a') as ofile: - ofile.write("{}\n".format(json.dump(obj=results, fp=ofile))) + for result in results: + with open(self.output_path, 'a') as ofile: + ofile.write("{}\n".format(json.dumps(obj=result))) def _add_known_log(self): """ @@ -572,6 +611,8 @@ def known_content(self): parser.add_argument('secret_key', type=str, help='Secret key generated by Azure application', action='store') parser.add_argument('--config', metavar='config', type=str, help='Path to YAML config file', action='store', dest='config') + parser.add_argument('--interactive-subscriber', action='store_true', + help='Manually (un)subscribe to audit log feeds', dest='interactive_subscriber') parser.add_argument('--general', action='store_true', help='Retrieve General content', dest='general') parser.add_argument('--exchange', action='store_true', help='Retrieve Exchange content', dest='exchange') parser.add_argument('--azure_ad', action='store_true', help='Retrieve Azure AD content', dest='azure_ad') @@ -610,6 +651,12 @@ def known_content(self): args = parser.parse_args() argsdict = vars(args) + if argsdict['interactive_subscriber']: + subscriber = AuditLogSubscriber.AuditLogSubscriber( + tenant_id=argsdict['tenant_id'], secret_key=argsdict['secret_key'], client_key=argsdict['client_key']) + subscriber.interactive() + quit(0) + content_types = [] if argsdict['general']: content_types.append('Audit.General') diff --git a/Source/GUI.py b/Source/GUI.py index 2615146..6053d79 100644 --- a/Source/GUI.py +++ b/Source/GUI.py @@ -5,11 +5,18 @@ from kivy.clock import Clock from kivy.properties import StringProperty import os +import sys import json import time import logging import threading import datetime +root_path = os.path.split(__file__)[0] +if getattr(sys, 'frozen', False): + icon_path = os.path.join(sys._MEIPASS, os.path.join(root_path, "icon.ico")) +else: + icon_path = os.path.join(root_path, "icon.ico") +Config.set('kivy','window_icon', icon_path) Config.set('graphics', 'resizable', False) Config.set('graphics', 'width', '450') Config.set('graphics', 'height', '600') @@ -154,9 +161,10 @@ def guid_example(self): def build(self): + self.icon = icon_path self.theme_cls.theme_style = "Dark" from UX import MainWidget - Builder.load_file(os.path.join(os.path.split(__file__)[0], 'UX/MainWidget.kv')) + Builder.load_file(os.path.join(root_path, 'UX/MainWidget.kv')) self.root_widget = MainWidget.MainWidget() prefix = self.root_widget.ids.tab_widget prefix.ids.config_widget.ids.clear_known_content.disabled = not os.path.exists('known_content') @@ -231,14 +239,16 @@ def save_settings(self): settings['gl_address'] = prefix.ids.config_widget.ids.graylog_ip.text settings['gl_port'] = prefix.ids.config_widget.ids.graylog_port.text settings['debug_logging'] = prefix.ids.config_widget.ids.log_switch.active - with open('gui_settings.json', 'w') as ofile: + settings_file = os.path.join(root_path, 'gui_settings.json') + with open(settings_file, 'w') as ofile: json.dump(settings, ofile) def load_settings(self): - if not os.path.exists('gui_settings.json'): + settings_file = os.path.join(root_path, 'gui_settings.json') + if not os.path.exists(settings_file): return - with open('gui_settings.json', 'r') as ofile: + with open(settings_file, 'r') as ofile: settings = json.load(ofile) prefix = self.root_widget.ids.tab_widget self.tenant_id = settings['tenant_id'] diff --git a/Source/Interfaces/PRTGInterface.py b/Source/Interfaces/PRTGInterface.py index 024d8a5..463b488 100644 --- a/Source/Interfaces/PRTGInterface.py +++ b/Source/Interfaces/PRTGInterface.py @@ -14,20 +14,19 @@ def __init__(self, config=None, **kwargs): self.config = config self.results = collections.defaultdict(collections.deque) - def _send_message(self, message, content_type, **kwargs): + def _send_message(self, msg, content_type, **kwargs): for channel in self.config['channels']: if content_type not in channel['filters']: continue - self._filter_result(message=message, content_type=content_type, channel=channel) + self._filter_result(msg=msg, content_type=content_type, channel=channel) - def _filter_result(self, message, content_type, channel): + def _filter_result(self, msg, content_type, channel): - for filter_rule in channel['filters'][content_type]: - for filter_key, filter_value in filter_rule.items(): - if filter_key not in message or filter_value.lower() != message[filter_key].lower(): - return - self.results[channel['name']].append(message) + for filter_key, filter_value in channel['filters'][content_type].items(): + if filter_key not in msg or filter_value.lower() != msg[filter_key].lower(): + return + self.results[channel['name']].append(msg) def output(self): try: diff --git a/Source/icon.ico b/Source/icon.ico new file mode 100644 index 0000000..7fadbf9 Binary files /dev/null and b/Source/icon.ico differ diff --git a/requirements.txt b/Source/requirements.txt similarity index 66% rename from requirements.txt rename to Source/requirements.txt index e034ddb..67dee77 100644 Binary files a/requirements.txt and b/Source/requirements.txt differ diff --git a/Windows/GUI - Office Audit Log Collector.exe b/Windows/GUI-OfficeAuditLogCollector-v1.1.exe similarity index 80% rename from Windows/GUI - Office Audit Log Collector.exe rename to Windows/GUI-OfficeAuditLogCollector-v1.1.exe index bffa194..6273b11 100644 Binary files a/Windows/GUI - Office Audit Log Collector.exe and b/Windows/GUI-OfficeAuditLogCollector-v1.1.exe differ diff --git a/Windows/Office Audit Log Collector.exe b/Windows/Office Audit Log Collector.exe deleted file mode 100644 index c752b96..0000000 Binary files a/Windows/Office Audit Log Collector.exe and /dev/null differ diff --git a/Windows/Office Audit Log Subscriber.exe b/Windows/OfficeAuditLogCollector-V1.1.exe similarity index 95% rename from Windows/Office Audit Log Subscriber.exe rename to Windows/OfficeAuditLogCollector-V1.1.exe index 7decd73..552d6fa 100644 Binary files a/Windows/Office Audit Log Subscriber.exe and b/Windows/OfficeAuditLogCollector-V1.1.exe differ