From 981a08c0f5e27df2740530a76625100a1f3e75b1 Mon Sep 17 00:00:00 2001 From: deoren Date: Mon, 6 Aug 2018 08:04:27 -0500 Subject: [PATCH] Template INI files, cmdline config path support - repo-provided INI files have been renamed to *.tmpl in order to prevent them from being used as-is or risk clobbering local/custom config files if user opts to install directly from Git repo. - Adjust log formatters to include line number for messages - Replace explicit exit calls from within our library function calls. - Adjust system-wide config path from /etc/DOMAIN/PROJECT to just /etc/PROJECT in order to better match the norm for /etc content layout. - Improve exception handling for config file parsing issues - Provide support for user-level config files - cmdline - $HOME/.config/PROJECT - Fall back to logging warning/error messages to the console if settings object is not provided to constructor. Adjust filter for handler later once the settings object is properly constructed. refs WhyAskWhy/mysql2sqlite-dev#2 refs WhyAskWhy/mysql2sqlite#3 --- mysql2sqlite.py | 107 ++++++++++++++---- ...neral.ini => mysql2sqlite_general.ini.tmpl | 0 mysql2sqlite_lib.py | 105 +++++++++++------ ...eries.ini => mysql2sqlite_queries.ini.tmpl | 0 4 files changed, 159 insertions(+), 53 deletions(-) rename mysql2sqlite_general.ini => mysql2sqlite_general.ini.tmpl (100%) rename mysql2sqlite_queries.ini => mysql2sqlite_queries.ini.tmpl (100%) diff --git a/mysql2sqlite.py b/mysql2sqlite.py index e0b49f8..3cbba95 100644 --- a/mysql2sqlite.py +++ b/mysql2sqlite.py @@ -23,6 +23,7 @@ # parse command line arguments, 'sys.argv' import argparse +import configparser import logging import logging.handlers import os @@ -30,13 +31,15 @@ import sqlite3 import sys +from collections import OrderedDict + app_name = 'mysql2sqlite' # TODO: Configure formatter to log function/class info -syslog_formatter = logging.Formatter('%(name)s - %(levelname)s - %(funcName)s - %(message)s') -file_formatter = logging.Formatter('%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s') -stdout_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(funcName)s - %(message)s') +syslog_formatter = logging.Formatter('%(name)s - L%(lineno)d - %(levelname)s - %(funcName)s - %(message)s') +file_formatter = logging.Formatter('%(asctime)s - %(name)s - L%(lineno)d - %(funcName)s - %(levelname)s - %(message)s') +stdout_formatter = logging.Formatter('%(asctime)s - %(name)s - L%(lineno)d - %(levelname)s - %(funcName)s - %(message)s') # Grab root logger and set initial logging level root_logger = logging.getLogger() @@ -82,6 +85,34 @@ log.debug("Logging initialized for %s", __name__) +######################################################## +# Collect command-line arguments (e.g., passed by Cron) +######################################################## + +parser = argparse.ArgumentParser( + # Borrow docstring for this module + description=__doc__.strip() + ) + +parser.add_argument( + '--config_file_dir', + action='store', + required=False, + help='The directory path containing general and query config files.') + +try: + log.info('Parsing commandline options') + args = parser.parse_args() +except argparse.ArgumentError as error: + log.exception("Unable to parse command-line arguments: %s", error) + sys.exit(1) + +if args.config_file_dir is not None: + cmdline_config_file_dir = args.config_file_dir +else: + cmdline_config_file_dir = "" + + ######################################## # Modules - Third party ######################################## @@ -125,26 +156,35 @@ script_name = os.path.basename(sys.argv[0]) # Read in configuration file. Attempt to read local copy first, then -# fall back to using the copy provided by SVN+Symlinks +# fall back to using the global config file. + +# TODO: Add support for command-line option -# TODO: Replace with command-line options -default_config_file_dir = '/etc/whyaskwhy.org/mysql2sqlite/config' +general_config_file = 'mysql2sqlite_general.ini' +query_config_file = 'mysql2sqlite_queries.ini' -general_config_file = {} -general_config_file['name'] = 'mysql2sqlite_general.ini' -general_config_file['local'] = os.path.join(script_path, general_config_file['name']) -general_config_file['global'] = os.path.join(default_config_file_dir, general_config_file['name']) +# Listed in in order of precedence: first match in list wins +# https://stackoverflow.com/a/28231217 +# https://www.blog.pythonlibrary.org/2016/03/24/python-201-ordereddict/ +config_file_paths = OrderedDict({ + 'cmdline_config_file_dir': cmdline_config_file_dir, + 'local_config_file_dir': script_path, + 'user_config_file_dir': os.path.expanduser('~/.config/mysql2sqlite'), + 'default_config_file_dir': '/etc/mysql2sqlite', +}) -queries_config_file = {} -queries_config_file['name'] = 'mysql2sqlite_queries.ini' -queries_config_file['local'] = os.path.join(script_path, queries_config_file['name']) -queries_config_file['global'] = os.path.join(default_config_file_dir, queries_config_file['name']) +general_config_file_candidates = [] +query_config_file_candidates = [] +for key in reversed(config_file_paths): + + general_config_file_candidates.append( + os.path.join(config_file_paths[key], general_config_file)) + + query_config_file_candidates.append( + os.path.join(config_file_paths[key], query_config_file)) # Prefer the local copy over the "global" one by loading it last (where the # second config file overrides or "shadows" settings from the first) -general_config_file_candidates = [general_config_file['global'], general_config_file['local']] - -queries_config_file_candidates = [queries_config_file['global'], queries_config_file['local']] # Generate configuration setting options log.debug( @@ -153,17 +193,42 @@ log.debug( "Passing in these query config file locations for evalution: %s", - queries_config_file_candidates) + query_config_file_candidates) # Generate configuration setting options log.info('Parsing config files') -general_settings = m2slib.GeneralSettings(general_config_file_candidates) -query_settings = m2slib.QuerySettings(queries_config_file_candidates) + +# Apply handler early so that console logging is enabled prior to parsing +# configuration files. The provided filter configuration allows logging +# warning and error messages only while the settings object has yet to be +# defined. +app_logger.addHandler(console_handler) +console_handler.addFilter(m2slib.ConsoleFilterFunc(settings=None)) + +try: + general_settings = m2slib.GeneralSettings(general_config_file_candidates) +except configparser.NoSectionError as error: + log.exception("Error parsing configuration file: %s", error) + sys.exit(1) +except IOError as error: + log.exception("Error reading configuration file: %s", error) + sys.exit(1) + +try: + query_settings = m2slib.QuerySettings(query_config_file_candidates) +except configparser.NoSectionError as error: + log.exception("Error parsing configuration file: %s", error) + sys.exit(1) +except IOError as error: + log.exception("Error reading configuration file: %s", error) + sys.exit(1) + # Now that the settings object has been properly created, lets use it to # finish configuring console logging for the main application logger. +console_handler.removeFilter(m2slib.ConsoleFilterFunc) console_handler.addFilter(m2slib.ConsoleFilterFunc(settings=general_settings)) -app_logger.addHandler(console_handler) + #################################################################### # Troubleshooting config file flag boolean conversion diff --git a/mysql2sqlite_general.ini b/mysql2sqlite_general.ini.tmpl similarity index 100% rename from mysql2sqlite_general.ini rename to mysql2sqlite_general.ini.tmpl diff --git a/mysql2sqlite_lib.py b/mysql2sqlite_lib.py index eff36c1..4ae9a09 100644 --- a/mysql2sqlite_lib.py +++ b/mysql2sqlite_lib.py @@ -68,6 +68,19 @@ # Classes ####################################################### +# TODO: Consider replacing class/functionality here with Python 3.2+ support for +# mapping API +# +# This support allows accessing a ConfigParser instance as a single +# dictionary with separate nested dictionaries for each section. In short, the +# two classes listed here do not appear to be needed any longer? +# +# Well, except for perhaps string to boolean conversion? +# +# https://pymotw.com/3/configparser/ +# https://docs.python.org/3.5/library/configparser.html#mapping-protocol-access + + class GeneralSettings(object): """ @@ -79,18 +92,22 @@ def __init__(self, config_file_list): self.log = log.getChild(self.__class__.__name__) - try: - parser = configparser.SafeConfigParser() - processed_files = parser.read(config_file_list) + parser = configparser.ConfigParser() + processed_files = parser.read(config_file_list) - except configparser.ParsingError as error: - self.log.exception("Unable to parse config file: %s", error) - sys.exit(1) - - else: + if processed_files: self.log.debug("CONFIG: Config files processed: %s", processed_files) + else: + self.log.error("Failure to read config files; " + "See provided templates, modify and place in one of the " + "supported locations: %s", + ", ".join(config_file_list)) + raise IOError("Failure to read config files; " + "See provided templates, modify and place in one of the " + "supported locations: ", + ", ".join(config_file_list)) # Begin building object by creating dictionary member attributes # from config file sections/values. @@ -123,7 +140,7 @@ def __init__(self, config_file_list): except configparser.NoSectionError as error: self.log.exception("Unable to parse config file: %s", error) - sys.exit(1) + raise class QuerySettings(object): @@ -136,17 +153,26 @@ def __init__(self, config_file_list): self.log = log.getChild(self.__class__.__name__) - try: - parser = configparser.SafeConfigParser() - processed_files = parser.read(config_file_list) + parser = configparser.SafeConfigParser() + processed_files = parser.read(config_file_list) - except configparser.ParsingError as error: - self.log.exception("Unable to parse config file: %s", error) - sys.exit(1) + # We've reached this point if no errors were thrown attempting + # to read the list of config files. We now need to count the + # number of parsed files and if zero, attempt to resolve why. - else: + if processed_files: self.log.debug("CONFIG: Config files processed: %s", processed_files) + else: + self.log.error("Failure to read config files; " + "See provided templates, modify and place in one of the " + "supported locations: %s", + ", ".join(config_file_list)) + + raise IOError("Failure to read config files; " + "See provided templates, modify and place in one of the " + "supported locations: ", + ", ".join(config_file_list)) # Setup an empty dictionary that we'll then populate with nested # dictionaries @@ -164,8 +190,10 @@ def __init__(self, config_file_list): except configparser.NoSectionError as error: - self.log.exception("Unable to parse config file: %s", error) - sys.exit(1) + self.log.exception( + "Unable to parse '%s' section of config file: %s", + section, error) + raise class ConsoleFilterFunc(logging.Filter): @@ -179,21 +207,34 @@ def __init__(self, settings): #print("Just proving that this function is being called") def filter(self, record): - if self.settings.flags['display_console_error_messages'] and record.levelname == 'ERROR': - #print("Error messages enabled") - return True - if self.settings.flags['display_console_warning_messages'] and record.levelname == 'WARNING': - #print("Warning messages enabled") - return True - if self.settings.flags['display_console_info_messages'] and record.levelname == 'INFO': - #print("Info messages enabled") - return True - if self.settings.flags['display_console_debug_messages'] and record.levelname == 'DEBUG': - #print("Debug messages enabled") - return True + + # If filter is not passed a settings object then fall back + # to default values. This may occur if the configuration files are + # not able to be read for one reason or another. In that situation + # we want the error output to be as verbose as possible. + if self.settings: + if self.settings.flags['display_console_error_messages'] and record.levelname == 'ERROR': + #print("Error messages enabled") + return True + if self.settings.flags['display_console_warning_messages'] and record.levelname == 'WARNING': + #print("Warning messages enabled") + return True + if self.settings.flags['display_console_info_messages'] and record.levelname == 'INFO': + #print("Info messages enabled") + return True + if self.settings.flags['display_console_debug_messages'] and record.levelname == 'DEBUG': + #print("Debug messages enabled") + return True + else: + #print("No matches") + return False else: - #print("No matches") - return False + # Go with hard-coded default of displaying warning and error + # messages until the settings object has been defined. + if record.levelname == 'ERROR': + return True + if record.levelname == 'WARNING': + return True ####################################################### # Functions diff --git a/mysql2sqlite_queries.ini b/mysql2sqlite_queries.ini.tmpl similarity index 100% rename from mysql2sqlite_queries.ini rename to mysql2sqlite_queries.ini.tmpl