Template INI files, cmdline config path support

- repo-provided INI files have been renamed to *.tmpl in order to prevent them from being used as-is or risk clobbering local/custom config files if user opts to install directly from Git repo. - Adjust log formatters to include line number for messages - Replace explicit exit calls from within our library function calls. - Adjust system-wide config path from /etc/DOMAIN/PROJECT to just /etc/PROJECT in order to better match the norm for /etc content layout. - Improve exception handling for config file parsing issues - Provide support for user-level config files - cmdline - $HOME/.config/PROJECT - Fall back to logging warning/error messages to the console if settings object is not provided to constructor. Adjust filter for handler later once the settings object is properly constructed. refs WhyAskWhy/mysql2sqlite-dev#2 refs WhyAskWhy#3
deoren · Aug 8, 2018 · 981a08c · 981a08c
1 parent c58268a
commit 981a08c
Show file tree

Hide file tree

Showing 4 changed files with 159 additions and 53 deletions.
diff --git a/mysql2sqlite.py b/mysql2sqlite.py
@@ -23,20 +23,23 @@
 
 # parse command line arguments, 'sys.argv'
 import argparse
+import configparser
 import logging
 import logging.handlers
 import os
 import os.path
 import sqlite3
 import sys
 
+from collections import OrderedDict
+
 
 app_name = 'mysql2sqlite'
 
 # TODO: Configure formatter to log function/class info
-syslog_formatter = logging.Formatter('%(name)s - %(levelname)s - %(funcName)s - %(message)s')
-file_formatter = logging.Formatter('%(asctime)s - %(name)s - %(funcName)s - %(levelname)s - %(message)s')
-stdout_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(funcName)s - %(message)s')
+syslog_formatter = logging.Formatter('%(name)s - L%(lineno)d - %(levelname)s - %(funcName)s - %(message)s')
+file_formatter = logging.Formatter('%(asctime)s - %(name)s - L%(lineno)d - %(funcName)s - %(levelname)s - %(message)s')
+stdout_formatter = logging.Formatter('%(asctime)s - %(name)s - L%(lineno)d - %(levelname)s - %(funcName)s - %(message)s')
 
 # Grab root logger and set initial logging level
 root_logger = logging.getLogger()
@@ -82,6 +85,34 @@
 log.debug("Logging initialized for %s", __name__)
 
 
+########################################################
+# Collect command-line arguments (e.g., passed by Cron)
+########################################################
+
+parser = argparse.ArgumentParser(
+    # Borrow docstring for this module
+    description=__doc__.strip()
+    )
+
+parser.add_argument(
+    '--config_file_dir',
+    action='store',
+    required=False,
+    help='The directory path containing general and query config files.')
+
+try:
+    log.info('Parsing commandline options')
+    args = parser.parse_args()
+except argparse.ArgumentError as error:
+    log.exception("Unable to parse command-line arguments: %s", error)
+    sys.exit(1)
+
+if args.config_file_dir is not None:
+    cmdline_config_file_dir = args.config_file_dir
+else:
+    cmdline_config_file_dir = ""
+
+
 ########################################
 # Modules - Third party
 ########################################
@@ -125,26 +156,35 @@
 script_name = os.path.basename(sys.argv[0])
 
 # Read in configuration file. Attempt to read local copy first, then
-# fall back to using the copy provided by SVN+Symlinks
+# fall back to using the global config file.
+
+# TODO: Add support for command-line option
 
-# TODO: Replace with command-line options
-default_config_file_dir = '/etc/whyaskwhy.org/mysql2sqlite/config'
+general_config_file = 'mysql2sqlite_general.ini'
+query_config_file = 'mysql2sqlite_queries.ini'
 
-general_config_file = {}
-general_config_file['name'] = 'mysql2sqlite_general.ini'
-general_config_file['local'] = os.path.join(script_path, general_config_file['name'])
-general_config_file['global'] = os.path.join(default_config_file_dir, general_config_file['name'])
+# Listed in in order of precedence: first match in list wins
+# https://stackoverflow.com/a/28231217
+# https://www.blog.pythonlibrary.org/2016/03/24/python-201-ordereddict/
+config_file_paths = OrderedDict({
+    'cmdline_config_file_dir': cmdline_config_file_dir,
+    'local_config_file_dir':  script_path,
+    'user_config_file_dir':  os.path.expanduser('~/.config/mysql2sqlite'),
+    'default_config_file_dir': '/etc/mysql2sqlite',
+})
 
-queries_config_file = {}
-queries_config_file['name'] = 'mysql2sqlite_queries.ini'
-queries_config_file['local'] = os.path.join(script_path, queries_config_file['name'])
-queries_config_file['global'] = os.path.join(default_config_file_dir, queries_config_file['name'])
+general_config_file_candidates = []
+query_config_file_candidates = []
+for key in reversed(config_file_paths):
+
+     general_config_file_candidates.append(
+         os.path.join(config_file_paths[key], general_config_file))
+
+     query_config_file_candidates.append(
+         os.path.join(config_file_paths[key], query_config_file))
 
 # Prefer the local copy over the "global" one by loading it last (where the
 # second config file overrides or "shadows" settings from the first)
-general_config_file_candidates = [general_config_file['global'], general_config_file['local']]
-
-queries_config_file_candidates = [queries_config_file['global'], queries_config_file['local']]
 
 # Generate configuration setting options
 log.debug(
@@ -153,17 +193,42 @@
 
 log.debug(
     "Passing in these query config file locations for evalution: %s",
-    queries_config_file_candidates)
+    query_config_file_candidates)
 
 # Generate configuration setting options
 log.info('Parsing config files')
-general_settings = m2slib.GeneralSettings(general_config_file_candidates)
-query_settings = m2slib.QuerySettings(queries_config_file_candidates)
+
+# Apply handler early so that console logging is enabled prior to parsing
+# configuration files. The provided filter configuration allows logging
+# warning and error messages only while the settings object has yet to be
+# defined.
+app_logger.addHandler(console_handler)
+console_handler.addFilter(m2slib.ConsoleFilterFunc(settings=None))
+
+try:
+    general_settings = m2slib.GeneralSettings(general_config_file_candidates)
+except configparser.NoSectionError as error:
+    log.exception("Error parsing configuration file: %s", error)
+    sys.exit(1)
+except IOError as error:
+    log.exception("Error reading configuration file: %s", error)
+    sys.exit(1)
+
+try:
+    query_settings = m2slib.QuerySettings(query_config_file_candidates)
+except configparser.NoSectionError as error:
+    log.exception("Error parsing configuration file: %s", error)
+    sys.exit(1)
+except IOError as error:
+    log.exception("Error reading configuration file: %s", error)
+    sys.exit(1)
+
 
 # Now that the settings object has been properly created, lets use it to
 # finish configuring console logging for the main application logger.
+console_handler.removeFilter(m2slib.ConsoleFilterFunc)
 console_handler.addFilter(m2slib.ConsoleFilterFunc(settings=general_settings))
-app_logger.addHandler(console_handler)
+
 
 ####################################################################
 # Troubleshooting config file flag boolean conversion

diff --git a/mysql2sqlite_general.ini → mysql2sqlite_general.ini.tmpl b/mysql2sqlite_general.ini → mysql2sqlite_general.ini.tmpl
diff --git a/mysql2sqlite_lib.py b/mysql2sqlite_lib.py
@@ -68,6 +68,19 @@
 # Classes
 #######################################################
 
+# TODO: Consider replacing class/functionality here with Python 3.2+ support for
+# mapping API
+#
+# This support allows accessing a ConfigParser instance as a single
+# dictionary with separate nested dictionaries for each section. In short, the
+# two classes listed here do not appear to be needed any longer?
+#
+# Well, except for perhaps string to boolean conversion?
+#
+# https://pymotw.com/3/configparser/
+# https://docs.python.org/3.5/library/configparser.html#mapping-protocol-access
+
+
 class GeneralSettings(object):
 
     """
@@ -79,18 +92,22 @@ def __init__(self, config_file_list):
 
         self.log = log.getChild(self.__class__.__name__)
 
-        try:
-            parser = configparser.SafeConfigParser()
-            processed_files = parser.read(config_file_list)
+        parser = configparser.ConfigParser()
+        processed_files = parser.read(config_file_list)
 
-        except configparser.ParsingError as error:
-            self.log.exception("Unable to parse config file: %s", error)
-            sys.exit(1)
-
-        else:
+        if processed_files:
             self.log.debug("CONFIG: Config files processed: %s",
                 processed_files)
+        else:
+            self.log.error("Failure to read config files; "
+                "See provided templates, modify and place in one of the "
+                "supported locations: %s",
+                ", ".join(config_file_list))
 
+            raise IOError("Failure to read config files; "
+                "See provided templates, modify and place in one of the "
+                "supported locations: ",
+                ", ".join(config_file_list))
 
         # Begin building object by creating dictionary member attributes
         # from config file sections/values.
@@ -123,7 +140,7 @@ def __init__(self, config_file_list):
         except configparser.NoSectionError as error:
 
             self.log.exception("Unable to parse config file: %s", error)
-            sys.exit(1)
+            raise
 
 class QuerySettings(object):
 
@@ -136,17 +153,26 @@ def __init__(self, config_file_list):
 
         self.log = log.getChild(self.__class__.__name__)
 
-        try:
-            parser = configparser.SafeConfigParser()
-            processed_files = parser.read(config_file_list)
+        parser = configparser.SafeConfigParser()
+        processed_files = parser.read(config_file_list)
 
-        except configparser.ParsingError as error:
-            self.log.exception("Unable to parse config file: %s", error)
-            sys.exit(1)
+        # We've reached this point if no errors were thrown attempting
+        # to read the list of config files. We now need to count the
+        # number of parsed files and if zero, attempt to resolve why.
 
-        else:
+        if processed_files:
             self.log.debug("CONFIG: Config files processed: %s",
                 processed_files)
+        else:
+            self.log.error("Failure to read config files; "
+                "See provided templates, modify and place in one of the "
+                "supported locations: %s",
+                ", ".join(config_file_list))
+
+            raise IOError("Failure to read config files; "
+                "See provided templates, modify and place in one of the "
+                "supported locations: ",
+                ", ".join(config_file_list))
 
         # Setup an empty dictionary that we'll then populate with nested
         # dictionaries
@@ -164,8 +190,10 @@ def __init__(self, config_file_list):
 
             except configparser.NoSectionError as error:
 
-                self.log.exception("Unable to parse config file: %s", error)
-                sys.exit(1)
+                self.log.exception(
+                    "Unable to parse '%s' section of config file: %s",
+                    section, error)
+                raise
 
 class ConsoleFilterFunc(logging.Filter):
 
@@ -179,21 +207,34 @@ def __init__(self, settings):
         #print("Just proving that this function is being called")
 
     def filter(self, record):
-        if self.settings.flags['display_console_error_messages'] and record.levelname == 'ERROR':
-            #print("Error messages enabled")
-            return True
-        if self.settings.flags['display_console_warning_messages'] and record.levelname == 'WARNING':
-            #print("Warning messages enabled")
-            return True
-        if self.settings.flags['display_console_info_messages'] and record.levelname == 'INFO':
-            #print("Info messages enabled")
-            return True
-        if self.settings.flags['display_console_debug_messages'] and record.levelname == 'DEBUG':
-            #print("Debug messages enabled")
-            return True
+
+        # If filter is not passed a settings object then fall back
+        # to default values. This may occur if the configuration files are
+        # not able to be read for one reason or another. In that situation
+        # we want the error output to be as verbose as possible.
+        if self.settings:
+            if self.settings.flags['display_console_error_messages'] and record.levelname == 'ERROR':
+                #print("Error messages enabled")
+                return True
+            if self.settings.flags['display_console_warning_messages'] and record.levelname == 'WARNING':
+                #print("Warning messages enabled")
+                return True
+            if self.settings.flags['display_console_info_messages'] and record.levelname == 'INFO':
+                #print("Info messages enabled")
+                return True
+            if self.settings.flags['display_console_debug_messages'] and record.levelname == 'DEBUG':
+                #print("Debug messages enabled")
+                return True
+            else:
+                #print("No matches")
+                return False
         else:
-            #print("No matches")
-            return False
+            # Go with hard-coded default of displaying warning and error
+            # messages until the settings object has been defined.
+            if record.levelname == 'ERROR':
+                return True
+            if record.levelname == 'WARNING':
+                return True
 
 #######################################################
 # Functions

diff --git a/mysql2sqlite_queries.ini → mysql2sqlite_queries.ini.tmpl b/mysql2sqlite_queries.ini → mysql2sqlite_queries.ini.tmpl