From b4bab234c2fa33987305f468b66624e7603e3dd3 Mon Sep 17 00:00:00 2001 From: deoren Date: Fri, 9 Feb 2018 03:45:14 -0600 Subject: [PATCH] Initial import of exported SVN content (r83) refs WhyAskWhy/automated-tickets#1 --- TODO.txt | 161 ++++++++ automated_tickets.ini | 129 +++++++ automated_tickets.py | 306 ++++++++++++++++ automated_tickets_import.sql | 101 +++++ automated_tickets_lib.py | 586 ++++++++++++++++++++++++++++++ create_users.sql | 54 +++ database_schema.sql | 107 ++++++ helpful_queries.sql | 149 ++++++++ template_new_automated_ticket.sql | 67 ++++ 9 files changed, 1660 insertions(+) create mode 100644 TODO.txt create mode 100644 automated_tickets.ini create mode 100644 automated_tickets.py create mode 100644 automated_tickets_import.sql create mode 100644 automated_tickets_lib.py create mode 100644 create_users.sql create mode 100644 database_schema.sql create mode 100644 helpful_queries.sql create mode 100644 template_new_automated_ticket.sql diff --git a/TODO.txt b/TODO.txt new file mode 100644 index 0000000..3c55ad5 --- /dev/null +++ b/TODO.txt @@ -0,0 +1,161 @@ + +The goal is to move the event entries into a database and refactor the code back to a single script. Any user-customizable behavior will be placed into a configuration file, at least for the time being. At some point we could look at moving those settings into the database itself. + +##################################################################################################### + Scratch notes +##################################################################################################### + + +Future milestones +============================================== + +Tables: + +* UNKNOWN (not sure what this would be called) +** wiki page name +** message subject +** message body +*** perhaps not if we build the templae some other way +** message header +** message footer +** + +* contacts +** would be needed to allow for multiple email notifcations per event +** would also be needed in cases where recipients are CC'd instead of BCC'd + +* events +** bulk of the content would go here +** columns +*** priority +**** 1:1 correlation to Redmine ticket priority values +*** description +**** meant for humans looking at db entry +*** title +*** date_occurs +*** time_occurs +**** UTC time and then convert to localtime (so that Python can handle the + daylight savings conversion) +*** contact_id? +**** how to link the tables? Should I? +***** one-to-many relationship here? +**** foreign key to contacts table +*** template_id + + +* templates +** vs flat-files, though in our case we use a Redmine include statement to pull in a wiki page + +* notifications +** ties together contacts, events, ... ? + +* meta +** meta.db_last_modified +*** logging when any other table was last updated +** meta.db_revision +*** auto-increment for every table update +** meta.db_schema_version +*** would be used in order to help migrate from one schema version to another (add columns, rename older one, etc.) +** ? + +Configuration file: + +* Queries +** currently read-only queries should be sufficient as I would use another + process to get data into the database + +* Flags +** to disable/enable specific classes of event notifications +*** e.g., disable student worker tickets for times when they're not there +*** this should probably be set directly in the database (a column in the events table) +** to control feedback (display debug, warning, info, errors, ...) + +* Database credentials + + + + +##################################################################################################### + TODO +##################################################################################################### + + * Create 'meta' table for logging when any other table was last updated + (presumably we won't be wholesale nuking the existing tables on + each run by this point) + + - meta.db_last_modified + - meta.db_revision (auto-increment for every update to database) + + * Create TRIGGER to handle updating meta.db_last_modified column when + any other table is modified + + - possibly have this same TRIGGER handle updating the meta.db_revision + column too + + +##################################################################################################### + Google Keep scratch notes +##################################################################################################### + +Purpose: + +Send reminders for events. The goal is to support upcoming, current and recently passed dates and times. + +The primary purpose is to replace an older hard-coded script that is used to generate automated tickets, but it could be used as a reporting tool as well. + +For example, you could have an event reminder type of EndOfLife (eol, end-of-life, end_of_life, etc.) and query based on that type. You would then see that MariaDB 10.0 expires in 2019-03 and that 10.1 expires in 2020-10. This is assuming that we had one or more MariaDB installations we wanted to be aware of expiring soon. + +Tags would be more flexible though, so that would likely be implemented over fixed categories. + +Early builds would just have a DB manually updated and a script (likely Python) that performs the notification work. We would implement email notifications first, eventually adding support for XMPP notifications. + +Re DB schema, I am unsure about the tables and columns per table, but I see the need to support (or be able to calculate) these values: + +* event notification schedule: before or after event occurs? +* event date +* event time (default to a sane value if not specified? +* reminder sent (would need some way to allow for multiple entries per event: email, xmpp, text, allowing for a mix of them +* event description +* event documentation reference +* notification template support, but likely outside the database. Perhaps in the database to allow for modification via future web UI +** should be able to create templates separately and associate same template with multiple events (hence a true template) +* event type (multiple selection possible): tags support +* contact name +* contact email +* contact phone +* contact xmpp +* notification type: email +* notification type: xmpp +* notification type: text +* notification type: web API submission (e.g., Redmine WS API for submitting tickets) +* notification type: redmine_email +* redmine category +* redmine project +* redmine status +* redmine due date + +The Redmine project, status, category and other details could be contained in a custom footer (for redmine email functionality) + +Early version would run as an hourly cron job, so any functionality to send reminder "0 minutes before start" would not act as expected. That functionality would need to be implemented as either an early notification or a post-event notification. + + +Some of the things that the scripts do now: + +Daily, skip weekends +Monday, Friday +Friday +2x a month, specific dates +Once month + +I could set the frequency type in events table and use script call to request specific types of events. For example, the existing daily entry would call script with argument for daily tasks. Then the weekly task would do the same and so on. + +A standard template could be crafted since we are using an include reference in the body of the email. + +The data for the table could be pretty much what the tasks table has now. In a lot of ways all I would be doing is combining the separate scripts into one and getting input data out of script where it does not belong. + + +Ignoring earlier comments, but probably the 0.2 revision (the 0.1 being nearly a 1:1 conversion of the scripts to one script and a db backend, but with pulling directly from db page vs using Redmine include macro) will include support for multiple secondary wiki pages. The first one pulls in the main content, additional pages would be pulled in as content to go in the place of placeholders that would be inserted into the primary Automated Ticket include pages. + +For example, where the main page might offer a minimum of summary content and directions and instead rely on the include macro, the 0.2 revision (including the wiki page updates to support it) would pull content from the additional (likely referenced as secondary later on) pages and insert for each placeholder. The primary Automated Ticket pages would no longer (or as seldom as necessary) use the include macro to pull in other content. + +My initial impression is that this would require at least one more table, this one to hold wiki page references. The entries would tie back to a specific event and would note whether the table entry was for a primary page or a secondary inclusion page. diff --git a/automated_tickets.ini b/automated_tickets.ini new file mode 100644 index 0000000..c4f894e --- /dev/null +++ b/automated_tickets.ini @@ -0,0 +1,129 @@ +# Purpose: +# +# Expose general configuration options so values do not have to be +# hardcoded in the main script. + + + +############################################################ +# "toggle" options to broadly enable/disable behavior. Primarily used to +# control the types of output messages used to assist when first configuring +# or testing the script. For example, you might wish to see the output to +# know all is well before enabling the script as part of a cron job and +# then turning off all output +[flags] +############################################################ + +# Disables sending notifications and any other possibly destructive action +testing_mode = true + +# Set to true for verbose output as the script executes. Useful for troubleshooting. +display_debug_messages = true + +# Less verbose output. Enabling this provides status details. +display_info_messages = true + +# Output warning messages which indicate that a desired step was not +# completed as expected, but the issue does not appear to be severe +# enough to stop the script from completing in an "acceptable" manner. +# See the 'fail_on_warnings' option in case you would rather not +# continue operation when warning conditions are encountered +# TODO: Add support for this +display_warning_messages = true + +# Output error messages which block proper operation of the script +# TODO: Add support for this +display_error_messages = true + +# If enabled, this treats all warnings as failures and exits immediately with +# one or more messages indicating that this option is enabled and what warning +# condition occured +# TODO: Add support for this +fail_on_warnings = false + +# If enabled, events/tasks that are flagged as an intern_task are processed +# like all other matching events. If false, then those events are filtered out +process_intern_events = true + +# If enabled, any '{{include(PAGE_NAME_HERE)}}' macro calls found when pulling +# the wiki page contents from the first page will be expanded (macro calls +# replaced by the include pages) so that the final content is self-contained +# with no further direct dependencies on external pages. "Related pages" +# sections are prefectly acceptable, but not commonplace among the include +# pages intended for ticket generation. +expand_include_macros_in_wiki_pages = true + + +############################################################################## +# Used by the MySQL database connector module +[mysqldb_config] +############################################################################## + +# Read-only access to Redmine wiki and event_reminders database tables is sufficient +user = events_ro + +# Password for the MySQL database account. Reminder: read-only access +# is sufficient +password = ChangeM3 + +# MySQL database host. If you use a stunnel or other forwarded connection you +# will likely wish to set this to 127.0.0.1 and override the default port +# for the 'port' setting +host = 127.0.0.1 + +# If this script runs on the same box as the MySQL/MariaDB server then you +# probably want to enter 3306 here, otherwise if using stunnel or another +# port forwarding setup you will likely need to enter an alternate port. +port = 3306 + +# The database which holds event reminder entries. +events_database = event_reminders + +# The Redmine database holding the wiki pages which this script will pull +# from in order to generate "automated" tickets with pre-filled content +redmine_database = help_redmine_db + +# Specific to the upstream MySQL Connector module > 2.0.5. This forces all +# warning conditions to trigger exceptions, which if uncaught will terminate +# the script. In normal operation you shouldn't receive any exceptions, so +# this is enabled by default to help catch configuration or script errors +raise_on_warnings = true + + +[email] + +# These values are used if an event_reminders entry doesn't have a value recorded +# Note: These are helper values that are not yet directly referenced by Python +# code, but instead referenced by the enabled_event_table_entries query in +# this file. +default_from_address = automated-tickets@relay.example.com +default_to_address = automated-tickets@localhost + + +############################################################################## +# Used by the MySQL database connector module +[queries] + +# Pull wiki page contents from Redmine database +# The wiki_pages.title value is the name of the page as shown in the URL +# The project.identifier value is the project "shortname", shown in the URL +wiki_page_contents = SELECT wiki_contents.text FROM wiki_contents INNER JOIN wiki_pages ON wiki_pages.id = wiki_contents.id INNER JOIN wikis ON wikis.id = wiki_pages.wiki_id INNER JOIN projects ON projects.id = wikis.project_id WHERE wiki_pages.title = '{}' AND projects.identifier = '{}' + +# The query needed to pull event table entries. As is, this query does not limit +# the returned results by event schedule or whether the flag is set for +# processing "intern" tasks. That is handled programatically by the script +event_table_entries = SELECT IF(email_to_address IS NOT NULL, email_to_address, '${email:default_to_address}') AS email_to_address, IF(email_from_address IS NOT NULL, email_from_address, '${email:default_from_address}') AS email_from_address, email_subject_prefix, redmine_wiki_page_name, redmine_wiki_page_project_shortname, redmine_new_issue_project, redmine_new_issue_category, redmine_new_issue_status, IF(redmine_new_issue_due_after_days IS NOT NULL, CURRENT_DATE() + INTERVAL redmine_new_issue_due_after_days DAY, NULL) AS redmine_new_issue_due_after_days, redmine_new_issue_priority, event_schedule FROM events WHERE enabled = 1 + +############################################################################## + + +# FIXME: Come up with a better name. Perhaps have a separate section per +# notification type? If so, then 'email', 'xmpp', ... and this section +# can go away. +[notification_servers] + +email_server_ip_or_fqdn = localhost + +#email_server_username + +#email_server_password diff --git a/automated_tickets.py b/automated_tickets.py new file mode 100644 index 0000000..4f28619 --- /dev/null +++ b/automated_tickets.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 + +""" +Purpose: Parse collection of events and generate notifications for any events +matching the requested schedule. +""" + + + +####################################################### +# Module Imports +####################################################### + +# +# Standard Library +# + +# parse command line arguments, 'sys.argv' +import argparse + +import os +import os.path +import sys + + + + +########################################################################## +# This controls output display prior to user configurable values +# being read in and honored. Uncomment and disable/enable as needed +# for debugging purposes +#automated_tickets_lib.DISPLAY_DEBUG_MESSAGES = True +#automated_tickets_lib.DISPLAY_INFO_MESSAGES = True +#automated_tickets_lib.DISPLAY_WARNING_MESSAGES = True +#automated_tickets_lib.DISPLAY_ERROR_MESSAGES = True +# +# Basic import so that we can manipulate global variables +import automated_tickets_lib +########################################################################## + + + +# Our custom classes +from automated_tickets_lib import Settings + +# Consants +from automated_tickets_lib import DATE_LABEL + +# Our custom "print" fuctions +from automated_tickets_lib import print_debug, print_error, print_info, print_warning + +from automated_tickets_lib import get_events, get_wiki_page_contents, send_notification + +from automated_tickets_lib import get_include_calls, get_included_wiki_pages + + +######################################################## +# Collect command-line arguments passed by Cron +######################################################## + +parser = argparse.ArgumentParser( + description='Check for applicable events and generate notices for matches' + ) + +# This will need to be broken up later by the script +# WARNING: This needs to be kept in sync with DATE_LABEL within +# the library file. +parser.add_argument( + '--event_schedule', + action='store', + required=True, + choices=[ + 'daily', + 'twice_week', + 'weekly_monday', + 'weekly_tuesday', + 'weekly_wednesday', + 'weekly_thursday', + 'weekly_friday', + 'weekly', + 'twice_month', + 'monthly', + 'twice_year', + 'yearly' + ] +) + +# NOTE: Probably want to leave this as not required and fall back to checking +# for the config file in the same location as this script. If it is not found +# THEN we can throw an error. +parser.add_argument('--config_file', action='store', required=False) + +try: + args = parser.parse_args() +except Exception as e: + print_error("Unable to parse command-line arguments: {}".format(e), "argparse") + sys.exit(1) + +# NOTE: The command-line options parser enforces specific values and requires +# that at least one of those values is present. +event_schedule = args.event_schedule + +if args.config_file: + global_config_file = args.config_file +else: + # FIXME: + # The configuration parser will skip over requests for + # non-existant files, so it SHOULD be safe for now to + # set this location to an empty string. + global_config_file = "" + + +####################################################### +# CONSTANTS - Modify INI config files instead +####################################################### + +# Where this script is being called from. We will try to load local copies of all +# dependencies from this location first before falling back to default +# locations in order to support having all of the files bundled together for +# testing and portable use. +script_path = os.path.dirname(os.path.realpath(__file__)) + +# The name of this script. It is used as needed by error/debug messages +script_name = os.path.basename(sys.argv[0]) + +# Read in configuration file. Attempt to read local copy first, then +# fall back to using the copy specified on the command-line. We have +# hard-coded the name of the local file, but the user is free to specify +# a custom name/path using the command-line option. + +config_file = {} +config_file['name'] = 'automated_tickets.ini' +config_file['local'] = os.path.join(script_path, config_file['name']) + +# This location is (optionally) specified on the command-line. If it is not +# specified, then an empty string is set instead. The Settings class will +# confirm the file is actually present and complain if it is not. +config_file['global'] = global_config_file + +# Prefer a local copy over a "global" one by loading it last (where the +# second config file overrides or "shadows" settings from the first). If +# a local copy does not exist, then the one specified on the command-line +# will be used. If that one does not exist, then this script will throw +# an error and quit. +config_file_candidates = [config_file['global'], config_file['local']] + +# Generate configuration setting options +print_debug( + "Passing in these config file locations for evalution: {}".format( + config_file_candidates), "CONFIG") + +settings = Settings(config_file_candidates) + +# Controls status messages for each minor step of the process +# (e.g., pulling data, writing data, what flags are enabled, etc.) +DISPLAY_DEBUG_MESSAGES = settings.flags['display_debug_messages'] +automated_tickets_lib.DISPLAY_DEBUG_MESSAGES = DISPLAY_DEBUG_MESSAGES + +# Controls status messages for each major block +# (e.g., finishing table updates, reporting rows affected, etc.) +DISPLAY_INFO_MESSAGES = settings.flags['display_info_messages'] +automated_tickets_lib.DISPLAY_INFO_MESSAGES = DISPLAY_INFO_MESSAGES + +DISPLAY_WARNING_MESSAGES = settings.flags['display_warning_messages'] +automated_tickets_lib.DISPLAY_WARNING_MESSAGES = DISPLAY_WARNING_MESSAGES + +DISPLAY_ERROR_MESSAGES = settings.flags['display_error_messages'] +automated_tickets_lib.DISPLAY_ERROR_MESSAGES = DISPLAY_ERROR_MESSAGES + + + +# Troubleshooting config file flag boolean conversion +for key, value in list(settings.flags.items()): + print_debug("key: '{}' value: '{}' type of value: '{}'".format(key, value, type(value)), "MySQL") + + +# Generate list of matching events from database based on requested event +# schedule (daily, weekly, etc.) +events = [] +events = get_events(settings, event_schedule) + +message = {} + +for event in events: + + # FIXME: Perform string substitution on object instantiation where all + # validity checks can be grouped together. This can simply be a reference + # to those pre-computed values. + # FIXME: Reintroduce support for multiple destination email addresses + message['envelope'] = "From: {}\nTo: {}\nSubject: {} ({})\n".format( + event.email_from_address, event.email_to_address, + + # Formatting the prefix string before then using the result in + # the larger format string we're building here + # NOTE: Explicitly lowering the case of the dictionary key values + # pulled from the events table entry in order to properly reference + # the associated value. + # FIXME: Add try/except blocks around expansion so that any KeyError + # exceptions can be caught and processing can continue. + event.email_subject_prefix.format(DATE_LABEL[event.event_schedule].lower()), + event.event_schedule.lower() + ) + + print_debug(message['envelope'], "Email envelope details") + + # If this task has a known due date ... + if event.redmine_new_issue_due_date: + message['footer'] = "\nProject: {}\nCategory: {}\nStatus: {}\nPriority: {}\nDue date: {}\n".format( + event.redmine_new_issue_project, + event.redmine_new_issue_category, + event.redmine_new_issue_status, + event.redmine_new_issue_priority, + event.redmine_new_issue_due_date + ) + else: + message['footer'] = "\nProject: {}\nCategory: {}\nStatus: {}\nPriority: {}\n".format( + event.redmine_new_issue_project, + event.redmine_new_issue_category, + event.redmine_new_issue_status, + event.redmine_new_issue_priority + ) + + # Get the raw contents of the wiki page associated with the event + wiki_page_contents = get_wiki_page_contents( + settings, + event.redmine_wiki_page_name, + event.redmine_wiki_page_project_shortname, + settings.mysqldb_config['redmine_database'] + ) + + + # Optionally expand any include macro calls so that a full expanded + # (dependency free) page is used as the body of the message + if settings.flags['expand_include_macros_in_wiki_pages']: + + # Check wiki_page_contents for include macro calls and build a + # list of included pages to fetch the content from. + wiki_page_macro_calls = get_include_calls( + wiki_page_contents, + event.redmine_wiki_page_project_shortname + ) + + while wiki_page_macro_calls: + + print_debug(wiki_page_macro_calls) + + # proceed with getting the list of included pages + included_wiki_pages = [] + included_wiki_pages = get_included_wiki_pages( + wiki_page_macro_calls, + event.redmine_wiki_page_project_shortname) + + # for every included page, lets grab the contents + while included_wiki_pages: + wiki_page_to_process = included_wiki_pages.pop() + + included_wiki_page_contents = get_wiki_page_contents( + settings, + wiki_page_to_process, + event.redmine_wiki_page_project_shortname, + settings.mysqldb_config['redmine_database']) + + # At this point we have a page name which was included by the + # initial page and we also have the contents of that page + search_value = '{{include(%s:%s)}}' % ( + event.redmine_wiki_page_project_shortname, + wiki_page_to_process) + + print_debug(search_value, "Search string for include call") + + wiki_page_contents = wiki_page_contents.replace(search_value, included_wiki_page_contents) + + # After we have processed all initial include pages, check again to + # see if pulling in the content from those included pages resulted + # in us finding more include macro calls. + wiki_page_macro_calls = get_include_calls( + wiki_page_contents, + event.redmine_wiki_page_project_shortname + ) + + + # Use wiki page contents as the message body. This is either the fully + # expanded content after include macro calls have been processed or the + # original page content if the expansion option has been disabled in the + # automated_tickets.ini config file. + message['body'] = wiki_page_contents + + + # FIXME: Revisit this? + message['header'] = "" + + # Note: + # + # The spacing should be EXACTLY as shown here. Having one space + # between the envelope and header content results in Redmine adding + # header values (Message-Id for example) directly into the OP + email_message = "{}{}\n{}\n{}\n".format( + message['envelope'], + message['header'], + message['body'], + message['footer'] + ) + + # Send notification + send_notification(settings, event.email_from_address, event.email_to_address, email_message) + diff --git a/automated_tickets_import.sql b/automated_tickets_import.sql new file mode 100644 index 0000000..15499aa --- /dev/null +++ b/automated_tickets_import.sql @@ -0,0 +1,101 @@ +/* + + Purpose: + + Import all existing Automated Ticket entries as of 2017-03-28 + + Notes: + + * If 0 is used for redmine_new_issue_due_after_days, then the due date + is set for the same day, otherwise it is generated as ticket + creation date + the number of days specified + + * We do not specify a the from/to column values; the read query + will use a default value for each from the config file + + * We are relying on each task to default to enabled + + * Tasks are not "intern" tasks by default + + * Priority is set to 'Normal' by default + + + TODO: + + * Evaluate all categories for Automated Tickets and adjust as necessary + ** Example: "Checks - Equipment" might be a better fit for weekly + maintenance on the staff laptops, or, possibly for the public terminals + +*/ + +-- Variables, because I'm feeling lazy +SET @ds = 'desktop-support'; +SET @ss = 'server-support'; + +SET @ds_email = 'ds-automated-reminders@help.example.com'; +SET @ss_email = 'ss-automated-reminders@help.example.com'; + + +-- Here we are inserting a row with the redmine_new_issue_due_after_days +-- value set to 0. This results in the read query subsituting that value +-- for the current date so that tickets have the due date set to the +-- same day the ticket was generated. We are also overriding the default +-- priority. +INSERT INTO `event_reminders`.`events` + ( + `intern_task`, + `email_from_address`, + `email_subject_prefix`, + `redmine_wiki_page_name`, + `redmine_wiki_page_project_shortname`, + `redmine_new_issue_project`, + `redmine_new_issue_category`, + `redmine_new_issue_due_after_days`, + `redmine_new_issue_priority`, + `event_schedule`, + `comments` + ) +VALUES + + (1,@ds_email,'Check email for {}','AutomatedTicketsEmailCheck','sysdocs',@ds,'Checks - Email',0,'High','daily','Candidate for future 2x, 3x daily task'), + + (1,@ds_email,'Pickup/Deliver mail for {}','AutomatedTicketsMailCheck','sysdocs',@ds,'Checks - Dept Email',0,'Normal','daily','no particular priority'), + + (1,@ds_email,'Equipment Checks for {}','AutomatedTicketsEquipmentCheck','sysdocs',@ds,'Checks - Equipment',0,'Normal','twice_week','no particular priority'), + + (1,@ds_email,'Inspect Public terminals and clean/repair as necessary for {}','AutomatedTicketsCleaningPublicTerminals','sysdocs',@ds,'Checks - Equipment',0,'Normal','weekly','no particular priority'), + + (1,@ds_email,'Inspect Staff Laptops and bags for {}','AutomatedTicketsCleaningLaptops','sysdocs',@ds,'Laptop Maintenance',0,'Normal','weekly','no particular priority'), + + (1,@ds_email,'Cleanup work areas for {}','AutomatedTicketsWorkAreaCleanup','sysdocs',@ds,'Cleanup/Sort',0,'Normal','twice_week','no particular priority'), + + (1,@ds_email,'Reimage Staff Laptops for {}','AutomatedTicketsReimagingStaffLaptops','sysdocs',@ds,'Laptop Maintenance',0,'Normal','weekly','no particular priority'), + + (1,@ds_email,'Check storage supplies for {}','AutomatedTicketsCheckStorageSupplies','sysdocs',@ds,'Checks - Storage',0,'Normal','weekly','no particular priority'), + + (1,@ds_email,'Process voicemails for {}','AutomatedTicketsVoicemailCheck','sysdocs',@ds,'Checks - Voicemail',0,'Urgent','weekly','Used to be daily task'), + + (0,@ds_email,'Refresh prototype base images for {}','AutomatedTicketsUpdateBaseImages','sysdocs',@ds,'Image: Create or update base, ISO, etc',21,'Normal','monthly','Due date of 21 days later attempts to flag around the time that second Patch Tuesday updates are released'), + (0,@ds_email,'Generate new rollout image for RBD Circulation Laptops {}','AutomatedTicketsUpdateBaseImages','sysdocs',@ds,'Circ Laptops',21,'Normal','monthly','This task is dependent on us to refresh the base image and generate a new rollout image first before they can do their work'), + + (1,@ds_email,'Marcia''s laptop - Apply updates for {}','AutomatedTicketsMaricaLaptop','sysdocs',@ds,'On-Demand Updates',7,'Normal','monthly','Ticket is generated monthly, but Marcia brings laptop in less often than that'), + + (1,@ds_email,'Consumable supplies check for {}','AutomatedTicketsConsumablesCheck','sysdocs',@ds,'Inventory Management',7,'Normal','monthly','Historically no due date, but setting to creation + a reasonable time frame'), + + (1,@ds_email,'Graveyard Processing for {}','AutomatedTicketsSurplusProcessing','sysdocs',@ds,'Surplus',7,'Normal','monthly','Historically no due date, but setting to creation + a reasonable time frame'), + + (1,@ds_email,'Inventory check for {}','AutomatedTicketsInventoryCheck','sysdocs',@ds,'Inventory Management',7,'Normal','twice_month','Candidate for increased frequency'), + + (1,@ds_email,'Reimage RBD Circulation Laptops {}','AutomatedTicketsCirculatingStudentLaptops','sysdocs',@ds,'Circ Laptops',21,'Normal','monthly','This task is dependent on us to refresh the base image and generate a new rollout image first before they can do their work'), + + (1,@ds_email,'Verify dry-erase department calendar {}','AutomatedTicketsSystemsDryEraseCalendar','sysdocs',@ds,'Cleanup/Sort',7,'Normal','monthly','Historically no due date'), + + (1,@ds_email,'Process Renewals list for {}: Look for upcoming expirations, update prior reports if unacknowledged','AutomatedTicketsCheckRenewalDates','sysdocs',@ds,'Inventory Management',0,'Urgent','monthly','Historically no due date, normal priority'), + + (0,@ss_email,'mssql.example.com - Microsoft Updates for {}','AutomatedTicketsManualPatching','sysdocs',@ss,'Patch',21,'High','monthly','Historically no due date, normal priority'), + + (0,@ss_email,'storageserver1.example.com - Microsoft Updates for {}','AutomatedTicketsManualPatching','sysdocs',@ss,'Patch',21,'High','monthly','Historically no due date, normal priority'), + + (0,@ss_email,'loanserver.example.com - Microsoft Updates for {}','AutomatedTicketsManualPatching','sysdocs',@ss,'Patch',21,'High','monthly','Historically no due date, normal priority') + +; diff --git a/automated_tickets_lib.py b/automated_tickets_lib.py new file mode 100644 index 0000000..f0cc228 --- /dev/null +++ b/automated_tickets_lib.py @@ -0,0 +1,586 @@ +#!/usr/bin/env python3 + +""" +Library module used by the automated_tickets.py script. Not intended for direct use. +""" + + + +######################################## +# Modules - Standard Library +######################################## + +import configparser +import datetime +import os +import re +import smtplib +import sys + +######################################## +# Modules - Third party +######################################## + +# Third-party module +# https://pypi.python.org/pypi/MySQL-python/1.2.5 +# apt-get install python-mysqldb +#import MySQLdb + +# Upstream module, recommended by MariaDB documentation +# https://dev.mysql.com/downloads/repo/apt/ +# apt-get install mysql-connector-python +import mysql.connector as mysql + + + +if __name__ == "__main__": + sys.exit("This module is meant to be imported, not executed directly.") + +####################################################### +# Variables, constants +####################################################### + + +DATE = datetime.date.today() +TODAY = DATE.strftime('%Y-%m-%d') + +# Disable various modes by default. Will be overriden by main script that +# imports this module +DISPLAY_DEBUG_MESSAGES = True +DISPLAY_INFO_MESSAGES = True + +# Going to assume we want these by default, we can override in config file +DISPLAY_WARNING_MESSAGES = True +DISPLAY_ERROR_MESSAGES = True + +# Ref #4: Background coloring provided by ASCII escape sequences +BACKGROUND_COLORS = { + 'DEBUG': '\033[95m', + 'OKBLUE': '\033[94m', + 'OKGREEN': '\033[92m', + 'WARNING': '\033[93m', + 'FAIL': '\033[91m', + 'ENDC': '\033[0m', + 'BOLD': '\033[1m', + 'UNDERLINE': '\033[4m', +} + + +# TODO: Am I using these or SQL generated date values? +date = datetime.date.today() +WEEK = date.strftime('week %U') +WEEK_YEAR = date.strftime('week %U, %Y') +MONTH = date.strftime('%B') +MONTH_YEAR = date.strftime('%B %Y') +YEAR = date.strftime('%Y') + +# NOTE: If we use datetime.date.today() or date as set above +# we will get the same result as what we're doing here +TODAY = date.strftime('%Y-%m-%d') + +# These entries match up with the event schedule "keywords". For example, +# when daily events are requested, active daily events will result in +# notifications generated which have subject lines that include the value +# that is paired with the keyword below. +DATE_LABEL = { + 'daily':TODAY, + 'twice_week':TODAY, + 'weekly':TODAY, + 'weekly_monday':TODAY, + 'weekly_tuesday':TODAY, + 'weekly_wednesday':TODAY, + 'weekly_thursday':TODAY, + 'weekly_friday':TODAY, + 'twice_month':TODAY, + 'monthly':MONTH_YEAR, + 'twice_year':MONTH_YEAR, + 'yearly':YEAR +} + + + +####################################################### +# Classes +####################################################### + +class Event(object): + """ + Represents an event from the event_reminders table + """ + + def __init__(self, event): + + print_debug("{}".format(event), "Event class, input tuple") + + # Expand out incoming tuple without hard-coding specific index values + (self.email_to_address, + self.email_from_address, + self.email_subject_prefix, + self.redmine_wiki_page_name, + self.redmine_wiki_page_project_shortname, + self.redmine_new_issue_project, + self.redmine_new_issue_category, + self.redmine_new_issue_status, + self.redmine_new_issue_due_date, + self.redmine_new_issue_priority, + self.event_schedule + ) = event + + print_debug("new instance of object created", "Event class") + + +class Settings(object): + + """ + Represents the user configurable settings retrieved from + an external config file + """ + + def __init__(self, config_file_list): + + try: + parser = configparser.SafeConfigParser() + + # Enable Extended Interpolation - allow values from one section + # to be referenced from another section. + parser._interpolation = configparser.ExtendedInterpolation() + + # Attempt to read and parse a list of filenames, returning a list + # of filenames which were successfully parsed. If none of the + # filenames exist, the configparser instance will contain an empty + # dataset. + processed_files = parser.read(config_file_list) + + print_debug("Config files processed: {}".format(processed_files), "CONFIG") + + # FIXME: We can either pass a verified list of files to the parser + # OR we can verify the number of processed files is 1 or greater. + if len(processed_files) < 1: + + # Just raise the standard parsing error exception instead + # of trying to handle a missing file differently than one + # with parsing errors. We may change this later if found + # to be too confusing. + raise configparser.ParsingError(config_file_list) + + except configparser.ParsingError as err: + + error_message = "Unable to parse config file: {}".format(err) + print_error(error_message, "CONFIG") + sys.exit() + + + # Begin building object by creating dictionary member attributes + # from config file sections/values. + + self.flags = {} + self.mysqldb_config = {} + self.queries = {} + + # Likely will be removed at some point + self.notification_servers = {} + + # Not directly referenced yet, but exposing for future use + self.email = {} + + try: + # Grab all values from section as tuple pairs and convert + # to dictionaries for easy reference + + # Not directly referenced yet, but exposing for future use + self.email = dict(parser.items('email')) + + self.flags = dict(parser.items('flags')) + self.mysqldb_config = dict(parser.items('mysqldb_config')) + self.queries = dict(parser.items('queries')) + + # FIXME: This name will likely need adjusting later + # to match whatever new section name is chosen for the config file + self.notification_servers = dict(parser.items('notification_servers')) + + # FIXME: Is there a better to handle this? + # This is a one-off boolean flag from a separate section + self.mysqldb_config['raise_on_warnings'] = \ + parser.getboolean('mysqldb_config', 'raise_on_warnings') + + # Convert text "boolean" flag values to true boolean values + for key in self.flags: + self.flags[key] = parser.getboolean('flags', key) + + print_debug("{} has a value of {} and a type of {}".format( + key, + self.flags[key], + type(self.flags[key])), "CONFIG") + + except configparser.NoSectionError as err: + + error_message = "{}: Unable to parse config file: {}".format("CONFIG", err) + print_error(error_message) + sys.exit(error_message) + + +####################################################### +# Functions +####################################################### + +def open_db_connection(settings, database): + + """ + Open a connection to the database and return a cursor object + """ + + + #################################################################### + # Open connections to databases + #################################################################### + + try: + + print_debug("""MySQL connection details: + + user: {} + password: {} + host: {} + port: {} + database: {} + raise_on_warnings: {} + raise_on_warnings (type): {} + """.format(settings.mysqldb_config['user'], + settings.mysqldb_config['password'], + settings.mysqldb_config['host'], + settings.mysqldb_config['port'], + database, + settings.mysqldb_config['raise_on_warnings'], + type(settings.mysqldb_config['raise_on_warnings'])), "MySQL") + + mysql_connection = mysql.connect( + user=settings.mysqldb_config['user'], + password=settings.mysqldb_config['password'], + host=settings.mysqldb_config['host'], + port=settings.mysqldb_config['port'], + database=database, + raise_on_warnings=settings.mysqldb_config['raise_on_warnings'] + ) + + except mysql.Error as error: + error_message = "Unable to connect to database: {}".format(error) + print_error(error_message, "MySQL") + sys.exit("MySQL: {}".format(error_message)) + + + return mysql_connection + + +def get_wiki_page_contents(settings, wiki_page_name, wiki_page_project, wiki_page_database): + + """ + Retrieve contents of the specified Redmine wiki page for inclusion in notification + """ + + #################################################################### + # Create cursor object so that we can interact with the database + #################################################################### + + mysql_connection = open_db_connection(settings, wiki_page_database) + + # Cursor for the MySQL copy of the database + mysql_cursor = mysql_connection.cursor() + + # Dynamically create the select query used to pull data from MySQL table + # See automated_tickets.ini for the available queries + try: + + # We're filtering events on the event schedule (daily, monthly etc.) + # and also on whether the event is an intern task AND whether the + # configuration file has enabled processing those tasks. + + query = settings.queries['wiki_page_contents'].format( + wiki_page_name, wiki_page_project) + + print_debug(query, "Wiki page retrieval query") + + mysql_cursor.execute(query) + + except Exception as e: + print_error("Unable to execute wiki page retrieval query: {} ".format(e), "MySQL") + sys.exit() + + try: + # Grab first element of returned tuple, ignore everything else + wiki_page_content = mysql_cursor.fetchone()[0] + + except Exception as e: + + # FIXME: Is there a Plan B for wiki page lookup failures? + print_error("Unable to retrieve wiki page content: {} ".format(e), "MySQL") + sys.exit() + + if wiki_page_content is not None: + + # Since the fetchone call returned a list of tuples, strip out just + # the value and return it without the tuple or list enclosure. + return wiki_page_content + + else: + + return "Unable to retrieve content from {}:{}".format( + wiki_page_project, wiki_page_name + ) + +def get_include_calls(wiki_page_contents, wiki_page_project): + + """ + Parse the contents of the initial wiki page and pull out every instance + where the Redmine include macro is called. We'll use this list to + find included wiki pages and then later to search/replace the include + calls with the content of the wiki pages that the macro calls were + pulling into the original wiki page. + """ + + # This pattern is intended to match the entire include macro call, + # including the macro itself. + # + # NOTE: I could not get the syntax right to support using .format() + # so I fell back to using classic string formatting + # TODO: Consider moving this to external config file for easy maintenace + include_macro_pattern = r'{{include\(%s:[a-zA-Z0-9 _\-\']+\)}}' % (wiki_page_project) + + wiki_page_macro_calls = [] + wiki_page_macro_calls = re.findall(include_macro_pattern, wiki_page_contents) + + print_debug(wiki_page_macro_calls, "List of include macro calls") + + return wiki_page_macro_calls + + +def get_included_wiki_pages(wiki_page_macro_calls, wiki_page_project): + + """ + Accepts a list of include page macro call strings and pulls out the + names of the pages that are being included by the primary wiki page. + This list of pages will be sourced and used to replace the original + Redmine include macro calls so that when finished, a complete page + (without any further include calls) is used to generate notifications. + """ + + # This is meant to match just the page name that is being included + # + # NOTE: I could not get the syntax right to support using .format() + # so I fell back to using classic string formatting + # TODO: Consider moving this to external config file for easy maintenace + included_page_pattern = r'{{include\(%s:([a-zA-Z0-9 _\-\']+)\)}}' % (wiki_page_project) + + wiki_page_names = [] + + for match in wiki_page_macro_calls: + included_page = re.search(included_page_pattern, match) + + if included_page: + # Append the first parenthesized subgroup of the match + # The equivalent value appears to be 'included_page.groups()[0]' + wiki_page_names.append(included_page.group(1)) + else: + # This function should ONLY be called if there were include + # macro calls in the primary wiki page. If this situation + # occurred then there is a bug somewhere and we need to know + # about it. + print_error('No matches found', 'wiki_page_names search') + sys.exit() + + print_debug("{}".format(wiki_page_names), "List of included wiki pages") + + return wiki_page_names + + + +def get_events(settings, event_schedule): + + """ + Builds a list of Event objects representing rows in the event_reminders db + """ + + #################################################################### + # Create cursor object so that we can interact with the database + #################################################################### + + mysql_connection = open_db_connection(settings, settings.mysqldb_config['events_database']) + + # Cursor for the MySQL copy of the database + mysql_cursor = mysql_connection.cursor() + + # Dynamically create the select query used to pull data from MySQL table + # See automated_tickets.ini for the available queries + + + # Base query that filters just on the event schedule type. We may + # further constrain depending on what configuration settings have + # been toggled. + base_query = "{} AND event_schedule = '{}'".format( + settings.queries['event_table_entries'], + event_schedule) + + # Check configuration setting to determine if we need to filter out + # "intern" or student worker events. + if not settings.flags['process_intern_events']: + query = "{} AND event_schedule = '{}' AND intern_task = 0".format( + settings.queries['event_table_entries'], + event_schedule) + else: + # Use just the base query then + query = base_query + + try: + mysql_cursor.execute(query) + + except Exception as e: + print_error("Unable to query event_reminders table: {} ".format(e), "MySQL") + + print_debug("Pulling data from {} MySQL table ...".format('events'), "MySQL") + + events = [] + for event in mysql_cursor.fetchall(): + + # Collect a list of all events we need to take action for + events.append(Event(event)) + + #################################################################### + # Cleanup + #################################################################### + + # Close database connections + print_debug("Closing database connection ...", "MySQL") + mysql_connection.close() + + return events + +# Used by get_full_file_path() function, freestanding function +# in case it needs to be used elsewhere +def file_exists(full_path_to_file): + """Verify that the file exists and is readable.""" + + return bool(os.access(full_path_to_file, os.R_OK)) + +def file_can_be_modified(full_path_to_file): + """Verify that the file exists and is writable.""" + + return bool(os.access(full_path_to_file, os.W_OK)) + +# FIXME: Consider tossing this function since I'm not using it +def get_full_file_path(local_dir, global_dir, file_name): + """ + Returns the full path to an external resource. If the file can be found + locally (same directory as this script) that full path will be returned, + otherwise the full path will be built from the "normal" location + """ + + local_file = os.path.join(local_dir, file_name) + + global_file = os.path.join(global_dir, file_name) + + # TODO: Consider adding a hook to each function to provide this value + # for debug output (troubleshooting) + name_of_this_function = sys._getframe().f_code.co_name + + print_debug("local file is {}".format(local_file), name_of_this_function) + print_debug("global file is {}".format(global_file), name_of_this_function) + + # Attempt to reference local file + if file_exists(local_file): + return local_file + + elif file_exists(global_file): + return global_file + + else: + # FIXME: Raise exception instead? + error_message = "[!] Unable to verify access for '{}' at '{}' or '{}'. Exiting ..." + sys.exit(error_message.format(file_name, local_file, global_file)) + + +def print_debug(message, prefix=""): + """Prints message if the DISPLAY_DEBUG_MESSAGES flag is set""" + + if DISPLAY_DEBUG_MESSAGES: + + if len(prefix) > 0: + prefix = "{}:".format(prefix) + + print("{}[d] {} {}{}".format( + BACKGROUND_COLORS['DEBUG'], + prefix, + # Explicitly convert passed message to string for display + str(message), + BACKGROUND_COLORS['ENDC'])) + +def print_info(message, prefix=""): + """Prints message if the DISPLAY_INFO_MESSAGES flag is set""" + + if DISPLAY_INFO_MESSAGES: + + if len(prefix) > 0: + prefix = "{}:".format(prefix) + + print("{}[i] {} {}{}".format( + BACKGROUND_COLORS['BOLD'], + prefix, + # Explicitly convert passed message to string for display + str(message), + BACKGROUND_COLORS['ENDC'])) + +def print_warning(message, prefix=""): + """Prints warning message to console""" + + if DISPLAY_WARNING_MESSAGES: + + if len(prefix) > 0: + prefix = "{}:".format(prefix) + + print("{}[w] {} {}{}".format( + BACKGROUND_COLORS['WARNING'], + prefix, + # Explicitly convert passed message to string for display + str(message), + BACKGROUND_COLORS['ENDC'])) + +def print_error(message, prefix=""): + """Prints warning message to console""" + + if DISPLAY_ERROR_MESSAGES: + + if len(prefix) > 0: + prefix = "{}:".format(prefix) + + print("{}[!] {} {}{}".format( + BACKGROUND_COLORS['FAIL'], + prefix, + # Explicitly convert passed message to string for display + str(message), + BACKGROUND_COLORS['ENDC'])) + +# FIXME: Add the from_address and to_address values onto the message object +def send_notification(settings, from_address, to_address, message): + """ + Long term, this should be an entry point to a validation and notification + chain of functions, supporting email, text, XMPP and other types + of notifications. For now, only email notifications are supported. + """ + + print_debug(message, "Notification") + + email_server = settings.notification_servers['email_server_ip_or_fqdn'] + email_debug_filename = 'email.txt' + + if settings.flags['testing_mode']: + + # Fill in details from this run at the end of the file. It is up to + # the caller to prune the old file if they wish to have the new + # results go to a clean file. + with open(email_debug_filename, "a") as fh: + fh.writelines(message) + + else: + + server = smtplib.SMTP(email_server) + #server.set_debuglevel(1) + server.sendmail(from_address, to_address, message) + server.quit() diff --git a/create_users.sql b/create_users.sql new file mode 100644 index 0000000..5348771 --- /dev/null +++ b/create_users.sql @@ -0,0 +1,54 @@ + +-- Purpose: CREATE USER accounts and GRANT permissions to work with event_reminders DATABASE + +-- This should already be taken care of by importing database_schema.sql +-- CREATE DATABASE event_reminders CHARACTER SET utf8; + + +-- The below examples all use a UNIX socket for connections. MySQL specifies +-- users as a username/hostname pair, so to have an account accessed by a +-- remote client IP, you will need to specify that remote IP at creation time. +-- https://dev.mysql.com/doc/refman/5.5/en/account-names.html +-- +-- Update the CREATE USER and GRANT examples below to reference the remote IP +-- instead of localhost if the account needs to be accessible remotely. + + +-- Create read-write user for existing database. Useful for admin work now +-- and for use by the future web app (though best practice likely dictates +-- creating a separate account just for the web app +CREATE USER 'events_rw'@'localhost' IDENTIFIED BY 'ChangeM3'; +GRANT ALL PRIVILEGES ON event_reminders.* TO 'events_rw'@'localhost'; + + + + +-- Create read-only user for (existing) database. Useful for creating backups +-- and for scripted access. This account needs not only read access to the +-- event_reminders database, but the Redmine database containing wiki pages +-- that are referenced by this project. +CREATE USER 'events_ro'@'localhost' IDENTIFIED BY 'ChangeM3'; +GRANT SELECT,LOCK TABLES ON event_reminders.* TO 'events_ro'@'localhost'; + +-- The following permissions are needed to pull wiki page contents from +-- the Redmine database. See the query in the automated_tickets.ini config +-- file for how these columns are used. +GRANT SELECT(text) ON help_redmine_db.wiki_contents TO 'events_ro'@'localhost'; +GRANT SELECT(id) ON help_redmine_db.wiki_contents TO 'events_ro'@'localhost'; + +GRANT SELECT(id) ON help_redmine_db.wiki_pages TO 'events_ro'@'localhost'; +GRANT SELECT(wiki_id) ON help_redmine_db.wiki_pages TO 'events_ro'@'localhost'; +GRANT SELECT(title) ON help_redmine_db.wiki_pages TO 'events_ro'@'localhost'; + +GRANT SELECT(id) ON help_redmine_db.wikis TO 'events_ro'@'localhost'; +GRANT SELECT(project_id) ON help_redmine_db.wikis TO 'events_ro'@'localhost'; + +GRANT SELECT(identifier) ON help_redmine_db.projects TO 'events_ro'@'localhost'; +GRANT SELECT(id) ON help_redmine_db.projects TO 'events_ro'@'localhost'; + +FLUSH PRIVILEGES; + + + + + diff --git a/database_schema.sql b/database_schema.sql new file mode 100644 index 0000000..b43d75c --- /dev/null +++ b/database_schema.sql @@ -0,0 +1,107 @@ +/* + + Purpose: Collection of SQL statements to create initial database for events + + WARNING: Requires MySQL >= 5.5.3 due to length of table/field comments + + References: + + http://dev.mysql.com/doc/refman/5.5/en/create-table.html + http://dev.mysql.com/doc/refman/5.5/en/create-view.html + http://www.mysqltutorial.org/create-sql-views-mysql.aspx + + Notes: + + * View column comments via: + show full columns from TABLE_NAME; + show create TABLE_NAME; + + * A comment for a column can be specified with the COMMENT option, up to + 1024 characters long (255 characters before MySQL 5.5.3). + + * A comment for the table, up to 2048 characters long (60 characters + before MySQL 5.5.3). + +*/ + + +-- Create database for event entries. Our script will parse those entries and +-- generate notifications. The first design will be strictly to replace the +-- existing crude/hard-coded scripts, each dedicated to a specific time period. +CREATE DATABASE event_reminders CHARACTER SET utf8; + +-- Emphasizing what database we're working with +USE event_reminders; + +-- Milestone one database schema design +-- +-- All columns are near 1:1 entries from the old "list of dictionaries" setup +-- that I used in the various scripts used to generate automated tickets. +-- Later revisions will see many columns from this table moved into separate +-- tables. + + + +CREATE TABLE `event_reminders`.`events` +( + `id` int(11) NOT NULL auto_increment, + + `enabled` TINYINT(1) NOT NULL DEFAULT 1 + COMMENT "If set to 1, then email notifications will be generated. Later designs might incorporate other actions for enabled entries. If set to 0, no actions will be taken for the entry.", + + `intern_task` TINYINT(1) NOT NULL DEFAULT 0 + COMMENT "Whether this event is something a student worker or intern handles. The default value is 0, or not an intern task. This value is used as a filter so that we can turn on/off events/tasks for times when students are not available to perform the task.", + + /* Matching the same field type/length as the email_addresses table */ + `email_to_address` varchar(255) NULL + COMMENT "FIXME: For automated tickets this address will usually be the same for all events. If this is left blank the the default set in the config file applies.", + + /* Matching the same field type/length as the email_addresses table */ + -- Note: The old one-script-per-schedule approach used different sender addresses depending on the destination project + `email_from_address` varchar(255) NULL + COMMENT "FIXME: For automated tickets this address will be the same for all events. This should be moved to its own table in the next milestone.", + + `email_subject_prefix` text NOT NULL + COMMENT "The prefix for notifications related to this event. The script referencing this table will append an auto-generated suffix to denote the date or date range.", + + /* Matching the same field type/length as the wiki_pages table */ + `redmine_wiki_page_name` varchar(255) NOT NULL + COMMENT "The name of the wiki page (without project prefix) whose text will be inserted into the body of the email notification.", + + /* Matching the same field type/length as the projects table */ + `redmine_wiki_page_project_shortname` varchar(255) NOT NULL + COMMENT "The tag or project identifier displayed in the project URL. Used to match the wiki page whose text will be pulled for email notification.", + + /* Matching the same field type/length as the projects table */ + `redmine_new_issue_project` varchar(255) NOT NULL + COMMENT "The full name of the project where the ticket generated from the email notification will be routed", + + /* Matching the same field type/length as issue_categories table */ + `redmine_new_issue_category` varchar(60) NOT NULL + COMMENT "The full category name for the routed ticket.", + + /* Matching the same field/type length as issue_statuses table */ + `redmine_new_issue_status` varchar(30) NOT NULL DEFAULT 'Assigned' + COMMENT "The status that should be set for newly created tickets (the workflow must allow for this initial status)", + + /* Custom approach to handling due dates. At some future milestone we may need to add support for "trigger" dates + to complement this setting */ + `redmine_new_issue_due_after_days` smallint NULL + COMMENT "Supported values: The number of days after the ticket is generated when it should be due. Whole, positive numbers only.", + + /* Matching the same field type as enumerations table */ + `redmine_new_issue_priority` varchar(30) NOT NULL DEFAULT 'Normal' + COMMENT "Supported values: Real date value or lowercase 'today'. That keyword is replaced as part of the retrieval query.", + + /* Fixed options that reflect entries in the /etc/cron.d/automated_tickets file */ + `event_schedule` varchar(30) NOT NULL + COMMENT "Specific keywords that the controller script uses to calcuate due dates for newly generated tickets. Supported values are found in the DATE_LABEL dictionary. A few examples: daily, weekly, twice_month, twice_year", + + `last_modified` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `comments` text NULL, + PRIMARY KEY (`id`) +) + ENGINE=InnoDB + DEFAULT CHARSET=utf8 + COMMENT="Milestone one design for automated_tickets project. This table represents various events and tasks that we should receive notifications for." +; diff --git a/helpful_queries.sql b/helpful_queries.sql new file mode 100644 index 0000000..e4f025f --- /dev/null +++ b/helpful_queries.sql @@ -0,0 +1,149 @@ +-- Purpose: Useful queries for helping to fill out options in config file + + + + +-- The wiki id is not the same value as the project id. Projects may +-- have a wiki, or they may not and as a new wiki is created the wiki_id +-- number is incremented to track the new wiki. The 'wikis' table ties +-- together the project id with the wiki id, and then the 'wiki_pages' +-- table contains the name of the pages. The actual page content resides +-- in the 'wiki_contents' table. + + + +-- Summary of events, useful for updating documentation manually +SELECT + CASE + WHEN intern_task = 1 THEN 'Yes' + WHEN intern_task = 0 THEN 'No' + END AS 'Student task', + CASE + WHEN enabled = 1 THEN 'Yes' + WHEN enabled = 0 THEN 'No' + END AS 'Enabled', + SUBSTRING(email_subject_prefix FROM 1 FOR 60) AS 'Ticket title template (truncated)', + redmine_wiki_page_name AS 'Wiki page', + event_schedule AS 'Frequency', + redmine_new_issue_due_after_days AS 'Due in X days' +FROM + events +ORDER BY + enabled, + redmine_new_issue_project, + event_schedule +; + +-- "Dashboard" query +SELECT + wp.title AS 'Wiki page name', + p.name AS 'Project Name', + p.identifier as 'Project shortname' +FROM + projects AS p +INNER JOIN wikis AS w ON + w.project_id = p.id +INNER JOIN wiki_pages AS wp ON + wp.wiki_id = w.id +ORDER BY p.name, wp.title; + + +-- The query responsible for pulling wiki page content from a specified +-- wiki page within a specified project (identified by project shortname) +SET @project_shortname = 'docs'; +SET @wiki_page_title = 'Creating_a_new_Git_repository'; +SELECT + wiki_contents.text +FROM + wiki_contents +INNER JOIN wiki_pages ON + wiki_pages.id = wiki_contents.id +INNER JOIN wikis ON + wikis.id = wiki_pages.wiki_id +INNER JOIN projects ON + projects.id = wikis.project_id +WHERE + wiki_pages.title = @wiki_page_title +AND + projects.identifier = @project_shortname; + + +-- Various details that will be needed for filling out the configuration file +-- and for general troubleshooting later on (ex: search/replace include macro) +SELECT + p.name AS 'Project Name', + p.id as 'Project id', + p.identifier as 'Project shortname', + w.id AS 'Wiki id', + wp.title AS 'Wiki page name', + wp.id AS 'Wiki page id' +FROM + projects AS p +INNER JOIN wikis AS w ON + -- tie together wikis table with projects table + w.project_id = p.id +INNER JOIN wiki_pages AS wp ON + wp.wiki_id = w.id +ORDER BY p.name, wp.title; + + +-- List the name of each wiki page in the specified project that uses +-- the '{{include()}}' Redmine macro. Leave off the trailing AND +-- bit in order to list all wiki pages that use the include macro. +SET @wiki_project_shortname = 'docs'; +SELECT wiki_pages.title +FROM wiki_contents +INNER JOIN wiki_pages ON wiki_pages.id = wiki_contents.id +INNER JOIN wikis ON wikis.id = wiki_pages.wiki_id +INNER JOIN projects ON projects.id = wikis.project_id +WHERE wiki_contents.text LIKE '%include%' AND projects.identifier = @wiki_project_shortname; + + +-- List all enabled events +SELECT + id, + enabled, + email_to_address, + email_from_address, + email_subject_prefix, + event_schedule +FROM events +WHERE enabled = 1; + + +-- List all events that are overriding either the TO or FROM address +SELECT + id, + enabled, + email_to_address, + email_from_address, + email_subject_prefix, + event_schedule +FROM + events +WHERE + (email_to_address IS NOT NULL) OR (email_from_address IS NOT NULL) +ORDER BY + event_schedule; + + +-- Retrieve members of a specific group so that they can be added as Watchers +-- via inclusion in the CC list of a newly created ticket +-- +-- http://www.redmine.org/boards/1/topics/19470 +/* + Notes: + + * Groups are included in the users table + * Users have a type. Known types are below: + ** GroupNonMember + ** GroupAnonymous + ** Group + ** User + ** AnonymousUser + * I should match against 'Group' type + * The users.id value will need to be used against the groups_users table + to map a group name to a group id and then look up users who are members + of that group id in order to build a list of group members for a group. + +*/ diff --git a/template_new_automated_ticket.sql b/template_new_automated_ticket.sql new file mode 100644 index 0000000..f112d62 --- /dev/null +++ b/template_new_automated_ticket.sql @@ -0,0 +1,67 @@ +/* + + Purpose: + + Template for creating new automated tickets. + + Notes: + + * If 0 is used for redmine_new_issue_due_after_days, then the due date + is set for the same day, otherwise it is generated as ticket + creation date + the number of days specified + + * We do not specify a the from/to column values; the read query + will use a default value for each from the config file + + * We are relying on each task to default to enabled + + * Tasks are not "intern" tasks by default + + * Priority is set to 'Normal' by default + +*/ + +-- Variables, because I'm feeling lazy +SET @ds = 'desktop-support'; +SET @ss = 'server-support'; + +SET @ds_email = 'ds-automated-reminders@help.example.com'; +SET @ss_email = 'ss-automated-reminders@help.example.com'; + + +-- Here we are inserting a row with the redmine_new_issue_due_after_days +-- value set to 0. This results in the read query subsituting that value +-- for the current date so that tickets have the due date set to the +-- same day the ticket was generated. We are also overriding the default +-- priority. +INSERT INTO `event_reminders`.`events` + ( + `intern_task`, + `email_from_address`, + `email_subject_prefix`, + `redmine_wiki_page_name`, + `redmine_wiki_page_project_shortname`, + `redmine_new_issue_project`, + `redmine_new_issue_category`, + `redmine_new_issue_due_after_days`, + `redmine_new_issue_priority`, + `event_schedule`, + `comments` + ) +VALUES + + -- Template entry for a Desktop Support task + -- + -- * 'sysdocs' is the usual value for 'WIKI_PROJECT_SHORT_NAME_HERE' + -- * Valid values for FREQUENCY_HERE are (all lowercase): daily, twice_week, weekly, weekly_monday, weekly_tuesday, weekly_wednesday, weekly_thursday, weekly_friday, twice_month, monthly, twice_year, yearly + -- See the DATE_LABEL dictionary for an authoratative list. + (1,@ds_email,'SUBJECT_LINE_TEMPLATE_HERE for {}','WIKI_PAGE_NAME_HERE','WIKI_PROJECT_SHORT_NAME_HERE',@ds,'CATEGORY_NAME_HERE',DUE_AFTER_DAYS_HERE_AS_UNQUOTED_NUMBER,'PRIORITY_HERE','frequency_here','OPTIONAL_COMMENT_HERE_REMOVE_THIS_IF_NOT_USING'), + + -- Template entry for a Server Support task + -- + -- * 'sysdocs' is the usual value for 'WIKI_PROJECT_SHORT_NAME_HERE' + -- * Valid values for FREQUENCY_HERE are (all lowercase): daily, twice_week, weekly, weekly_monday, weekly_tuesday, weekly_wednesday, weekly_thursday, weekly_friday, twice_month, monthly, twice_year, yearly + -- See the DATE_LABEL dictionary for an authoratative list. + (0,@ss_email,'SUBJECT_LINE_TEMPLATE_HERE for {}','WIKI_PAGE_NAME_HERE','WIKI_PROJECT_SHORT_NAME_HERE',@ss,'CATEGORY_NAME_HERE',DUE_AFTER_DAYS_HERE_AS_UNQUOTED_NUMBER,'PRIORITY_HERE','frequency_here','OPTIONAL_COMMENT_HERE_REMOVE_THIS_IF_NOT_USING') + +;