Merge dockerized rsyslog. config split into multiple files.

Dockerfile

@@ -1,18 +1,19 @@
 FROM centos:centos7
-MAINTAINER The BitScout Community <community@TBA>
+MAINTAINER The ViaQ Community <community@TBA>
 
 EXPOSE 10514
 
 ENV SYSLOG_LISTEN_PORT=10514 \
     ES_HOST=bitscout-elasticsearch \
     ES_PORT=9200
 
-RUN yum install -y rsyslog rsyslog-elasticsearch rsyslog-gssapi \
-    rsyslog-mmjsonparse rsyslog-mmsnmptrapd && \
+RUN curl https://copr.fedorainfracloud.org/coprs/portante/rsyslog-8.17/repo/epel-7/portante-rsyslog-8.17-epel-7.repo > /etc/yum.repos.d/portante-rsyslog-v8.17-epel-7.repo && \
+    yum install -y rsyslog rsyslog-elasticsearch rsyslog-gssapi \
+    rsyslog-mmjsonparse && \
     yum clean all
 
-ADD rsyslog.conf /etc/rsyslog.conf
-ADD rsyslog.d/* /etc/rsyslog.d/
+VOLUME /data
+
 ADD run.sh /usr/sbin/
 WORKDIR /var/lib/rsyslog
 

README.md

@@ -1,2 +1,17 @@
 # docker-rsyslog
-BitScout rsyslog docker container - implements the aggregator/formatter
+ViaQ rsyslog docker container - implements the aggregator/formatter
+
+## Environmental variables:
+`ES_HOST` must be FQDN of ElasticSearch server.  
+`ES_PORT` must be the port on which the ElasticSearch server is listening.  
+`SYSLOG_LISTEN_PORT` the port this rsyslog instance is listening for. both TCP and UDP.  
+
+## External rsyslog config
+In order to add own rsyslog configuration file please add the configuration files to a local directory and map in to `/data` docker volume.  
+The following files are taken form the local directory:  
+`rsyslog.conf, rsyslog.d/*.conf`  
+In case `rsyslog.conf` exists, the default `rsyslog.d/*.conf` is removed and not used in the container.
+
+## Running:
+docker run -d -p $syslog_listen_port:$syslog_listen_port/tcp -p $syslog_listen_port:$syslog_listen_port/udp -v $local_dir:/data -u $uid -e ES_HOST=$elasticsearchhost -e ES_PORT=$port -e SYSLOG_LISTEN_PORT=$syslog_listen_port -e NORMALIZER_NAME=container-rsyslog8.17 -e NORMALIZER_IP=$normalizer_ip -e LOGSTASH_PREFIX=v2016.03.10.0-viaq- --name $appname $image
+

build-image.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 set -ex
-prefix=${PREFIX:-${1:-bitscout/}}
+prefix=${PREFIX:-${1:-viaq/}}
 version=${VERSION:-${2:-latest}}
 docker build -t "${prefix}rsyslog:${version}" .
 

rsyslog.conf

@@ -3,147 +3,46 @@
 # For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
 # If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html
 
-#### MODULES ####
-
-# only using rsyslog in aggregator mode
-#module(load="imjournal" StateFile="/var/lib/rsyslog/imjournal.state")
-
-# Provides UDP syslog reception
-module(load="imudp")
-input(type="imudp" port="%SYSLOG_LISTEN_PORT%")
-
-# Provides TCP syslog reception
-module(load="imptcp")
-input(type="imptcp" port="%SYSLOG_LISTEN_PORT%")
-
-# ElasticSearch output module
-module(load="omelasticsearch")
-
-# Parsing CEE JSON messages
-module(load="mmjsonparse")
-
-# Reformating SNMP trap messages module
-module(load="mmsnmptrapd")
-
-# Ensures we have UTF-8 encoded payloads
-#module(load="mmutf8fix")
-
-
 #### GLOBAL DIRECTIVES ####
 
 global(
     # Where to place auxiliary files
     workDirectory="/var/lib/rsyslog"
     # perf-dept: we want fully qualified domain names for common logging
-    preserveFQDN="on")
+    preserveFQDN="on"
+    # Try to avoid any message truncation
+    maxMessageSize="65536")
 
-# main_queue not available in latest el7 rsyslog (7.4) only in 7.5 and later
-# main_queue(
-#     # Beef up the internal message queue
-#     queue.size="131072"
-#     # 90% of QueueSize
-#     queue.discardmark="117964"
-#     # If we reach the discard mark, we'll throw out notice, info, and debug messages
-#     queue.discardseverity="5")
+main_queue(
+    # 4 worker threads in the main queue
+    queue.workerthreads="4"
+    # Beaf up the internal message queue
+    queue.size="131072"
+    # 90% of QueueSize
+    queue.discardmark="117964"
+    # If we reach the discard mark, we'll throw out notice, info, and debug messages
+    queue.discardseverity="5")
 
-# this is for index names to be like: bitscout-YYYY.MM.DD
-# WARNING: any rsyslog collecting host MUST be running UTC
-#          if the proper index is to be chosen to hold the
-#          log entry. If you are running EDT, e.g., then
-#          the previous day's index will be chosen even
-#          though the UTC value is the current day, because
-#          the pattern logic does not convert "timereported"
-#          to a UTC value before pulling data out of it.
-template(name="bitscout-index-pattern" type="list") {
-    constant(value="bitscout-")
-    property(name="timereported" dateFormat="rfc3339" position.from="1" position.to="4")
-    constant(value=".")
-    property(name="timereported" dateFormat="rfc3339" position.from="6" position.to="7")
-    constant(value=".")
-    property(name="timereported" dateFormat="rfc3339" position.from="9" position.to="10")
-    }
-# this is for formatting our syslog data in JSON with @timestamp using a "hierarchical" metdata namespace
-template(name="com-redhat-rsyslog-hier"
-         type="list") {
-    constant(value="{")
-    constant(value="\"@timestamp\":\"")               property(name="timereported" dateFormat="rfc3339")
-    constant(value="\",\"@version\":\"2015.09.24-0")
-    constant(value="\",\"message\":\"")               property(name="msg" format="json")
-    constant(value="\",\"hostname\":\"")              property(name="hostname")
-    constant(value="\",\"level\":\"")                 property(name="syslogseverity-text")
-    constant(value="\",\"pid\":\"")                   property(name="procid")
-    constant(value="\",\"rsyslog\": {")
-    constant(value="\"facility\":\"")                 property(name="syslogfacility-text")
-    constant(value="\",\"programname\":\"")           property(name="programname")
-    constant(value="\",\"fromhost\":\"")              property(name="fromhost")
-    constant(value="\",\"fromhost-ip\":\"")           property(name="fromhost-ip")
-    constant(value="\",\"timegenerated\":\"")         property(name="timegenerated" dateFormat="rfc3339")
-    constant(value="\",\"protocol-version\":\"")      property(name="protocol-version")
-    constant(value="\",\"structured-data\":\"")       property(name="structured-data")
-    constant(value="\",\"app-name\":\"")              property(name="app-name")
-    constant(value="\",\"msgid\":\"")                 property(name="msgid")
-    constant(value="\",\"inputname\":\"")             property(name="inputname")
-    constant(value="\",\"_cee\":")                    property(name="$!all-json")
-    constant(value="} }")
-    }
 
-# Use default timestamp format
-$ActionFileDefaultTemplate RSYSLOG_FileFormat
+# Include loading modules and basic formatting
+$IncludeConfig /etc/rsyslog.d/00-load-modules-and-formatting.conf
 
-# Include all config files in /etc/rsyslog.d/
-$IncludeConfig /etc/rsyslog.d/*.conf
+# Include templates for destination index name and for data model
+$IncludeConfig /etc/rsyslog.d/10-viaq-templates.conf
 
-#### RULES ####
+# Include Pipeline metadata information
+$IncludeConfig /etc/rsyslog.d/20-viaq-pipeline-metadata.conf
 
-# Ensure message is a properly formatted UTF-8 sequence
-#action(type="mmutf8fix" mode="utf-8")
+# Include top-level fields modifications
+$IncludeConfig /etc/rsyslog.d/25-viaq-common.conf
 
-# Reformat any SNMP trap messages (legacy format required)
-*.* :mmsnmptrapd:
+# Include fields in systemd section
+$IncludeConfig /etc/rsyslog.d/30-viaq-systemd.conf
 
-# Parse any CEE JSON messages
-action(type="mmjsonparse")
+# Include fields in rsyslog section
+$IncludeConfig /etc/rsyslog.d/40-viaq-rsyslog-section.conf
 
 # Index into elasticsearch directly in a hierarchical metadata namespace
-action(
-    type="omelasticsearch"
-    server="%ES_HOST%"
-    serverport="%ES_PORT%"
-    template="com-redhat-rsyslog-hier"
-    searchIndex="bitscout-index-pattern"
-    dynSearchIndex="on"
-    searchType="rsyslog"
-    bulkmode="on"
-    queue.type="linkedlist"
-    queue.size="5000"
-    queue.dequeuebatchsize="300"
-    action.resumeretrycount="-1")
-
-# Prevent local elasticsearch logs from flowing into other log files (they are
-# already being logged to files). The local ElasticSearch instance is the
-# "test" one, which has a "test-elasticsearch" tag, but we don't need to
-# explicitly check for that since no logs from any elasticsearch instance need
-# to be logged locally.
-:programname, isequal, "elasticsearch"  stop
-
-# Log anything (except mail) of level info or higher.
-# Don't log private authentication messages!
-*.info;mail.none;authpriv.none;cron.none                /var/log/messages
-
-# The authpriv file has restricted access.
-authpriv.*                                              /var/log/secure
-
-# Log all the mail messages in one place.
-mail.*                                                  -/var/log/maillog
-
-# Log cron stuff
-cron.*                                                  /var/log/cron
-
-# Everybody gets emergency messages
-*.emerg                                                 :omusrmsg:*
+$IncludeConfig /etc/rsyslog.d/50-elasticsearch-output.conf
 
-# Save news errors of level crit and higher in a special file.
-uucp,news.crit                                          /var/log/spooler
 
-# Save boot messages also to boot.log
-local7.*                                                /var/log/boot.log

rsyslog.d/00-load-modules-and-formatting.conf

@@ -0,0 +1,30 @@
+#### MODULES ####
+
+# Emit internal rsyslog counters
+module(load="impstats" format="cee" interval="60")
+
+# Provides UDP syslog reception
+module(load="imudp")
+input(type="imudp" port="%SYSLOG_LISTEN_PORT%")
+
+# Provides TCP syslog reception
+module(load="imptcp")
+input(type="imptcp" port="%SYSLOG_LISTEN_PORT%")
+
+# ElasticSearch output module
+module(load="omelasticsearch")
+
+# Parsing CEE JSON messages
+module(load="mmjsonparse")
+
+# Ensures we have UTF-8 encoded payloads
+module(load="mmutf8fix")
+
+#### RULES ####
+
+# Ensure message is a properly formatted UTF-8 sequence
+action(type="mmutf8fix" mode="utf-8")
+
+# Parse any CEE JSON messages
+action(type="mmjsonparse")
+

rsyslog.d/10-viaq-templates.conf

@@ -0,0 +1,40 @@
+# Templates
+#
+# this is for index names to be like: logstash-YYYY.MM.DD
+# WARNING: any rsyslog collecting host MUST be running UTC
+#          if the proper index is to be chosen to hold the
+#          log entry. If you are running EDT, e.g., then
+#          the previous day's index will be chosen even
+#          though the UTC value is the current day, because
+#          the pattern logic does not convert "timereported"
+#          to a UTC value before pulling data out of it.
+
+
+template(name="viaq-index-pattern" type="list") {
+    constant(value="%LOGSTASH_PREFIX%")
+    property(name="timereported" dateFormat="rfc3339" position.from="1" position.to="4")
+    constant(value=".")
+    property(name="timereported" dateFormat="rfc3339" position.from="6" position.to="7")
+    constant(value=".")
+    property(name="timereported" dateFormat="rfc3339" position.from="9" position.to="10")
+    }
+
+# this is for formatting our syslog data in JSON with @timestamp using a "hierarchical" metdata namespace
+
+template(name="com-redhat-rsyslog-hier"
+         type="list") {
+    constant(value="{")
+    constant(value="\"@timestamp\":\"")               property(name="timegenerated" dateFormat="rfc3339")
+    constant(value="\",\"message\":\"")               property(name="$.msg" format="json")
+    constant(value="\",\"hostname\":\"")              property(name="$.hostname")
+#    constant(value="\",\"ipaddr4\":\"")               property(name="$.ipaddr4")
+    constant(value="\",\"level\":\"")                 property(name="$.level")
+    constant(value="\",\"pid\":\"")                   property(name="$.pid")
+    constant(value="\",\"tags\":\"")                  property(name="$.tags")
+    constant(value="\",\"service\":\"")               property(name="$.service")
+    constant(value="\",\"CEE\":")                     property(name="$!all-json")
+    constant(value=",\"systemd\":")                   property(name="$.systemd")
+    constant(value=",\"rsyslog\":")                   property(name="$.rsyslog")
+    constant(value=",\"pipeline_metadata\":")         property(name="$.pipeline_metadata")
+    constant(value="}\n")
+    }

rsyslog.d/20-viaq-pipeline-metadata.conf

@@ -0,0 +1,17 @@
+# Pipeline metadata related fields
+
+# If there was some pipeline_metadata coming from the collector - we want to preserve it
+if strlen($!pipeline_metadata) > 0 then {
+    set $.pipeline_metadata = $!pipeline_metadata;
+    unset $!pipeline_metadata;
+}
+
+# Add information about the normalizer
+template(name="timegeneratedrfc3339" type="string" string="%timegenerated:::date-rfc3339%")
+set $.pipeline_metadata!normalizer!received_at = exec_template("timegeneratedrfc3339");
+set $.pipeline_metadata!normalizer!inputname = $inputname;
+set $.pipeline_metadata!normalizer!ipaddr4 = "%NORMALIZER_IP%";
+set $.pipeline_metadata!normalizer!name = "%NORMALIZER_NAME%";
+set $.pipeline_metadata!version = "2016.03.10.0";
+
+