rsyslog.conf

# rsyslog configuration file

# For more information see /usr/share/doc/rsyslog-*/rsyslog_conf.html
# If you experience problems, see http://www.rsyslog.com/doc/troubleshoot.html

#### MODULES ####

# only using rsyslog in aggregator mode
#module(load="imjournal" StateFile="/var/lib/rsyslog/imjournal.state")

# Provides UDP syslog reception
module(load="imudp")
input(type="imudp" port="%SYSLOG_LISTEN_PORT%")

# Provides TCP syslog reception
module(load="imptcp")
input(type="imptcp" port="%SYSLOG_LISTEN_PORT%")

# ElasticSearch output module
module(load="omelasticsearch")

# Parsing CEE JSON messages
module(load="mmjsonparse")

# Reformating SNMP trap messages module
module(load="mmsnmptrapd")

# Ensures we have UTF-8 encoded payloads
#module(load="mmutf8fix")


#### GLOBAL DIRECTIVES ####

global(
    # Where to place auxiliary files
    workDirectory="/var/lib/rsyslog"
    # perf-dept: we want fully qualified domain names for common logging
    preserveFQDN="on")

# main_queue not available in latest el7 rsyslog (7.4) only in 7.5 and later
# main_queue(
#     # Beef up the internal message queue
#     queue.size="131072"
#     # 90% of QueueSize
#     queue.discardmark="117964"
#     # If we reach the discard mark, we'll throw out notice, info, and debug messages
#     queue.discardseverity="5")

# this is for index names to be like: bitscout-YYYY.MM.DD
# WARNING: any rsyslog collecting host MUST be running UTC
#          if the proper index is to be chosen to hold the
#          log entry. If you are running EDT, e.g., then
#          the previous day's index will be chosen even
#          though the UTC value is the current day, because
#          the pattern logic does not convert "timereported"
#          to a UTC value before pulling data out of it.
template(name="bitscout-index-pattern" type="list") {
    constant(value="bitscout-")
    property(name="timereported" dateFormat="rfc3339" position.from="1" position.to="4")
    constant(value=".")
    property(name="timereported" dateFormat="rfc3339" position.from="6" position.to="7")
    constant(value=".")
    property(name="timereported" dateFormat="rfc3339" position.from="9" position.to="10")
    }
# this is for formatting our syslog data in JSON with @timestamp using a "hierarchical" metdata namespace
template(name="com-redhat-rsyslog-hier"
         type="list") {
    constant(value="{")
    constant(value="\"@timestamp\":\"")               property(name="timereported" dateFormat="rfc3339")
    constant(value="\",\"@version\":\"2015.09.24-0")
    constant(value="\",\"message\":\"")               property(name="msg" format="json")
    constant(value="\",\"hostname\":\"")              property(name="hostname")
    constant(value="\",\"level\":\"")                 property(name="syslogseverity-text")
    constant(value="\",\"pid\":\"")                   property(name="procid")
    constant(value="\",\"rsyslog\": {")
    constant(value="\"facility\":\"")                 property(name="syslogfacility-text")
    constant(value="\",\"programname\":\"")           property(name="programname")
    constant(value="\",\"fromhost\":\"")              property(name="fromhost")
    constant(value="\",\"fromhost-ip\":\"")           property(name="fromhost-ip")
    constant(value="\",\"timegenerated\":\"")         property(name="timegenerated" dateFormat="rfc3339")
    constant(value="\",\"protocol-version\":\"")      property(name="protocol-version")
    constant(value="\",\"structured-data\":\"")       property(name="structured-data")
    constant(value="\",\"app-name\":\"")              property(name="app-name")
    constant(value="\",\"msgid\":\"")                 property(name="msgid")
    constant(value="\",\"inputname\":\"")             property(name="inputname")
    constant(value="\",\"_cee\":")                    property(name="$!all-json")
    constant(value="} }")
    }

# Use default timestamp format
$ActionFileDefaultTemplate RSYSLOG_FileFormat

# Include all config files in /etc/rsyslog.d/
$IncludeConfig /etc/rsyslog.d/*.conf

#### RULES ####

# Ensure message is a properly formatted UTF-8 sequence
#action(type="mmutf8fix" mode="utf-8")

# Reformat any SNMP trap messages (legacy format required)
*.* :mmsnmptrapd:

# Parse any CEE JSON messages
action(type="mmjsonparse")

# Index into elasticsearch directly in a hierarchical metadata namespace
action(
    type="omelasticsearch"
    server="%ES_HOST%"
    serverport="%ES_PORT%"
    template="com-redhat-rsyslog-hier"
    searchIndex="bitscout-index-pattern"
    dynSearchIndex="on"
    searchType="rsyslog"
    bulkmode="on"
    queue.type="linkedlist"
    queue.size="5000"
    queue.dequeuebatchsize="300"
    action.resumeretrycount="-1")

# Prevent local elasticsearch logs from flowing into other log files (they are
# already being logged to files). The local ElasticSearch instance is the
# "test" one, which has a "test-elasticsearch" tag, but we don't need to
# explicitly check for that since no logs from any elasticsearch instance need
# to be logged locally.
:programname, isequal, "elasticsearch"  stop

# Log anything (except mail) of level info or higher.
# Don't log private authentication messages!
*.info;mail.none;authpriv.none;cron.none                /var/log/messages

# The authpriv file has restricted access.
authpriv.*                                              /var/log/secure

# Log all the mail messages in one place.
mail.*                                                  -/var/log/maillog

# Log cron stuff
cron.*                                                  /var/log/cron

# Everybody gets emergency messages
*.emerg                                                 :omusrmsg:*

# Save news errors of level crit and higher in a special file.
uucp,news.crit                                          /var/log/spooler

# Save boot messages also to boot.log
local7.*                                                /var/log/boot.log