chore: move python code from conf to files

Improvement for dev experience move python to .py files outside of .conf

package/Dockerfile

@@ -59,6 +59,7 @@ RUN poetry export --format requirements.txt | pip3 install --user -r /dev/stdin
 
 COPY package/etc/syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY package/etc/conf.d /etc/syslog-ng/conf.d
+COPY package/etc/pylib /etc/syslog-ng/pylib
 COPY package/etc/context_templates /etc/syslog-ng/context_templates
 COPY package/etc/test_parsers /etc/syslog-ng/test_parsers
 COPY package/etc/local_config /etc/syslog-ng/local_config

package/etc/conf.d/conflib/_splunk/fix_dns.conf

@@ -1,45 +1,6 @@
-python {
-
-"""
-simple syslog-ng Python parser example
-resolves IP to hostname
-value pair names are hard-coded
-"""
-import re
-import socket
-
-class FixHostResolver(object):
-
-    def parse(self, log_message):
-        """
-        Resolves IP to hostname
-        """
-
-
-        # try to resolve the IP address
-        try:
-            ipaddr = log_message['SOURCEIP'].decode('utf-8')
-
-            hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr)
-            #print(ipaddr)
-            #print(hostname)
-            parts=str(hostname).split('.')
-            name = parts[0]
-            #print(name)
-            if len(parts)>1:
-                log_message['HOST'] = name
-        except:
-            pass
-
-        # return True, other way message is dropped
-        return True
-
-};
-
-
 parser p_fix_host_resolver {
     python(
-        class("FixHostResolver")
+        class("parser_fix_dns.FixHostResolver")
     );
 };
 

package/etc/conf.d/conflib/syslog/app-syslog-leef.conf

@@ -6,89 +6,9 @@ template t_leef_event {
         template("${.leef.event}");
 };
 
-
-python {
-
-import re
-import binascii
-class leef_kv(object):
-
-
-    def init(self, options):
-        self.regex = r"( ?(?:[A-Z]{2,4}T|HAEC|IDLW|MSK|NT|UTC|THA))"
-        return True
-
-    def parse(self, log_message):
-
-        try:
-            msg = log_message['MESSAGE'].decode("utf-8")
-            # All LEEF message are | separated super structures
-            structure = msg.split('|')
-            # Indexed fields for Splunk
-
-            log_message['fields.leef_version'] = structure[0][5:]
-            log_message['fields.leef_vendor'] = structure[1]
-            log_message['fields.leef_product'] = structure[2]
-            log_message['fields.leef_product_version'] = structure[3]
-            log_message['fields.leef_EventID'] = structure[4]
-            #We just want the event field
-            event = structure[len(structure)-1]
-            log_message['.leef.event'] = event
-            # V1 will always use tab
-            if structure[0][5:].startswith("1"):
-                separator="\t"
-                lv = "1"
-                pairs = event.split(separator)
-                if len(pairs)<4:
-                    separator='|'
-                    pairs = structure[5:]
-                    event = "\t".join(pairs)
-                    log_message['.leef.event'] = event
-            else:
-                lv = "2"
-                #V2 messages should always provide the sep but some fail do comply
-                #with the format spec if they don't assume tab
-                if len(structure) == 6 or not structure[5]:
-                    separator="\t"
-                    pairs = event.split(separator)
-                else:
-                    separator=structure[5]
-                    if separator.startswith("0"):
-                        separator = separator[1:]
-                    pairs = event.split(separator)
-
-
-            if separator.startswith("x"):
-                hex_sep = f"0{separator.lower()}"
-            else:
-                hex_sep = f'0x{binascii.b2a_hex(separator.encode("utf-8")).decode("utf-8").lower()}'
-            if structure[0][5:].startswith("1"):
-                log_message['.splunk.sourcetype'] = f"LEEF:{lv}"
-            else:
-                log_message['.splunk.sourcetype'] = f"LEEF:{lv}:{hex_sep}"
-            log_message['.splunk.source'] = f"{structure[1]}:{structure[2]}"
-            log_message['fields.sc4s_vendor_product'] = f"{structure[1]}_{structure[2]}"
-
-
-            for p in pairs:
-                f,v = p.split("=", 1)
-                if f == "devTime":
-                    log_message[".leef." + f] = re.sub(self.regex, "" , v, 0, re.MULTILINE)
-                else:
-                    log_message[".leef." + f] = v
-        except Exception as e:
-            log_message['fields.leef_exception']=str(e)
-            pass
-
-        # return True, other way message is dropped
-        return True
-
-};
-
-
 parser p_leef_kv {
     python(
-        class("leef_kv")
+        class("parser_leef.leef_kv")
     );
 };
 

package/etc/pylib/parser_fix_dns.py

@@ -0,0 +1,32 @@
+"""
+simple syslog-ng Python parser example
+resolves IP to hostname
+value pair names are hard-coded
+"""
+import re
+import socket
+
+
+class FixHostResolver(object):
+    def parse(self, log_message):
+        """
+        Resolves IP to hostname
+        """
+
+        # try to resolve the IP address
+        try:
+            ipaddr = log_message["SOURCEIP"].decode("utf-8")
+
+            hostname, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr)
+            # print(ipaddr)
+            # print(hostname)
+            parts = str(hostname).split(".")
+            name = parts[0]
+            # print(name)
+            if len(parts) > 1:
+                log_message["HOST"] = name
+        except:
+            pass
+
+        # return True, other way message is dropped
+        return True

package/etc/pylib/parser_leef.py

@@ -0,0 +1,73 @@
+import re
+import binascii
+
+
+class leef_kv(object):
+    def init(self, options):
+        self.regex = r"( ?(?:[A-Z]{2,4}T|HAEC|IDLW|MSK|NT|UTC|THA))"
+        return True
+
+    def parse(self, log_message):
+
+        try:
+            msg = log_message["MESSAGE"].decode("utf-8")
+            # All LEEF message are | separated super structures
+            structure = msg.split("|")
+            # Indexed fields for Splunk
+
+            log_message["fields.leef_version"] = structure[0][5:]
+            log_message["fields.leef_vendor"] = structure[1]
+            log_message["fields.leef_product"] = structure[2]
+            log_message["fields.leef_product_version"] = structure[3]
+            log_message["fields.leef_EventID"] = structure[4]
+            # We just want the event field
+            event = structure[len(structure) - 1]
+            log_message[".leef.event"] = event
+            # V1 will always use tab
+            if structure[0][5:].startswith("1"):
+                separator = "\t"
+                lv = "1"
+                pairs = event.split(separator)
+                if len(pairs) < 4:
+                    separator = "|"
+                    pairs = structure[5:]
+                    event = "\t".join(pairs)
+                    log_message[".leef.event"] = event
+            else:
+                lv = "2"
+                # V2 messages should always provide the sep but some fail do comply
+                # with the format spec if they don't assume tab
+                if len(structure) == 6 or not structure[5]:
+                    separator = "\t"
+                    pairs = event.split(separator)
+                else:
+                    separator = structure[5]
+                    if separator.startswith("0"):
+                        separator = separator[1:]
+                    pairs = event.split(separator)
+
+            if separator.startswith("x"):
+                hex_sep = f"0{separator.lower()}"
+            else:
+                hex_sep = f'0x{binascii.b2a_hex(separator.encode("utf-8")).decode("utf-8").lower()}'
+            if structure[0][5:].startswith("1"):
+                log_message[".splunk.sourcetype"] = f"LEEF:{lv}"
+            else:
+                log_message[".splunk.sourcetype"] = f"LEEF:{lv}:{hex_sep}"
+            log_message[".splunk.source"] = f"{structure[1]}:{structure[2]}"
+            log_message["fields.sc4s_vendor_product"] = f"{structure[1]}_{structure[2]}"
+
+            for p in pairs:
+                f, v = p.split("=", 1)
+                if f == "devTime":
+                    log_message[".leef." + f] = re.sub(
+                        self.regex, "", v, 0, re.MULTILINE
+                    )
+                else:
+                    log_message[".leef." + f] = v
+        except Exception as e:
+            log_message["fields.leef_exception"] = str(e)
+            pass
+
+        # return True, other way message is dropped
+        return True

package/sbin/entrypoint.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 function join_by { local d=$1; shift; local f=$1; shift; printf %s "$f" "${@/#/$d}"; }
 
+export PYTHONPATH=/etc/syslog-ng/pylib
+
 export SC4S_LISTEN_STATUS_PORT=${SC4S_LISTEN_STATUS_PORT:=8080}
 
 # These path variables allow for a single entrypoint script to be utilized for both Container and BYOE runtimes

pyproject.toml

@@ -28,6 +28,7 @@ pre-commit = "^2.16.0"
 typing-extensions = "*"
 mkdocs-include-dir-to-nav = "^1.2.0"
 docker-compose = "^1.29.2"
+black = "^22.1.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]