From bf9f97d92dcba5b82d4b08e179d4aa6276a95d68 Mon Sep 17 00:00:00 2001 From: mstopa-splunk <139441697+mstopa-splunk@users.noreply.github.com> Date: Thu, 2 Nov 2023 10:01:51 +0100 Subject: [PATCH] feat: add FixFQDNResolver (#2253) --- docs/configuration.md | 3 ++- .../etc/conf.d/conflib/_splunk/fix_dns.conf | 10 +++++++-- .../conf.d/sources/source_syslog/plugin.jinja | 12 +++++++++-- .../conf.d/sources/source_syslog/plugin.py | 1 + package/etc/pylib/parser_fix_dns.py | 21 ++++++++++++++++++- .../conf.d/sources/source_syslog/plugin.jinja | 12 +++++++++-- tests/docker-compose.yml | 1 + 7 files changed, 52 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 6f5beea743..2e73abb96f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -8,6 +8,7 @@ and variables needed to properly configure SC4S for your environment. | Variable | Values | Description | |----------|---------------|-------------| | SC4S_USE_REVERSE_DNS | yes or no(default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | +| SC4S_REVERSE_DNS_KEEP_FQDN | yes or no(default) | don't extract hostname from FQDN, pass the full domain name to HOST instead | | SC4S_CONTAINER_HOST | string | variable passed to the container to identify the actual log host for container implementations | * NOTE: Do _not_ configure HEC Acknowledgement when deploying the HEC token on the Splunk side; the underlying syslog-ng http @@ -312,7 +313,7 @@ In some cases the host value is not present in an event (or an IP address is in who require a true hostname be attached to each event, SC4S provides an optional facility to perform a reverse IP to name lookup. If the variable `SC4S_USE_REVERSE_DNS` is set to "yes", SC4S will first check `host.csv` and replace the value of `host` with the value specified that matches the incoming IP address. -If a value is not found in `host.csv` then a reverse DNS lookup will be attempted against the configured nameserver. +If a value is not found in `host.csv` then a reverse DNS lookup will be attempted against the configured nameserver. In this case, SC4S by default extracts only the hostname from FQDN (`example.domain.com` -> `example`). If `SC4S_REVERSE_DNS_KEEP_FQDN` variable is set to "yes", full domain name will be assigned to the host field. The IP address will only be used as the host value as a last resort. * NOTE: Use of this variable can have a significant impact on performance if the reverse DNS facility (typically a caching diff --git a/package/etc/conf.d/conflib/_splunk/fix_dns.conf b/package/etc/conf.d/conflib/_splunk/fix_dns.conf index 95e7bb9295..2cff5875ca 100644 --- a/package/etc/conf.d/conflib/_splunk/fix_dns.conf +++ b/package/etc/conf.d/conflib/_splunk/fix_dns.conf @@ -1,6 +1,12 @@ -parser p_fix_host_resolver { +parser p_fix_hostname_resolver { python( - class("parser_fix_dns.FixHostResolver") + class("parser_fix_dns.FixHostnameResolver") + ); +}; + +parser p_fix_fqdn_resolver { + python( + class("parser_fix_dns.FixFQDNResolver") ); }; diff --git a/package/etc/conf.d/sources/source_syslog/plugin.jinja b/package/etc/conf.d/sources/source_syslog/plugin.jinja index 0264a6b337..89caf04801 100644 --- a/package/etc/conf.d/sources/source_syslog/plugin.jinja +++ b/package/etc/conf.d/sources/source_syslog/plugin.jinja @@ -208,7 +208,11 @@ source s_{{ port_id }} { {%- if use_reverse_dns == True %} if { filter(f_host_is_nil_or_ip); - parser(p_fix_host_resolver); + {%- if reverse_dns_keep_fqdn == True %} + parser(p_fix_fqdn_resolver); + {%- else %} + parser(p_fix_hostname_resolver); + {%- endif %} }; {%- endif %} }; @@ -410,7 +414,11 @@ source s_{{ port_id }} { {%- if use_reverse_dns == True %} if { filter(f_host_is_nil_or_ip); - parser(p_fix_host_resolver); + {%- if reverse_dns_keep_fqdn == True %} + parser(p_fix_fqdn_resolver); + {%- else %} + parser(p_fix_hostname_resolver); + {%- endif %} }; {%- endif %} }; diff --git a/package/etc/conf.d/sources/source_syslog/plugin.py b/package/etc/conf.d/sources/source_syslog/plugin.py index a1c7bbac8e..950b58c944 100755 --- a/package/etc/conf.d/sources/source_syslog/plugin.py +++ b/package/etc/conf.d/sources/source_syslog/plugin.py @@ -63,6 +63,7 @@ def normalize_env_variable_input(env_variable: str): store_raw_message=normalize_env_variable_input("SC4S_SOURCE_STORE_RAWMSG"), port_id=port_id, use_reverse_dns=normalize_env_variable_input("SC4S_USE_REVERSE_DNS"), + reverse_dns_keep_fqdn=normalize_env_variable_input("SC4S_REVERSE_DNS_KEEP_FQDN"), use_udp_log_iw=normalize_env_variable_input("SC4S_SOURCE_UDP_IW_USE"), use_namecache=normalize_env_variable_input("SC4S_USE_NAME_CACHE"), use_vpscache=normalize_env_variable_input("SC4S_USE_VPS_CACHE"), diff --git a/package/etc/pylib/parser_fix_dns.py b/package/etc/pylib/parser_fix_dns.py index 838bce9205..aff6f66e89 100644 --- a/package/etc/pylib/parser_fix_dns.py +++ b/package/etc/pylib/parser_fix_dns.py @@ -15,7 +15,7 @@ class LogParser: pass -class FixHostResolver(LogParser): +class FixHostnameResolver(LogParser): def parse(self, log_message): """ Resolves IP to hostname @@ -33,5 +33,24 @@ def parse(self, log_message): except Exception: return False + # return True, other way message is dropped + return True + + +class FixFQDNResolver(LogParser): + def parse(self, log_message): + """ + Resolves IP to FQDN + """ + + # try to resolve the IP address + try: + ipaddr = log_message.get_as_str("SOURCEIP", "", repr="internal") + + fqdn, aliaslist, ipaddrlist = socket.gethostbyaddr(ipaddr) + log_message["HOST"] = str(fqdn) + except Exception: + return False + # return True, other way message is dropped return True \ No newline at end of file diff --git a/package/lite/etc/conf.d/sources/source_syslog/plugin.jinja b/package/lite/etc/conf.d/sources/source_syslog/plugin.jinja index 79745fa801..4f11dc0585 100644 --- a/package/lite/etc/conf.d/sources/source_syslog/plugin.jinja +++ b/package/lite/etc/conf.d/sources/source_syslog/plugin.jinja @@ -196,7 +196,11 @@ source s_{{ port_id }} { {%- if use_reverse_dns == True %} if { filter(f_host_is_nil_or_ip); - parser(p_fix_host_resolver); + {%- if reverse_dns_keep_fqdn == True %} + parser(p_fix_fqdn_resolver); + {%- else %} + parser(p_fix_hostname_resolver); + {%- endif %} }; {%- endif %} }; @@ -372,7 +376,11 @@ source s_{{ port_id }} { {%- if use_reverse_dns == True %} if { filter(f_host_is_nil_or_ip); - parser(p_fix_host_resolver); + {%- if reverse_dns_keep_fqdn == True %} + parser(p_fix_fqdn_resolver); + {%- else %} + parser(p_fix_hostname_resolver); + {%- endif %} }; {%- endif %} }; diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 471e61fb1a..887cf9b193 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -84,6 +84,7 @@ services: - SC4S_LISTEN_SIMPLE_BRO_SYSLOG_TCP_PORT=6502 - SC4S_LISTEN_DEFAULT_TCP_PORT=514,6504,6505,6506 #SC4S_USE_REVERSE_DNS=YES + #SC4S_REVERSE_DNS_KEEP_FQDN=YES splunk: image: docker.io/splunk/splunk:latest hostname: splunk