From 6d08d16d38bd851d88b71c8aea774024c4c8296e Mon Sep 17 00:00:00 2001 From: Ryan Faircloth Date: Fri, 1 Oct 2021 16:27:07 -0400 Subject: [PATCH] feat: add support for IBM datapower (#1279) --- docs/sources/IBM/index.md | 46 +++++++++++++++++ mkdocs.yml | 1 + .../conflib/net_source/app-ibm_datapower.conf | 37 ++++++++++++++ .../vendor_product_by_source.conf.example | 4 ++ .../vendor_product_by_source.csv.example | 3 +- tests/test_ibm_datapower.py | 51 +++++++++++++++++++ 6 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 docs/sources/IBM/index.md create mode 100644 package/etc/conf.d/conflib/net_source/app-ibm_datapower.conf create mode 100644 tests/test_ibm_datapower.py diff --git a/docs/sources/IBM/index.md b/docs/sources/IBM/index.md new file mode 100644 index 0000000000..47693a1256 --- /dev/null +++ b/docs/sources/IBM/index.md @@ -0,0 +1,46 @@ +# Vendor - IBM + +## Product - Data power + +| Ref | Link | +|----------------|---------------------------------------------------------------------------------------------------------| +| Splunk Add-on | https://splunkbase.splunk.com/app/4662/ | + +### Sourcetypes + +| sourcetype | notes | +|----------------|---------------------------------------------------------------------------------------------------------| +| ibm:datapower:syslog | Common sourcetype | +| ibm:datapower:* | * is taken from the event sourcetype | + | + +### Index Configuration + +| key | source | index | notes | +|----------------|----------------|----------------|----------------| +| ibm_datapower | na | inifraops | none | + +### Filter type + +Requires dedicated port or vendor_product_by_source configuration + +### Options + + +| Variable | default | description | +|----------------|----------------|----------------| +| SC4S_LISTEN_IBM_DATAPOWER_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | +| SC4S_LISTEN_IBM_DATAPOWER_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | +| SC4S_ARCHIVE_IBM_DATAPOWER | no | Enable archive to disk for this specific source | +| SC4S_DEST_IBM_DATAPOWER_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | + + +### Verification + +An active site will generate frequent events use the following search to check for new events + +Verify timestamp, and host values match as expected + +``` +index= (sourcetype=cef source="ibm:datapower*") +``` diff --git a/mkdocs.yml b/mkdocs.yml index c25a105ae4..fd5befda0a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -55,6 +55,7 @@ nav: - Fortinet: sources/Fortinet/index.md - HAProxy: sources/HAProxy/index.md - HPe: sources/HPe/index.md + - IBM: sources/IBM/index.md - Imperva: sources/Imperva/index.md - InfoBlox: sources/InfoBlox/index.md - ISC: sources/ISC/index.md diff --git a/package/etc/conf.d/conflib/net_source/app-ibm_datapower.conf b/package/etc/conf.d/conflib/net_source/app-ibm_datapower.conf new file mode 100644 index 0000000000..686f4b3796 --- /dev/null +++ b/package/etc/conf.d/conflib/net_source/app-ibm_datapower.conf @@ -0,0 +1,37 @@ +block parser ibm-datapower-parser() { + channel { + rewrite { + r_set_splunk_dest_default( + index('infraops') + sourcetype('ibm:datapower:syslog') + vendor_product("ibm_datapower") + template('t_msg_only') + ); + }; + # + if { + filter { + message( + '\[[^\]]*\]\[([^\]]*)\]\[[^\]]*\]\s' + flags(store-matches) + ); + }; + rewrite{ + r_set_splunk_dest_update( + sourcetype('ibm:datapower:$1') + meta_key("$1") + ); + }; + }; + }; +}; + +application ibm-datapower[sc4s-network-source] { + filter { + ( + "${.netsource.sc4s_vendor_product}" eq "ibm_datapower" + or "${SOURCE}" eq "s_IBM_DATAPOWER" + ) + }; + parser { ibm-datapower-parser(); }; +}; diff --git a/package/etc/context_templates/vendor_product_by_source.conf.example b/package/etc/context_templates/vendor_product_by_source.conf.example index 1e48bbd0e0..9fa2d8059b 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf.example +++ b/package/etc/context_templates/vendor_product_by_source.conf.example @@ -91,4 +91,8 @@ filter f_vmware_esx { filter f_dell_cmc { host("test-dell-cmc-" type(string) flags(prefix)) +}; + +filter f_ibm_datapower { + host("test-ibmdp-" type(string) flags(prefix)) }; \ No newline at end of file diff --git a/package/etc/context_templates/vendor_product_by_source.csv.example b/package/etc/context_templates/vendor_product_by_source.csv.example index 70425f4d89..0ae8272b37 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv.example +++ b/package/etc/context_templates/vendor_product_by_source.csv.example @@ -20,4 +20,5 @@ f_tzfixny,sc4s_time_zone,"America/New_York" f_cisco_esa,sc4s_vendor_product,"cisco_esa" f_sophos_webappliance,sc4s_vendor_product,"sophos_webappliance" f_vmware_esx,sc4s_vendor_product,"vmware_esx" -f_dell_cmc,sc4s_vendor_product,"dell_poweredge_cmc" \ No newline at end of file +f_dell_cmc,sc4s_vendor_product,"dell_poweredge_cmc" +f_ibm_datapower,sc4s_vendor_product,"ibm_datapower" \ No newline at end of file diff --git a/tests/test_ibm_datapower.py b/tests/test_ibm_datapower.py new file mode 100644 index 0000000000..04918f8ccd --- /dev/null +++ b/tests/test_ibm_datapower.py @@ -0,0 +1,51 @@ +# Copyright 2019 Splunk, Inc. +# +# Use of this source code is governed by a BSD-2-clause-style +# license that can be found in the LICENSE-BSD2 file or at +# https://opensource.org/licenses/BSD-2-Clause + +from jinja2 import Environment + +from .sendmessage import * +from .splunkutils import * +from .timeutils import * + +import pytest + +env = Environment() + + +testdata = [ + "{{ mark }}{{ bsd }} {{ host }} [APIGWTEBN][0x88c00002][apigw][error] source-https(APIGWTEBN_https_443): trans(3168147)[11.11.11.16]: The request URL 'https://11.11.11.11/' is not routed to any API collection.", +] + + +@pytest.mark.parametrize("event", testdata) +def test_ibm_datapower( + record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event +): + host = "test-ibmdp-" + get_host_key + + dt = datetime.datetime.now() + iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt) + + # Tune time functions + epoch = epoch[:-7] + + mt = env.from_string(event + "\n") + message = mt.render(mark="<166>", bsd=bsd, host=host) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search index=infraops _time={{ epoch }} sourcetype="ibm:datapower:apigw" (host="{{ host }}" OR "{{ host }}")' + ) + search = st.render(epoch=epoch, host=host) + + resultCount, eventCount = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", resultCount) + record_property("message", message) + + assert resultCount == 1