feat: add support for IBM datapower (#1279)

docs/sources/IBM/index.md

@@ -0,0 +1,46 @@
+# Vendor - IBM
+
+## Product - Data power
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | https://splunkbase.splunk.com/app/4662/                                                      |
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| ibm:datapower:syslog        | Common sourcetype                                                                                                 |
+| ibm:datapower:*        | * is taken from the event sourcetype                                                                                                 |
+                                |
+
+### Index Configuration
+
+| key            | source     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| ibm_datapower      | na     | inifraops          | none          |
+
+### Filter type
+
+Requires dedicated port or vendor_product_by_source configuration
+
+### Options
+
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_IBM_DATAPOWER_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_LISTEN_IBM_DATAPOWER_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_ARCHIVE_IBM_DATAPOWER | no | Enable archive to disk for this specific source |
+| SC4S_DEST_IBM_DATAPOWER_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source |
+
+
+### Verification
+
+An active site will generate frequent events use the following search to check for new events
+
+Verify timestamp, and host values match as expected
+
+```
+index=<asconfigured> (sourcetype=cef source="ibm:datapower*")
+```

mkdocs.yml

@@ -55,6 +55,7 @@ nav:
       - Fortinet: sources/Fortinet/index.md
       - HAProxy: sources/HAProxy/index.md
       - HPe: sources/HPe/index.md
+      - IBM: sources/IBM/index.md
       - Imperva: sources/Imperva/index.md
       - InfoBlox: sources/InfoBlox/index.md
       - ISC: sources/ISC/index.md

package/etc/conf.d/conflib/net_source/app-ibm_datapower.conf

@@ -0,0 +1,37 @@
+block parser ibm-datapower-parser() {    
+    channel {
+        rewrite {
+            r_set_splunk_dest_default(
+                index('infraops')
+                sourcetype('ibm:datapower:syslog')
+                vendor_product("ibm_datapower")
+                template('t_msg_only')
+            );                  
+        };  
+        #
+        if {
+            filter {
+                message(
+                    '\[[^\]]*\]\[([^\]]*)\]\[[^\]]*\]\s'
+                    flags(store-matches)
+                );
+            };
+            rewrite{
+                r_set_splunk_dest_update(
+                    sourcetype('ibm:datapower:$1')
+                    meta_key("$1")
+                ); 
+            };
+        };  
+    };
+};
+
+application ibm-datapower[sc4s-network-source] {
+	filter { 
+        (
+            "${.netsource.sc4s_vendor_product}" eq "ibm_datapower"
+            or "${SOURCE}" eq "s_IBM_DATAPOWER"
+        )
+    };	
+    parser { ibm-datapower-parser(); };   
+};

package/etc/context_templates/vendor_product_by_source.conf.example

@@ -91,4 +91,8 @@ filter f_vmware_esx {
 
 filter f_dell_cmc {
     host("test-dell-cmc-" type(string) flags(prefix))
+};
+
+filter f_ibm_datapower {
+    host("test-ibmdp-" type(string) flags(prefix))
 };

package/etc/context_templates/vendor_product_by_source.csv.example

@@ -20,4 +20,5 @@ f_tzfixny,sc4s_time_zone,"America/New_York"
 f_cisco_esa,sc4s_vendor_product,"cisco_esa"
 f_sophos_webappliance,sc4s_vendor_product,"sophos_webappliance"
 f_vmware_esx,sc4s_vendor_product,"vmware_esx"
-f_dell_cmc,sc4s_vendor_product,"dell_poweredge_cmc"
+f_dell_cmc,sc4s_vendor_product,"dell_poweredge_cmc"
+f_ibm_datapower,sc4s_vendor_product,"ibm_datapower"

tests/test_ibm_datapower.py

@@ -0,0 +1,51 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+from .timeutils import *
+
+import pytest
+
+env = Environment()
+
+
+testdata = [
+    "{{ mark }}{{ bsd }} {{ host }} [APIGWTEBN][0x88c00002][apigw][error] source-https(APIGWTEBN_https_443): trans(3168147)[11.11.11.16]: The request URL 'https://11.11.11.11/' is not routed to any API collection.",
+]
+
+
+@pytest.mark.parametrize("event", testdata)
+def test_ibm_datapower(
+    record_property, setup_wordlist, get_host_key, setup_splunk, setup_sc4s, event
+):
+    host = "test-ibmdp-" + get_host_key
+
+    dt = datetime.datetime.now()
+    iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
+
+    # Tune time functions
+    epoch = epoch[:-7]
+
+    mt = env.from_string(event + "\n")
+    message = mt.render(mark="<166>", bsd=bsd, host=host)
+
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string(
+        'search index=infraops _time={{ epoch }} sourcetype="ibm:datapower:apigw" (host="{{ host }}" OR "{{ host }}")'
+    )
+    search = st.render(epoch=epoch, host=host)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1