Netapp ontap audit ems support

.github/workflows/ci-lite.yaml

@@ -227,6 +227,7 @@ jobs:
           TEST_SC4S_ACTIVATE_EXAMPLES: "yes"
           SC4S_DEBUG_CONTAINER: "yes"
           SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes"
+          SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes"
           SC4S_USE_VPS_CACHE: "yes"
     steps:
       - name: Checkout

.github/workflows/ci-main.yaml

@@ -227,6 +227,7 @@ jobs:
           TEST_SC4S_ACTIVATE_EXAMPLES: "yes"
           SC4S_DEBUG_CONTAINER: "yes"
           SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes"
+          SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes"
           SC4S_USE_VPS_CACHE: "yes"
 
     steps:

docs/sources/vendor/NetApp/ontap.md

@@ -3,7 +3,7 @@
 ## Key facts
 
 * MSG Format based filter
-* Legacy BSD Format default port 514
+* Netapp Ontap messages are not distinctive. So, either configure known Netapp Ontap hosts in SC4S, or open unique ports for Netapp Ontap devices
 
 ## Links
 
@@ -16,11 +16,49 @@
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| netapp:ems  | None |
+| ontap:ems  | This sourcetype will be assinged only when the environment variable `SC4S_NETAPP_ONTAP_NEW_FORMAT` is not set or is set to 'no'. By default it is unset |
+| netapp:ontap:audit  | This sourcetype will be assinged only when the environment variable `SC4S_NETAPP_ONTAP_NEW_FORMAT` is set to 'yes' |
+| netapp:ontap:ems  | This sourcetype will be assinged only when the environment variable `SC4S_NETAPP_ONTAP_NEW_FORMAT` is set to 'yes' |
 
 ## Sourcetype and Index Configuration
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| netapp_ontap      | netapp:ems     | infraops          | none          |
+| netapp_ontap      | ontap:ems     | infraops          | none          |
+| netapp_ontap_audit      | netapp:ontap:audit     | infraops          | none          |
+| netapp_ontap_ems      | netapp:ontap:ems     | infraops          | none          |
 
+## Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_NETAPP_ONTAP_NEW_FORMAT      | empty string      | (empty/yes) Set to "yes" for the applying the latest changes. Make sure to configure your system to send the logs to a specific port or have a hostname-based configuration |
+
+## Parser Configuration
+1. Through sc4s-vps
+```c
+#/opt/sc4s/local/config/app-parsers/app-vps-netapp_ontap.conf
+#File name provided is a suggestion it must be globally unique
+
+application app-vps-test-netapp_ontap[sc4s-vps] {
+    filter {
+        host("netapp-ontap-" type(string) flags(prefix))
+        or (
+            message("netapp-ontap-" type(string) flags(prefix))
+            and program("netapp-ontap-" type(string) flags(prefix))
+        )
+    };
+    parser {
+        p_set_netsource_fields(
+            vendor('netapp')
+            product('ontap')
+        );
+    };
+};
+```
+
+2. or through unique port
+```
+# /opt/sc4s/env_file 
+SC4S_LISTEN_NETAPP_ONTAP_UDP_PORT=5005
+```

docs/upgrade.md

@@ -18,6 +18,9 @@ For a step by step guide [see here](./v3_upgrade.md).
 
 You may need to migrate legacy log paths or version 1 app-parsers for version 2. To do this, open an issue and attach the original configuration and a compressed pcap of sample data for testing. We will evaluate whether to include the source in an upcoming release.
 
+### Upgrade from <3.33.0
+In NetApp ONTAP, the ontap:ems sourcetype has been updated to netapp:ontap:audit, so old logs are now classified under netapp:ontap:audit. Additionally, a new netapp:ontap:ems sourcetype has been introduced. If you upgrade and want these new changes, ensure that you set `SC4S_NETAPP_ONTAP_NEW_FORMAT` environment variable to `yes` and configure your system to send the logs to a specific port or have a hostname-based configuration in place for proper log onboarding into Splunk.
+
 ### Upgrade from <2.23.0
 
 * In VMware vSphere, update the ESX and vCenter sourcetype for add-on compatibility.

package/etc/conf.d/conflib/netsource/app-netsource-netapp_ontap.conf

@@ -0,0 +1,55 @@
+block parser app-netsource-netapp_ontap() {
+    channel {
+        rewrite {
+            r_set_splunk_dest_default(
+                index("infraops")
+                vendor("netapp")
+                product("ontap")
+            );
+        };
+
+        if {
+            parser {
+                regexp-parser(
+                    prefix(".tmp.")
+                    patterns('^[A-Za-z0-9\-\_\.]+: [0-9a-f]+\.[0-9a-f]+ [0-9a-f]+ [A-Z][a-z][a-z] (?<timestamp>[A-Z][a-z][a-z] \d\d \d\d\d\d \d\d:\d\d:\d\d [+-]?\d{1,2}:\d\d)')
+                );
+                date-parser-nofilter(
+                    format(
+                        '%b %d %Y %H:%M:%S %z',
+                    )
+                    template("${.tmp.timestamp}")
+                );
+            };
+
+            rewrite {
+                set('$PROGRAM: $MESSAGE', value(MESSAGE));
+                set('$PROGRAM', value(HOST));
+                unset(value(PROGRAM));
+            };
+
+            rewrite {
+                r_set_splunk_dest_update_v2(
+                    sourcetype('netapp:ontap:audit')
+                    class('audit')
+                );
+            };
+        } else {
+            rewrite {
+                r_set_splunk_dest_update_v2(
+                    sourcetype('netapp:ontap:ems')
+                    class('ems')
+                );
+            };
+        };
+    };
+};
+
+application app-netsource-netapp_ontap[sc4s-network-source] {
+	filter {
+        match("netapp", value('.netsource.sc4s_vendor'), type(string))
+        and match("ontap", value('.netsource.sc4s_product'), type(string))
+        and "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes"
+    };	
+    parser { app-netsource-netapp_ontap(); };
+};

package/etc/conf.d/conflib/syslog/app-syslog-netapp_ontap.conf

@@ -37,7 +37,8 @@ block parser app-syslog-netapp_ontap() {
 };
 application app-syslog-netapp_ontap[sc4s-syslog] {
 	filter {
-        program('^[A-Za-z0-9\-\_\.]+$');
+        program('^[A-Za-z0-9\-\_\.]+$')
+        and not "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes";
     };	
     parser { app-syslog-netapp_ontap(); };
 };

package/etc/test_parsers/app-vps-test-netapp_ontap.conf

@@ -0,0 +1,15 @@
+application app-vps-test-netapp_ontap[sc4s-vps] {
+    filter {
+        host("netapp-ontap-" type(string) flags(prefix))
+        or (
+            message("netapp-ontap-" type(string) flags(prefix))
+            and program("netapp-ontap-" type(string) flags(prefix))
+        )
+    };
+    parser {
+        p_set_netsource_fields(
+            vendor('netapp')
+            product('ontap')
+        );
+    };
+};

package/lite/etc/addons/netapp/app-netsource-netapp_ontap.conf

@@ -0,0 +1,55 @@
+block parser app-netsource-netapp_ontap() {
+    channel {
+        rewrite {
+            r_set_splunk_dest_default(
+                index("infraops")
+                vendor("netapp")
+                product("ontap")
+            );
+        };
+
+        if {
+            parser {
+                regexp-parser(
+                    prefix(".tmp.")
+                    patterns('^[A-Za-z0-9\-\_\.]+: [0-9a-f]+\.[0-9a-f]+ [0-9a-f]+ [A-Z][a-z][a-z] (?<timestamp>[A-Z][a-z][a-z] \d\d \d\d\d\d \d\d:\d\d:\d\d [+-]?\d{1,2}:\d\d)')
+                );
+                date-parser-nofilter(
+                    format(
+                        '%b %d %Y %H:%M:%S %z',
+                    )
+                    template("${.tmp.timestamp}")
+                );
+            };
+
+            rewrite {
+                set('$PROGRAM: $MESSAGE', value(MESSAGE));
+                set('$PROGRAM', value(HOST));
+                unset(value(PROGRAM));
+            };
+
+            rewrite {
+                r_set_splunk_dest_update_v2(
+                    sourcetype('netapp:ontap:audit')
+                    class('audit')
+                );
+            };
+        } else {
+            rewrite {
+                r_set_splunk_dest_update_v2(
+                    sourcetype('netapp:ontap:ems')
+                    class('ems')
+                );
+            };
+        };
+    };
+};
+
+application app-netsource-netapp_ontap[sc4s-network-source] {
+	filter {
+        match("netapp", value('.netsource.sc4s_vendor'), type(string))
+        and match("ontap", value('.netsource.sc4s_product'), type(string))
+        and "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes"
+    };	
+    parser { app-netsource-netapp_ontap(); };
+};

package/lite/etc/addons/netapp/app-syslog-netapp_ontap.conf

@@ -37,7 +37,8 @@ block parser app-syslog-netapp_ontap() {
 };
 application app-syslog-netapp_ontap[sc4s-syslog] {
 	filter {
-        program('^[A-Za-z0-9\-\_\.]+$');
+        program('^[A-Za-z0-9\-\_\.]+$')
+        and not "`SC4S_NETAPP_ONTAP_NEW_FORMAT`" eq "yes";
     };	
     parser { app-syslog-netapp_ontap(); };
 };

tests/test_netapp.py

@@ -17,16 +17,16 @@
 
 
 testdata = [
-    "{{ mark }}{{ bsd }} {{ host }}: {{ host }}: 0000001e.0794c163 055b6737 {{ device_time }} [kern_audit:info:2385] 8503ea0000ba6b71 :: nodea:ontapi :: 10.10.10.10:41464 :: nodea-esx:usera :: clone-create :: Error: Missing input: source-path; Missing input: volume",
+    "{{ mark }}{{ bsd }} {{ host }}: {{ host }}: 00000030.00c8f1e2 11e5347f {{ device_time }} [kern_audit3167] 8004b7000021e73b:4005f7000021e73d :: cluster:ssh :: 0.0.0.0:32879 :: cluster:admin :: qos statistics volume performance show -rows 20 -iter 1 :: Pending",
 ]
 
-
+# <14>Oct 3 11:36:46 host: host: 00000030.00c8f1e2 11e5347f Thu Oct 03 2024 11:36:44 -06:00 [kern_audit3167] 8004b7000021e73b:4005f7000021e73d :: cluster:ssh :: 0.0.0.0:32879 :: cluster:admin :: qos statistics volume performance show -rows 20 -iter 1 :: Pending
 @pytest.mark.addons("netapp")
 @pytest.mark.parametrize("event", testdata)
-def test_netapp(
+def test_netapp_ontap_audit(
     record_property,  get_host_key, setup_splunk, setup_sc4s, event
 ):
-    host = get_host_key
+    host = "netapp-ontap-" + get_host_key
 
     dt = datetime.datetime.now(datetime.timezone.utc)
     _, bsd, _, _, _, _, epoch = time_operations(dt)
@@ -37,12 +37,12 @@ def test_netapp(
     device_time = dt.strftime("%a %b %d %Y %H:%M:%S +00:00")
 
     mt = env.from_string(event + "\n")
-    message = mt.render(mark="<166>", bsd=bsd, host=host, device_time=device_time)
+    message = mt.render(mark="<14>", bsd=bsd, host=host, device_time=device_time)
 
     sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
 
     st = env.from_string(
-        'search index=infraops _time={{ epoch }} sourcetype="ontap:ems" (host="{{ host }}" OR "{{ host }}")'
+        'search index=infraops _time={{ epoch }} sourcetype="netapp:ontap:audit"'
     )
     search = st.render(epoch=epoch, host=host)
 
@@ -53,3 +53,38 @@ def test_netapp(
     record_property("message", message)
 
     assert result_count == 1
+
+
+# Netapp Ontap EMS event in rfc5424 format
+# <5>1 2024-10-03T07:54:02-06:00 host program - wafl.scan.done - Completed Volume Footprint Estimator Scan on volume vm_unix002_0d@vserver:27902083bf98-11e9-87fe-00a098b15eb6
+@pytest.mark.addons("netapp")
+def test_netapp_ontap_ems_rfc5424(
+        record_property,  get_host_key, setup_splunk, setup_sc4s
+):
+    host = "netapp-ontap-" + get_host_key
+
+    dt = datetime.datetime.now(datetime.timezone.utc)
+    iso, _, _, _, _, _, epoch = time_operations(dt)
+
+    # Tune time functions
+    epoch = epoch[:-3]
+
+    mt = env.from_string(
+        '{{ mark }} {{ iso }} {{ host }} program - wafl.scan.done - Completed Volume Footprint Estimator Scan on volume vm_unix002_0d@vserver:27902083bf98-11e9-87fe-00a098b15eb6'
+    )
+    message = mt.render(mark="<5>1", iso=iso, host=host)
+
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string(
+        'search _time={{ epoch }} index=infraops sourcetype="netapp:ontap:ems"'
+    )
+    search = st.render(epoch=epoch, host=host)
+
+    result_count, _ = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", result_count)
+    record_property("message", message)
+
+    assert result_count == 1