feat(netiq): Add support for novell netiq syslog (#1252)

* feat(netiq): Add support for novel netiq syslog

closes #1200

* feat: Improve zscaler parsing performance

* chore: remove early parse of json

* chore: don't use generic json parser if program is set

docs/sources/McAfee/index.md

@@ -51,7 +51,7 @@ You must create a certificate for the SC4S server to receive encrypted syslog fr
 
 Uncomment the following line in `/lib/systemd/system/sc4s.service` to allow the docker container to use the certificate:
 
-`Environment="SC4S_TLS_DIR=-v /opt/sc4s/tls:/etc/syslog-ng/tls:z"`
+`Environment="SC4S_TLS_DIR=-v   :/etc/syslog-ng/tls:z"`
 
 ### Troubleshooting
 from the command line of the SC4S host, run this: `openssl s_client -connect localhost:6514`

docs/sources/Novell/index.md

@@ -0,0 +1,44 @@
+# Vendor - Novell
+
+## Product - NetIQ
+
+| Ref            | Link                                                                                                    |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| Splunk Add-on  | None                                                               |
+| Product Manual | unknown |
+
+### Sourcetypes
+
+| sourcetype     | notes                                                                                                   |
+|----------------|---------------------------------------------------------------------------------------------------------|
+| novell:netiq     |  none  |
+
+### Sourcetype and Index Configuration
+
+| key            | sourcetype     | index          | notes          |
+|----------------|----------------|----------------|----------------|
+| novell_netiq    | novell_netiq    | netauth          | None     |
+
+### Filter type
+
+MSGParser
+
+
+### Options
+
+| Variable       | default        | description    |
+|----------------|----------------|----------------|
+| SC4S_LISTEN_NOVELL_NETIQ_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_LISTEN_NOVELL_NETIQ_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_ARCHIVE_NOVELL_NETIQ | no | Enable archive to disk for this specific source |
+| SC4S_DEST_NOVELL_NETIQ_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+
+### Verification
+
+Use the following search to validate events are present
+
+```
+index=netauth sourcetype=novel:netiq
+```
+
+Verify timestamp, and host values match as expected

mkdocs.yml

@@ -66,6 +66,7 @@ nav:
       - Microsoft: sources/Microsoft/index.md
       - NetApp: sources/NetApp/index.md
       - Netmotion: sources/Netmotion/index.md
+      - Novell: sources/Novell/indexmd
       - Nix: sources/nix/index.md
       - OSSEC: sources/Ossec/index.md
       - "Palo Alto Networks": sources/PaloaltoNetworks/index.md

package/etc/conf.d/conflib/json/app-novell_netiq.conf

@@ -0,0 +1,27 @@
+block parser novel-netiq-parser() {    
+    channel {
+
+        rewrite {
+                r_set_splunk_dest_default(
+                    vendor_product('novell_netiq'),
+                    index('netauth'),
+                    source('novell:netiq:${.json.component}'),
+                    sourcetype('novell:netiq')                                    
+                );
+        };
+
+        parser {
+            date-parser-nofilter(format(
+                        '%a, %d %b %Y %H:%M:%S %z',
+                    )
+                template("${.json.timeStamp}")
+            );
+        };
+    };   
+};
+application novel-netiq[json] {
+	filter{
+        "${.json.appName}" eq "Novell Access Manager";
+    };
+    parser { novel-netiq-parser(); };   
+};

package/etc/conf.d/conflib/json/app-zscaler_lss.conf

@@ -0,0 +1,72 @@
+block parser zscaler_lss-parser() {    
+ channel {
+
+        if {
+            filter {
+                match('.' value('.json.ClientZEN'))
+                and match('.' value('.json.AppGroup'))
+                and match('.' value('.json.Application'))
+            };
+            rewrite {
+                r_set_splunk_dest_default(
+                    index("netproxy")
+                    sourcetype('zscalerlss-zpa-app')
+                    vendor_product("zscaler_lss")
+                );                                          
+            };           
+        } elif {
+            filter {
+                match('.' value('.json.Exporter'))
+                and match('.' value('.json.Customer'))
+                and match('.' value('.json.ConnectionID'))
+            };
+            rewrite {
+                r_set_splunk_dest_default(
+                    index("netproxy")
+                    sourcetype('zscalerlss-zpa-bba')
+                    vendor_product("zscaler_lss")
+                );                                          
+            };                       
+        } elif {
+            filter {
+                match('.' value('.json.Connector'))
+                and match('.' value('.json.Customer'))
+                and match('.' value('.json.ConnectorGroup'))
+            };
+            rewrite {
+                r_set_splunk_dest_default(
+                    index("netproxy")
+                    sourcetype('zscalerlss-zpa-connector')
+                    vendor_product("zscaler_lss")
+                );                                          
+            };                       
+        } elif {
+            filter {
+                match('.' value('.json.SAMLAttributes'))
+                and match('.' value('.json.Customer'))
+            };
+            rewrite {
+                r_set_splunk_dest_default(
+                    index("netproxy")
+                    sourcetype('zscalerlss-zpa-auth')
+                    vendor_product("zscaler_lss")
+                );                                          
+            };                       
+        };
+        parser {
+                #.jsonLog.Timestamp Mar 04 20:37:53 2020
+            date-parser(
+                format('%a %b %d %H:%M:%S %Y',
+                       '%a %b %d %k:%M:%S %Y')
+                template("${.json.LogTimestamp}")
+                flags(guess-timezone)
+            );
+        };
+
+
+   };
+};
+application zscaler_lss[json] {
+	parser { zscaler_lss-parser(); };   
+};
+

package/etc/conf.d/conflib/raw/app-syslog_json.conf

@@ -8,11 +8,6 @@ block parser syslog_json-parser() {
             set("$1", value("MESSAGE"));
             set("rfc3164_json", value("fields.sc4s_syslog_format"));
         };                       
-        parser {
-            json-parser(
-                prefix('.json.')
-            );
-        };
    };
 };
 application syslog_json[sc4s-raw-syslog] {

package/etc/conf.d/conflib/syslog/app-json.conf

@@ -0,0 +1,32 @@
+block parser json-group-parser() {    
+    channel {
+        rewrite {
+            r_set_splunk_dest_default(
+                sourcetype('json')
+                vendor_product("generic_json")
+            ); 
+            set("json", value("fields.sc4s_syslog_format"));        
+            set("t_msg_trim", value(".splunk.sc4s_template"));
+
+        };  
+        parser {
+            json-parser(
+                prefix('.json.')
+            );
+        };             
+
+        if {
+            parser { app-parser(topic(json)); };
+        };   
+        rewrite {
+            groupunset(values('.json.*'));
+        };
+   };
+};
+application json-group[sc4s-syslog] {
+	filter { 
+        "${PROGRAM}" eq "" 
+        and message('{' type(string) flags(prefix));
+    };	
+    parser { json-group-parser(); };   
+};

tests/test_novell.py

@@ -0,0 +1,54 @@
+# Copyright 2019 Splunk, Inc.
+#
+# Use of this source code is governed by a BSD-2-clause-style
+# license that can be found in the LICENSE-BSD2 file or at
+# https://opensource.org/licenses/BSD-2-Clause
+import random
+
+from jinja2 import Environment
+
+from .sendmessage import *
+from .splunkutils import *
+from .timeutils import *
+import pytest
+
+env = Environment()
+
+testdata = [
+    '{{mark}}{{ bsd }} {{ host }} {"appName":"Novell Access Manager","timeStamp":"{{device_time}}","eventId":"002E0009","subTarget":"c7620505dc4b61cca7665cf1c092ea9980af164691cc5adf88d104dfff18a315","stringValue1":"https://login-test.authbridge-nonprod.XXXgroup.com/nidp/saml2/metadata","stringValue2":"https://obp-sso-tst2.xxx.wbctestau.xxxx.com.au/oam/fed","stringValue3":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36-SCCrow32z","numericValue1":0,"numericValue2":0,"numericValue3":0,"originator":"C423618A1F3FB8F2","component":"nidp","data":"MTAuOTcuMTQ0LjE1Ng==","description":"NIDS: Provided an authentication to a remote consumer","message":"[Tue, 15 Jun 2021 02:35:28 +1000]  [Novell Access Manager\\\\nidp]: AMDEVICEID#C423618A1F3FB8F2: AMAUTHID#c7620505dc4b61cca7665cf1c092ea9980af164691cc5adf88d104dfff18a315: Provided an authentication to a remote consumer on behalf of user: [cn=xxxxx,ou=users,o=data]. Authentication Type: [https://login-test.authbridge-nonprod.XXXgroup.com/nidp/saml2/metadata] Authenticating Entity Name: [https://obp-sso-tst2.xxx.xxx.XXX.com.au/oam/fed] Contract Class or Method Name: [Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36-SCCrow32z] Client IP Address: [10.0.0.0]","target":"cn=xxx,ou=users,o=data"}',
+]
+# Tue, 15 Jun 2021 02:35:28 +1000
+
+
+@pytest.mark.parametrize("event", testdata)
+def test_data_access_manager(
+    record_property, setup_wordlist, setup_splunk, setup_sc4s, event
+):
+    host = "{}-{}".format(random.choice(setup_wordlist), random.choice(setup_wordlist))
+
+    dt = datetime.datetime.now(datetime.timezone.utc)
+    iso, bsd, time, date, tzoffset, tzname, epoch = time_operations(dt)
+    # Tune time functions
+    epoch = epoch[:-7]
+    device_time = dt.strftime("%a, %d %b %Y %H:%M:%S +0000")
+
+    mt = env.from_string(event + "\n")
+    message = mt.render(mark="<132>", bsd=bsd, host=host, device_time=device_time)
+
+    sendsingle(message, setup_sc4s[0], setup_sc4s[1][514])
+
+    st = env.from_string(
+        'search index=netauth _time={{ epoch }} sourcetype="novell:netiq"'
+    )
+
+    message1 = mt.render(mark="", bsd="", host="", app="ossec")
+    message1 = message1.lstrip()
+    search = st.render(epoch=epoch, host=host, message=message1)
+
+    resultCount, eventCount = splunk_single(setup_splunk, search)
+
+    record_property("host", host)
+    record_property("resultCount", resultCount)
+    record_property("message", message)
+
+    assert resultCount == 1