feat!: Replace vendor_product with split fields (#1427)

This PR is a BREAKING CHANGE. Existing uses of meta_key have been removed. Vendor_product field will become vendor and product fields. Class will be optionally used when the third level of lookup is required.
splunk · Feb 3, 2022 · f36f94f · f36f94f
1 parent 5cadf80
commit f36f94f
Show file tree

Hide file tree

Showing 154 changed files with 540 additions and 379 deletions.
diff --git a/docs/destinations.md b/docs/destinations.md
@@ -40,7 +40,7 @@ SC4S_DEST_SPLUNK_HEC_OTHER_MODE=SELECT
 #filename:
 application sc4s-lp-cisco_ios_dest_fmt_other{{ source }}[sc4s-lp-dest-select-d_fmt_hec_OTHER] {
     filter {
-        match('CISCO_IOS' value('.dest_key'))
+        'CISCO_IOS' eq "${fields.sc4s_vendor}_${fields.sc4s_product}"
         #Match any cisco event that is not like "%ACL-7-1234"
         and not message('^%[^\-]+-7-');
     };    

diff --git a/docs/sources/Dell_EMC/index.md b/docs/sources/Dell_EMC/index.md
@@ -20,7 +20,7 @@
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| dell_emc_powerswitch_n      | all       | netops          | none          |
+| dellemc_powerswitch_n      | all       | netops          | none          |
 
 ### Filter type
 
@@ -36,10 +36,10 @@ Message Format
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_DELL_DELL_EMC_POWERSWITCH_N_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
-| SC4S_LISTEN_DELL_DELL_EMC_POWERSWITCH_N_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers |
-| SC4S_ARCHIVE_DELL_DELL_EMC_POWERSWITCH_N | no | Enable archive to disk for this specific source |
-| SC4S_DEST_DELL_DELL_EMC_POWERSWITCH_N_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_DELLEMC_POWERSWITCH_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_LISTEN_DELLEMC_POWERSWITCH_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_ARCHIVE_DELLEMC_POWERSWITCH | no | Enable archive to disk for this specific source |
+| SC4S_DEST_DELLEMC_POWERSWITCH_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Verification
 

diff --git a/docs/sources/InfoBlox/index.md b/docs/sources/InfoBlox/index.md
@@ -23,11 +23,11 @@ Warning: Despite the TA indication this data source is CIM compliant the all ver
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| infoblox_dns      | infoblox:dns       | netdns          | none          |
-| infoblox_dhcp    | infoblox:dhcp      | netipam          | none          |
-| infoblox_threat    | infoblox:threatprotect      | netids          | none          |
-| infoblox_audit    | infoblox:audit      | netops          | none          |
-| infoblox_fallback    | infoblox:port      | netops          | none          |
+| infoblox_nios_dns      | infoblox:dns       | netdns          | none          |
+| infoblox_nios_dhcp    | infoblox:dhcp      | netipam          | none          |
+| infoblox_nios_threat    | infoblox:threatprotect      | netids          | none          |
+| infoblox_nios_audit    | infoblox:audit      | netops          | none          |
+| infoblox_nios_fallback    | infoblox:port      | netops          | none          |
 
 ### Filter type
 
@@ -43,10 +43,10 @@ Must be identified by host or ip assignment. Update the filter `f_infoblox` or c
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_INFOBLOX_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
-| SC4S_LISTEN_INFOBLOX_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers |
-| SC4S_ARCHIVE_INFOBLOX | no | Enable archive to disk for this specific source |
-| SC4S_DEST_INFOBLOX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_INFOBLOX_NIOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_LISTEN_INFOBLOX_NIOS_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_ARCHIVE_INFOBLOX_NIOS | no | Enable archive to disk for this specific source |
+| SC4S_DEST_INFOBLOX_NIOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Verification
 

diff --git a/docs/sources/Ossec/index.md b/docs/sources/Ossec/index.md
@@ -17,7 +17,7 @@
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| ossec    | ossec    | main          | None     |
+| ossec_ossec    | ossec    | main          | None     |
 
 ### Filter type
 
@@ -34,10 +34,10 @@ IP, Netmask or Host
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_OSSEC_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
-| SC4S_LISTEN_OSSEC_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers |
-| SC4S_ARCHIVE_OSSEC | no | Enable archive to disk for this specific source |
-| SC4S_DEST_OSSEC_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_OSSEC_OSSEC_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_LISTEN_OSSEC_OSSEC_UDP_PORT      | empty string      | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_ARCHIVE_OSSEC_OSSEC | no | Enable archive to disk for this specific source |
+| SC4S_DEST_OSSEC_OSSEC_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Verification
 

diff --git a/docs/sources/PaloaltoNetworks/index.md b/docs/sources/PaloaltoNetworks/index.md
@@ -25,14 +25,14 @@
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| pan_log      | pan:log       | netops          | none          |
-| pan_globalprotect | pan:pan_globalprotect | netfw | none |
-| pan_traffic    | pan:traffic      | netfw          | none          |
-| pan_threat    | pan:threat      | netproxy          | none          |
-| pan_system    | pan:system      | netops          | none          |
-| pan_config    | pan:config      | netops          | none          |
-| hipmatch    | pan:hipmatch      | netops          | none          |
-| pan_correlation    | pan:correlation      | netops          | none          |
+| pan_panos_log      | pan:log       | netops          | none          |
+| pan_panos_globalprotect | pan:pan_globalprotect | netfw | none |
+| pan_tpanos_raffic    | pan:traffic      | netfw          | none          |
+| pan_panos_threat    | pan:threat      | netproxy          | none          |
+| pan_panos_system    | pan:system      | netops          | none          |
+| pan_panos_config    | pan:config      | netops          | none          |
+| pan_panos_hipmatch    | pan:hipmatch      | netops          | none          |
+| pan_panos_correlation    | pan:correlation      | netops          | none          |
 
 ### Filter type
 
@@ -51,10 +51,10 @@ MSG Parse: This filter parses message content
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_LISTEN_PULSE_PALOALTO_PANOS_RFC6587_PORT      | empty string      | Enable a TCP using IETF Framing (RFC6587) port for this specific vendor product using a comma-separated list of port numbers |
-| SC4S_LISTEN_PALOALTO_PANOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
-| SC4S_ARCHIVE_PALOALTO_PANOS | no | Enable archive to disk for this specific source |
-| SC4S_DEST_PALOALTO_PANOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_LISTEN_PULSE_PAN_PANOS_RFC6587_PORT      | empty string      | Enable a TCP using IETF Framing (RFC6587) port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_LISTEN_PAN_PANOS_TCP_PORT      | empty string      | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers |
+| SC4S_ARCHIVE_PAN_PANOS | no | Enable archive to disk for this specific source |
+| SC4S_DEST_PAN_PANOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Verification
 

diff --git a/docs/sources/Tanium/index.md b/docs/sources/Tanium/index.md
@@ -21,7 +21,7 @@ The source is understood to require a valid certificate.
 
 | key            | index      | notes          |
 |----------------|------------|----------------|
-| tanium     | epintel          | none          |
+| tanium_syslog     | epintel          | none          |
 
 ### Filter type
 
@@ -32,8 +32,8 @@ timestamp: When present the field ``Client-Time-UTC`` will be used as the time s
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_ARCHIVE_TANIUM | no | Enable archive to disk for this specific source |
-| SC4S_DEST_TANIUM_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_ARCHIVE_TANIUM_SYSLOG | no | Enable archive to disk for this specific source |
+| SC4S_DEST_TANIUM_SYSLOG_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 | SC4S_SOURCE_TLS_ENABLE | no | This must be set to yes so that SC4S listens for encrypted syslog from ePO
 
 ### Additional setup

diff --git a/docs/sources/Tintri/index.md b/docs/sources/Tintri/index.md
@@ -14,14 +14,14 @@ The source is understood to require a valid certificate.
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| TINTRI | none |
+| tintri | none |
 
 
 ### Index Configuration
 
 | key            | index      | notes          |
 |----------------|------------|----------------|
-| TINTRI     | infraops          | none          |
+| tintri_syslog     | infraops          | none          |
 
 ### Filter type
 
@@ -31,8 +31,8 @@ MSG Parse: This filter parses message content generic linux logs will use the os
 
 | Variable       | default        | description    |
 |----------------|----------------|----------------|
-| SC4S_ARCHIVE_TINTRI | no | Enable archive to disk for this specific source |
-| SC4S_DEST_TINTRI_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
+| SC4S_ARCHIVE_TINTRI_SYSLOG | no | Enable archive to disk for this specific source |
+| SC4S_DEST_TINTRI_SYSLOG_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | 
 
 ### Additional setup
 

diff --git a/docs/sources/VMWare/index.md b/docs/sources/VMWare/index.md
@@ -77,9 +77,9 @@ index=<asconfigured> (sourcetype=cef source="carbonblack:protection:cef")
 
 | key            | sourcetype     | index          | notes          |
 |----------------|----------------|----------------|----------------|
-| vmware_esx      | vmware:vsphere:esx | main          | none          |
-| vmware_nsx      | vmware:vsphere:nsx | main          | none          |
-| vmware_vcenter      | vmware:vsphere:vcenter | main          | none          |
+| vmware_vsphere_esx      | vmware:vsphere:esx | main          | none          |
+| vmware_vsphere_nsx      | vmware:vsphere:nsx | main          | none          |
+| vmware_vsphere_vcenter      | vmware:vsphere:vcenter | main          | none          |
 
 ### Filter type
 

diff --git a/docs/sources/Zscaler/index.md b/docs/sources/Zscaler/index.md
@@ -17,20 +17,20 @@ the IP or host name of the SC4S instance and port 514
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| zscalernss-alerts  | Requires format customization add ``\tvendor=Zscaler\tproduct=alerts`` immediately prior to the ``\n`` in the NSS Alert Web format. See Zscaler manual for more info. |
-| zscalernss-dns  | Requires format customization  add ``\tvendor=Zscaler\tproduct=dns`` immediately prior to the ``\n`` in the NSS DNS format. See Zscaler manual for more info. |
-| zscalernss-web  | None    |
-| zscalernss-fw  | Requires format customization add ``\tvendor=Zscaler\tproduct=fw`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. |
+| zscaler_nss_alerts  | Requires format customization add ``\tvendor=Zscaler\tproduct=alerts`` immediately prior to the ``\n`` in the NSS Alert Web format. See Zscaler manual for more info. |
+| zscaler_nss_dns  | Requires format customization  add ``\tvendor=Zscaler\tproduct=dns`` immediately prior to the ``\n`` in the NSS DNS format. See Zscaler manual for more info. |
+| zscaler_nss_web  | None    |
+| zscaler_nss_fw  | Requires format customization add ``\tvendor=Zscaler\tproduct=fw`` immediately prior to the ``\n`` in the Firewall format. See Zscaler manual for more info. |
 
 
 ### Sourcetype and Index Configuration
 
 | key                 | sourcetype             | index    | notes   |
 |---------------------|------------------------|----------|---------|
-| zscaler_alerts      | zscalernss-alerts      | main     | none    |
-| zscaler_dns         | zscalernss-dns         | netdns   | none    |
-| zscaler_fw          | zscalernss-fw          | netfw    | none    |
-| zscaler_web         | zscalernss-web         | netproxy | none    |
+| zscaler_nss_alerts      | zscalernss-alerts      | main     | none    |
+| zscaler_nss_dns         | zscalernss-dns         | netdns   | none    |
+| zscaler_nss_fw          | zscalernss-fw          | netfw    | none    |
+| zscaler_nss_web         | zscalernss-web         | netproxy | none    |
 | zscaler_zia_audit   | zscalernss-zia-audit   | netops   | none    |
 | zscaler_zia_sandbox | zscalernss-zia-sandbox | main     | none    |
 
@@ -80,10 +80,10 @@ the IP or host name of the SC4S instance and port 514
 
 | sourcetype     | notes                                                                                                   |
 |----------------|---------------------------------------------------------------------------------------------------------|
-| zscalerlss-zpa-app  | None |
-| zscalerlss-zpa-auth  | None |
-| zscalerlss-zpa-bba  | None |
-| zscalerlss-zpa-connector  | None |
+| zscaler_lss-app  | None |
+| zscaler_lss-auth  | None |
+| zscaler_lss-bba  | None |
+| zscaler_lss-connector  | None |
 
 
 ### Sourcetype and Index Configuration

diff --git a/docs/sources/index.md b/docs/sources/index.md
@@ -67,7 +67,8 @@ block parser alcatel_switch-parser() {
             r_set_splunk_dest_default(
                 index('netops')
                 sourcetype('alcatel:switch')
-                vendor_product("alcatel_switch")
+                vendor('alcatel')
+                product('switch')
                 template('t_hdr_msg')
             );              
         };       
@@ -97,7 +98,9 @@ block parser dell_poweredge_cmc-parser() {
             r_set_splunk_dest_default(
                 index('infraops')
                 sourcetype('dell:poweredge:cmc:syslog')
-                vendor_product("dell_poweredge_cmc")
+                vendor('dell')
+                product('poweredge')
+                class('cmc')
             );              
         };       
    };
@@ -126,7 +129,7 @@ block parser cisco_ios_debug-postfilter() {
         #In this case the outcome is drop the event other logic such as adding indexed fields or editing the message is possible
         rewrite { 
             r_set_splunk_dest_update(
-                vendor_product('null_queue')
+                vendor('null') product('queue')
             );
         };
    };

diff --git a/docs/upgrade.md b/docs/upgrade.md
@@ -36,4 +36,28 @@ See the [release information](https://github.com/splunk/splunk-connect-for-syslo
 * Internal metrics will now use the _metrics index by default update vendor_product key 'sc4s_metrics' to change the index
 * Deprecated use of vendor_product_by_source for null queue or dropping events see See [Filtering events from output](https://splunk.github.io/splunk-connect-for-syslog/main/sources/) this use will be removed in v3
 * Deprecated use of `SPLUNK_HEC_ALT_DESTS` this variable is no longer used and will be ignored
-* Deprecated use of `SC4S_DEST_GLOBAL_ALTERNATES` this variable will be removed in future major versions see Destinations section in configuration
+* Deprecated use of `SC4S_DEST_GLOBAL_ALTERNATES` this variable will be removed in future major versions see Destinations section in configuration
+* Corrected Vendor/Product keys *BREAKING* Please see source doc pages and revise configuration as part of upgrade
+    * Zscaler (multiple changes)
+    * dell_emc_powerswitch_n
+    * F5_BIGIP
+    * INFOBLOX
+    * Dell RSA SecureID
+    * ubiquiti
+    * SC4S will now use "splunk as the vendor value, "sc4s" as the product
+    * Fireye HX
+    * Juniper
+    * ossec
+    * Palo Alto Networks
+    * Pulse Connect
+    * ricoh
+    * tanium
+    * tintri
+    * Vmware esx,vcenter,nsx,horizon
+* Internal Changes
+    * `.dest_key` field is no longer used
+    * `sc4s_vendor_product` is read only and will be removed
+    * `sc4s_vendor` new contains "vendor" portion of vendor_product
+    * `sc4s_vendor_product` new contains "product" portion of vendor product
+    * `sc4s_class` new contains additional data previously concatenated to vendor_product
+    * removed `meta_key`
diff --git a/package/etc/conf.d/conflib/_splunk/splunk_context.conf b/package/etc/conf.d/conflib/_splunk/splunk_context.conf
@@ -3,16 +3,16 @@ block parser p_add_context_splunk(key('')) {
 
         parser {
             add-contextual-data(
-                selector("${fields.sc4s_vendor_product}"),
+                selector("${fields.sc4s_vendor}_${fields.sc4s_product}"),
                 database("conf.d/local/context/splunk_metadata.csv"),
                 prefix(".splunk."),
                 ignore-case(yes)        
             ); 
         } ;
-        if ("${.meta_key}" ne "" and "${fields.sc4s_vendor_product}" ne "${.meta_key}" ){
+        if ("${fields.sc4s_class}" ne ""){
             parser {
                 add-contextual-data(
-                    selector("${.meta_key}"),
+                    selector("${fields.sc4s_vendor}_${fields.sc4s_product}_${fields.sc4s_class}"),
                     database("conf.d/local/context/splunk_metadata.csv"),
                     prefix(".splunk."),
                     ignore-case(yes)