From e792c77cd102e82b6b6da57c37d534715d593f13 Mon Sep 17 00:00:00 2001 From: rfaircloth-splunk Date: Sat, 5 Feb 2022 15:28:08 -0500 Subject: [PATCH] feat: Support app-parser for vps Current vendor product by source config uses two files that must be kept in sync this solution allows using one or more files (customer choice) to define mapping. --- docs/sources/Cisco/index.md | 125 +++++++++--------- docs/sources/index.md | 11 +- mkdocs.yml | 2 +- package/Dockerfile | 1 + .../_common/p_vendor_product_by_source.conf | 9 -- .../conf.d/conflib/_splunk/splunkfields.conf | 10 ++ .../app-netsource-aruba_clearpass.conf | 1 + .../app-netsource-brocade_syslog.conf | 3 +- .../app-netsource-buffalo_terastation.conf | 3 +- .../app-netsource-checkpoint_fw.conf | 3 +- .../netsource/app-netsource-cisco_esa.conf | 3 +- .../netsource/app-netsource-cisco_meraki.conf | 3 +- .../netsource/app-netsource-cisco_wsa.conf | 3 +- .../app-netsource-cisco_wsa_11_7.conf | 7 +- .../app-netsource-cisco_wsa_splunk.conf | 1 + .../app-netsource-dell_poweredge_cmc.conf | 3 +- .../app-netsource-dell_rsa_secureid.conf | 3 +- .../app-netsource-ibm_datapower.conf | 3 +- .../netsource/app-netsource-infoblox.conf | 3 +- .../app-netsource-proofpoint_pps.conf | 3 +- .../app-netsource-ubiquiti_unifi.conf | 1 + .../app-netsource-dell_switch_n.conf | 2 +- .../_splunk => enrich}/splunk_context.conf | 0 .../enrich/vendor_product_by_source.conf | 27 ++++ .../vendor_product_by_source.conf.example | 95 ------------- .../vendor_product_by_source.csv.example | 23 ---- package/etc/syslog-ng.conf | 7 +- .../app-vps-test-aruba_clearpass.conf | 16 +++ .../app-vps-test-brocade_syslog.conf | 16 +++ .../test_parsers/app-vps-test-cisco_esa.conf | 16 +++ .../app-vps-test-cisco_meraki.conf | 16 +++ .../test_parsers/app-vps-test-cisco_wsa.conf | 16 +++ .../app-vps-test-cisco_wsa11_7.conf | 16 +++ .../app-vps-test-cisco_wsa_recommended.conf | 16 +++ .../test_parsers/app-vps-test-dell_cmc.conf | 16 +++ .../app-vps-test-dell_rsa_secureid.conf | 16 +++ .../test_parsers/app-vps-test-f5_bigip.conf | 16 +++ .../app-vps-test-ibm_datapower.conf | 16 +++ .../app-vps-test-infoblox_nios.conf | 16 +++ .../app-vps-test-mikrotik_routeros.conf | 16 +++ .../app-vps-test-pfsense_firewall.conf | 16 +++ .../app-vps-test-proofpoint_pps.conf | 16 +++ .../app-vps-test-schneider_apc.conf | 16 +++ .../app-vps-test-sophos_webappliance.conf | 16 +++ .../app-vps-test-spectracom_ntp.conf | 16 +++ .../app-vps-test-symantec_dlp.conf | 16 +++ .../app-vps-test-ubiquiti_unifi_fw.conf | 16 +++ .../app-vps-test-vmware_vcenter.conf | 16 +++ package/sbin/entrypoint.sh | 2 + 49 files changed, 484 insertions(+), 209 deletions(-) delete mode 100644 package/etc/conf.d/conflib/_common/p_vendor_product_by_source.conf rename package/etc/conf.d/conflib/{netsource => syslog}/app-netsource-dell_switch_n.conf (94%) rename package/etc/conf.d/{conflib/_splunk => enrich}/splunk_context.conf (100%) create mode 100644 package/etc/conf.d/enrich/vendor_product_by_source.conf create mode 100644 package/etc/test_parsers/app-vps-test-aruba_clearpass.conf create mode 100644 package/etc/test_parsers/app-vps-test-brocade_syslog.conf create mode 100644 package/etc/test_parsers/app-vps-test-cisco_esa.conf create mode 100644 package/etc/test_parsers/app-vps-test-cisco_meraki.conf create mode 100644 package/etc/test_parsers/app-vps-test-cisco_wsa.conf create mode 100644 package/etc/test_parsers/app-vps-test-cisco_wsa11_7.conf create mode 100644 package/etc/test_parsers/app-vps-test-cisco_wsa_recommended.conf create mode 100644 package/etc/test_parsers/app-vps-test-dell_cmc.conf create mode 100644 package/etc/test_parsers/app-vps-test-dell_rsa_secureid.conf create mode 100644 package/etc/test_parsers/app-vps-test-f5_bigip.conf create mode 100644 package/etc/test_parsers/app-vps-test-ibm_datapower.conf create mode 100644 package/etc/test_parsers/app-vps-test-infoblox_nios.conf create mode 100644 package/etc/test_parsers/app-vps-test-mikrotik_routeros.conf create mode 100644 package/etc/test_parsers/app-vps-test-pfsense_firewall.conf create mode 100644 package/etc/test_parsers/app-vps-test-proofpoint_pps.conf create mode 100644 package/etc/test_parsers/app-vps-test-schneider_apc.conf create mode 100644 package/etc/test_parsers/app-vps-test-sophos_webappliance.conf create mode 100644 package/etc/test_parsers/app-vps-test-spectracom_ntp.conf create mode 100644 package/etc/test_parsers/app-vps-test-symantec_dlp.conf create mode 100644 package/etc/test_parsers/app-vps-test-ubiquiti_unifi_fw.conf create mode 100644 package/etc/test_parsers/app-vps-test-vmware_vcenter.conf diff --git a/docs/sources/Cisco/index.md b/docs/sources/Cisco/index.md index a86b0e2f3e..d4c86bfddb 100644 --- a/docs/sources/Cisco/index.md +++ b/docs/sources/Cisco/index.md @@ -33,7 +33,7 @@ Unknown this product is unsupported by Cisco | SC4S_LISTEN_CISCO_ACE_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_ACE_TCP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_ACE | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_ACE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_ACE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -47,8 +47,8 @@ index= sourcetype=cisco:ace | stats count by host | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | https://splunkbase.splunk.com/app/1811/ | -| Product Manual | https://community.cisco.com/t5/security-documents/acs-5-x-configuring-the-external-syslog-server/ta-p/3143143 | +| Splunk Add-on | | +| Product Manual | | ### Sourcetypes @@ -84,7 +84,7 @@ EXTRACT-acs_message_header2 = ^CSCOacs_\S+\s+(?\S+)\s+(? sourcetype=cisco:acs ``` -Verify timestamp, and host values match as expected +Verify timestamp, and host values match as expected ## Product - ASA AND FTD (Firepower) @@ -102,9 +102,9 @@ Including Legacy FWSM and PIX | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on for ASA (No long supports FWSM and PIX) | https://splunkbase.splunk.com/app/1620/ | -| Cisco eStreamer for Splunk | https://splunkbase.splunk.com/app/1629/ | -| Product Manual | https://www.cisco.com/c/en/us/td/docs/security/asa/asa82/configuration/guide/config/monitor_syslog.html | +| Splunk Add-on for ASA (No long supports FWSM and PIX) | | +| Cisco eStreamer for Splunk | | +| Product Manual | | ### Sourcetypes @@ -114,7 +114,7 @@ Including Legacy FWSM and PIX | cisco:ftd | cisco FTD Firepower will also use this source type except those noted below | | cisco:fwsm | Splunk has | | cisco:pix | cisco PIX will also use this source type except those noted below | -| cisco:firepower:syslog | FTD Unified events see https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide.pdf | +| cisco:firepower:syslog | FTD Unified events see | ### Sourcetype and Index Configuration @@ -135,11 +135,11 @@ MSG Parse: This filter parses message content * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_metadata.csv file and set the index and sourcetype as required for the data source. * Follow vendor configuration steps per Product Manual above ensure: - * Log Level is 6 "Informational" - * Protocol is TCP/IP - * permit-hostdown is on - * device-id is hostname and included - * timestamp is included + * Log Level is 6 "Informational" + * Protocol is TCP/IP + * permit-hostdown is on + * device-id is hostname and included + * timestamp is included ### Options @@ -148,7 +148,7 @@ MSG Parse: This filter parses message content | SC4S_LISTEN_CISCO_ASA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_ASA_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_ASA | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_ASA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_ASA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -158,14 +158,14 @@ Use the following search to validate events are present index= sourcetype=cisco:asa ``` -Verify timestamp, and host values match as expected +Verify timestamp, and host values match as expected ## Product - Cisco Email Security Appliance (ESA) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | https://splunkbase.splunk.com/app/1761/ | -| Product Manual | https://www.cisco.com/c/en/us/td/docs/security/esa/esa14-0/user_guide/b_ESA_Admin_Guide_14-0.pdf | +| Splunk Add-on | | +| Product Manual | | ### Sourcetypes @@ -205,7 +205,7 @@ IP, Netmask or Host * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * ESA Follow vendor configuration steps per Product Manual. * Ensure host and timestamp are included. -* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf `` update the host or ip mask for ``f_cisco_esa`` to identiy the esa events. +* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf`` update the host or ip mask for ``f_cisco_esa`` to identiy the esa events. ### Options @@ -214,7 +214,7 @@ IP, Netmask or Host | SC4S_LISTEN_CISCO_ESA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_ESA_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_ESA | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_ESA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_ESA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -260,7 +260,7 @@ PATTERN MATCH | SC4S_LISTEN_CISCO_CIMC_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_CIMC_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_CIMC | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_CIMC_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_CIMC_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -280,17 +280,17 @@ Cisco Network Products of multiple types share common logging characteristics th * Cisco APIC/ACI * Cisco IOS * Cisco IOS-XR -* Cisco IOS-XE +* Cisco IOS-XE * Cisco NX-OS * Cisco FX-OS | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | https://splunkbase.splunk.com/app/1467/ | -| IOS Manual | https://www.cisco.com/c/en/us/td/docs/switches/lan/catalyst2960/software/release/12-2_55_se/configuration/guide/scg_2960/swlog.html | -| NX-OS Manual | https://www.cisco.com/c/en/us/td/docs/switches/datacenter/nexus9000/sw/6-x/system_management/configuration/guide/b_Cisco_Nexus_9000_Series_NX-OS_System_Management_Configuration_Guide/sm_5syslog.html| -| Cisco ACI | https://community.cisco.com/legacyfs/online/attachments/document/technote-aci-syslog_external-v1.pdf | -| Cisco WLC & AP | https://www.cisco.com/c/en/us/support/docs/wireless/4100-series-wireless-lan-controllers/107252-WLC-Syslog-Server.html#anc8 | +| Splunk Add-on | | +| IOS Manual | | +| NX-OS Manual | | +| Cisco ACI | | +| Cisco WLC & AP | | ### Sourcetypes @@ -314,25 +314,25 @@ Cisco Network Products of multiple types share common logging characteristics th * Install the Splunk Add-on on the search head(s) for the user communities interested in this data source. If SC4S is exclusively used the addon is not required on the indexer. * Review and update the splunk_metadata.csv file and set the index and sourcetype as required for the data source. * IOS Follow vendor configuration steps per Product Manual above ensure: - * Ensure a reliable NTP server is set and synced - * Log Level is 6 "Informational" - * Protocol is TCP/IP - * permit-hostdown is on - * device-id is hostname and included - * timestamp is included + * Ensure a reliable NTP server is set and synced + * Log Level is 6 "Informational" + * Protocol is TCP/IP + * permit-hostdown is on + * device-id is hostname and included + * timestamp is included * NX-OS Follow vendor configuration steps per Product Manual above ensure: - * Ensure a reliable NTP server is set and synced - * Log Level is 6 "Informational" user may select alternate levels by module based on use cases - * Protocol is TCP/IP - * device-id is hostname and included - * timestamp is included and milisecond accuracy selected + * Ensure a reliable NTP server is set and synced + * Log Level is 6 "Informational" user may select alternate levels by module based on use cases + * Protocol is TCP/IP + * device-id is hostname and included + * timestamp is included and milisecond accuracy selected * ACI Logging configuration of the ACI product often varies by use case. - * Ensure NTP sync is configured and active - * Ensure proper host names are configured + * Ensure NTP sync is configured and active + * Ensure proper host names are configured * WLC - * Ensure NTP sync is configured and active - * Ensure proper host names are configured - * For security use cases per AP logging is required + * Ensure NTP sync is configured and active + * Ensure proper host names are configured + * For security use cases per AP logging is required ### Options @@ -341,7 +341,7 @@ Cisco Network Products of multiple types share common logging characteristics th | SC4S_LISTEN_CISCO_IOS_UDP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_IOS_TCP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_IOS | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_IOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_IOS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -355,8 +355,8 @@ index= sourcetype=cisco:ios | stats count by host | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | https://splunkbase.splunk.com/app/1915/ | -| Product Manual | https://www.cisco.com/c/en/us/td/docs/security/ise/2-6/Cisco_ISE_Syslogs/Cisco_ISE_Syslogs/Cisco_ISE_Syslogs_chapter_00.html | +| Splunk Add-on | | +| Product Manual | | ### Sourcetypes @@ -385,7 +385,7 @@ PATTERN MATCH | SC4S_LISTEN_CISCO_ISE_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers expecting RFC5424 format | | SC4S_LISTEN_CISCO_ISE_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers expecting RFC5424 format | | SC4S_ARCHIVE_CISCO_ISE | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_ISE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_ISE_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -395,14 +395,14 @@ Use the following search to validate events are present index= sourcetype=cisco:ise:syslog ``` -Verify timestamp, and host values match as expected +Verify timestamp, and host values match as expected ## Product - Meraki Product Line (MR, MS, MX, MV) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | https://splunkbase.splunk.com/app/3018/ | -| Product Manual | https://documentation.meraki.com/zGeneral_Administration/Monitoring_and_Reporting/Syslog_Server_Overview_and_Configuration | +| Splunk Add-on | | +| Product Manual | | ### Sourcetypes @@ -433,7 +433,7 @@ IP, Netmask, Host or Port | SC4S_LISTEN_CISCO_MERAKI_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers expecting RFC5424 format | | SC4S_LISTEN_CISCO_MERAKI_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers expecting RFC5424 format | | SC4S_ARCHIVE_CISCO_MERAKI | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_MERAKI_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_MERAKI_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -443,13 +443,13 @@ Use the following search to validate events are present index= sourcetype=merkai ``` -Verify timestamp, and host values match as expected +Verify timestamp, and host values match as expected -## Product - Cisco TelePresence Video Communication Server (TVCS) +## Product - Cisco TelePresence Video Communication Server (TVCS) | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Product Manual | https://www.cisco.com/c/en/us/products/unified-communications/telepresence-video-communication-server-vcs/index.html | +| Product Manual | | ### Sourcetypes @@ -475,7 +475,7 @@ Source side unknown | SC4S_LISTEN_CISCO_TVCS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_TVCS_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_TVCS | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_TVCS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_TVCS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | | SC4S_LISTEN_CISCO_TVCS_LEGACY_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers expecting RFC3164 format | | SC4S_LISTEN_CISCO_TVCS_LEGACY_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers expecting RFC3164 format | | SC4S_ARCHIVE_CISCO_TVCS_LEGACY | no | Enable archive to disk for this specific source | @@ -524,7 +524,7 @@ PATTERN MATCH | SC4S_LISTEN_CISCO_UCM_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_UCM_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_UCM | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_UCM_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_UCM_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -570,7 +570,7 @@ PATTERN MATCH | SC4S_LISTEN_CISCO_UCS_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_UCS_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_UCS | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_UCS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_UCS_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -616,7 +616,7 @@ PATTERN MATCH | SC4S_LISTEN_CISCO_UCS_HX_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_UCS_HX_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_UCS_HX | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_UCS_HX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_UCS_HX_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification @@ -632,10 +632,10 @@ Verify timestamp, and host values match as expected | Ref | Link | |----------------|---------------------------------------------------------------------------------------------------------| -| Splunk Add-on | https://splunkbase.splunk.com/app/1747/ | -| Product Manual | https://www.cisco.com/c/en/us/td/docs/security/wsa/wsa11-7/user_guide/b_WSA_UserGuide_11_7.html | +| Splunk Add-on | | +| Product Manual | | -* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf `` update the host or ip mask for ``f_cisco_wsa`` to identiy the wsa squid events prior to WSA v11.7 and ``f_cisco_wsa11_7`` to identify the squid events since WSA v11.7. Update the host or ip mask for ``f_cisco_wsa_w3crecommended`` to identify the wsa w3c events since WSA v12.5. +* Update ``vi /opt/sc4s/local/context/vendor_product_by_source.conf`` update the host or ip mask for ``f_cisco_wsa`` to identiy the wsa squid events prior to WSA v11.7 and ``f_cisco_wsa11-7`` to identify the squid events since WSA v11.7. Update the host or ip mask for ``f_cisco_wsa_w3crecommended`` to identify the wsa w3c events since WSA v12.5. ### Sourcetypes @@ -643,6 +643,7 @@ Verify timestamp, and host values match as expected | cisco:wsa:squid | The access logs of Cisco IronPort WSA version prior to 11.7 record Web Proxy client history in squid. | | cisco:wsa:squid:new | The access logs of Cisco IronPort WSA version since 11.7 record Web Proxy client history in squid. | | cisco:wsa:w3c:recommended | The access logs of Cisco IronPort WSA version since 12.5 record Web Proxy client history in W3C. | + ### Sourcetype and Index Configuration | key | sourcetype | index | notes | @@ -669,7 +670,7 @@ IP, Netmask or Host | SC4S_LISTEN_CISCO_WSA_TCP_PORT | empty string | Enable a TCP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_LISTEN_CISCO_WSA_UDP_PORT | empty string | Enable a UDP port for this specific vendor product using a comma-separated list of port numbers | | SC4S_ARCHIVE_CISCO_WSA | no | Enable archive to disk for this specific source | -| SC4S_DEST_CISCO_WSA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | +| SC4S_DEST_CISCO_WSA_HEC | no | When Splunk HEC is disabled globally set to yes to enable this specific source | ### Verification diff --git a/docs/sources/index.md b/docs/sources/index.md index 6c60f89154..d886a115b0 100644 --- a/docs/sources/index.md +++ b/docs/sources/index.md @@ -1,8 +1,15 @@ # Introduction -When using Splunk Connect for Syslog to onboard a data source, the SC4S filter (or "log path") performs the operations that are traditionally performed at index-time by the corresponding Technical Add-on installed there. These index-time operations include linebreaking, sourcetype setting and timestamping. For this reason, if a data source is exclusively onboarded using SC4S then you will not need to install its corresponding Add-On on the indexers. You must, however, install the Add-on on the search head(s) for the user communities interested in this data source. +When using Splunk Connect for Syslog to onboard a data source, the syslog-ng "app-parser" performs the operations that are traditionally performed at index-time by the corresponding Technical Add-on installed there. These index-time operations include linebreaking, source/sourcetype setting and timestamping. For this reason, if a data source is exclusively onboarded using SC4S then you will not need to install its corresponding Add-On on the indexers. You must, however, install the Add-on on the search head(s) for the user communities interested in this data source. -SC4S "unique" filters are based either on the port upon which events arrive or the hostname/CIDR block from which they are sent. The "soup" filters run for events that arrive on port 514 (default for syslog), and contain regex and other syslog-specific parsers to identify events from a specific source, apply the correct sourcetype, and set other metadata. Data sources which generate events that are not unique enough to accurately identify with soup filters _must_ employ the "unique" filters (port/hostname/CIDR block) instead -- the soup filters are unavailable for these sources. +SC4S is designed to process "syslog" refering to IETF RFC standards 5424, legacy BSD syslog, RFC3164 (Not a standard document), and may "almost" syslog formats. +When possible data sources are identified and processed based on characteristics of the event that make them unique as compared to other events for example. Cisco devices using IOS will include " : %" followed by a string. While Arista EOS devices will use a valid RFC3164 header with a value in the "PROGRAM" position with "%" as the first char in the "MESSAGE" portion. This allows two similar event structures to be processed correct. + +When identification by message content alone is not possible for example the "sshd" program field is commonly used across vendors additional "hint" or guidance configuration allows SC4S to better classify events. The hints can be applied by +definition of a specific port which will be used as a property of the event or by configuration of a host name/ip pattern. For example "VMWARE VSPHERE" products have a number of "PROGRAM" fields which can be used to identify vmware specific events in the syslog stream and these can be properly sourcetyped automatically however because "sshd" is not uniuqe it will be treated as generic "os:nix" events until further configuration is applied. The administrator can take one of two actions to refine the processing for vmware + +* Define a specific port for vmware and reconfigure sources to use the defined port "SC4S_LISTEN_VMWARE_VSPHERE_TCP=9000". Any events arriving on port 9000 will now have a metadata field attached ".netsource.sc4s_vendor_product=VMWARE_VSPHERE" +* Define a "app-parser" to apply the metadata field by using a syslog-ng filter to apply the metadata field. ## Supporting previously unknown sources. diff --git a/mkdocs.yml b/mkdocs.yml index 457ce78ca1..ab08332a25 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,7 +36,7 @@ nav: - Development: "developing/index.md" - Destinations: "destinations.md" - Sources: - - About: sources/index.md + - Read First: sources/index.md - Alcatel: sources/Alcatel/index.md - Alsid: sources/Alsid/index.md - Arista: sources/Arista/index.md diff --git a/package/Dockerfile b/package/Dockerfile index 633bcd5211..95ce964a4c 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -60,6 +60,7 @@ RUN poetry export --format requirements.txt | pip3 install --user -r /dev/stdin COPY package/etc/syslog-ng.conf /etc/syslog-ng/syslog-ng.conf COPY package/etc/conf.d /etc/syslog-ng/conf.d COPY package/etc/context_templates /etc/syslog-ng/context_templates +COPY package/etc/test_parsers /etc/syslog-ng/test_parsers COPY package/etc/local_config /etc/syslog-ng/local_config COPY package/etc/local_config /etc/syslog-ng/local_config COPY package/sbin/entrypoint.sh / diff --git a/package/etc/conf.d/conflib/_common/p_vendor_product_by_source.conf b/package/etc/conf.d/conflib/_common/p_vendor_product_by_source.conf deleted file mode 100644 index 0abb13cce2..0000000000 --- a/package/etc/conf.d/conflib/_common/p_vendor_product_by_source.conf +++ /dev/null @@ -1,9 +0,0 @@ -parser vendor_product_by_source { - add-contextual-data( - selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.conf")), - database("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.csv") - ignore-case(yes) - prefix(".netsource.") - ); -}; - diff --git a/package/etc/conf.d/conflib/_splunk/splunkfields.conf b/package/etc/conf.d/conflib/_splunk/splunkfields.conf index e5b032063c..c7fa78a044 100644 --- a/package/etc/conf.d/conflib/_splunk/splunkfields.conf +++ b/package/etc/conf.d/conflib/_splunk/splunkfields.conf @@ -78,3 +78,13 @@ block rewrite r_set_splunk_dest_update( }; }; + +block rewrite r_set_splunk_vps( + #While the following is not used it remains to prevent breaking changes in content + vendor("${.netsource.sc4s_vendor}") + product("${.netsource.sc4s_product}") + ) { + set("`vendor`", value(".netsource.sc4s_vendor") condition('`vendor`' ne "")); + set("`product`", value(".netsource.sc4s_product") condition('`product`' ne "")); + set("`vendor`_`product`", value(".netsource.sc4s_vendor_product")); +}; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-aruba_clearpass.conf b/package/etc/conf.d/conflib/netsource/app-netsource-aruba_clearpass.conf index c5badd5646..7fd959627e 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-aruba_clearpass.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-aruba_clearpass.conf @@ -17,6 +17,7 @@ block parser app-netsource-aruba_clearpass() { application app-netsource-aruba_clearpass[sc4s-network-source] { filter { program('CPPM_' type(string) flags(prefix)) + or ( "${.netsource.sc4s_vendor}" eq "aruba" and "${.netsource.sc4s_product}" eq "clearpass") or "${.netsource.sc4s_vendor_product}" eq "aruba_clearpass" or "${SOURCE}" eq "s_ARUBA_CLEARPASS" ; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-brocade_syslog.conf b/package/etc/conf.d/conflib/netsource/app-netsource-brocade_syslog.conf index 6ba4dcf4f6..83f0c26048 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-brocade_syslog.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-brocade_syslog.conf @@ -18,7 +18,8 @@ block parser app-netsource-brocade_syslog() { }; application app-netsource-brocade_syslog[sc4s-network-source] { filter { - "${.netsource.sc4s_vendor_product}" eq "brocade_syslog" + ( "${.netsource.sc4s_vendor}" eq "brocade" and "${.netsource.sc4s_product}" eq "syslog") + or "${.netsource.sc4s_vendor_product}" eq "brocade_syslog" or "${SOURCE}" eq "s_BROCADE" }; parser { app-netsource-brocade_syslog(); }; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-buffalo_terastation.conf b/package/etc/conf.d/conflib/netsource/app-netsource-buffalo_terastation.conf index e8ac7260fa..224c156521 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-buffalo_terastation.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-buffalo_terastation.conf @@ -14,7 +14,8 @@ block parser app-netsource-buffalo_terastation() { }; application app-netsource-buffalo_terastation[sc4s-network-source] { filter { - "${.netsource.sc4s_vendor_product}" eq "buffalo_terastation" + ( "${.netsource.sc4s_vendor}" eq "buffalo" and "${.netsource.sc4s_product}" eq "terastation") + or "${.netsource.sc4s_vendor_product}" eq "buffalo_terastation" or "${SOURCE}" eq "s_BUFFALO_TERASTATION" ; }; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-checkpoint_fw.conf b/package/etc/conf.d/conflib/netsource/app-netsource-checkpoint_fw.conf index 1c6cbdf4fe..1fced5d620 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-checkpoint_fw.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-checkpoint_fw.conf @@ -16,7 +16,8 @@ block parser app-netsource-checkpoint_fw() { }; application app-netsource-checkpoint_fw[sc4s-network-source] { filter { - "${.netsource.sc4s_vendor_product}" eq "checkpoint_fw" + ( "${.netsource.sc4s_vendor}" eq "checkpoint" and "${.netsource.sc4s_product}" eq "fw") + or "${.netsource.sc4s_vendor_product}" eq "checkpoint_fw" or "${SOURCE}" eq "s_CHECKPOINT_FW" ; }; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_esa.conf b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_esa.conf index bbda7bfd14..0bbcd1bd80 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_esa.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_esa.conf @@ -149,7 +149,8 @@ application app-netsource-cisco_esa[sc4s-network-source] { filter { not "${fields.sc4s_vendor_product}" eq "cisco_esa" and ( - "${.netsource.sc4s_vendor_product}" eq "cisco_esa" + ( "${.netsource.sc4s_vendor}" eq "cisco" and "${.netsource.sc4s_product}" eq "esa") + or "${.netsource.sc4s_vendor_product}" eq "cisco_esa" or "${SOURCE}" eq "s_CISCO_ESA" ) }; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_meraki.conf b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_meraki.conf index 06020fc5e3..55aa99fe68 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_meraki.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_meraki.conf @@ -15,7 +15,8 @@ block parser app-netsource-cisco_meraki() { }; application app-netsource-cisco_meraki[sc4s-network-source] { filter { - "${.netsource.sc4s_vendor_product}" eq "cisco_meraki" + ( "${.netsource.sc4s_vendor}" eq "cisco" and "${.netsource.sc4s_product}" eq "meraki") + or "${.netsource.sc4s_vendor_product}" eq "cisco_meraki" or "${SOURCE}" eq "s_CISCO_MERAKI" }; parser { app-netsource-cisco_meraki(); }; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa.conf b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa.conf index 29b0a50663..096c59267e 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa.conf @@ -50,7 +50,8 @@ block parser app-netsource-cisco_wsa() { }; application app-netsource-cisco_wsa[sc4s-network-source] { filter { - "${.netsource.sc4s_vendor_product}" eq "cisco_wsa" + ( "${.netsource.sc4s_vendor}" eq "cisco" and "${.netsource.sc4s_product}" eq "wsa") + or "${.netsource.sc4s_vendor_product}" eq "cisco_wsa" or ( "${SOURCE}" eq "s_CISCO_WSA" and "${.netsource.sc4s_vendor_product}" eq "" diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_11_7.conf b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_11_7.conf index bf6d00aac6..edc03ac250 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_11_7.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_11_7.conf @@ -50,10 +50,11 @@ block parser app-netsource-cisco_wsa_11_7() { }; application app-netsource-cisco_wsa_11_7[sc4s-network-source] { filter { - "${.netsource.sc4s_vendor_product}" eq "cisco_wsa11_7" + "${.netsource.sc4s_vendor_product}" eq "cisco_wsa11-7" or ( - "${SOURCE}" eq "s_CISCO_WSA" - and "${.netsource.sc4s_vendor_product}" eq "cisco_wsa11_7" + ( "${.netsource.sc4s_vendor}" eq "cisco" and "${.netsource.sc4s_product}" eq "wsa11-7") + or "${SOURCE}" eq "s_CISCO_WSA" + and "${.netsource.sc4s_vendor_product}" eq "cisco_wsa11-7" ) }; parser { app-netsource-cisco_wsa_11_7(); }; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_splunk.conf b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_splunk.conf index 6ea09b212c..0468ad6fda 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_splunk.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-cisco_wsa_splunk.conf @@ -53,6 +53,7 @@ block parser app-netsource-cisco_wsa_splunk() { application app-netsource-cisco_wsa_splunk[sc4s-network-source] { filter { "${.netsource.sc4s_vendor_product}" eq "cisco_wsa_recommended" + or ( "${.netsource.sc4s_vendor}" eq "cisco" and "${.netsource.sc4s_product}" eq "wsa_recommended") or ( "${SOURCE}" eq "s_CISCO_WSA" and "${.netsource.sc4s_vendor_product}" eq "cisco_wsa_recommended" diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-dell_poweredge_cmc.conf b/package/etc/conf.d/conflib/netsource/app-netsource-dell_poweredge_cmc.conf index fbec93ea1b..058c7e6368 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-dell_poweredge_cmc.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-dell_poweredge_cmc.conf @@ -15,7 +15,8 @@ block parser app-netsource-dell_poweredge_cmc() { }; application app-netsource-dell_poweredge_cmc[sc4s-network-source] { filter { - ("${.netsource.sc4s_vendor_product}" eq "dell_poweredge_cmc" + ( "${.netsource.sc4s_vendor}" eq "dell" and "${.netsource.sc4s_product}" eq "poweredge_cmc") + or ("${.netsource.sc4s_vendor_product}" eq "dell_poweredge_cmc" or "${SOURCE}" eq "s_DELL_POWEREDGE_CMC") and "${fields.sc4s_vendor_product}" eq "" }; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-dell_rsa_secureid.conf b/package/etc/conf.d/conflib/netsource/app-netsource-dell_rsa_secureid.conf index e2d57b9b0c..6052f05dcd 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-dell_rsa_secureid.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-dell_rsa_secureid.conf @@ -91,7 +91,8 @@ block parser app-netsource-dell_rsa_secureid() { }; application app-netsource-dell_rsa_secureid[sc4s-network-source] { filter { - "${.netsource.sc4s_vendor_product}" eq "dell_rsa_secureid" + ( "${.netsource.sc4s_vendor}" eq "dell" and "${.netsource.sc4s_product}" eq "rsa_secureid") + or "${.netsource.sc4s_vendor_product}" eq "dell_rsa_secureid" or "${SOURCE}" eq "s_DELL_RSA_SECUREID" }; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-ibm_datapower.conf b/package/etc/conf.d/conflib/netsource/app-netsource-ibm_datapower.conf index e632bdc7e9..fa92067da0 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-ibm_datapower.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-ibm_datapower.conf @@ -34,7 +34,8 @@ block parser app-netsource-ibm_datapower() { application app-netsource-ibm_datapower[sc4s-network-source] { filter { ( - "${.netsource.sc4s_vendor_product}" eq "ibm_datapower" + ( "${.netsource.sc4s_vendor}" eq "ibm" and "${.netsource.sc4s_product}" eq "datapower") + or "${.netsource.sc4s_vendor_product}" eq "ibm_datapower" or "${SOURCE}" eq "s_IBM_DATAPOWER" ) }; diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-infoblox.conf b/package/etc/conf.d/conflib/netsource/app-netsource-infoblox.conf index b8a283d85a..00d45957a4 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-infoblox.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-infoblox.conf @@ -71,7 +71,8 @@ block parser app-netsource-infoblox() { application app-netsource-infoblox[sc4s-network-source] { filter { ( - "${.netsource.sc4s_vendor_product}" eq "infoblox" + ( "${.netsource.sc4s_vendor}" eq "infoblox" and "${.netsource.sc4s_product}" eq "nios") + or "${.netsource.sc4s_vendor_product}" eq "infoblox" or "${SOURCE}" eq "s_INFOBLOX" ) and not message('CEF:0', type('string') flags(prefix)) diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-proofpoint_pps.conf b/package/etc/conf.d/conflib/netsource/app-netsource-proofpoint_pps.conf index 06c616640d..a991c95fd7 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-proofpoint_pps.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-proofpoint_pps.conf @@ -38,7 +38,8 @@ block parser app-netsource-proofpoint_pps() { application app-netsource-proofpoint_pps[sc4s-network-source] { filter { ( - "${.netsource.sc4s_vendor_product}" eq "proofpoint_pps" + ( "${.netsource.sc4s_vendor}" eq "proofpoint" and "${.netsource.sc4s_product}" eq "pps") + or "${.netsource.sc4s_vendor_product}" eq "proofpoint_pps" or "${.netsource.sc4s_vendor_product}" eq "proofpoint_pps_filter" or "${.netsource.sc4s_vendor_product}" eq "proofpoint_pps_sendmail" ) diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-ubiquiti_unifi.conf b/package/etc/conf.d/conflib/netsource/app-netsource-ubiquiti_unifi.conf index ead4c9d0a5..bbd61be759 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-ubiquiti_unifi.conf +++ b/package/etc/conf.d/conflib/netsource/app-netsource-ubiquiti_unifi.conf @@ -44,6 +44,7 @@ block parser app-netsource-ubiquiti_unifi() { application app-netsource-ubiquiti_unifi[sc4s-network-source] { filter { "${.netsource.sc4s_vendor_product}" eq "ubiquiti_unifi_fw" + or ( "${.netsource.sc4s_vendor}" eq "ubiquiti" and "${.netsource.sc4s_product}" eq "unifi") or ( "${SOURCE}" eq "s_UBIQUITI_UNIFI" and "${fields.sc4s_vendor_product}" eq "" diff --git a/package/etc/conf.d/conflib/netsource/app-netsource-dell_switch_n.conf b/package/etc/conf.d/conflib/syslog/app-netsource-dell_switch_n.conf similarity index 94% rename from package/etc/conf.d/conflib/netsource/app-netsource-dell_switch_n.conf rename to package/etc/conf.d/conflib/syslog/app-netsource-dell_switch_n.conf index e1007bb5ce..5be6c12057 100644 --- a/package/etc/conf.d/conflib/netsource/app-netsource-dell_switch_n.conf +++ b/package/etc/conf.d/conflib/syslog/app-netsource-dell_switch_n.conf @@ -31,7 +31,7 @@ block parser app-netsource-dell_switch_n() { }; }; -application app-netsource-dell_switch_n[sc4s-network-source] { +application app-netsource-dell_switch_n[sc4s-syslog] { parser { app-netsource-dell_switch_n(); }; }; diff --git a/package/etc/conf.d/conflib/_splunk/splunk_context.conf b/package/etc/conf.d/enrich/splunk_context.conf similarity index 100% rename from package/etc/conf.d/conflib/_splunk/splunk_context.conf rename to package/etc/conf.d/enrich/splunk_context.conf diff --git a/package/etc/conf.d/enrich/vendor_product_by_source.conf b/package/etc/conf.d/enrich/vendor_product_by_source.conf new file mode 100644 index 0000000000..5f5ad4096a --- /dev/null +++ b/package/etc/conf.d/enrich/vendor_product_by_source.conf @@ -0,0 +1,27 @@ +parser vendor_product_by_source { + channel { + if { + parser { + app-parser(topic(sc4s-vps)); + }; + } elif { + parser { + add-contextual-data( + selector(filters("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.conf")), + database("`syslog-ng-sysconfdir`/conf.d/local/context/vendor_product_by_source.csv") + ignore-case(yes) + prefix(".netsource.") + ); + }; + if { + filter { + "${.netsource.sc4s_vendor_product}" eq "" + }; + rewrite{ + set('${.netsource.sc4s_vendor}_${.netsource.sc4s_product}' value('.netsource.sc4s_vendor_product}')); + }; + }; + }; + }; +}; + diff --git a/package/etc/context_templates/vendor_product_by_source.conf.example b/package/etc/context_templates/vendor_product_by_source.conf.example index 745aa02720..dd1d191f7f 100644 --- a/package/etc/context_templates/vendor_product_by_source.conf.example +++ b/package/etc/context_templates/vendor_product_by_source.conf.example @@ -1,99 +1,4 @@ -filter f_test_test { - host("testvp-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_aruba_clearpass { - host("aruba-cp-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; - -filter f_brocade_syslog { - host("test_brocade-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; - -filter f_citrix_netscaler { - host("test_ctitrixns-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_dell_rsa_secureid { - host("test_rsasecureid*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_juniper_netscreen { - host("jnpns-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_cisco_meraki { - host("testcm-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_cisco_wsa{ - host("cisco-wsa-*" type(glob)) -}; -filter f_cisco_wsa_w3crecommended{ - host("cisco-wsaw3c-*" type(glob)) -}; -filter f_cisco_wsa11_7{ - host("cisco-wsa11-7" type(string) flags(prefix)) - or host("cisco_wsa11_7" type(string) flags(prefix)) -}; -filter f_f5_bigip { - host("test-f5-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_infoblox { - host("infoblox-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_pfsense { - host("pfsense-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_proofpoint_pps { - host("pps-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_schneider_apc { - host("test_apc-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_ubiquiti_unifi_fw { - host("usg-*" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; filter f_tzfixny { host("tzfny-*" type(glob)) #or netmask(xxx.xxx.xxx.xxx/xx) }; -filter f_cisco_esa { - host("cisco_esa" type(glob)) - #or netmask(xxx.xxx.xxx.xxx/xx) -}; -filter f_spectracom_ntp { - #Source uses invalid format can not use host must use IP or port - netmask(169.254.100.1/24) -}; -filter f_sophos_webappliance { - host("test-sophos-webapp-" type(string) flags(prefix)) -}; - -filter f_vmware_esx { - host("testvmwe-" type(string) flags(prefix)) -}; - -filter f_dell_cmc { - host("test-dell-cmc-" type(string) flags(prefix)) -}; - -filter f_ibm_datapower { - host("test-ibmdp-" type(string) flags(prefix)) -}; - -filter f_mikrotik_routeros { - host("test-mrtros-" type(string) flags(prefix)) -}; - -filter f_symantec_dlp { - host("test-dlp-" type(string) flags(prefix)) -}; diff --git a/package/etc/context_templates/vendor_product_by_source.csv.example b/package/etc/context_templates/vendor_product_by_source.csv.example index 2a28427e0b..8fb17468ad 100644 --- a/package/etc/context_templates/vendor_product_by_source.csv.example +++ b/package/etc/context_templates/vendor_product_by_source.csv.example @@ -1,24 +1 @@ -f_aruba_clearpass,sc4s_vendor_product,"aruba_clearpass" -f_test_test,sc4s_vendor_product,"test_test" -f_brocade_syslog,sc4s_vendor_product,"brocade_syslog" -f_cisco_meraki,sc4s_vendor_product,"cisco_meraki" -f_cisco_wsa,sc4s_vendor_product,"cisco_wsa" -f_cisco_wsa11_7,sc4s_vendor_product,"cisco_wsa11_7" -f_cisco_wsa_w3crecommended,sc4s_vendor_product,"cisco_wsa_recommended" -f_dell_rsa_secureid,sc4s_vendor_product,"dell_rsa_secureid" -f_f5_bigip,sc4s_vendor_product,"f5_bigip" -f_infoblox,sc4s_vendor_product,"infoblox" -f_pfsense,sc4s_vendor_product,"pfsense" -f_proofpoint_pps,sc4s_vendor_product,"proofpoint_pps" -f_schneider_apc,sc4s_vendor_product,"schneider_apc" -f_spectracom_ntp,sc4s_vendor_productm",spectracom_ntp" -f_ubiquiti_unifi_fw,sc4s_vendor_product,"ubiquiti_unifi_fw" f_tzfixny,sc4s_time_zone,"America/New_York" -f_cisco_esa,sc4s_vendor_product,"cisco_esa" -f_sophos_webappliance,sc4s_vendor_product,"sophos_webappliance" -f_vmware_esx,sc4s_vendor,"vmware" -f_vmware_esx,sc4s_product,"vcenter" -f_dell_cmc,sc4s_vendor_product,"dell_poweredge_cmc" -f_ibm_datapower,sc4s_vendor_product,"ibm_datapower" -f_mikrotik_routeros,sc4s_vendor_product,"mikrotik_routeros" -f_symantec_dlp,sc4s_vendor_product,"symantec_dlp" \ No newline at end of file diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 0b443cde25..d84b2c72d8 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -28,12 +28,15 @@ global_options(); @include "conf.d/templates/*.conf" @include "conf.d/conflib/*.conf" @include "conf.d/conflib/*/*.conf" +@include "conf.d/local/config/app_parsers/*.conf" +@include "conf.d/local/config/app_parsers/*/*.conf" @include "conf.d/filters/*/*.conf" @include "conf.d/local/config/filters/*.conf" -@include "conf.d/local/config/app_parsers/*.conf" -@include "conf.d/local/config/app_parsers/*/*.conf" +@include "conf.d/enrich/*.conf" +@include "conf.d/enrich/*/*.conf" + @include "conf.d/sources/*.conf" @include "conf.d/sources/*/*.conf" diff --git a/package/etc/test_parsers/app-vps-test-aruba_clearpass.conf b/package/etc/test_parsers/app-vps-test-aruba_clearpass.conf new file mode 100644 index 0000000000..2bd79db8b3 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-aruba_clearpass.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-brocade_syslog() { + channel { + rewrite { + r_set_splunk_vps( + vendor('brocade') + product('syslog') + ); + }; + }; +}; +application app-vps-test-brocade_syslog[sc4s-vps] { + filter { + host("test_brocade-*" type(glob)) + }; + parser { app-vps-test-brocade_syslog(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-brocade_syslog.conf b/package/etc/test_parsers/app-vps-test-brocade_syslog.conf new file mode 100644 index 0000000000..29687b2989 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-brocade_syslog.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-aruba_clearpass() { + channel { + rewrite { + r_set_splunk_vps( + vendor('aruba') + product('clearpass') + ); + }; + }; +}; +application app-vps-test-aruba_clearpass[sc4s-vps] { + filter { + host("aruba-cp-*" type(glob)) + }; + parser { app-vps-test-aruba_clearpass(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-cisco_esa.conf b/package/etc/test_parsers/app-vps-test-cisco_esa.conf new file mode 100644 index 0000000000..77d1ead8ba --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-cisco_esa.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-cisco_esa() { + channel { + rewrite { + r_set_splunk_vps( + vendor('cisco') + product('esa') + ); + }; + }; +}; +application app-vps-test-cisco_esa[sc4s-vps] { + filter { + "${HOST}" eq "cisco_esa" + }; + parser { app-vps-test-cisco_esa(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-cisco_meraki.conf b/package/etc/test_parsers/app-vps-test-cisco_meraki.conf new file mode 100644 index 0000000000..b68dd7205a --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-cisco_meraki.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-cisco_meraki() { + channel { + rewrite { + r_set_splunk_vps( + vendor('cisco') + product('meraki') + ); + }; + }; +}; +application app-vps-test-cisco_meraki[sc4s-vps] { + filter { + host("^testcm-") + }; + parser { app-vps-test-cisco_meraki(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-cisco_wsa.conf b/package/etc/test_parsers/app-vps-test-cisco_wsa.conf new file mode 100644 index 0000000000..0821796565 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-cisco_wsa.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-cisco_wsa() { + channel { + rewrite { + r_set_splunk_vps( + vendor('cisco') + product('wsa') + ); + }; + }; +}; +application app-vps-test-cisco_wsa[sc4s-vps] { + filter { + host('^cisco-wsa-') + }; + parser { app-vps-test-cisco_wsa(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-cisco_wsa11_7.conf b/package/etc/test_parsers/app-vps-test-cisco_wsa11_7.conf new file mode 100644 index 0000000000..186d6831ee --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-cisco_wsa11_7.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-cisco_wsa11-7() { + channel { + rewrite { + r_set_splunk_vps( + vendor('cisco') + product('wsa11-7') + ); + }; + }; +}; +application app-vps-test-cisco_wsa11-7[sc4s-vps] { + filter { + host('^cisco-wsa11-7-') + }; + parser { app-vps-test-cisco_wsa11-7(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-cisco_wsa_recommended.conf b/package/etc/test_parsers/app-vps-test-cisco_wsa_recommended.conf new file mode 100644 index 0000000000..76d8dfb9a9 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-cisco_wsa_recommended.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-cisco_wsa_recommended() { + channel { + rewrite { + r_set_splunk_vps( + vendor('cisco') + product('wsa_recommended') + ); + }; + }; +}; +application app-vps-test-cisco_wsa_recommended[sc4s-vps] { + filter { + host('^cisco-wsaw3c-') + }; + parser { app-vps-test-cisco_wsa_recommended(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-dell_cmc.conf b/package/etc/test_parsers/app-vps-test-dell_cmc.conf new file mode 100644 index 0000000000..aa2b38ad85 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-dell_cmc.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-dell_cmc() { + channel { + rewrite { + r_set_splunk_vps( + vendor('dell') + product('poweredge_cmc') + ); + }; + }; +}; +application app-vps-test-dell_cmc[sc4s-vps] { + filter { + host("test-dell-cmc-" type(string) flags(prefix)) + }; + parser { app-vps-test-dell_cmc(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-dell_rsa_secureid.conf b/package/etc/test_parsers/app-vps-test-dell_rsa_secureid.conf new file mode 100644 index 0000000000..ccaf57df97 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-dell_rsa_secureid.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-dell_rsa_secureid() { + channel { + rewrite { + r_set_splunk_vps( + vendor('dell') + product('rsa_secureid') + ); + }; + }; +}; +application app-vps-test-dell_rsa_secureid[sc4s-vps] { + filter { + host("test_rsasecureid*" type(glob)) + }; + parser { app-vps-test-dell_rsa_secureid(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-f5_bigip.conf b/package/etc/test_parsers/app-vps-test-f5_bigip.conf new file mode 100644 index 0000000000..3ceb0ca5ad --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-f5_bigip.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-f5_bigip() { + channel { + rewrite { + r_set_splunk_vps( + vendor('f5') + product('bigip') + ); + }; + }; +}; +application app-vps-test-f5_bigip[sc4s-vps] { + filter { + "${HOST}" eq "f5_bigip" + }; + parser { app-vps-test-f5_bigip(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-ibm_datapower.conf b/package/etc/test_parsers/app-vps-test-ibm_datapower.conf new file mode 100644 index 0000000000..a8e5c0fc8e --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-ibm_datapower.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-ibm_datapower() { + channel { + rewrite { + r_set_splunk_vps( + vendor('ibm') + product('datapower') + ); + }; + }; +}; +application app-vps-test-ibm_datapower[sc4s-vps] { + filter { + host("^test-ibmdp-") + }; + parser { app-vps-test-ibm_datapower(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-infoblox_nios.conf b/package/etc/test_parsers/app-vps-test-infoblox_nios.conf new file mode 100644 index 0000000000..ad940e4571 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-infoblox_nios.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-infoblox_nios() { + channel { + rewrite { + r_set_splunk_vps( + vendor('infoblox') + product('nios') + ); + }; + }; +}; +application app-vps-test-infoblox_nios[sc4s-vps] { + filter { + host("infoblox-*" type(glob)) + }; + parser { app-vps-test-infoblox_nios(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-mikrotik_routeros.conf b/package/etc/test_parsers/app-vps-test-mikrotik_routeros.conf new file mode 100644 index 0000000000..a0c2eca199 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-mikrotik_routeros.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-mikrotik_routeros() { + channel { + rewrite { + r_set_splunk_vps( + vendor('mikrotik') + product('routeros') + ); + }; + }; +}; +application app-vps-test-mikrotik_routeros[sc4s-vps] { + filter { + host("test-mrtros-" type(string) flags(prefix)) + }; + parser { app-vps-test-mikrotik_routeros(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-pfsense_firewall.conf b/package/etc/test_parsers/app-vps-test-pfsense_firewall.conf new file mode 100644 index 0000000000..3d6d5d5594 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-pfsense_firewall.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-pfsense_firewall() { + channel { + rewrite { + r_set_splunk_vps( + vendor('pfsense') + product('firewall') + ); + }; + }; +}; +application app-vps-test-pfsense_firewall[sc4s-vps] { + filter { + "${HOST}" eq "pfsense_firewall" + }; + parser { app-vps-test-pfsense_firewall(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-proofpoint_pps.conf b/package/etc/test_parsers/app-vps-test-proofpoint_pps.conf new file mode 100644 index 0000000000..90a8e23c8f --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-proofpoint_pps.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-proofpoint_pps() { + channel { + rewrite { + r_set_splunk_vps( + vendor('proofpoint') + product('pps') + ); + }; + }; +}; +application app-vps-test-proofpoint_pps[sc4s-vps] { + filter { + host("pps-*" type(glob)) + }; + parser { app-vps-test-proofpoint_pps(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-schneider_apc.conf b/package/etc/test_parsers/app-vps-test-schneider_apc.conf new file mode 100644 index 0000000000..42aee51d07 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-schneider_apc.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-schneider_apc() { + channel { + rewrite { + r_set_splunk_vps( + vendor('schneider') + product('apc') + ); + }; + }; +}; +application app-vps-test-schneider_apc[sc4s-vps] { + filter { + host("test_apc-*" type(glob)) + }; + parser { app-vps-test-schneider_apc(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-sophos_webappliance.conf b/package/etc/test_parsers/app-vps-test-sophos_webappliance.conf new file mode 100644 index 0000000000..b00dc99ed5 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-sophos_webappliance.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-sophos_webappliance() { + channel { + rewrite { + r_set_splunk_vps( + vendor('sophos') + product('webappliance') + ); + }; + }; +}; +application app-vps-test-sophos_webappliance[sc4s-vps] { + filter { + host("test-sophos-webapp-" type(string) flags(prefix)) + }; + parser { app-vps-test-sophos_webappliance(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-spectracom_ntp.conf b/package/etc/test_parsers/app-vps-test-spectracom_ntp.conf new file mode 100644 index 0000000000..9a49ac512f --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-spectracom_ntp.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-spectracom_ntp() { + channel { + rewrite { + r_set_splunk_vps( + vendor('spectracom') + product('ntp') + ); + }; + }; +}; +application app-vps-test-spectracom_ntp[sc4s-vps] { + filter { + netmask(169.254.100.1/24) + }; + parser { app-vps-test-spectracom_ntp(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-symantec_dlp.conf b/package/etc/test_parsers/app-vps-test-symantec_dlp.conf new file mode 100644 index 0000000000..6092b73349 --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-symantec_dlp.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-symantec_dlp() { + channel { + rewrite { + r_set_splunk_vps( + vendor('symantec') + product('dlp') + ); + }; + }; +}; +application app-vps-test-symantec_dlp[sc4s-vps] { + filter { + host("test-dlp-" type(string) flags(prefix)) + }; + parser { app-vps-test-symantec_dlp(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-ubiquiti_unifi_fw.conf b/package/etc/test_parsers/app-vps-test-ubiquiti_unifi_fw.conf new file mode 100644 index 0000000000..389e4f803c --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-ubiquiti_unifi_fw.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-ubiquiti_unifi_fw() { + channel { + rewrite { + r_set_splunk_vps( + vendor('ubiquiti') + product('unifi') + ); + }; + }; +}; +application app-vps-test-ubiquiti_unifi_fw[sc4s-vps] { + filter { + host("usg-*" type(glob)) + }; + parser { app-vps-test-ubiquiti_unifi_fw(); }; +}; diff --git a/package/etc/test_parsers/app-vps-test-vmware_vcenter.conf b/package/etc/test_parsers/app-vps-test-vmware_vcenter.conf new file mode 100644 index 0000000000..4ddd8ad92e --- /dev/null +++ b/package/etc/test_parsers/app-vps-test-vmware_vcenter.conf @@ -0,0 +1,16 @@ +block parser app-vps-test-vmware_vcenter() { + channel { + rewrite { + r_set_splunk_vps( + vendor('vmware') + product('vcenter') + ); + }; + }; +}; +application app-vps-test-vmware_vcenter[sc4s-vps] { + filter { + host("testvmwe-" type(string) flags(prefix)) + }; + parser { app-vps-test-vmware_vcenter(); }; +}; diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index c8ea805036..a7d3afcb52 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -66,6 +66,7 @@ mkdir -p $SC4S_VAR/log/ mkdir -p $SC4S_ETC/conf.d/local/context/ mkdir -p $SC4S_ETC/conf.d/merged/context/ mkdir -p $SC4S_ETC/conf.d/local/config/ +mkdir -p $SC4S_ETC/conf.d/local/config/app_parsers/ mkdir -p $SC4S_ETC/local_config/ cp -f $SC4S_ETC/context_templates/* $SC4S_ETC/conf.d/local/context @@ -81,6 +82,7 @@ fi if [ "$TEST_SC4S_ACTIVATE_EXAMPLES" == "yes" ] then for file in $SC4S_ETC/conf.d/local/context/*.example ; do cp --verbose -n $file ${file%.example}; done + cp -f $SC4S_ETC/test_parsers/* $SC4S_ETC/conf.d/local/config/app_parsers/ fi for file in $SC4S_ETC/conf.d/local/context/*.example ; do touch ${file%.example}; done touch $SC4S_ETC/conf.d/local/context/splunk_metadata.csv