diff --git a/docs/configuration.md b/docs/configuration.md index d16b870d60..e3dcdf0f61 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -7,8 +7,6 @@ and variables needed to properly configure SC4S for your environment. | Variable | Values | Description | |----------|---------------|-------------| -| SC4S_DEST_SPLUNK_HEC_DEFAULT_URL | url | URL(s) of the Splunk endpoint, can be a single URL space separated list | -| SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN | string | Splunk HTTP Event Collector Token | | SC4S_USE_REVERSE_DNS | yes or no(default) | use reverse DNS to identify hosts when HOST is not valid in the syslog header | | SC4S_CONTAINER_HOST | string | variable passed to the container to identify the actual log host for container implementations | @@ -35,10 +33,8 @@ loss and or proxy outages. Note: the follow variables are lower case | Variable | Values | Description | |----------|---------------|-------------| -| SC4S_DEST_SPLUNK_HEC_GLOBAL | yes | Send events to Splunk using HEC. This applies _only_ to the primary HEC destination. | | SC4S_DEST_SPLUNK_HEC_CIPHER_SUITE | comma separated list | Open SSL cipher suite list | | SC4S_DEST_SPLUNK_HEC_SSL_VERSION | comma separated list | Open SSL version list | -| SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | | SC4S_DEST_SPLUNK_HEC_WORKERS | numeric | Number of destination workers (default: 10 threads). This should rarely need to be changed; consult sc4s community for advice on appropriate setting in extreme high- or low-volume environments. | | SC4S_DEST_SPLUNK_INDEXED_FIELDS | facility,
severity,
container,
loghost,
destport,
fromhostip,
proto

none | List of sc4s indexed fields that will be included with each event in Splunk (default is the entire list except "none"). Two other indexed fields, `sc4s_vendor_product` and `sc4s_syslog_format`, will also appear along with the fields selected via the list and cannot be turned on or off individually. If no indexed fields are desired (including the two internal ones), set the value to the single value of "none". When setting this variable, separate multiple entries with commas and do not include extra spaces.

This list maps to the following indexed fields that will appear in all Splunk events:
facility: sc4s_syslog_facility
severity: sc4s_syslog_severity
container: sc4s_container
loghost: sc4s_loghost
dport: sc4s_destport
fromhostip: sc4s_fromhostip
proto: sc4s_proto @@ -52,77 +48,7 @@ for the alternate HEC destination `d_hec_FOO` to 24, set `SC4S_DEST_SPLUNK_HEC_F Additional trusted (private) Certificate authorities may be trusted by appending each PEM formatted certificate to `/opt/sc4s/tls/trusted.pem` -## Configuration of Alternate Destinations -In addition to the standard HEC destination that is used to send events to Splunk, alternate destinations can be created and configured -in SC4S. All alternate destinations (including alternate HEC destinations discussed below) are configured using the environment -variables below. Global and/or source-specific forms of the variables below can be used to send data to additional and/or alternate -destinations. - -* NOTE: The administrator is responsible for ensuring that any non-HEC alternate destinations are configured in the -local mount tree, and that the underlying syslog-ng process in SC4S properly parses them. - -* NOTE: Do not include the primary HEC destination (`d_hec`) in any list of alternate destinations. The configuration of the primary HEC -destination is configured separately from that of the alternates below. However, _alternate_ HEC destinations (e.g. `d_hec_FOO`) should be -configured below, just like any other user-supplied destination. - -| Variable | Values | Description | -|----------|---------------|-------------| -| SC4S_DEST_GLOBAL_ALTERNATES | Comma or space-separated list of destinations | Send all sources to alternate destinations | -| SC4S_DEST_<VENDOR_PRODUCT>_ALTERNATES | Comma or space-separated list of syslog-ng destinations | Send specific sources to alternate syslog-ng destinations using the VENDOR_PRODUCT syntax, e.g. `SC4S_DEST_CISCO_ASA_ALTERNATES` | - -## Configuration of Filtered Alternate Destinations (Advanced) - -Though source-specific forms of the variables configured above will limit configured alternate destinations to a specific data source, there -are cases where even more granularity is desired within a specific data source (e.g. to send all Cisco ASA "debug" traffic to Cisco Prime for -analysis). This extra traffic may or may not be needed in Splunk. To accommodate this use case, Filtered Alternate Destinations allow a -filter to be supplied to redirect a _portion_ of a given source's traffic to a list of alternate destinations (and, optionally, to prevent -matching events from being sent to Splunk). Again, these are configured through environment variables similar -to the ones above: - -| Variable | Values | Description | -|----------|---------------|-------------| -| SC4S_DEST_<VENDOR_PRODUCT>_ALT_FILTER | syslog-ng filter | Filter to determine which events are sent to alternate destination(s) | -| SC4S_DEST_<VENDOR_PRODUCT>_FILTERED_ALTERNATES | Comma or space-separated list of syslog-ng destinations | Send filtered events to alternate syslog-ng destinations using the VENDOR_PRODUCT syntax, e.g. `SC4S_DEST_CISCO_ASA_FILTERED_ALTERNATES` | - - -* NOTE: This is an advanced capability, and filters and destinations using proper syslog-ng syntax must be constructed prior to utilizing -this feature. - -* NOTE: Unlike the standard alternate destinations configured above, the regular "mainline" destinations (including the primary HEC -destination or configured archive destination (`d_hec` or `d_archive`)) are _not_ included for events matching the configured alternate -destination filter. If an event matches the filter, the list of filtered alternate destinations completely replaces any mainline destinations -including defaults and global or source-based standard alternate destinations. Be sure to include them in the filtered destination list if -desired. - -* HINT: Since the filtered alternate destinations completely replace the mainline destinations (including HEC to Splunk), a filter that -matches all traffic can be used with a destination list that does _not_ include the standard HEC destination to effectively turn off HEC -for a given data source. - -## Creation of Additional Splunk HEC Destinations - -Additional Splunk HEC destinations can be dynamically created through environment variables. When set, the destinations will be -created with the `DESTID` appended, for example: `d_hec_fmt_FOO`. These destinations can then be specified for use (along with any other -destinations created locally) either globally or per source. See the "Alternate Destination Use" in the next section for details. - -| Variable | Values | Description | -|----------|---------------|-------------| -| SPLUNK_HEC_ALT_DESTS | Comma or space-separated UPPER case list of destination IDs | Destination IDs are UPPER case, single-word friendly strings used to identify the new destinations which will be named with the `DESTID` appended, for example `d_hec_FOO` | -| SC4S_DEST_SPLUNK_HEC_<DESTID>_URL | url | Example: `SC4S_DEST_SPLUNK_HEC_FOO_URL=https://splunk:8088` `DESTID` must be a member of the list specified in `SPLUNK_HEC_ALT_DESTS` configured above | -| SC4S_DEST_SPLUNK_HEC_<DESTID>_TOKEN | string | Example: `SC4S_DEST_SPLUNK_HEC_FOO_TOKEN=` `DESTID` must be a member of the list specified in `SPLUNK_HEC_ALT_DESTS` configured above | - -* NOTE: The `DESTID` specified in the `URL` and `TOKEN` variables above _must_ match the `DESTID` entries enumerated in the -`SPLUNK_HEC_ALT_DESTS` list. For each `DESTID` value specified in `SPLUNK_HEC_ALT_DESTS` there must be a corresponding `URL` and `TOKEN` -variable set as well. Failure to do so will cause destinations to be created without proper HEC parameters which will result in connection -failure. - -* NOTE: Alternate HEC destinations, such as `d_hec_FOO` from this example, must be included in `SC4S_DEST_GLOBAL_ALTERNATES`. - -* NOTE: Additional Splunk HEC destinations will _not_ be tested at startup. It is the responsibility of the admin to ensure that additional destinations -are provisioned with the correct URL(s) and tokens to ensure proper connectivity. - -* NOTE: The disk and CPU requirements will increase proportionally depending on the number of additional HEC destinations in use (e.g. each HEC -destination will have its own disk buffer by default). ## Configuration of timezone for legacy sources diff --git a/docs/destinations.md b/docs/destinations.md new file mode 100644 index 0000000000..d0439e71da --- /dev/null +++ b/docs/destinations.md @@ -0,0 +1,137 @@ + +# SC4S Destination Configuration + +Splunk Connect for Syslog can be configured to utilize any destination available in +syslog-ng OSE. The configuration system provides ease of use helpers to manage configuration +for the three most common destination needs, Splunk HEC, RFC5424 Syslog, and Legacy BSD Syslog. + +In the getting started guide you configured the Splunk HEC "DEFAULT" destination to receive all traffic by default. The "DEFAULT" destination should be configured to accept all events to ensure that at least one +destination has the event to avoid data loss due to misconfiguration. The following example demonstrates configuration of a second HEC destination where only "selected" data will be sent. + +## Example 1 Send all events +```bash +#Note "OTHER" should be a meaningful name +SC4S_DEST_SPLUNK_HEC_OTHER_URL=https://splunk:8088 +SC4S_DEST_SPLUNK_HEC_OTHER_TOKEN=${SPLUNK_HEC_TOKEN} +SC4S_DEST_SPLUNK_HEC_OTHER_TLS_VERIFY=no +SC4S_DEST_SPLUNK_HEC_OTHER_MODE=GLOBAL +``` + +## Example 2 Send only cisco IOS Events +```bash +#Note "OTHER" should be a meaningful name +SC4S_DEST_SPLUNK_HEC_OTHER_URL=https://splunk:8088 +SC4S_DEST_SPLUNK_HEC_OTHER_TOKEN=${SPLUNK_HEC_TOKEN} +SC4S_DEST_SPLUNK_HEC_OTHER_TLS_VERIFY=no +SC4S_DEST_SPLUNK_HEC_OTHER_MODE=SELECT +SC4S_DEST_CISCO_IOS_ALTERNATES=d_fmt_hec_OTHER +``` + +## Example 3 Send only cisco IOS events that are not debug +```bash +#Note "OTHER" should be a meaningful name +SC4S_DEST_SPLUNK_HEC_OTHER_URL=https://splunk:8088 +SC4S_DEST_SPLUNK_HEC_OTHER_TOKEN=${SPLUNK_HEC_TOKEN} +SC4S_DEST_SPLUNK_HEC_OTHER_TLS_VERIFY=no +SC4S_DEST_SPLUNK_HEC_OTHER_MODE=SELECT +``` + +```c +#filename: +application sc4s-lp-cisco_ios_dest_fmt_other{{ source }}[sc4s-lp-dest-select-d_fmt_hec_OTHER] { + filter { + match('CISCO_IOS' value('.dest_key')) + #Match any cisco event that is not like "%ACL-7-1234" + and not message('^%[^\-]+-7-'); + }; +}; + +``` + +# Supported Simple Destination configurations + +SC4S Supports the following destination configurations via configuration. Any custom destination +can be supported (defined by syslog-ng OSE) + +* Splunk HTTP Event Collector (HEC) +* RFC5424 format without frames i.e. ```<166>1 2022-02-02T14:59:55.000+00:00 kinetic-charlie - - - - %FTD-6-430003: DeviceUUID: ``` +* RFC5424 format with frames also known as RFC6587 ```123 <166>1 2022-02-02T14:59:55.000+00:00 kinetic-charlie - - - - %FTD-6-430003: DeviceUUID: ``` +* RFC3194 (BSD format) ```<134>Feb 2 13:43:05.000 horse-ammonia CheckPoint[26203]:``` + +## HEC Destination Configuration + +| Variable | Values | Description | +|----------|---------------|-------------| +| SC4S_DEST_SPLUNK_HEC_<ID>_URL | url | URL(s) of the Splunk endpoint, can be a single URL space separated list | +| SC4S_DEST_SPLUNK_HEC_<ID>_TOKEN | string | Splunk HTTP Event Collector Token | +| SC4S_DEST_SPLUNK_HEC_<ID>_MODE | string | "GLOBAL" or "SELECT" | +| SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY | yes(default) or no | verify HTTP(s) certificate | + +## Syslog Standard destination. + +Note: in many cases destinations incorrectly assert "syslog" support. IETF standards RFC5424, RFC5425, RFC6587 define the use of "syslog" as a network protocol. Often the actual configuration required is Legacy BSD syslog which is NOT a standard and was documented "historically" in RFC3194 see BSD Destination section. + +| Variable | Values | Description | +|----------|---------------|-------------| +| SC4S_DEST_SYSLOG_<ID>_HOST | fqdn or ip | the FQDN or IP of the target | +| SC4S_DEST_SYSLOG_<ID>_PORT | number | 601 (default when framed) 514 (default when not framed) | +| SC4S_DEST_SYSLOG_<ID>_IETF | yes,no | default "yes" use IETF Standard frames | +| SC4S_DEST_SYSLOG_<ID>_MODE | string | "GLOBAL" or "SELECT" | + +## BSD legacy destination (Non standard) + +Note: in many cases destinations incorrectly assert "syslog" support. IETF standards RFC5424, RFC5425, RFC6587 define the use of "syslog" as a network protocol. Often the actual configuration required is Legacy BSD syslog which is NOT a standard and was documented "historically" in RFC3194 see BSD Destination section. + +| Variable | Values | Description | +|----------|---------------|-------------| +| SC4S_DEST_BSD_<ID>_HOST | fqdn or ip | the FQDN or IP of the target | +| SC4S_DEST_BSD_<ID>_PORT | number | default 514 | +| SC4S_DEST_BSD_<ID>_TRANSPORT | tcp,udp,tls | default tcp | +| SC4S_DEST_BSD_<ID>_MODE | string | "GLOBAL" or "SELECT" | + + +## Configuration of Alternate Destinations + +In addition to the standard HEC destination that is used to send events to Splunk, alternate destinations can be created and configured +in SC4S. All alternate destinations (including alternate HEC destinations discussed below) are configured using the environment +variables below. Global and/or source-specific forms of the variables below can be used to send data to additional and/or alternate +destinations. + +* NOTE: The administrator is responsible for ensuring that any non-HEC alternate destinations are configured in the +local mount tree, and that the underlying syslog-ng process in SC4S properly parses them. + +* NOTE: Do not include the primary HEC destination (`d_fmt_hec`) in any list of alternate destinations. The configuration of the primary HEC +destination is configured separately from that of the alternates below. However, _alternate_ HEC destinations (e.g. `d_fmt_hec_FOO`) should be +configured below, just like any other user-supplied destination. + +| Variable | Values | Description | +|----------|---------------|-------------| +| SC4S_DEST_<VENDOR_PRODUCT>_ALTERNATES | Comma or space-separated list of syslog-ng destinations | Send specific sources to alternate syslog-ng destinations using the VENDOR_PRODUCT syntax, e.g. `SC4S_DEST_CISCO_ASA_ALTERNATES=d_syslog_foo` | + +## Configuration of Filtered Alternate Destinations (Advanced) + +Though source-specific forms of the variables configured above will limit configured alternate destinations to a specific data source, there +are cases where even more granularity is desired within a specific data source (e.g. to send all Cisco ASA "debug" traffic to Cisco Prime for +analysis). This extra traffic may or may not be needed in Splunk. To accommodate this use case, Filtered Alternate Destinations allow a +filter to be supplied to redirect a _portion_ of a given source's traffic to a list of alternate destinations (and, optionally, to prevent +matching events from being sent to Splunk). Again, these are configured through environment variables similar +to the ones above: + +| Variable | Values | Description | +|----------|---------------|-------------| +| SC4S_DEST_<VENDOR_PRODUCT>_ALT_FILTER | syslog-ng filter | Filter to determine which events are sent to alternate destination(s) | +| SC4S_DEST_<VENDOR_PRODUCT>_FILTERED_ALTERNATES | Comma or space-separated list of syslog-ng destinations | Send filtered events to alternate syslog-ng destinations using the VENDOR_PRODUCT syntax, e.g. `SC4S_DEST_CISCO_ASA_FILTERED_ALTERNATES` | + + +* NOTE: This is an advanced capability, and filters and destinations using proper syslog-ng syntax must be constructed prior to utilizing +this feature. + +* NOTE: Unlike the standard alternate destinations configured above, the regular "mainline" destinations (including the primary HEC +destination or configured archive destination (`d_hec` or `d_archive`)) are _not_ included for events matching the configured alternate +destination filter. If an event matches the filter, the list of filtered alternate destinations completely replaces any mainline destinations +including defaults and global or source-based standard alternate destinations. Be sure to include them in the filtered destination list if +desired. + +* HINT: Since the filtered alternate destinations completely replace the mainline destinations (including HEC to Splunk), a filter that +matches all traffic can be used with a destination list that does _not_ include the standard HEC destination to effectively turn off HEC +for a given data source. \ No newline at end of file diff --git a/docs/upgrade.md b/docs/upgrade.md index 07ca35019d..3c84db8037 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -34,4 +34,6 @@ See the [release information](https://github.com/splunk/splunk-connect-for-syslo * New images will no longer be published to docker.io please review curent getting started docs and update the sc4s.service file accordingly * Internal metrics will now use "multi" format by default if using unsupported versions of Splunk <8.1 see configuration doc to revert to "event" or "single" format. * Internal metrics will now use the _metrics index by default update vendor_product key 'sc4s_metrics' to change the index -* Deprecated use of vendor_product_by_source for null queue or dropping events see See [Filtering events from output](https://splunk.github.io/splunk-connect-for-syslog/main/sources/) this use will be removed in v3 \ No newline at end of file +* Deprecated use of vendor_product_by_source for null queue or dropping events see See [Filtering events from output](https://splunk.github.io/splunk-connect-for-syslog/main/sources/) this use will be removed in v3 +* Deprecated use of `SPLUNK_HEC_ALT_DESTS` this variable is no longer used and will be ignored +* Deprecated use of `SC4S_DEST_GLOBAL_ALTERNATES` this variable will be removed in future major versions see Destinations section in configuration \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 6f785d5b50..2ae0d52d73 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,9 +31,10 @@ nav: - "Docker Desktop + Compose (MacOS)": "gettingstarted/docker-compose-MacOS.md" - "Bring your own Envionment": "gettingstarted/byoe-rhel8.md" - "Quickstart Guide": "gettingstarted/quickstart_guide.md" - - Configuration: "configuration.md" - Architectural Considerations: "architecture.md" + - Configuration: "configuration.md" - Development: "developing/index.md" + - Destinations: "destinations.md" - Sources: - About: sources/index.md - Alcatel: sources/Alcatel/index.md diff --git a/package/etc/conf.d/conflib/app-lp-archive/app-lp-global_archive.conf b/package/etc/conf.d/conflib/app-lp-archive/app-lp-global_archive.conf new file mode 100644 index 0000000000..511647d1aa --- /dev/null +++ b/package/etc/conf.d/conflib/app-lp-archive/app-lp-global_archive.conf @@ -0,0 +1,6 @@ +#This "app" parser will always match unless disabled by SC4S_ARCHIVE_GLOBAL=no +application app-lp-global_archive[sc4s-lp-archive] { + filter { + '$(if ("`SC4S_ARCHIVE_GLOBAL`" ne "") "`SC4S_ARCHIVE_GLOBAL`" "no")' eq "yes" + }; +}; \ No newline at end of file diff --git a/package/etc/conf.d/conflib/app-lp_dest_archive/plugin.conf b/package/etc/conf.d/conflib/app-lp_dest_archive/plugin.conf new file mode 100644 index 0000000000..ee21c4753c --- /dev/null +++ b/package/etc/conf.d/conflib/app-lp_dest_archive/plugin.conf @@ -0,0 +1,4 @@ + +@module confgen context(root) name(app-lp_dest_archive) exec("`SC4S_ETC`/conf.d/conflib/app-lp_dest_archive/plugin.py") + +app-lp_dest_archive() \ No newline at end of file diff --git a/package/etc/conf.d/conflib/app-lp_dest_archive/plugin.jinja b/package/etc/conf.d/conflib/app-lp_dest_archive/plugin.jinja new file mode 100644 index 0000000000..048eacae01 --- /dev/null +++ b/package/etc/conf.d/conflib/app-lp_dest_archive/plugin.jinja @@ -0,0 +1,6 @@ + +application sc4s-lp-dest-archive-{{ source }}[sc4s-lp-archive] { + filter { + match('{{ source }}' value('.dest_key')) + }; +}; diff --git a/package/etc/conf.d/conflib/app-lp_dest_archive/plugin.py b/package/etc/conf.d/conflib/app-lp_dest_archive/plugin.py new file mode 100755 index 0000000000..06ff722f08 --- /dev/null +++ b/package/etc/conf.d/conflib/app-lp_dest_archive/plugin.py @@ -0,0 +1,22 @@ +#! /usr/bin/env python3 +import os +import jinja2 +import re + +plugin_path = os.path.dirname(os.path.abspath(__file__)) + +templateLoader = jinja2.FileSystemLoader(searchpath=plugin_path) +templateEnv = jinja2.Environment(loader=templateLoader) +tm = templateEnv.get_template("plugin.jinja") + + +regexfa = r"SC4S_DEST_(.*)(?') }; parser { - app-parser(topic({{ topic }}-almost-syslog)); + app-parser(topic(sc4s-almost-syslog)); }; } elif { filter{ @@ -120,7 +120,7 @@ source s_{{ port_id }} { } elif { #Try raw parsers remind customers these vendors need to improve parser { - app-parser(topic({{ topic }}-raw-syslog)); + app-parser(topic(sc4s-raw-syslog)); }; } else {}; }; @@ -161,7 +161,7 @@ source s_{{ port_id }} { "${fields.sc4s_vendor_product}" eq "" }; parser { - app-parser(topic({{ topic }}-syslog)); + app-parser(topic(sc4s-syslog)); }; }; if { @@ -184,7 +184,7 @@ source s_{{ port_id }} { match("^rfc" value("fields.sc4s_syslog_format")); }; parser { - app-parser(topic({{ topic }}-network-source)); + app-parser(topic(sc4s-network-source)); }; }; @@ -234,7 +234,7 @@ source s_{{ port_id }} { }; if { parser { - app-parser(topic({{ topic }}-postfilter)); + app-parser(topic(sc4s-postfilter)); }; }; @@ -337,7 +337,7 @@ source s_{{ port_id }} { }; if { parser { - app-parser(topic({{ topic }}-syslog)); + app-parser(topic(sc4s-syslog)); }; }; @@ -358,7 +358,7 @@ source s_{{ port_id }} { parser(vendor_product_by_source); if { parser { - app-parser(topic({{ topic }}-network-source)); + app-parser(topic(sc4s-network-source)); }; }; diff --git a/package/etc/conf.d/sc4slib/source_syslog/plugin.py b/package/etc/conf.d/sources/source_syslog/plugin.py similarity index 100% rename from package/etc/conf.d/sc4slib/source_syslog/plugin.py rename to package/etc/conf.d/sources/source_syslog/plugin.py diff --git a/package/etc/syslog-ng.conf b/package/etc/syslog-ng.conf index 79c756ad76..0b443cde25 100644 --- a/package/etc/syslog-ng.conf +++ b/package/etc/syslog-ng.conf @@ -36,9 +36,15 @@ global_options(); @include "conf.d/local/config/app_parsers/*/*.conf" @include "conf.d/sources/*.conf" +@include "conf.d/sources/*/*.conf" @include "conf.d/local/config/sources/*.conf" +@include "conf.d/local/config/sources/*/*.conf" + @include "conf.d/destinations/*.conf" +@include "conf.d/destinations/*/*.conf" + @include "conf.d/local/config/destinations/*.conf" +@include "conf.d/local/config/destinations/*/*.conf" # Group 0 log paths run before all others for high level activity like null queue @include "conf.d/log_paths/0/*.conf" @@ -47,4 +53,5 @@ global_options(); @include "conf.d/local/config/log_paths/*.conf" #Stock log paths +@include "conf.d/log_paths/2/*/*.conf" @include "conf.d/log_paths/2/*.conf" diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh index 57c1819418..c8ea805036 100755 --- a/package/sbin/entrypoint.sh +++ b/package/sbin/entrypoint.sh @@ -25,15 +25,6 @@ export SC4S_VAR=${SC4S_VAR:=/var/lib/syslog-ng} export SC4S_BIN=${SC4S_BIN:=/usr/bin} export SC4S_SBIN=${SC4S_SBIN:=/usr/sbin} -# The unique port environment variables associated with SC4S_LISTEN__6587_PORT will be renamed to -# SC4S_LISTEN__RFC6587_PORT to indicate compliance with the RFC. -# This compatibility block will be removed in version 2.0 -for var in `env | awk -F "=" '{print $1}' | grep "_6587_"`; do - export `echo $var | sed -n -e 's/_6587_PORT/_RFC6587_PORT/p'`=${!var} -done - -export SC4S_DESTS_ALTERNATES=$(env | grep -v FILTERED_ALTERNATES | grep _ALTERNATES= | grep -v SC4S_DEST_GLOBAL_ALTERNATES | cut -d= -f2 | sort | uniq | paste -s -d, -) -[ -z "$SC4S_DESTS_ALTERNATES" ] && unset SC4S_DESTS_ALTERNATES export SC4S_DESTS_FILTERED_ALTERNATES=$(env | grep _FILTERED_ALTERNATES= | grep -v SC4S_DEST_GLOBAL_FILTERED_ALTERNATES | cut -d= -f2 | sort | uniq | paste -s -d, -) [ -z "$SC4S_DESTS_FILTERED_ALTERNATES" ] && unset SC4S_DESTS_FILTERED_ALTERNATES @@ -145,9 +136,6 @@ fi export SOURCE_SIMPLE_SET=$(printenv | grep '^SC4S_LISTEN_SIMPLE_.*_PORT=.' | sed 's/^SC4S_LISTEN_SIMPLE_//;s/_..._PORT\=.*//;s/_[^_]*_PORT\=.*//' | sort | uniq | xargs echo | sed 's/ /,/g' | tr '[:upper:]' '[:lower:]' ) export SOURCE_ALL_SET=$(printenv | grep '^SC4S_LISTEN_.*_PORT=.' | grep -v "disabled" | sed 's/^SC4S_LISTEN_//;s/_..._PORT\=.*//;s/_[^_]*_PORT\=.*//' | sort | uniq | xargs echo | sed 's/ /,/g' | tr '[:lower:]' '[:upper:]' ) -export DEST_ARCHIVE_PATTERN=$(printenv | grep ARC | grep yes | sed 's/SC4S_DEST_//' | sed 's/_ARCHIVE=yes//' | sort | uniq | xargs echo | sed 's/ /|/g') -export DEST_HEC_PATTERN=$(printenv | grep ARC | grep yes | sed 's/SC4S_DEST_//' | sed 's/_HEC=yes//' | sort | uniq | xargs echo | sed 's/ /|/g') - syslog-ng --no-caps --preprocess-into=- | grep vendor_product | grep set | grep -v 'set(.\$' | sed 's/^ *//' | grep 'value("fields.sc4s_vendor_product"' | grep -v "\`vendor_product\`" | sed s/^set\(// | cut -d',' -f1 | sed 's/\"//g' >/tmp/keys syslog-ng --no-caps --preprocess-into=- | grep 'meta_key(.' | sed 's/^ *meta_key(.//' | sed "s/')//" >>/tmp/keys rm -f $SC4S_ETC/conf.d/local/context/splunk_metadata.csv.example >/dev/null || true diff --git a/tests/Dockerfile.nc b/tests/Dockerfile.nc new file mode 100644 index 0000000000..f303ac044d --- /dev/null +++ b/tests/Dockerfile.nc @@ -0,0 +1,3 @@ +FROM registry.access.redhat.com/ubi8:8.5-214 + +RUN dnf install -y nc \ No newline at end of file diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 3a26969167..1cbc8e61b9 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -31,10 +31,18 @@ services: tty: true links: - splunk + - nc environment: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=${SPLUNK_HEC_TOKEN} - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + - SC4S_DEST_SYSLOG_NC_HOST=nc + - SC4S_DEST_SYSLOG_NC_PORT=2514 + - SC4S_DEST_SYSLOG_NC_MODE=SELECT + - SC4S_DEST_SYSLOG_NC_IETF=no + # - SC4S_DEST_SPLUNK_HEC_SECOND_URL=https://splunk:8088 + # - SC4S_DEST_SPLUNK_HEC_SECOND_TOKEN=${SPLUNK_HEC_TOKEN} + # - SC4S_DEST_SPLUNK_HEC_SECOND_TLS_VERIFY=no - SC4S_LISTEN_PFSENSE_TCP_PORT=6000 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 @@ -43,7 +51,8 @@ services: - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 #- SC4S_ARCHIVE_GLOBAL=yes - - SC4S_DEST_SPECTRACOM_NTP_ARCHIVE=yes + - SC4S_DEST_ALCATEL_SWITCH_ARCHIVE=yes + - SC4S_DEST_ALCATEL_SWITCH_ALTERNATES=d_syslog_nc - SC4S_SOURCE_STORE_RAWMSG=yes - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes - SC4S_SOURCE_LISTEN_UDP_SOCKETS=2 @@ -74,6 +83,12 @@ services: - SPLUNK_APPS_URL=https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz volumes: - splunk-var:/opt/splunk/var + nc: + build: + context: .. + dockerfile: tests/Dockerfile.nc + hostname: nc + command: nc -lkv 2514 volumes: splunk-var: