feat: pluggable modules (#2242)

* create mechanism of enabling and disabling pluggable modules

* setup tests for sc4s lite

* make for sc4s lite available all the topcis that supporting sc4s

* k8s setup for pluggable modules

* add commented example to helm of pluggable modules

* create pluggable modules

* add docs for pluggable modules

* test

.github/workflows/ci-lite.yaml

@@ -224,7 +224,7 @@ jobs:
             --splunk_host=splunk \
             --sc4s_host=sc4s \
             --junitxml=test-results/test.xml \
-            -n 14  tests/test_linux_syslog.py  tests/test_common_lite.py
+            -n 14 -m "lite or addons"
   release:
     name: Release
     runs-on: ubuntu-latest

charts/splunk-connect-for-syslog/templates/addon-configmap.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.sc4s.addons }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "splunk-connect-for-syslog.fullname" . }}-addons
+  labels:
+    {{- include "splunk-connect-for-syslog.labels" . | nindent 4 }}
+data:
+{{ toYaml .Values.sc4s.addons | indent 2 }}
+{{- end }}

charts/splunk-connect-for-syslog/templates/statefulset.yaml

@@ -201,6 +201,11 @@ spec:
             mountPath: /etc/syslog-ng/conf.d/configmap/config/app_parsers
             readOnly: true
           {{- end }}
+          {{- if .Values.sc4s.addons }}
+          - name: addons
+            mountPath: /etc/syslog-ng/conf.d/configmap/addons/
+            readOnly: true
+          {{- end }}
           livenessProbe:
             httpGet:
               path: /healthz
@@ -233,7 +238,12 @@ spec:
         - name: config
           configMap:
             name: {{ include "splunk-connect-for-syslog.fullname" . }}-config
-        {{- end }}               
+        {{- end }}
+        {{- if .Values.sc4s.addons }}
+        - name: addons
+          configMap:
+            name: {{ include "splunk-connect-for-syslog.fullname" . }}-addons
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/splunk-connect-for-syslog/values.yaml

@@ -55,6 +55,89 @@ sc4s:
 #          parser { app-postfilter-cisco_asa_metadata(); };
 #        };
 
+#   addons:
+#     config.yaml: |-
+#       ---
+#       addons:
+#         - Example addon
+#         - barracuda
+#         - cisco
+#         - paloalto
+#         - vmware
+#         - dell
+#         - zscaler
+#         - checkpoint
+#         - kaspersky
+#         - mcafee
+#         - ibm
+#         - tanium
+#         - arista
+#         - ubiquiti
+#         - forcepoint
+#         - netapp
+#         - clearswift
+#         - cylance
+#         - cyberark
+#         - avi
+#         - darktrace
+#         - beyondtrust
+#         - qumulo
+#         - liveaction
+#         - juniper
+#         - hp
+#         - broadcom
+#         - aruba
+#         - f5
+#         - imperva
+#         - fireeye
+#         - trendmicro
+#         - novell
+#         - brocade
+#         - ossec
+#         - vectra
+#         - microsoft
+#         - mikrotik
+#         - nutanix
+#         - proofpoint
+#         - tenable
+#         - pulse
+#         - stealthwatch
+#         - netscout
+#         - purestorage
+#         - spectracom
+#         - xypro
+#         - raritan
+#         - ricoh
+#         - avaya
+#         - secureauth
+#         - infoblox
+#         - schneider
+#         - isc
+#         - fortinet
+#         - solace
+#         - radware
+#         - riverbed
+#         - pfsense
+#         - github
+#         - buffalo
+#         - varonis
+#         - alsid
+#         - sophos
+#         - netmotion
+#         - tintri
+#         - alcatel
+#         - cohesity
+#         - ruckus
+#         - polycom
+#         - citrix
+#         - haproxy
+#         - a10networks
+#         - microfocus
+#         - syslogng
+#         - wallix
+#         - thycotic
+#         - tim
+
 image:
   repository: ghcr.io/splunk/splunk-connect-for-syslog/container3
   pullPolicy: IfNotPresent

docs/lite.md

@@ -10,8 +10,11 @@ The addition of Pluggable Modular Parsers offers users the flexibility to incorp
 ### SC4S Lite
 The primary component of the system, SC4S Lite is built upon the SC4S, providing a lightweight, high-performance sc4s  solution, the current SC4S is very complex and packed with too many oob parser support.
 
-### Pluggable Modular Parsers
-The Pluggable Modular Parsers provide the ability to customize and extend the data processing logic of SC4S Lite. Users can create and integrate custom parsers to suit their specific needs, enhancing the flexibility of data processing.
+### Pluggable Modules
+Pluggable modules it's **predefined modules**, that you can **only** enable/disable (can't create or update modulew) by changing config file.
+Each pluggable module representing set of parsers for each vendor that supporting SC4S.
+
+[More detail guide here](pluggable_modules.md)
 
 ### Splunk Enterprise or Splunk Cloud
 The Splunk platform is the destination for the syslog data. Splunk allows for comprehensive analysis, searching, and visualization of the processed data.

docs/pluggable_modules.md

@@ -0,0 +1,57 @@
+# Pluggable modules guide:
+
+Pluggable modules **it's predefined modules**, that you can **only** *enable/disable* by changing config file.
+After updating this config file with enabled addons you need to restart sc4s.
+
+This config file it's a yaml file with list of addons ([whole list of addons here](https://github.com/splunk/splunk-connect-for-syslog/blob/main/package/lite/etc/config.yaml)):
+```
+---
+addons:
+    - cisco
+    - paloalto
+    - dell
+```
+
+You don't need to rebuild docker image, you **need mount custom config** into */etc/syslog-ng/config.yaml*.
+
+
+## docker-compose:
+
+1. [Read guide](./gettingstarted/docker-compose.md) how to use *docker-compose* for SC4S
+
+2. Use *SC4S Lite image* instead of *SC4S* in docker-compose.yaml
+```
+image: ghcr.io/splunk/splunk-connect-for-syslog/container3lite
+```
+
+3. *Mount config file* with addons to */etc/syslog-ng/config.yaml*:
+
+```
+volumes:
+    - /path/to/your/config.yaml:/etc/syslog-ng/config.yaml
+```
+
+
+
+## k8s:
+
+1. [Read guide](./gettingstarted/k8s-microk8s.md) how to use *k8s* for SC4S
+
+2. Use *SC4S Lite image* instead of *SC4S*  in values.yaml:
+```
+image:
+  repository: ghcr.io/splunk/splunk-connect-for-syslog/container3lite
+```
+
+3. Mount config file. Add *addons* section on *sc4s* section of values.yaml:
+
+```
+sc4s:
+    addons:
+        config.yaml: |-
+            ---
+            addons:
+                - cisco
+                - paloalto
+                - dell
+```

mkdocs.yml

@@ -54,6 +54,9 @@ nav:
       - Basic Onboarding: "sources/base"
       - Known Vendors: "sources/vendor"
   - Performance: "performance.md"
+  - SC4S Lite (Experimental):
+      - Intro: "lite.md"
+      - Pluggable modules: "pluggable_modules.md"
   - Troubleshooting:
       - SC4S Startup and Validation: "troubleshooting/troubleshoot_SC4S_server.md"
       - SC4S Logging and Troubleshooting Resources: "troubleshooting/troubleshoot_resources.md"

package/Dockerfile.lite

@@ -68,12 +68,15 @@ RUN python3 -m venv /var/lib/python-venv \
     && poetry export --format requirements.txt | /var/lib/python-venv/bin/pip3 --no-cache-dir install -r /dev/stdin \
     && apk del build-base python3-dev libffi-dev
 
-COPY package/etc/syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
+COPY package/lite/etc/syslog-ng.conf.jinja /etc/syslog-ng/syslog-ng.conf.jinja
 
-COPY package/etc/conf.d/conflib/fallback /etc/syslog-ng/conf.d/conflib/fallback
-COPY package/etc/conf.d/conflib/raw /etc/syslog-ng/conf.d/conflib/raw
 COPY package/etc/conf.d/conflib/_splunk /etc/syslog-ng/conf.d/conflib/_splunk
 COPY package/etc/conf.d/conflib/_common /etc/syslog-ng/conf.d/conflib/_common
+COPY package/etc/conf.d/conflib/app-fix-invalid-program /etc/syslog-ng/conf.d/conflib/app-fix-invalid-program
+COPY package/etc/conf.d/conflib/app-hec-debug /etc/syslog-ng/conf.d/conflib/app-hec-debug
+COPY package/etc/conf.d/conflib/app-lp_dest_archive /etc/syslog-ng/conf.d/conflib/app-lp_dest_archive
+COPY package/etc/conf.d/conflib/app-lp_dest_selected_alts /etc/syslog-ng/conf.d/conflib/app-lp_dest_selected_alts
+COPY package/etc/conf.d/conflib/app-lp-archive /etc/syslog-ng/conf.d/conflib/app-lp-archive
 
 
 #COPY package/etc/conf.d /etc/syslog-ng/conf.d
@@ -84,16 +87,16 @@ COPY package/etc/conf.d/plugin /etc/syslog-ng/conf.d/plugin
 COPY package/etc/conf.d/sc4slib /etc/syslog-ng/conf.d/sc4slib
 COPY package/etc/conf.d/sources /etc/syslog-ng/conf.d/sources
 
-COPY package/lite/etc/conf.d/conflib/raw/app-raw-bsd_nopri.conf /etc/syslog-ng/conf.d/conflib/raw/app-raw-bsd_nopri.conf
-COPY package/lite/etc/conf.d/log_paths/2/lp_dest_alts_global/plugin.jinja /etc/syslog-ng/conf.d/log_paths/2/lp_dest_alts_global/plugin.jinja
-COPY package/lite/etc/conf.d/plugin/app_parser_topics.conf /etc/syslog-ng/conf.d/plugin/app_parser_topics.conf
-COPY package/lite/etc/conf.d/sources/source_syslog/plugin.jinja /etc/syslog-ng/conf.d/sources/source_syslog/plugin.jinja
+COPY package/lite/etc/conf.d/conflib /etc/syslog-ng/conf.d/conflib
 
 COPY package/etc/context_templates /etc/syslog-ng/context_templates
 COPY package/etc/local_config /etc/syslog-ng/local_config
 COPY package/etc/pylib /etc/syslog-ng/pylib
 COPY package/etc/test_parsers /etc/syslog-ng/test_parsers
 
+COPY package/lite/etc/config.yaml /etc/syslog-ng/config.yaml
+COPY package/lite/etc/addons /etc/syslog-ng/addons
+
 COPY package/sbin/entrypoint.sh /
 COPY package/sbin/healthcheck.sh /
 

package/etc/pylib/config_generator/__main__.py

@@ -0,0 +1,29 @@
+from argparse import ArgumentParser
+from pathlib import Path
+
+from .addons import load_addons
+from .config import load_addons_config
+from .template_generator import template_generator
+
+
+def parse_cli_args():
+    cli_parser = ArgumentParser()
+    cli_parser.add_argument("--config", type=Path)
+    return cli_parser.parse_args()
+
+
+def generate_syslogng_config() -> None:
+    cli_args = parse_cli_args()
+    config = load_addons_config(cli_args.config)
+    addons = load_addons(config.addons_path)
+
+    syslogng_config = template_generator(
+        config.syslog_path,
+        config=config,
+        addons=sorted(addons, key=lambda addon: addon.path),
+    )
+    print(syslogng_config)
+
+
+if __name__ == "__main__":
+    generate_syslogng_config()

package/etc/pylib/config_generator/addons.py

@@ -0,0 +1,50 @@
+from dataclasses import dataclass
+from logging import getLogger
+from pathlib import Path
+
+import yaml
+
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class AddonMetadata:
+    name: str
+
+
+@dataclass
+class Addon:
+    path: Path
+    metadata: AddonMetadata
+
+
+def load_addons(addons_directory: Path) -> list[Addon]:
+    addons: list[Addon] = []
+
+    for potential_addon in addons_directory.iterdir():
+        addon_full_path = addons_directory / potential_addon
+
+        if (
+            addon_full_path.is_dir() and
+            (addon_full_path / "addon_metadata.yaml").exists()
+        ):
+            try:
+                metadata = load_addon_metadata(addon_full_path)
+                addons.append(Addon(path=addon_full_path, metadata=metadata))
+            except Exception as e:
+                logger.error(f"Skipping invalid addon {potential_addon}")
+                raise e
+
+    return addons
+
+
+def load_addon_metadata(addon_path: Path) -> AddonMetadata:
+    with open(addon_path / "addon_metadata.yaml", "r") as file_stream:
+        try:
+            metadata = yaml.safe_load(file_stream)
+            return AddonMetadata(name=metadata["name"])
+        except yaml.YAMLError:
+            logger.error(f"Metadata file of {addon_path} should be valid yaml")
+        except KeyError:
+            logger.error(f"Missing metadata in {addon_path}")

package/etc/pylib/config_generator/config.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from logging import getLogger
+from os.path import expandvars
+from pathlib import Path
+
+import yaml
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class Config:
+    addons: list[str]
+    addons_path: Path = Path(expandvars("${SC4S_ETC}/addons"))
+    syslog_path: Path = Path(expandvars("${SC4S_ETC}/syslog-ng.conf.jinja"))
+
+
+def load_addons_config(config_path: Path) -> Config:
+    with open(config_path, "r") as file_stream:
+        try:
+            raw_config = yaml.safe_load(file_stream)
+            return Config(**raw_config)
+        except yaml.YAMLError:
+            logger.error("Config should be correct yaml")
+        except KeyError:
+            logger.error("Field is missing in config")

package/etc/pylib/config_generator/template_generator.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+
+import jinja2
+
+
+def template_generator(template_path: Path, **kwargs) -> str:
+    env = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(template_path.parent),
+        autoescape=False,
+    )
+    template = env.get_template(template_path.name)
+    return template.render(**kwargs)

package/lite/etc/addons/a10networks/addon_metadata.yaml

@@ -0,0 +1,2 @@
+---
+name: "a10networks"