From d4e4e7926033e1a0f0d221fa6911f092d96489b4 Mon Sep 17 00:00:00 2001
From: Ilya <138466237+ikheifets-splunk@users.noreply.github.com>
Date: Mon, 13 May 2024 04:44:22 +0200
Subject: [PATCH] feat: Edge Processor (#2328)

* feat: adopot EP HEC implementation for SC4S

* feat: implement mTLS for EP

* feat: automation k8s secret creation of mTLS

* docs: describe mTLS encryption

* docs: review edge processor documentation (#2402)

* docs: Edge Processor docs

* Update edge_processor.md

Hi, please take a look at my edits to make sure i did not accidentally misinterpret the meaning of the original text. I'll do another final pass after that.

* Update edge_processor.md

* Update edge_processor.md

Updated docs based on feedback

* Update edge_processor.md

updated line 40 based on feedback

* docs: fix logical mistakes

---------

Co-authored-by: Ilya Kheifets <ikheifets@splunk.com>

* fix: small improvements

* fix: make generic deploy certs

---------

Co-authored-by: JENNIFER WORTHINGTON <37708067+jenworthington@users.noreply.github.com>
---
 ansible/playbooks/microk8s.yml                |   1 +
 ansible/playbooks/microk8s_ha.yml             |   1 +
 ansible/tasks/mk8s/deploy_secrets.yml         |  25 ++++
 charts/splunk-connect-for-syslog/secrets.yaml |   0
 .../templates/statefulset.yaml                |  14 +++
 docs/edge_processor.md                        | 113 ++++++++++++++++++
 mkdocs.yml                                    |   6 +
 .../conf.d/conflib/_common/t_templates.conf   |   2 +-
 .../conf.d/destinations/dest_hec/plugin.jinja |   5 +
 .../conf.d/destinations/dest_hec/plugin.py    |   1 +
 package/sbin/entrypoint.sh                    |  11 +-
 11 files changed, 176 insertions(+), 3 deletions(-)
 create mode 100644 ansible/tasks/mk8s/deploy_secrets.yml
 create mode 100644 charts/splunk-connect-for-syslog/secrets.yaml
 create mode 100644 docs/edge_processor.md

diff --git a/ansible/playbooks/microk8s.yml b/ansible/playbooks/microk8s.yml
index b24e660321..19a03ec940 100644
--- a/ansible/playbooks/microk8s.yml
+++ b/ansible/playbooks/microk8s.yml
@@ -12,5 +12,6 @@
       openebs: true
   tasks:
     - include_tasks: ../tasks/mk8s/install_mk8s.yml
+    - include_tasks: ../tasks/mk8s/deploy_secrets.yml
     - include_tasks: ../tasks/mk8s/install_helm_repo.yml
     - include_tasks: ../tasks/mk8s/deploy_app.yml
diff --git a/ansible/playbooks/microk8s_ha.yml b/ansible/playbooks/microk8s_ha.yml
index ac5b4fdaef..c81ec4b5a4 100644
--- a/ansible/playbooks/microk8s_ha.yml
+++ b/ansible/playbooks/microk8s_ha.yml
@@ -12,6 +12,7 @@
       openebs: true
   tasks:
     - include_tasks: ../tasks/mk8s/install_mk8s.yml
+    - include_tasks: ../tasks/mk8s/deploy_secrets.yml
     - include_tasks: ../tasks/mk8s/install_helm_repo.yml
     - include_tasks: ../tasks/mk8s/deploy_app.yml
     - include_tasks: ../tasks/mk8s/get_registration_token.yml
diff --git a/ansible/tasks/mk8s/deploy_secrets.yml b/ansible/tasks/mk8s/deploy_secrets.yml
new file mode 100644
index 0000000000..878d88d033
--- /dev/null
+++ b/ansible/tasks/mk8s/deploy_secrets.yml
@@ -0,0 +1,25 @@
+---
+- name: Load k8s secrets
+  include_vars:
+    file: "{{ item }}"
+  with_first_found:
+    - files:
+      - /opt/ansible/resources/k8s_secrets.yaml
+      - /opt/charts/splunk-connect-for-syslog/secrets.yaml
+
+- name: Create mTLS secret
+  ansible.builtin.shell: |
+    microk8s kubectl apply -f - <<EOF
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: {{ hec_tls.secret }}
+    type: Opaque
+    data:
+      key.pem: {{ hec_tls.value.key | b64encode }}
+      cert.pem: {{ hec_tls.value.cert | b64encode }}
+      ca_cert.pem: {{ hec_tls.value.ca | b64encode }}
+    EOF
+  when:
+    - hec_tls is defined
+    - ('secret' in hec_tls) and ('value' in hec_tls)
diff --git a/charts/splunk-connect-for-syslog/secrets.yaml b/charts/splunk-connect-for-syslog/secrets.yaml
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/charts/splunk-connect-for-syslog/templates/statefulset.yaml b/charts/splunk-connect-for-syslog/templates/statefulset.yaml
index aca80bc49e..2eecb5b9cf 100644
--- a/charts/splunk-connect-for-syslog/templates/statefulset.yaml
+++ b/charts/splunk-connect-for-syslog/templates/statefulset.yaml
@@ -65,6 +65,10 @@ spec:
           - name: SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE
             value: "no"
           {{- end }}
+          {{- if .Values.splunk.hec_tls }}
+          - name: SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT
+            value: "/etc/syslog-ng/tls/hec"
+          {{- end }}
           {{- if .Values.sc4s.existingCert }}
           - name: SC4S_SOURCE_TLS_ENABLE
             value: "yes"
@@ -186,6 +190,11 @@ spec:
           - name: data
             mountPath: /var/lib/syslog-ng/
           {{- end }}
+          {{- if .Values.splunk.hec_tls }}
+          - name: hec-tls
+            mountPath: /etc/syslog-ng/tls/hec/
+            readOnly: true
+          {{- end }}
           {{- if .Values.sc4s.existingCert }}
           - name: tls
             mountPath: /etc/syslog-ng/tls/
@@ -224,6 +233,11 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       volumes:
+        {{- if .Values.splunk.hec_tls }}
+        - name: hec-tls
+          secret:
+            secretName: {{ .Values.splunk.hec_tls }}
+        {{- end }}
         {{- if .Values.sc4s.existingCert }}
         - name: tls
           secret:
diff --git a/docs/edge_processor.md b/docs/edge_processor.md
new file mode 100644
index 0000000000..169f986674
--- /dev/null
+++ b/docs/edge_processor.md
@@ -0,0 +1,113 @@
+# Edge Processor integration guide (Experimental)
+
+## Intro
+
+You can use the `Edge Processor` to:
+
+* Enrich log messages with extra data, such as adding a new field or overriding an index using `SPL2`.
+* Filter log messages using `SPL2`.
+* Send log messages to alternative destinations, for example, `AWS S3` or `Apache Kafka`.
+
+## How it works
+
+```mermaid
+stateDiagram
+    direction LR
+
+    SC4S: SC4S
+    EP: Edge Processor
+    Dest: Another destination
+    Device: Your device
+    S3: AWS S3
+    Instance: Instance
+    Pipeline: Pipeline with SPL2
+
+    Device --> SC4S: Syslog protocol
+    SC4S --> EP: HEC
+    state EP {
+      direction LR
+      Instance --> Pipeline
+    }
+    EP --> Splunk
+    EP --> S3
+    EP --> Dest
+```
+
+## Set up the Edge Processor for SC4S
+
+### Set up on Docker / Podman
+
+1. On the `env_file`, configure the HEC URL as IP of *managed instance*, that you registered on Edge Processor.
+2.  Add your HEC token. You can find your token in the Edge Processor "global settings" page. 
+
+```
+SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=http://x.x.x.x:8088
+SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+```
+
+### Kubernetes
+
+1. Set up the Edge Processor on your `values.yaml` HEC URL using the IP of *managed instance*, that you registered on Edge Processor.
+
+2. Provide the hec_token. You can find this token on the Edge Processor's "global settings" page.
+
+```
+splunk:
+  hec_url: "http://x.x.x.x:8088"
+  hec_token: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+```
+
+## mTLS encryption
+
+### Prepare your certificates
+
+Before setup, [generate mTLS certificates](https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/EdgeProcessor/SecureForwarders). Server mTLS certificates should be uploaded to `Edge Processor` and client certifcates should be used with `SC4S`.
+
+Rename the certificate files. SC4S requires the following names:
+
+  * `key.pem` - client certificate key
+  * `cert.pem` - client certificate
+  * `ca_cert.pem` - certificate authority
+
+### Docker / Podman
+
+  1. Use HTTPS in HEC url: `SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://x.x.x.x:8088`.
+  2. Move your clients mTLS certificates to `/opt/sc4s/tls/hec`.
+  3. Mount `/opt/sc4s/tls/hec` to `/etc/syslog-ng/tls/hec` using docker/podman volumes.
+  4. Define mounting mTLS point for HEC: `SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT=/etc/syslog-ng/tls/hec`.
+  5. Start or restart SC4S.
+
+### Kubernetes
+
+  1. Add the secret name of the mTLS certificates to the `values.yaml` file:
+
+```
+splunk:
+  hec_url: "https://x.x.x.x:8088"
+  hec_token: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+  hec_tls: "hec-tls-secret"
+```
+
+  2. Add your mTLS certificates to the `charts/splunk-connect-for-syslog/secrets.yaml` file:
+
+```
+hec_tls:
+  secret: "hec-tls-secret"
+  value:
+    key: |
+      -----BEGIN PRIVATE KEY-----
+      Exmaple key
+      -----END PRIVATE KEY-----
+    cert: |
+      -----BEGIN CERTIFICATE-----
+      Exmaple cert
+      -----END CERTIFICATE-----
+    ca: |
+      -----BEGIN CERTIFICATE-----
+      Example ca
+      -----END CERTIFICATE-----
+```
+
+  3. Encrypt your `secrets.yaml` using `ansible-vault encrypt charts/splunk-connect-for-syslog/secrets.yaml`.
+  4. Add the IP address for your cluster nodes to the inventory file `ansible/inventory/inventory_microk8s_ha.yaml`.
+  5. Deploy the Ansible playbook `ansible-playbook -i ansible/inventory/inventory_microk8s_ha.yaml ansible/playbooks/microk8s_ha.yml --ask-vault-pass`
diff --git a/mkdocs.yml b/mkdocs.yml
index 9f26aae99d..806ae86867 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -14,6 +14,11 @@ markdown_extensions:
   - sane_lists
   - codehilite
   - pymdownx.snippets
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
 
 
 theme:
@@ -57,6 +62,7 @@ nav:
   - SC4S Lite (Experimental):
       - Intro: "lite.md"
       - Pluggable modules: "pluggable_modules.md"
+  - Edge Processor (Experimental): "edge_processor.md"
   - Troubleshooting:
       - SC4S Startup and Validation: "troubleshooting/troubleshoot_SC4S_server.md"
       - SC4S Logging and Troubleshooting Resources: "troubleshooting/troubleshoot_resources.md"
diff --git a/package/etc/conf.d/conflib/_common/t_templates.conf b/package/etc/conf.d/conflib/_common/t_templates.conf
index da7738bf24..0dc09795b3 100644
--- a/package/etc/conf.d/conflib/_common/t_templates.conf
+++ b/package/etc/conf.d/conflib/_common/t_templates.conf
@@ -138,7 +138,7 @@ template t_JSON_5424_SDATA {
 
 template t_splunk_hec {
     template('$(format-json
-        time=$(if ("${.netsource.sc4s_use_recv_time}" eq "yes") "$R_UNIXTIME" "$S_UNIXTIME")
+        time=$(if ("${.netsource.sc4s_use_recv_time}" eq "yes") ${R_UNIXTIME} ${S_UNIXTIME})
         host=$(lowercase ${HOST})
         source=${.splunk.source:-SC4S}
         sourcetype=${.splunk.sourcetype:-sc4s:fallback}
diff --git a/package/etc/conf.d/destinations/dest_hec/plugin.jinja b/package/etc/conf.d/destinations/dest_hec/plugin.jinja
index ca5a6e7407..aa47b4b879 100644
--- a/package/etc/conf.d/destinations/dest_hec/plugin.jinja
+++ b/package/etc/conf.d/destinations/dest_hec/plugin.jinja
@@ -34,6 +34,11 @@ destination d_hec{{ dest_mode }}{{ altname }}{
     {%- endif %}
         tls(
             peer-verify({{ peer_verify }})
+        {% if tls_mount %}
+            key-file("{{ tls_mount }}/key.pem")
+            cert-file("{{ tls_mount }}/cert.pem")
+            ca-dir("{{ tls_mount }}")
+        {% endif %}
         {%- if cipher_suite %}
             cipher-suite("{{ cipher_suite }}")
         {%- endif %}
diff --git a/package/etc/conf.d/destinations/dest_hec/plugin.py b/package/etc/conf.d/destinations/dest_hec/plugin.py
index 269df2f23a..e75f9492c8 100755
--- a/package/etc/conf.d/destinations/dest_hec/plugin.py
+++ b/package/etc/conf.d/destinations/dest_hec/plugin.py
@@ -144,6 +144,7 @@ def hec_endpoint_collector(hec_path, url_hec):
             f"SC4S_DEST_SPLUNK_HEC_{group}_DISKBUFF_DISKBUFSIZE",
             int(disk_space / workers),
         ),
+        tls_mount=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TLS_MOUNT"),
         peer_verify=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TLS_VERIFY", "yes"),
         cipher_suite=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_CIPHER_SUITE"),
         ssl_version=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_SSL_VERSION"),
diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh
index 1a1ae69595..25ae354708 100755
--- a/package/sbin/entrypoint.sh
+++ b/package/sbin/entrypoint.sh
@@ -170,18 +170,25 @@ if [ "$SC4S_DEST_SPLUNK_HEC_GLOBAL" != "no" ]
 then
   HEC=$(echo $SC4S_DEST_SPLUNK_HEC_DEFAULT_URL | cut -d' ' -f 1)
   if [ "${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY}" == "no" ]; then export NO_VERIFY=-k ; fi
+  
+  if [ -n "${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}" ]; then
+    export HEC_TLS_OPTS="--cert ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/cert.pem  --key ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/key.pem --cacert ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/ca_cert.pem";
+  else
+    export HEC_TLS_OPTS="";
+  fi
+
   SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX=$(grep -Po '(?<=^splunk_sc4s_fallback,index,).*' -m1 $SC4S_ETC/conf.d/local/context/splunk_metadata.csv )
   export SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX:=main}
   SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=$(cat $SC4S_ETC/conf.d/local/context/splunk_metadata.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3)
   export SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX:=main}
 
-  if curl -s -S ${NO_VERIFY} "${HEC}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d "{\"event\": \"HEC TEST EVENT\", \"sourcetype\": \"sc4s:probe\", \"index\": \"${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}\"}" 2>&1 | grep -v '{"text":"Success"'
+  if curl -s -S ${NO_VERIFY} "${HEC}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d "{\"event\": \"HEC TEST EVENT\", \"sourcetype\": \"sc4s:probe\", \"index\": \"${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}\"}" ${HEC_TLS_OPTS} 2>&1 | grep -v -e '{"text":"Success"' -e '{"text": "Success"'
   then
     echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}. sourcetype=sc4s:fallback\nStartup will continue to prevent data loss if this is a transient failure."
     echo ""
   else
     echo -e "SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX} for sourcetype=sc4s:fallback..."
-    if curl -s -S ${NO_VERIFY} "${HEC}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d "{\"event\": \"HEC TEST EVENT\", \"sourcetype\": \"sc4s:probe\", \"index\": \"${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}\"}" 2>&1 | grep -v '{"text":"Success"'
+    if curl -s -S ${NO_VERIFY} "${HEC}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d "{\"event\": \"HEC TEST EVENT\", \"sourcetype\": \"sc4s:probe\", \"index\": \"${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}\"}" ${HEC_TLS_OPTS} 2>&1 | grep -v -e '{"text":"Success"' -e '{"text": "Success"'
       then
         echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue for index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}. sourcetype=sc4s:events \nStartup will continue to prevent data loss if this is a transient failure."
         echo ""