fix: add flower [run-int-tests] [run-ui-tests]

Signed-off-by: Ilya Kheifets <[email protected]>

charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl

@@ -54,6 +54,11 @@ app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-tr
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{- define "splunk-connect-for-snmp.worker.flower.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-flower
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
 
 {{/*
 Common labels
@@ -78,6 +83,11 @@ Common labels
 {{ include "splunk-connect-for-snmp.labels" . }}
 {{- end }}
 
+{{- define "splunk-connect-for-snmp.worker.flower.labels" -}}
+{{ include "splunk-connect-for-snmp.worker.flower.selectorLabels" . }}
+{{ include "splunk-connect-for-snmp.labels" . }}
+{{- end }}
+
 {{- define "environmental-variables" -}}
 - name: CONFIG_PATH
   value: /app/config/config.yaml

charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml

@@ -0,0 +1,69 @@
+{{- if .Values.flower.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower
+  labels:
+    {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }}
+
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "splunk-connect-for-snmp.serviceAccountName" . }}
+      securityContext:
+        fsGroup: 10001
+      containers:
+        - name: {{ .Chart.Name }}-flower
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10001
+            runAsGroup: 10001
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            [
+              "celery", "flower",
+            ]
+          env:
+            {{- include "environmental-variables" . | nindent 12 }}
+          ports:
+            - containerPort: 5555
+
+          volumeMounts:
+            - name: config
+              mountPath: "/app/config"
+              readOnly: true
+            - name: pysnmp-cache-volume
+              mountPath: "/.pysnmp/"
+              readOnly: false
+            - name: tmp
+              mountPath: "/tmp/"
+              readOnly: false
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "splunk-connect-for-snmp.name" . }}-config
+            items:
+              - key: "config.yaml"
+                path: "config.yaml"
+        - name: pysnmp-cache-volume
+          emptyDir: {}
+        - name: tmp
+          emptyDir: {}
+
+{{- end }}

charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml

@@ -0,0 +1,22 @@
+{{- if .Values.flower.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower
+  labels:
+    {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }}
+
+
+spec:
+  selector:
+    {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 4 }}
+  ports:
+    - protocol: TCP
+      port: {{ .Values.flower.port | default 80 }}
+      targetPort: 5555
+  type: LoadBalancer
+  {{- if .Values.flower.loadBalancerIP }}
+  externalIPs:
+    - {{ .Values.flower.loadBalancerIP }}
+  {{- end }}
+{{- end }}

charts/splunk-connect-for-snmp/values.schema.json

@@ -40,6 +40,21 @@
     "pysnmpDebug": {
       "type": "string"
     },
+    "flower": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "enabled": {
+          "type": "boolean"
+        },
+        "loadBalancerIP": {
+          "type": "string"
+        },
+        "port": {
+          "type": "integer"
+        }
+      }
+    },
     "UI": {
       "type": "object",
       "additionalProperties": false,

charts/splunk-connect-for-snmp/values.yaml

@@ -49,6 +49,11 @@ UI:
   # valuesFileDirectory directory regardless of valuesFileName proper configuration.
   keepSectionFiles: false
 
+flower:
+  enabled: false
+  port: 80
+  loadBalancerIP: ''
+
 ################################################################################
 # Splunk Cloud / Splunk Enterprise configuration.
 ################################################################################

docker_compose/.env

@@ -95,4 +95,6 @@ TRAP_LOG_LEVEL=INFO
 # Scheduler configuration
 SCHEDULER_LOG_LEVEL=INFO
 
+FLOWER_PORT=80
+
 #Secrets

docker_compose/docker-compose.yaml

@@ -234,6 +234,28 @@ services:
         reservations:
           cpus: ${WORKER_TRAP_CPU_RESERVATIONS:-0.25}
           memory: ${WORKER_TRAP_MEMORY_RESERVATIONS:-250M}
+  flower:
+    <<: [*dns_and_networks, *dependency_and_restart_policy]
+    image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest}
+    command: [celery, flower]
+    environment:
+      <<: [
+        *general_sc4snmp_data,
+        *splunk_general_setup,
+        *splunk_extended_setup,
+        *workers_general_setup,
+        *pysnmp_debug
+      ]
+    profiles: [debug]
+    ports:
+    - mode: host
+      protocol: tcp
+      published: ${FLOWER_PORT}
+      target: 5555
+    volumes:
+    - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro
+    - worker-flower-pysnmp-cache-volume:/.pysnmp/:rw
+    - worker-flower-tmp:/tmp/:rw
 volumes:
   snmp-mibserver-tmp:
   inventory-tmp:
@@ -248,3 +270,5 @@ volumes:
   worker-sender-pysnmp-cache-volume:
   worker-trap-tmp:
   worker-trap-pysnmp-cache-volume:
+  worker-flower-tmp:
+  worker-flower-pysnmp-cache-volume:

docs/internal/flower.md

@@ -0,0 +1,18 @@
+# Celery admin panel (internal)
+
+Main component of SC4SNMP architecture is Celery framework that can run background and periodic tasks (like polling and walk). It can be helpful to administrate Celery cluster using admin panel with [Flower](https://github.com/mher/flower).
+
+## k8s
+
+Just add `flower` section on `values.yaml`:
+
+```
+flower:
+    enabled: true
+    loadBalancerIP: x.x.x.x
+```
+
+
+## docker-compose
+
+Just use `--profile debug` flag to run optional `flower` service together with SC4SNMP: `docker-compose --profile debug up`

entrypoint.sh

@@ -25,6 +25,9 @@ celery)
     worker-sender)
         celery -A splunk_connect_for_snmp.poller worker -l "$LOG_LEVEL" -Q send --autoscale=6,"$WORKER_CONCURRENCY"
         ;;
+    flower)
+        celery -A splunk_connect_for_snmp.poller flower
+        ;;
     *)
         celery "$2"
         ;;

pyproject.toml

@@ -46,6 +46,7 @@ JSON-log-formatter ="^0.5.1"
 pysnmplib = {git = "https://github.com/pysnmp/pysnmp.git", branch = "main"}
 urllib3 = "^1.26.17"
 jsonschema = "4.23.0"
+flower = "^2.0.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"