fix: add flower

Signed-off-by: Ilya Kheifets <[email protected]>

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Changed
 - add IF-MIB.ifOperStatus to baseIF profile
 - allow to set subnets for docker network configuration from .env file
+- add admin panel for celery
 
 ### Fixed
 

charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl

@@ -54,6 +54,11 @@ app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-tr
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{- define "splunk-connect-for-snmp.worker.flower.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-flower
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
 
 {{/*
 Common labels
@@ -78,6 +83,11 @@ Common labels
 {{ include "splunk-connect-for-snmp.labels" . }}
 {{- end }}
 
+{{- define "splunk-connect-for-snmp.worker.flower.labels" -}}
+{{ include "splunk-connect-for-snmp.worker.flower.selectorLabels" . }}
+{{ include "splunk-connect-for-snmp.labels" . }}
+{{- end }}
+
 {{- define "environmental-variables" -}}
 - name: CONFIG_PATH
   value: /app/config/config.yaml

charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml

@@ -0,0 +1,69 @@
+{{- if .Values.flower.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower
+  labels:
+    {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }}
+
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "splunk-connect-for-snmp.serviceAccountName" . }}
+      securityContext:
+        fsGroup: 10001
+      containers:
+        - name: {{ .Chart.Name }}-flower
+          securityContext:
+            capabilities:
+              drop:
+                - ALL
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 10001
+            runAsGroup: 10001
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            [
+              "celery", "flower",
+            ]
+          env:
+            {{- include "environmental-variables" . | nindent 12 }}
+          ports:
+            - containerPort: 5555
+
+          volumeMounts:
+            - name: config
+              mountPath: "/app/config"
+              readOnly: true
+            - name: pysnmp-cache-volume
+              mountPath: "/.pysnmp/"
+              readOnly: false
+            - name: tmp
+              mountPath: "/tmp/"
+              readOnly: false
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "splunk-connect-for-snmp.name" . }}-config
+            items:
+              - key: "config.yaml"
+                path: "config.yaml"
+        - name: pysnmp-cache-volume
+          emptyDir: {}
+        - name: tmp
+          emptyDir: {}
+
+{{- end }}

charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml

@@ -0,0 +1,22 @@
+{{- if .Values.flower.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower
+  labels:
+    {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }}
+
+
+spec:
+  selector:
+    {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 4 }}
+  ports:
+    - protocol: TCP
+      port: {{ .Values.flower.port | default 80 }}
+      targetPort: 5555
+  type: LoadBalancer
+  {{- if .Values.flower.loadBalancerIP }}
+  externalIPs:
+    - {{ .Values.flower.loadBalancerIP }}
+  {{- end }}
+{{- end }}

charts/splunk-connect-for-snmp/values.schema.json

@@ -40,6 +40,21 @@
     "pysnmpDebug": {
       "type": "string"
     },
+    "flower": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "enabled": {
+          "type": "boolean"
+        },
+        "loadBalancerIP": {
+          "type": "string"
+        },
+        "port": {
+          "type": "integer"
+        }
+      }
+    },
     "UI": {
       "type": "object",
       "additionalProperties": false,

charts/splunk-connect-for-snmp/values.yaml

@@ -49,6 +49,11 @@ UI:
   # valuesFileDirectory directory regardless of valuesFileName proper configuration.
   keepSectionFiles: false
 
+flower:
+  enabled: false
+  port: 80
+  loadBalancerIP: ''
+
 ################################################################################
 # Splunk Cloud / Splunk Enterprise configuration.
 ################################################################################

docker_compose/docker-compose.yaml

@@ -234,6 +234,23 @@ services:
         reservations:
           cpus: ${WORKER_TRAP_CPU_RESERVATIONS:-0.25}
           memory: ${WORKER_TRAP_MEMORY_RESERVATIONS:-250M}
+  flower:
+    <<: [*dns_and_networks, *dependency_and_restart_policy]
+    image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest}
+    command: [celery, flower]
+    environment:
+      <<: [
+        *general_sc4snmp_data,
+        *splunk_general_setup,
+        *splunk_extended_setup,
+        *workers_general_setup,
+        *pysnmp_debug
+      ]
+    profiles: [debug]
+    volumes:
+    - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro
+    - worker-flower-pysnmp-cache-volume:/.pysnmp/:rw
+    - worker-flower-tmp:/tmp/:rw
 volumes:
   snmp-mibserver-tmp:
   inventory-tmp:
@@ -248,3 +265,5 @@ volumes:
   worker-sender-pysnmp-cache-volume:
   worker-trap-tmp:
   worker-trap-pysnmp-cache-volume:
+  worker-flower-tmp:
+  worker-flower-pysnmp-cache-volume:

docs/internal/flower.md

@@ -0,0 +1,18 @@
+# Celery admin panel (internal)
+
+Main component of SC4SNMP architecture is Celery framworks that can run background and periodic tasks (like polling and walk). By this reason super helpfull might be administrate Celery cluster using admin panel and we using for that [Flower](https://github.com/mher/flower).
+
+## k8s
+
+Just add `flower` section on `values.yaml`:
+
+```
+flower:
+    enabled: true
+    loadBalancerIP: x.x.x.x
+```
+
+
+## docker-compose
+
+Just use `--profile debug` flag to run optional `flower` service: `docker-compose --profiles debug up`

entrypoint.sh

@@ -25,6 +25,9 @@ celery)
     worker-sender)
         celery -A splunk_connect_for_snmp.poller worker -l "$LOG_LEVEL" -Q send --autoscale=6,"$WORKER_CONCURRENCY"
         ;;
+    flower)
+        celery -A splunk_connect_for_snmp.poller flower
+        ;;
     *)
         celery "$2"
         ;;

pyproject.toml

@@ -46,6 +46,7 @@ JSON-log-formatter ="^0.5.1"
 pysnmplib = {git = "https://github.com/pysnmp/pysnmp.git", branch = "main"}
 urllib3 = "^1.26.17"
 jsonschema = "4.23.0"
+flower = "^2.0.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"