From 9c1dfc85627896289098c8a2af22f2c1f5243f0d Mon Sep 17 00:00:00 2001 From: Ilya Kheifets Date: Tue, 10 Dec 2024 10:50:40 +0100 Subject: [PATCH] fix: add flower Signed-off-by: Ilya Kheifets --- CHANGELOG.md | 1 + .../templates/worker/_helpers.tpl | 10 +++ .../templates/worker/flower/deployment.yaml | 69 ++++++++++++++++++ .../templates/worker/flower/service.yaml | 22 ++++++ .../values.schema.json | 15 ++++ charts/splunk-connect-for-snmp/values.yaml | 5 ++ docker_compose/docker-compose.yaml | 19 +++++ docs/internal/flower.md | 18 +++++ entrypoint.sh | 3 + poetry.lock | 70 ++++++++++++++++++- pyproject.toml | 1 + 11 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml create mode 100644 charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml create mode 100644 docs/internal/flower.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3097a1221..6a1e85fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changed - add IF-MIB.ifOperStatus to baseIF profile - allow to set subnets for docker network configuration from .env file +- add admin panel for celery ### Fixed diff --git a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl index 7e3e6647e..04a225b5c 100644 --- a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl +++ b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl @@ -54,6 +54,11 @@ app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-tr app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.flower.selectorLabels" -}} +app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-flower +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + {{/* Common labels @@ -78,6 +83,11 @@ Common labels {{ include "splunk-connect-for-snmp.labels" . }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.flower.labels" -}} +{{ include "splunk-connect-for-snmp.worker.flower.selectorLabels" . }} +{{ include "splunk-connect-for-snmp.labels" . }} +{{- end }} + {{- define "environmental-variables" -}} - name: CONFIG_PATH value: /app/config/config.yaml diff --git a/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml b/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml new file mode 100644 index 000000000..e2704a83d --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml @@ -0,0 +1,69 @@ +{{- if .Values.flower.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower + labels: + {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }} + +spec: + replicas: 1 + selector: + matchLabels: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "splunk-connect-for-snmp.serviceAccountName" . }} + securityContext: + fsGroup: 10001 + containers: + - name: {{ .Chart.Name }}-flower + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + [ + "celery", "flower", + ] + env: + {{- include "environmental-variables" . | nindent 12 }} + ports: + - containerPort: 5555 + + volumeMounts: + - name: config + mountPath: "/app/config" + readOnly: true + - name: pysnmp-cache-volume + mountPath: "/.pysnmp/" + readOnly: false + - name: tmp + mountPath: "/tmp/" + readOnly: false + volumes: + - name: config + configMap: + name: {{ include "splunk-connect-for-snmp.name" . }}-config + items: + - key: "config.yaml" + path: "config.yaml" + - name: pysnmp-cache-volume + emptyDir: {} + - name: tmp + emptyDir: {} + +{{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml b/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml new file mode 100644 index 000000000..dc3d8f3ed --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.flower.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower + labels: + {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }} + + +spec: + selector: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 4 }} + ports: + - protocol: TCP + port: {{ .Values.flower.port | default 80 }} + targetPort: 5555 + type: LoadBalancer + {{- if .Values.flower.loadBalancerIP }} + externalIPs: + - {{ .Values.flower.loadBalancerIP }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/values.schema.json b/charts/splunk-connect-for-snmp/values.schema.json index c8036dca0..8a3eac876 100644 --- a/charts/splunk-connect-for-snmp/values.schema.json +++ b/charts/splunk-connect-for-snmp/values.schema.json @@ -40,6 +40,21 @@ "pysnmpDebug": { "type": "string" }, + "flower": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "loadBalancerIP": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + }, "UI": { "type": "object", "additionalProperties": false, diff --git a/charts/splunk-connect-for-snmp/values.yaml b/charts/splunk-connect-for-snmp/values.yaml index 96a4ce1b9..6edb801c6 100644 --- a/charts/splunk-connect-for-snmp/values.yaml +++ b/charts/splunk-connect-for-snmp/values.yaml @@ -49,6 +49,11 @@ UI: # valuesFileDirectory directory regardless of valuesFileName proper configuration. keepSectionFiles: false +flower: + enabled: false + port: 80 + loadBalancerIP: '' + ################################################################################ # Splunk Cloud / Splunk Enterprise configuration. ################################################################################ diff --git a/docker_compose/docker-compose.yaml b/docker_compose/docker-compose.yaml index 8df07ca20..ec04004fd 100644 --- a/docker_compose/docker-compose.yaml +++ b/docker_compose/docker-compose.yaml @@ -234,6 +234,23 @@ services: reservations: cpus: ${WORKER_TRAP_CPU_RESERVATIONS:-0.25} memory: ${WORKER_TRAP_MEMORY_RESERVATIONS:-250M} + flower: + <<: [*dns_and_networks, *dependency_and_restart_policy] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + command: [celery, flower] + environment: + <<: [ + *general_sc4snmp_data, + *splunk_general_setup, + *splunk_extended_setup, + *workers_general_setup, + *pysnmp_debug + ] + profiles: [debug] + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - worker-flower-pysnmp-cache-volume:/.pysnmp/:rw + - worker-flower-tmp:/tmp/:rw volumes: snmp-mibserver-tmp: inventory-tmp: @@ -248,3 +265,5 @@ volumes: worker-sender-pysnmp-cache-volume: worker-trap-tmp: worker-trap-pysnmp-cache-volume: + worker-flower-tmp: + worker-flower-pysnmp-cache-volume: diff --git a/docs/internal/flower.md b/docs/internal/flower.md new file mode 100644 index 000000000..ead90558e --- /dev/null +++ b/docs/internal/flower.md @@ -0,0 +1,18 @@ +# Celery admin panel (internal) + +Main component of SC4SNMP architecture is Celery framworks that can run background and periodic tasks (like polling and walk). By this reason super helpfull might be administrate Celery cluster using admin panel and we using for that [Flower](https://github.com/mher/flower). + +## k8s + +Just add `flower` section on `values.yaml`: + +``` +flower: + enabled: true + loadBalancerIP: x.x.x.x +``` + + +## docker-compose + +Just use `--profile debug` flag to run optional `flower` service: `docker-compose --profiles debug up` \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 9cd252155..89b78b8a1 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -25,6 +25,9 @@ celery) worker-sender) celery -A splunk_connect_for_snmp.poller worker -l "$LOG_LEVEL" -Q send --autoscale=6,"$WORKER_CONCURRENCY" ;; + flower) + celery -A splunk_connect_for_snmp.poller flower + ;; *) celery "$2" ;; diff --git a/poetry.lock b/poetry.lock index 08b63a35b..5df31030b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "amqp" @@ -417,6 +417,24 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "flower" +version = "2.0.1" +description = "Celery Flower" +optional = false +python-versions = ">=3.7" +files = [ + {file = "flower-2.0.1-py2.py3-none-any.whl", hash = "sha256:9db2c621eeefbc844c8dd88be64aef61e84e2deb29b271e02ab2b5b9f01068e2"}, + {file = "flower-2.0.1.tar.gz", hash = "sha256:5ab717b979530770c16afb48b50d2a98d23c3e9fe39851dcf6bc4d01845a02a0"}, +] + +[package.dependencies] +celery = ">=5.0.5" +humanize = "*" +prometheus-client = ">=0.8.0" +pytz = "*" +tornado = ">=5.0.0,<7.0.0" + [[package]] name = "ghp-import" version = "2.1.0" @@ -434,6 +452,20 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] +[[package]] +name = "humanize" +version = "4.10.0" +description = "Python humanize utilities" +optional = false +python-versions = ">=3.8" +files = [ + {file = "humanize-4.10.0-py3-none-any.whl", hash = "sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6"}, + {file = "humanize-4.10.0.tar.gz", hash = "sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978"}, +] + +[package.extras] +tests = ["freezegun", "pytest", "pytest-cov"] + [[package]] name = "idna" version = "3.7" @@ -1150,6 +1182,20 @@ files = [ {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, ] +[[package]] +name = "prometheus-client" +version = "0.21.1" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.8" +files = [ + {file = "prometheus_client-0.21.1-py3-none-any.whl", hash = "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301"}, + {file = "prometheus_client-0.21.1.tar.gz", hash = "sha256:252505a722ac04b0456be05c05f75f45d760c2911ffc45f2a06bcaed9f3ae3fb"}, +] + +[package.extras] +twisted = ["twisted"] + [[package]] name = "prompt-toolkit" version = "3.0.30" @@ -2063,6 +2109,26 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tornado" +version = "6.4.2" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">=3.8" +files = [ + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, + {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, + {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, + {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, +] + [[package]] name = "typing-extensions" version = "4.3.0" @@ -2317,4 +2383,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "8fc9e4f9972418fec948be2960d7454911de3e854f7740f16d89e0f454f080df" +content-hash = "e0d7310d84b21cc4d70e2af6162cf0d82a73a527eaabee82d031f8fbee193aeb" diff --git a/pyproject.toml b/pyproject.toml index 74d9aa567..5e3af6924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ JSON-log-formatter ="^0.5.1" pysnmplib = {git = "https://github.com/pysnmp/pysnmp.git", branch = "main"} urllib3 = "^1.26.17" jsonschema = "4.23.0" +flower = "^2.0.1" [tool.poetry.group.dev.dependencies] pytest = "^8.0.0"