From d9d44f5fceab15fdad1d163cab39089fe936f0b6 Mon Sep 17 00:00:00 2001 From: Ilya Kheifets Date: Tue, 10 Dec 2024 10:50:40 +0100 Subject: [PATCH] fix: add flower [run-int-tests] [run-ui-tests] Signed-off-by: Ilya Kheifets --- .../templates/worker/_helpers.tpl | 10 +++ .../templates/worker/flower/deployment.yaml | 69 ++++++++++++++++++ .../templates/worker/flower/service.yaml | 22 ++++++ .../values.schema.json | 15 ++++ charts/splunk-connect-for-snmp/values.yaml | 5 ++ docker_compose/.env | 2 + docker_compose/docker-compose.yaml | 24 +++++++ docs/internal/flower.md | 18 +++++ entrypoint.sh | 3 + poetry.lock | 70 ++++++++++++++++++- pyproject.toml | 1 + 11 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml create mode 100644 charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml create mode 100644 docs/internal/flower.md diff --git a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl index 7e3e6647e..04a225b5c 100644 --- a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl +++ b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl @@ -54,6 +54,11 @@ app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-tr app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.flower.selectorLabels" -}} +app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-flower +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + {{/* Common labels @@ -78,6 +83,11 @@ Common labels {{ include "splunk-connect-for-snmp.labels" . }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.flower.labels" -}} +{{ include "splunk-connect-for-snmp.worker.flower.selectorLabels" . }} +{{ include "splunk-connect-for-snmp.labels" . }} +{{- end }} + {{- define "environmental-variables" -}} - name: CONFIG_PATH value: /app/config/config.yaml diff --git a/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml b/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml new file mode 100644 index 000000000..e2704a83d --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml @@ -0,0 +1,69 @@ +{{- if .Values.flower.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower + labels: + {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }} + +spec: + replicas: 1 + selector: + matchLabels: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "splunk-connect-for-snmp.serviceAccountName" . }} + securityContext: + fsGroup: 10001 + containers: + - name: {{ .Chart.Name }}-flower + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + [ + "celery", "flower", + ] + env: + {{- include "environmental-variables" . | nindent 12 }} + ports: + - containerPort: 5555 + + volumeMounts: + - name: config + mountPath: "/app/config" + readOnly: true + - name: pysnmp-cache-volume + mountPath: "/.pysnmp/" + readOnly: false + - name: tmp + mountPath: "/tmp/" + readOnly: false + volumes: + - name: config + configMap: + name: {{ include "splunk-connect-for-snmp.name" . }}-config + items: + - key: "config.yaml" + path: "config.yaml" + - name: pysnmp-cache-volume + emptyDir: {} + - name: tmp + emptyDir: {} + +{{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml b/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml new file mode 100644 index 000000000..dc3d8f3ed --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.flower.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower + labels: + {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }} + + +spec: + selector: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 4 }} + ports: + - protocol: TCP + port: {{ .Values.flower.port | default 80 }} + targetPort: 5555 + type: LoadBalancer + {{- if .Values.flower.loadBalancerIP }} + externalIPs: + - {{ .Values.flower.loadBalancerIP }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/values.schema.json b/charts/splunk-connect-for-snmp/values.schema.json index c8036dca0..8a3eac876 100644 --- a/charts/splunk-connect-for-snmp/values.schema.json +++ b/charts/splunk-connect-for-snmp/values.schema.json @@ -40,6 +40,21 @@ "pysnmpDebug": { "type": "string" }, + "flower": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "loadBalancerIP": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + }, "UI": { "type": "object", "additionalProperties": false, diff --git a/charts/splunk-connect-for-snmp/values.yaml b/charts/splunk-connect-for-snmp/values.yaml index 96a4ce1b9..6edb801c6 100644 --- a/charts/splunk-connect-for-snmp/values.yaml +++ b/charts/splunk-connect-for-snmp/values.yaml @@ -49,6 +49,11 @@ UI: # valuesFileDirectory directory regardless of valuesFileName proper configuration. keepSectionFiles: false +flower: + enabled: false + port: 80 + loadBalancerIP: '' + ################################################################################ # Splunk Cloud / Splunk Enterprise configuration. ################################################################################ diff --git a/docker_compose/.env b/docker_compose/.env index c13ee915e..4eca26c9e 100644 --- a/docker_compose/.env +++ b/docker_compose/.env @@ -95,4 +95,6 @@ TRAP_LOG_LEVEL=INFO # Scheduler configuration SCHEDULER_LOG_LEVEL=INFO +FLOWER_PORT=80 + #Secrets diff --git a/docker_compose/docker-compose.yaml b/docker_compose/docker-compose.yaml index 8df07ca20..2b578c5f2 100644 --- a/docker_compose/docker-compose.yaml +++ b/docker_compose/docker-compose.yaml @@ -234,6 +234,28 @@ services: reservations: cpus: ${WORKER_TRAP_CPU_RESERVATIONS:-0.25} memory: ${WORKER_TRAP_MEMORY_RESERVATIONS:-250M} + flower: + <<: [*dns_and_networks, *dependency_and_restart_policy] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + command: [celery, flower] + environment: + <<: [ + *general_sc4snmp_data, + *splunk_general_setup, + *splunk_extended_setup, + *workers_general_setup, + *pysnmp_debug + ] + profiles: [debug] + ports: + - mode: host + protocol: tcp + published: ${FLOWER_PORT} + target: 5555 + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - worker-flower-pysnmp-cache-volume:/.pysnmp/:rw + - worker-flower-tmp:/tmp/:rw volumes: snmp-mibserver-tmp: inventory-tmp: @@ -248,3 +270,5 @@ volumes: worker-sender-pysnmp-cache-volume: worker-trap-tmp: worker-trap-pysnmp-cache-volume: + worker-flower-tmp: + worker-flower-pysnmp-cache-volume: diff --git a/docs/internal/flower.md b/docs/internal/flower.md new file mode 100644 index 000000000..5448fe2bf --- /dev/null +++ b/docs/internal/flower.md @@ -0,0 +1,18 @@ +# Celery admin panel (internal) + +Main component of SC4SNMP architecture is Celery framework that can run background and periodic tasks (like polling and walk). It can be helpful to administrate Celery cluster using admin panel with [Flower](https://github.com/mher/flower). + +## k8s + +Just add `flower` section on `values.yaml`: + +``` +flower: + enabled: true + loadBalancerIP: x.x.x.x +``` + + +## docker-compose + +Just use `--profile debug` flag to run optional `flower` service together with SC4SNMP: `docker-compose --profile debug up` \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 9cd252155..89b78b8a1 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -25,6 +25,9 @@ celery) worker-sender) celery -A splunk_connect_for_snmp.poller worker -l "$LOG_LEVEL" -Q send --autoscale=6,"$WORKER_CONCURRENCY" ;; + flower) + celery -A splunk_connect_for_snmp.poller flower + ;; *) celery "$2" ;; diff --git a/poetry.lock b/poetry.lock index 08b63a35b..5df31030b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "amqp" @@ -417,6 +417,24 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "flower" +version = "2.0.1" +description = "Celery Flower" +optional = false +python-versions = ">=3.7" +files = [ + {file = "flower-2.0.1-py2.py3-none-any.whl", hash = "sha256:9db2c621eeefbc844c8dd88be64aef61e84e2deb29b271e02ab2b5b9f01068e2"}, + {file = "flower-2.0.1.tar.gz", hash = "sha256:5ab717b979530770c16afb48b50d2a98d23c3e9fe39851dcf6bc4d01845a02a0"}, +] + +[package.dependencies] +celery = ">=5.0.5" +humanize = "*" +prometheus-client = ">=0.8.0" +pytz = "*" +tornado = ">=5.0.0,<7.0.0" + [[package]] name = "ghp-import" version = "2.1.0" @@ -434,6 +452,20 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] +[[package]] +name = "humanize" +version = "4.10.0" +description = "Python humanize utilities" +optional = false +python-versions = ">=3.8" +files = [ + {file = "humanize-4.10.0-py3-none-any.whl", hash = "sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6"}, + {file = "humanize-4.10.0.tar.gz", hash = "sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978"}, +] + +[package.extras] +tests = ["freezegun", "pytest", "pytest-cov"] + [[package]] name = "idna" version = "3.7" @@ -1150,6 +1182,20 @@ files = [ {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, ] +[[package]] +name = "prometheus-client" +version = "0.21.1" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.8" +files = [ + {file = "prometheus_client-0.21.1-py3-none-any.whl", hash = "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301"}, + {file = "prometheus_client-0.21.1.tar.gz", hash = "sha256:252505a722ac04b0456be05c05f75f45d760c2911ffc45f2a06bcaed9f3ae3fb"}, +] + +[package.extras] +twisted = ["twisted"] + [[package]] name = "prompt-toolkit" version = "3.0.30" @@ -2063,6 +2109,26 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tornado" +version = "6.4.2" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">=3.8" +files = [ + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, + {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, + {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, + {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, +] + [[package]] name = "typing-extensions" version = "4.3.0" @@ -2317,4 +2383,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "8fc9e4f9972418fec948be2960d7454911de3e854f7740f16d89e0f454f080df" +content-hash = "e0d7310d84b21cc4d70e2af6162cf0d82a73a527eaabee82d031f8fbee193aeb" diff --git a/pyproject.toml b/pyproject.toml index 74d9aa567..5e3af6924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ JSON-log-formatter ="^0.5.1" pysnmplib = {git = "https://github.com/pysnmp/pysnmp.git", branch = "main"} urllib3 = "^1.26.17" jsonschema = "4.23.0" +flower = "^2.0.1" [tool.poetry.group.dev.dependencies] pytest = "^8.0.0"