From d1b687d3e4398c2e2b5721a934c4da24587917bc Mon Sep 17 00:00:00 2001 From: Ilya Kheifets Date: Tue, 10 Dec 2024 10:50:40 +0100 Subject: [PATCH] fix: add flower Signed-off-by: Ilya Kheifets --- .../templates/worker/_helpers.tpl | 10 +++ .../templates/worker/flower/deployment.yaml | 66 +++++++++++++++++ .../templates/worker/flower/service.yaml | 22 ++++++ .../values.schema.json | 15 ++++ charts/splunk-connect-for-snmp/values.yaml | 5 ++ docker_compose/docker-compose.yaml | 19 +++++ entrypoint.sh | 3 + poetry.lock | 70 ++++++++++++++++++- pyproject.toml | 1 + 9 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml create mode 100644 charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml diff --git a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl index 7e3e6647e..04a225b5c 100644 --- a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl +++ b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl @@ -54,6 +54,11 @@ app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-tr app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.flower.selectorLabels" -}} +app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-flower +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + {{/* Common labels @@ -78,6 +83,11 @@ Common labels {{ include "splunk-connect-for-snmp.labels" . }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.flower.labels" -}} +{{ include "splunk-connect-for-snmp.worker.flower.selectorLabels" . }} +{{ include "splunk-connect-for-snmp.labels" . }} +{{- end }} + {{- define "environmental-variables" -}} - name: CONFIG_PATH value: /app/config/config.yaml diff --git a/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml b/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml new file mode 100644 index 000000000..3c023b2e1 --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml @@ -0,0 +1,66 @@ +{{- if .Values.flower.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower + labels: + {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }} + +spec: + replicas: 1 + selector: + matchLabels: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: flower + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + [ + "celery", "flower", + ] + env: + {{- include "environmental-variables" . | nindent 12 }} + ports: + - containerPort: 5555 + + volumeMounts: + - name: config + mountPath: "/app/config" + readOnly: true + - name: pysnmp-cache-volume + mountPath: "/.pysnmp/" + readOnly: false + - name: tmp + mountPath: "/tmp/" + readOnly: false + volumes: + - name: config + configMap: + name: {{ include "splunk-connect-for-snmp.name" . }}-config + items: + - key: "config.yaml" + path: "config.yaml" + - name: pysnmp-cache-volume + emptyDir: {} + - name: tmp + emptyDir: {} + +{{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml b/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml new file mode 100644 index 000000000..dc3d8f3ed --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.flower.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower + labels: + {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }} + + +spec: + selector: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 4 }} + ports: + - protocol: TCP + port: {{ .Values.flower.port | default 80 }} + targetPort: 5555 + type: LoadBalancer + {{- if .Values.flower.loadBalancerIP }} + externalIPs: + - {{ .Values.flower.loadBalancerIP }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/values.schema.json b/charts/splunk-connect-for-snmp/values.schema.json index c8036dca0..8a3eac876 100644 --- a/charts/splunk-connect-for-snmp/values.schema.json +++ b/charts/splunk-connect-for-snmp/values.schema.json @@ -40,6 +40,21 @@ "pysnmpDebug": { "type": "string" }, + "flower": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "loadBalancerIP": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + }, "UI": { "type": "object", "additionalProperties": false, diff --git a/charts/splunk-connect-for-snmp/values.yaml b/charts/splunk-connect-for-snmp/values.yaml index 96a4ce1b9..6edb801c6 100644 --- a/charts/splunk-connect-for-snmp/values.yaml +++ b/charts/splunk-connect-for-snmp/values.yaml @@ -49,6 +49,11 @@ UI: # valuesFileDirectory directory regardless of valuesFileName proper configuration. keepSectionFiles: false +flower: + enabled: false + port: 80 + loadBalancerIP: '' + ################################################################################ # Splunk Cloud / Splunk Enterprise configuration. ################################################################################ diff --git a/docker_compose/docker-compose.yaml b/docker_compose/docker-compose.yaml index 8df07ca20..ec04004fd 100644 --- a/docker_compose/docker-compose.yaml +++ b/docker_compose/docker-compose.yaml @@ -234,6 +234,23 @@ services: reservations: cpus: ${WORKER_TRAP_CPU_RESERVATIONS:-0.25} memory: ${WORKER_TRAP_MEMORY_RESERVATIONS:-250M} + flower: + <<: [*dns_and_networks, *dependency_and_restart_policy] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + command: [celery, flower] + environment: + <<: [ + *general_sc4snmp_data, + *splunk_general_setup, + *splunk_extended_setup, + *workers_general_setup, + *pysnmp_debug + ] + profiles: [debug] + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - worker-flower-pysnmp-cache-volume:/.pysnmp/:rw + - worker-flower-tmp:/tmp/:rw volumes: snmp-mibserver-tmp: inventory-tmp: @@ -248,3 +265,5 @@ volumes: worker-sender-pysnmp-cache-volume: worker-trap-tmp: worker-trap-pysnmp-cache-volume: + worker-flower-tmp: + worker-flower-pysnmp-cache-volume: diff --git a/entrypoint.sh b/entrypoint.sh index 9cd252155..89b78b8a1 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -25,6 +25,9 @@ celery) worker-sender) celery -A splunk_connect_for_snmp.poller worker -l "$LOG_LEVEL" -Q send --autoscale=6,"$WORKER_CONCURRENCY" ;; + flower) + celery -A splunk_connect_for_snmp.poller flower + ;; *) celery "$2" ;; diff --git a/poetry.lock b/poetry.lock index 08b63a35b..5df31030b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "amqp" @@ -417,6 +417,24 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "flower" +version = "2.0.1" +description = "Celery Flower" +optional = false +python-versions = ">=3.7" +files = [ + {file = "flower-2.0.1-py2.py3-none-any.whl", hash = "sha256:9db2c621eeefbc844c8dd88be64aef61e84e2deb29b271e02ab2b5b9f01068e2"}, + {file = "flower-2.0.1.tar.gz", hash = "sha256:5ab717b979530770c16afb48b50d2a98d23c3e9fe39851dcf6bc4d01845a02a0"}, +] + +[package.dependencies] +celery = ">=5.0.5" +humanize = "*" +prometheus-client = ">=0.8.0" +pytz = "*" +tornado = ">=5.0.0,<7.0.0" + [[package]] name = "ghp-import" version = "2.1.0" @@ -434,6 +452,20 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] +[[package]] +name = "humanize" +version = "4.10.0" +description = "Python humanize utilities" +optional = false +python-versions = ">=3.8" +files = [ + {file = "humanize-4.10.0-py3-none-any.whl", hash = "sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6"}, + {file = "humanize-4.10.0.tar.gz", hash = "sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978"}, +] + +[package.extras] +tests = ["freezegun", "pytest", "pytest-cov"] + [[package]] name = "idna" version = "3.7" @@ -1150,6 +1182,20 @@ files = [ {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, ] +[[package]] +name = "prometheus-client" +version = "0.21.1" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.8" +files = [ + {file = "prometheus_client-0.21.1-py3-none-any.whl", hash = "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301"}, + {file = "prometheus_client-0.21.1.tar.gz", hash = "sha256:252505a722ac04b0456be05c05f75f45d760c2911ffc45f2a06bcaed9f3ae3fb"}, +] + +[package.extras] +twisted = ["twisted"] + [[package]] name = "prompt-toolkit" version = "3.0.30" @@ -2063,6 +2109,26 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tornado" +version = "6.4.2" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">=3.8" +files = [ + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, + {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, + {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, + {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, +] + [[package]] name = "typing-extensions" version = "4.3.0" @@ -2317,4 +2383,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "8fc9e4f9972418fec948be2960d7454911de3e854f7740f16d89e0f454f080df" +content-hash = "e0d7310d84b21cc4d70e2af6162cf0d82a73a527eaabee82d031f8fbee193aeb" diff --git a/pyproject.toml b/pyproject.toml index 74d9aa567..5e3af6924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ JSON-log-formatter ="^0.5.1" pysnmplib = {git = "https://github.com/pysnmp/pysnmp.git", branch = "main"} urllib3 = "^1.26.17" jsonschema = "4.23.0" +flower = "^2.0.1" [tool.poetry.group.dev.dependencies] pytest = "^8.0.0"