diff --git a/CHANGELOG.md b/CHANGELOG.md index 3097a1221..6a1e85fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changed - add IF-MIB.ifOperStatus to baseIF profile - allow to set subnets for docker network configuration from .env file +- add admin panel for celery ### Fixed diff --git a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl index 7e3e6647e..04a225b5c 100644 --- a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl +++ b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl @@ -54,6 +54,11 @@ app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-tr app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.flower.selectorLabels" -}} +app.kubernetes.io/name: {{ include "splunk-connect-for-snmp.worker.name" . }}-flower +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + {{/* Common labels @@ -78,6 +83,11 @@ Common labels {{ include "splunk-connect-for-snmp.labels" . }} {{- end }} +{{- define "splunk-connect-for-snmp.worker.flower.labels" -}} +{{ include "splunk-connect-for-snmp.worker.flower.selectorLabels" . }} +{{ include "splunk-connect-for-snmp.labels" . }} +{{- end }} + {{- define "environmental-variables" -}} - name: CONFIG_PATH value: /app/config/config.yaml diff --git a/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml b/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml new file mode 100644 index 000000000..e2704a83d --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/flower/deployment.yaml @@ -0,0 +1,69 @@ +{{- if .Values.flower.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower + labels: + {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }} + +spec: + replicas: 1 + selector: + matchLabels: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "splunk-connect-for-snmp.serviceAccountName" . }} + securityContext: + fsGroup: 10001 + containers: + - name: {{ .Chart.Name }}-flower + securityContext: + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + [ + "celery", "flower", + ] + env: + {{- include "environmental-variables" . | nindent 12 }} + ports: + - containerPort: 5555 + + volumeMounts: + - name: config + mountPath: "/app/config" + readOnly: true + - name: pysnmp-cache-volume + mountPath: "/.pysnmp/" + readOnly: false + - name: tmp + mountPath: "/tmp/" + readOnly: false + volumes: + - name: config + configMap: + name: {{ include "splunk-connect-for-snmp.name" . }}-config + items: + - key: "config.yaml" + path: "config.yaml" + - name: pysnmp-cache-volume + emptyDir: {} + - name: tmp + emptyDir: {} + +{{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml b/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml new file mode 100644 index 000000000..dc3d8f3ed --- /dev/null +++ b/charts/splunk-connect-for-snmp/templates/worker/flower/service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.flower.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "splunk-connect-for-snmp.worker.fullname" . }}-flower + labels: + {{- include "splunk-connect-for-snmp.worker.flower.labels" . | nindent 4 }} + + +spec: + selector: + {{- include "splunk-connect-for-snmp.worker.flower.selectorLabels" . | nindent 4 }} + ports: + - protocol: TCP + port: {{ .Values.flower.port | default 80 }} + targetPort: 5555 + type: LoadBalancer + {{- if .Values.flower.loadBalancerIP }} + externalIPs: + - {{ .Values.flower.loadBalancerIP }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/splunk-connect-for-snmp/values.schema.json b/charts/splunk-connect-for-snmp/values.schema.json index c8036dca0..8a3eac876 100644 --- a/charts/splunk-connect-for-snmp/values.schema.json +++ b/charts/splunk-connect-for-snmp/values.schema.json @@ -40,6 +40,21 @@ "pysnmpDebug": { "type": "string" }, + "flower": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "loadBalancerIP": { + "type": "string" + }, + "port": { + "type": "integer" + } + } + }, "UI": { "type": "object", "additionalProperties": false, diff --git a/charts/splunk-connect-for-snmp/values.yaml b/charts/splunk-connect-for-snmp/values.yaml index 96a4ce1b9..6edb801c6 100644 --- a/charts/splunk-connect-for-snmp/values.yaml +++ b/charts/splunk-connect-for-snmp/values.yaml @@ -49,6 +49,11 @@ UI: # valuesFileDirectory directory regardless of valuesFileName proper configuration. keepSectionFiles: false +flower: + enabled: false + port: 80 + loadBalancerIP: '' + ################################################################################ # Splunk Cloud / Splunk Enterprise configuration. ################################################################################ diff --git a/docker_compose/docker-compose.yaml b/docker_compose/docker-compose.yaml index 8df07ca20..d17099be3 100644 --- a/docker_compose/docker-compose.yaml +++ b/docker_compose/docker-compose.yaml @@ -234,6 +234,28 @@ services: reservations: cpus: ${WORKER_TRAP_CPU_RESERVATIONS:-0.25} memory: ${WORKER_TRAP_MEMORY_RESERVATIONS:-250M} + flower: + <<: [*dns_and_networks, *dependency_and_restart_policy] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + command: [celery, flower] + environment: + <<: [ + *general_sc4snmp_data, + *splunk_general_setup, + *splunk_extended_setup, + *workers_general_setup, + *pysnmp_debug + ] + profiles: [debug] + ports: + - mode: host + protocol: tcp + published: 80 + target: 5555 + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - worker-flower-pysnmp-cache-volume:/.pysnmp/:rw + - worker-flower-tmp:/tmp/:rw volumes: snmp-mibserver-tmp: inventory-tmp: @@ -248,3 +270,5 @@ volumes: worker-sender-pysnmp-cache-volume: worker-trap-tmp: worker-trap-pysnmp-cache-volume: + worker-flower-tmp: + worker-flower-pysnmp-cache-volume: diff --git a/docs/internal/flower.md b/docs/internal/flower.md new file mode 100644 index 000000000..d1ed4b5e6 --- /dev/null +++ b/docs/internal/flower.md @@ -0,0 +1,18 @@ +# Celery admin panel (internal) + +Main component of SC4SNMP architecture is Celery framworks that can run background and periodic tasks (like polling and walk). By this reason super helpfull might be administrate Celery cluster using admin panel and we using for that [Flower](https://github.com/mher/flower). + +## k8s + +Just add `flower` section on `values.yaml`: + +``` +flower: + enabled: true + loadBalancerIP: x.x.x.x +``` + + +## docker-compose + +Just use `--profile debug` flag to run optional `flower` service together with SC4SNMP: `docker-compose --profile debug up` \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 9cd252155..89b78b8a1 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -25,6 +25,9 @@ celery) worker-sender) celery -A splunk_connect_for_snmp.poller worker -l "$LOG_LEVEL" -Q send --autoscale=6,"$WORKER_CONCURRENCY" ;; + flower) + celery -A splunk_connect_for_snmp.poller flower + ;; *) celery "$2" ;; diff --git a/poetry.lock b/poetry.lock index 08b63a35b..5df31030b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "amqp" @@ -417,6 +417,24 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "flower" +version = "2.0.1" +description = "Celery Flower" +optional = false +python-versions = ">=3.7" +files = [ + {file = "flower-2.0.1-py2.py3-none-any.whl", hash = "sha256:9db2c621eeefbc844c8dd88be64aef61e84e2deb29b271e02ab2b5b9f01068e2"}, + {file = "flower-2.0.1.tar.gz", hash = "sha256:5ab717b979530770c16afb48b50d2a98d23c3e9fe39851dcf6bc4d01845a02a0"}, +] + +[package.dependencies] +celery = ">=5.0.5" +humanize = "*" +prometheus-client = ">=0.8.0" +pytz = "*" +tornado = ">=5.0.0,<7.0.0" + [[package]] name = "ghp-import" version = "2.1.0" @@ -434,6 +452,20 @@ python-dateutil = ">=2.8.1" [package.extras] dev = ["flake8", "markdown", "twine", "wheel"] +[[package]] +name = "humanize" +version = "4.10.0" +description = "Python humanize utilities" +optional = false +python-versions = ">=3.8" +files = [ + {file = "humanize-4.10.0-py3-none-any.whl", hash = "sha256:39e7ccb96923e732b5c2e27aeaa3b10a8dfeeba3eb965ba7b74a3eb0e30040a6"}, + {file = "humanize-4.10.0.tar.gz", hash = "sha256:06b6eb0293e4b85e8d385397c5868926820db32b9b654b932f57fa41c23c9978"}, +] + +[package.extras] +tests = ["freezegun", "pytest", "pytest-cov"] + [[package]] name = "idna" version = "3.7" @@ -1150,6 +1182,20 @@ files = [ {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, ] +[[package]] +name = "prometheus-client" +version = "0.21.1" +description = "Python client for the Prometheus monitoring system." +optional = false +python-versions = ">=3.8" +files = [ + {file = "prometheus_client-0.21.1-py3-none-any.whl", hash = "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301"}, + {file = "prometheus_client-0.21.1.tar.gz", hash = "sha256:252505a722ac04b0456be05c05f75f45d760c2911ffc45f2a06bcaed9f3ae3fb"}, +] + +[package.extras] +twisted = ["twisted"] + [[package]] name = "prompt-toolkit" version = "3.0.30" @@ -2063,6 +2109,26 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tornado" +version = "6.4.2" +description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +optional = false +python-versions = ">=3.8" +files = [ + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1"}, + {file = "tornado-6.4.2-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946"}, + {file = "tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73"}, + {file = "tornado-6.4.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c"}, + {file = "tornado-6.4.2-cp38-abi3-win32.whl", hash = "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482"}, + {file = "tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38"}, + {file = "tornado-6.4.2.tar.gz", hash = "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b"}, +] + [[package]] name = "typing-extensions" version = "4.3.0" @@ -2317,4 +2383,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "8fc9e4f9972418fec948be2960d7454911de3e854f7740f16d89e0f454f080df" +content-hash = "e0d7310d84b21cc4d70e2af6162cf0d82a73a527eaabee82d031f8fbee193aeb" diff --git a/pyproject.toml b/pyproject.toml index 74d9aa567..5e3af6924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ JSON-log-formatter ="^0.5.1" pysnmplib = {git = "https://github.com/pysnmp/pysnmp.git", branch = "main"} urllib3 = "^1.26.17" jsonschema = "4.23.0" +flower = "^2.0.1" [tool.poetry.group.dev.dependencies] pytest = "^8.0.0"