feat: partial walk

splunk · Apr 26, 2024 · 94f3a47 · 94f3a47
1 parent 1ffd1ae
commit 94f3a47
Show file tree

Hide file tree

Showing 23 changed files with 293 additions and 45 deletions.
diff --git a/charts/splunk-connect-for-snmp/templates/inventory/job.yaml b/charts/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -50,6 +50,8 @@ spec:
             value: {{ .Values.scheduler.tasksExpiryTime | quote }}
           - name: CONFIG_FROM_MONGO
             value: {{ quote .Values.UI.enable | default "false" }}
+          - name: ENABLE_FULL_WALK
+            value: {{ .Values.poller.enableFullWalk | default "false" | quote }}
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/charts/splunk-connect-for-snmp/templates/ui/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/ui/_helpers.tpl
@@ -51,6 +51,8 @@ spec:
             value: {{ .Values.scheduler.logLevel | default "INFO" }}
           - name: CONFIG_FROM_MONGO
             value: {{ quote .Values.UI.enable | default "false" }}
+          - name: ENABLE_FULL_WALK
+            value: {{ .Values.poller.enableFullWalk | default "false" | quote }}
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/charts/splunk-connect-for-snmp/values.yaml b/charts/splunk-connect-for-snmp/values.yaml
@@ -226,6 +226,9 @@ poller:
   # https://splunk.github.io/splunk-connect-for-snmp/main/configuration/poller-configuration/#define-usernamesecrets
   usernameSecrets: []
 
+  # flag to enable polling full walk tree for devices
+  enableFullWalk: false
+
   # Here is where polling happens. Learn more on how to configure it here:
   # https://splunk.github.io/splunk-connect-for-snmp/main/configuration/poller-configuration/
 

diff --git a/integration_tests/splunk_test_utils.py b/integration_tests/splunk_test_utils.py
@@ -51,6 +51,13 @@ def splunk_single_search(service, search):
 
 
 inventory_template = """poller:
+  enableFullWalk: true
+  inventory: |
+    address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete
+"""
+
+inventory_template_no_walk = """poller:
+  enableFullWalk: false
   inventory: |
     address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete
 """
@@ -77,6 +84,7 @@ def splunk_single_search(service, search):
 
 TEMPLATE_MAPPING = {
     "inventory.yaml": inventory_template,
+    "inventory2.yaml": inventory_template_no_walk,
     "profiles.yaml": profiles_template,
     "scheduler_secrets.yaml": poller_secrets_template,
     "traps_secrets.yaml": traps_secrets_template,

diff --git a/integration_tests/test_poller_integration.py b/integration_tests/test_poller_integration.py
@@ -379,12 +379,12 @@ def setup_small_walk(request):
         },
     }
     update_profiles(profile)
-    update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,"], "inventory.yaml")
-    upgrade_helm(["inventory.yaml", "profiles.yaml"])
+    update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,"], "inventory2.yaml")
+    upgrade_helm(["inventory2.yaml", "profiles.yaml"])
     time.sleep(30)
     yield
-    update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,t"], "inventory.yaml")
-    upgrade_helm(["inventory.yaml"])
+    update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,t"], "inventory2.yaml")
+    upgrade_helm(["inventory2.yaml"])
     time.sleep(20)
 
 
@@ -410,6 +410,89 @@ def test_check_if_walk_scope_was_smaller(self, setup_splunk):
         assert metric_count > 0
 
 
+@pytest.fixture
+def setup_small_walk_with_full_walk_enabled(request):
+    trap_external_ip = request.config.getoption("trap_external_ip")
+    profile = {
+        "walk1": {
+            "condition": {"type": "walk"},
+            "varBinds": [yaml_escape_list(sq("IP-MIB"))],
+        },
+    }
+    update_profiles(profile)
+    update_file([f"{trap_external_ip},,2c,public,,,1810,walk1,f,"], "inventory.yaml")
+    upgrade_helm(["inventory.yaml", "profiles.yaml"])
+    time.sleep(30)
+    yield
+    update_file([f"{trap_external_ip},,2c,public,,,1810,walk1,f,t"], "inventory.yaml")
+    upgrade_helm(["inventory.yaml"])
+    time.sleep(20)
+
+
+@pytest.mark.usefixtures("setup_small_walk_with_full_walk_enabled")
+class TestSmallWalkWithFullWalkEnabled:
+    def test_check_if_full_walk_is_done_with_profile_set(self, setup_splunk):
+        time.sleep(30)
+        search_string = (
+            """| mpreview index=netmetrics earliest=-30s | search "TCP-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 1
+        )
+        assert result_count > 0
+        assert metric_count > 0
+        search_string = (
+            """| mpreview index=netmetrics earliest=-30s | search "IP-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 2
+        )
+        assert result_count > 0
+        assert metric_count > 0
+
+
+@pytest.fixture
+def setup_walk(request):
+    trap_external_ip = request.config.getoption("trap_external_ip")
+    update_file([f"{trap_external_ip},,2c,public,,,20,,f,"], "inventory2.yaml")
+    upgrade_helm(["inventory2.yaml", "profiles.yaml"])
+    time.sleep(30)
+    yield
+    update_file([f"{trap_external_ip},,2c,public,,,20,,f,t"], "inventory2.yaml")
+    upgrade_helm(["inventory2.yaml"])
+    time.sleep(20)
+
+
+@pytest.mark.usefixtures("setup_walk")
+class TestPartialWalk:
+    def test_check_if_partial_walk_is_done(self, setup_splunk):
+        time.sleep(20)
+        search_string = (
+            """| mpreview index=netmetrics earliest=-20s | search "TCP-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 1
+        )
+        assert result_count == 0
+        assert metric_count == 0
+        search_string = (
+            """| mpreview index=netmetrics earliest=-20s | search "IP-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 2
+        )
+        assert result_count == 0
+        assert metric_count == 0
+        search_string = (
+            """| mpreview index=netmetrics earliest=-20s | search "SNMPv2-MIB" """
+        )
+        result_count, metric_count = run_retried_single_search(
+            setup_splunk, search_string, 2
+        )
+        assert result_count > 0
+        assert metric_count > 0
+
+
 @pytest.fixture()
 def setup_v3_connection(request):
     trap_external_ip = request.config.getoption("trap_external_ip")

diff --git a/integration_tests/values.yaml b/integration_tests/values.yaml
@@ -65,6 +65,7 @@ scheduler:
   groups: |
     {}
 poller:
+  enableFullWalk: true
   usernameSecrets:
     - sv3poller
  #   - sc4snmp-hlab-sha-aes

diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -105,6 +105,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/.../manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml b/.../manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/...s/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/...s/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -104,6 +104,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/...tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/inventory/job.yaml b/...tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/...oscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/...oscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -104,6 +104,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "true"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/...red/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/configmap-backend.yaml b/...red/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/configmap-backend.yaml
@@ -53,6 +53,8 @@ data:
                 value: INFO
               - name: CONFIG_FROM_MONGO
                 value: "true"
+              - name: ENABLE_FULL_WALK
+                value: "false"
               volumeMounts:
                 - name: config
                   mountPath: "/app/config"

diff --git a/...d/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/...d/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -105,6 +105,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/.../manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/.../manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -105,6 +105,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml b/rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml
@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

diff --git a/...ifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/...ifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -105,6 +105,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"