feat: partial walk

charts/splunk-connect-for-snmp/templates/inventory/job.yaml

@@ -50,6 +50,8 @@ spec:
             value: {{ .Values.scheduler.tasksExpiryTime | quote }}
           - name: CONFIG_FROM_MONGO
             value: {{ quote .Values.UI.enable | default "false" }}
+          - name: ENABLE_FULL_WALK
+            value: {{ .Values.poller.enableFullWalk | default "false" | quote }}
           volumeMounts:
             - name: config
               mountPath: "/app/config"

charts/splunk-connect-for-snmp/templates/ui/_helpers.tpl

@@ -51,6 +51,8 @@ spec:
             value: {{ .Values.scheduler.logLevel | default "INFO" }}
           - name: CONFIG_FROM_MONGO
             value: {{ quote .Values.UI.enable | default "false" }}
+          - name: ENABLE_FULL_WALK
+            value: {{ .Values.poller.enableFullWalk | default "false" | quote }}
           volumeMounts:
             - name: config
               mountPath: "/app/config"

charts/splunk-connect-for-snmp/values.yaml

@@ -226,6 +226,9 @@ poller:
   # https://splunk.github.io/splunk-connect-for-snmp/main/configuration/poller-configuration/#define-usernamesecrets
   usernameSecrets: []
 
+  # flag to enable polling full walk tree for devices
+  enableFullWalk: false
+
   # Here is where polling happens. Learn more on how to configure it here:
   # https://splunk.github.io/splunk-connect-for-snmp/main/configuration/poller-configuration/
 

integration_tests/splunk_test_utils.py

@@ -51,6 +51,13 @@ def splunk_single_search(service, search):
 
 
 inventory_template = """poller:
+  enableFullWalk: true
+  inventory: |
+    address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete
+"""
+
+inventory_template_no_walk = """poller:
+  enableFullWalk: false
   inventory: |
     address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete
 """
@@ -77,6 +84,7 @@ def splunk_single_search(service, search):
 
 TEMPLATE_MAPPING = {
     "inventory.yaml": inventory_template,
+    "inventory2.yaml": inventory_template_no_walk,
     "profiles.yaml": profiles_template,
     "scheduler_secrets.yaml": poller_secrets_template,
     "traps_secrets.yaml": traps_secrets_template,

integration_tests/test_poller_integration.py

@@ -379,12 +379,12 @@ def setup_small_walk(request):
         },
     }
     update_profiles(profile)
-    update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,"], "inventory.yaml")
-    upgrade_helm(["inventory.yaml", "profiles.yaml"])
+    update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,"], "inventory2.yaml")
+    upgrade_helm(["inventory2.yaml", "profiles.yaml"])
     time.sleep(30)
     yield
-    update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,t"], "inventory.yaml")
-    upgrade_helm(["inventory.yaml"])
+    update_file([f"{trap_external_ip},,2c,public,,,20,walk1,f,t"], "inventory2.yaml")
+    upgrade_helm(["inventory2.yaml"])
     time.sleep(20)
 
 

integration_tests/values.yaml

@@ -65,6 +65,7 @@ scheduler:
   groups: |
     {}
 poller:
+  enableFullWalk: true
   usernameSecrets:
     - sv3poller
  #   - sc4snmp-hlab-sha-aes

rendered/manifests/tests/splunk-connect-for-snmp/templates/inventory/job.yaml

@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml

@@ -105,6 +105,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml

@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml

@@ -104,6 +104,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/inventory/job.yaml

@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml

@@ -104,6 +104,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/inventory/job.yaml

@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "true"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/ui/configmap-backend.yaml

@@ -53,6 +53,8 @@ data:
                 value: INFO
               - name: CONFIG_FROM_MONGO
                 value: "true"
+              - name: ENABLE_FULL_WALK
+                value: "false"
               volumeMounts:
                 - name: config
                   mountPath: "/app/config"

rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml

@@ -105,6 +105,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/inventory/job.yaml

@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml

@@ -105,6 +105,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/inventory/job.yaml

@@ -48,6 +48,8 @@ spec:
             value: "60"
           - name: CONFIG_FROM_MONGO
             value: "false"
+          - name: ENABLE_FULL_WALK
+            value: "false"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml

@@ -105,6 +105,12 @@ spec:
               value: "4"
             - name: PREFETCH_COUNT
               value: "30"
+            - name: RESOLVE_TRAP_ADDRESS
+              value: "false"
+            - name: MAX_DNS_CACHE_SIZE_TRAPS
+              value: "500"
+            - name: TTL_DNS_CACHE_TRAPS
+              value: "1800"
           volumeMounts:
             - name: config
               mountPath: "/app/config"

splunk_connect_for_snmp/common/inventory_processor.py

@@ -1,5 +1,7 @@
 import copy
+import logging
 import os
+import sys
 from contextlib import suppress
 from csv import DictReader
 from typing import List
@@ -29,6 +31,20 @@
     "security_engine",
     "securityEngine",
 ]
+ENABLE_FULL_WALK = human_bool(os.getenv("ENABLE_FULL_WALK", "false").lower())
+from splunk_connect_for_snmp.common.customised_json_formatter import (
+    CustomisedJSONFormatter,
+)
+
+formatter = CustomisedJSONFormatter()
+logger = logging.getLogger(__name__)
+logger.setLevel("DEBUG")
+
+# writing to stdout
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel("DEBUG")
+handler.setFormatter(formatter)
+logger.addHandler(handler)
 
 
 def transform_key_to_address(target):
@@ -88,22 +104,21 @@ def get_groups_keys(list_of_groups, group_name, inventory_group_port_mapping):
 
 
 class InventoryProcessor:
-    def __init__(self, group_manager: GroupsManager, logger, inventory_ui_collection):
+    def __init__(self, group_manager: GroupsManager, inventory_ui_collection):
         self.inventory_records: List[dict] = []
         self.group_manager = group_manager
-        self.logger = logger
         self.hosts_from_groups: dict = {}
         self.inventory_group_port_mapping: dict = {}
         self.single_hosts: List[dict] = []
         self.inventory_ui_collection = inventory_ui_collection
 
     def get_all_hosts(self):
         if CONFIG_FROM_MONGO:
-            self.logger.info("Loading inventory from inventory_ui collection")
+            logger.info("Loading inventory from inventory_ui collection")
             ir_reader = list(self.inventory_ui_collection.find({}, {"_id": 0}))
         else:
             with open(INVENTORY_PATH, encoding="utf-8") as csv_file:
-                self.logger.info(f"Loading inventory from {INVENTORY_PATH}")
+                logger.info(f"Loading inventory from {INVENTORY_PATH}")
                 ir_reader = list(DictReader(csv_file))
         for inventory_line in ir_reader:
             self.process_line(inventory_line)
@@ -115,7 +130,7 @@ def get_all_hosts(self):
             if was_present is None:
                 self.inventory_records.append(source_record)
             else:
-                self.logger.warning(
+                logger.warning(
                     f"Record: {host} has been already configured in group. Skipping..."
                 )
         return self.inventory_records, self.inventory_group_port_mapping
@@ -124,7 +139,7 @@ def process_line(self, source_record):
         address = source_record["address"]
         # Inventory record is commented out
         if address.startswith("#"):
-            self.logger.warning(f"Record: {address} is commented out. Skipping...")
+            logger.warning(f"Record: {address} is commented out. Skipping...")
         # Address is an IP address
         elif address[0].isdigit():
             self.single_hosts.append(source_record)
@@ -146,7 +161,7 @@ def get_group_hosts(self, source_object, group_name):
                     if key in ALLOWED_KEYS_VALUES:
                         host_group_object[key] = group_object[key]
                     else:
-                        self.logger.warning(
+                        logger.warning(
                             f"Key {key} is not allowed to be changed from the group level"
                         )
                 address = str(group_object["address"])
@@ -156,27 +171,26 @@ def get_group_hosts(self, source_object, group_name):
                 host_group_object["group"] = group_name
                 self.inventory_records.append(host_group_object)
         else:
-            self.logger.warning(
+            logger.warning(
                 f"Group {group_name} doesn't exist in the configuration. Treating {group_name} as a hostname"
             )
             self.single_hosts.append(source_object)
 
 
 class InventoryRecordManager:
-    def __init__(self, mongo_client, periodic_objects_collection, logger):
+    def __init__(self, mongo_client, periodic_objects_collection):
         self.targets_collection = mongo_client.sc4snmp.targets
         self.inventory_collection = mongo_client.sc4snmp.inventory
         self.attributes_collection = mongo_client.sc4snmp.attributes
         self.periodic_object_collection = periodic_objects_collection
-        self.logger = logger
 
     def delete(self, target):
         address, port = transform_key_to_address(target)
         self.periodic_object_collection.delete_all_tasks_of_host(target)
         self.inventory_collection.delete_one({"address": address, "port": port})
         self.targets_collection.delete_many({"address": target})
         self.attributes_collection.delete_many({"address": target})
-        self.logger.info(f"Deleting record: {target}")
+        logger.info(f"Deleting record: {target}")
 
     def update(
         self, inventory_record, new_source_record, runtime_profiles, expiry_time_changed
@@ -191,13 +205,13 @@ def update(
             upsert=True,
         )
         if status.matched_count == 0:
-            self.logger.info(f"New Record {inventory_record} {status.upserted_id}")
+            logger.info(f"New Record {inventory_record} {status.upserted_id}")
         elif status.modified_count == 1 and status.upserted_id is None:
-            self.logger.info(f"Modified Record {inventory_record}")
+            logger.info(f"Modified Record {inventory_record}")
         else:
-            self.logger.info(f"Unchanged Record {inventory_record}")
+            logger.info(f"Unchanged Record {inventory_record}")
             if expiry_time_changed:
-                self.logger.info(
+                logger.info(
                     f"Task expiry time was modified, generating new tasks for record {inventory_record}"
                 )
             else:
@@ -211,6 +225,8 @@ def update(
 
     def return_walk_profile(self, runtime_profiles, inventory_profiles):
         walk_profile = None
+        if ENABLE_FULL_WALK:
+            return None
         if inventory_profiles:
             walk_profiles = [
                 p
@@ -221,4 +237,6 @@ def return_walk_profile(self, runtime_profiles, inventory_profiles):
             if walk_profiles:
                 # if there's more than one walk profile, we're choosing the last one on the list
                 walk_profile = walk_profiles[-1]
+        if not walk_profile:
+            walk_profile = "WalkProfile"
         return walk_profile

splunk_connect_for_snmp/inventory/loader.py

@@ -183,12 +183,8 @@ def load():
     new_groups = groups_manager.return_collection()
 
     inventory_ui_collection = mongo_client.sc4snmp.inventory_ui
-    inventory_processor = InventoryProcessor(
-        groups_manager, logger, inventory_ui_collection
-    )
-    inventory_record_manager = InventoryRecordManager(
-        mongo_client, periodic_obj, logger
-    )
+    inventory_processor = InventoryProcessor(groups_manager, inventory_ui_collection)
+    inventory_record_manager = InventoryRecordManager(mongo_client, periodic_obj)
     if CONFIG_FROM_MONGO:
         logger.info(f"Loading inventory from inventory_ui collection")
     else:

splunk_connect_for_snmp/profiles/walk.yaml

@@ -0,0 +1,5 @@
+WalkProfile:
+  condition:
+    type: "walk"
+  varBinds:
+    - ['SNMPv2-MIB']