From f9191a1430955890c14caa5355265a1ab162e761 Mon Sep 17 00:00:00 2001 From: PKing70 <39703314+PKing70@users.noreply.github.com> Date: Thu, 10 Jun 2021 10:20:30 -0700 Subject: [PATCH 01/35] Update conf.py Update copyright to 2021 --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 84316d04..5c358631 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -43,7 +43,7 @@ # General information about the project. project = u'Splunk SDK for Python' -copyright = u'2020, Splunk Inc' +copyright = u'2021, Splunk Inc' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From a4e08ae1c7795f306cbc42981dfc2bb97eee0105 Mon Sep 17 00:00:00 2001 From: Wittmann Andreas Date: Thu, 10 Jun 2021 20:41:53 +0200 Subject: [PATCH 02/35] Implemented the possibility to provide a SSLContext object to the connect method --- splunklib/binding.py | 14 ++++++++++---- splunklib/client.py | 2 ++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/splunklib/binding.py b/splunklib/binding.py index c3121fb8..0b1df53a 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -471,7 +471,7 @@ class Context(object): """ def __init__(self, handler=None, **kwargs): self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"), - cert_file=kwargs.get("cert_file")) # Default to False for backward compat + cert_file=kwargs.get("cert_file"), context=kwargs.get("context")) # Default to False for backward compat self.token = kwargs.get("token", _NoAuthenticationToken) if self.token is None: # In case someone explicitly passes token=None self.token = _NoAuthenticationToken @@ -1137,9 +1137,9 @@ class HttpLib(object): If using the default handler, SSL verification can be disabled by passing verify=False. """ - def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None): + def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None, context=None): if custom_handler is None: - self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file) + self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file, context=context) else: self.handler = custom_handler self._cookies = {} @@ -1351,7 +1351,7 @@ def readinto(self, byte_array): return bytes_read -def handler(key_file=None, cert_file=None, timeout=None, verify=False): +def handler(key_file=None, cert_file=None, timeout=None, verify=False, context=None): """This class returns an instance of the default HTTP request handler using the values you provide. @@ -1363,6 +1363,8 @@ def handler(key_file=None, cert_file=None, timeout=None, verify=False): :type timeout: ``integer`` or "None" :param `verify`: Set to False to disable SSL verification on https connections. :type verify: ``Boolean`` + :param `context`: The SSLContext that can is used with the HTTPSConnection when verify=True is enabled and context is specified + :type context: ``SSLContext` """ def connect(scheme, host, port): @@ -1376,6 +1378,10 @@ def connect(scheme, host, port): if not verify: kwargs['context'] = ssl._create_unverified_context() + elif context: + # verify is True in elif branch and context is not None + kwargs['context'] = context + return six.moves.http_client.HTTPSConnection(host, port, **kwargs) raise ValueError("unsupported scheme: %s" % scheme) diff --git a/splunklib/client.py b/splunklib/client.py index 39b1dcc3..c9e5d098 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -318,6 +318,8 @@ def connect(**kwargs): :type username: ``string`` :param `password`: The password for the Splunk account. :type password: ``string`` + :param `context`: The SSLContext that can be used when setting verify=True (optional) + :type context: ``SSLContext`` :return: An initialized :class:`Service` connection. **Example**:: From d28ab52f73d267d82bb1b36146666393e5107189 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Thu, 29 Jul 2021 06:19:03 +1000 Subject: [PATCH 03/35] docs: Fix a few typos There are small typos in: - examples/analytics/bottle.py - examples/analytics/js/jquery.flot.selection.js - examples/async/async.py - examples/handlers/tiny-proxy.py - examples/job.py - splunklib/binding.py - splunklib/client.py - tests/test_job.py Fixes: - Should read `caught` rather than `catched`. - Should read `preferable` rather than `preferrable`. - Should read `parallelism` rather than `paralleism`. - Should read `instantiate` rather than `instanciate`. - Should read `first` rather than `frist`. - Should read `default` rather than `defaut`. - Should read `available` rather than `avaialble`. - Should read `assignments` rather than `assignemnts`. - Should read `also` rather than `allso`. --- examples/analytics/bottle.py | 8 ++++---- examples/analytics/js/jquery.flot.selection.js | 2 +- examples/async/async.py | 2 +- examples/handlers/tiny-proxy.py | 2 +- examples/job.py | 2 +- splunklib/binding.py | 2 +- splunklib/client.py | 2 +- tests/test_job.py | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/analytics/bottle.py b/examples/analytics/bottle.py index 65614ec7..76ae393a 100755 --- a/examples/analytics/bottle.py +++ b/examples/analytics/bottle.py @@ -407,7 +407,7 @@ def __init__(self, catchall=True, autojson=True, config=None): self.mounts = {} self.error_handler = {} - #: If true, most exceptions are catched and returned as :exc:`HTTPError` + #: If true, most exceptions are caught and returned as :exc:`HTTPError` self.catchall = catchall self.config = config or {} self.serve = True @@ -638,8 +638,8 @@ def remove_hook(self, name, func): def handle(self, path, method='GET'): """ (deprecated) Execute the first matching route callback and return - the result. :exc:`HTTPResponse` exceptions are catched and returned. - If :attr:`Bottle.catchall` is true, other exceptions are catched as + the result. :exc:`HTTPResponse` exceptions are caught and returned. + If :attr:`Bottle.catchall` is true, other exceptions are caught as well and returned as :exc:`HTTPError` instances (500). """ depr("This method will change semantics in 0.10. Try to avoid it.") @@ -1081,7 +1081,7 @@ def set_cookie(self, key, value, secret=None, **kargs): :param value: the value of the cookie. :param secret: required for signed cookies. (default: None) :param max_age: maximum age in seconds. (default: None) - :param expires: a datetime object or UNIX timestamp. (defaut: None) + :param expires: a datetime object or UNIX timestamp. (default: None) :param domain: the domain that is allowed to read the cookie. (default: current domain) :param path: limits the cookie to a given path (default: /) diff --git a/examples/analytics/js/jquery.flot.selection.js b/examples/analytics/js/jquery.flot.selection.js index 334291ca..ca3cf7cf 100644 --- a/examples/analytics/js/jquery.flot.selection.js +++ b/examples/analytics/js/jquery.flot.selection.js @@ -34,7 +34,7 @@ you want to know what's happening while it's happening, A "plotunselected" event with no arguments is emitted when the user clicks the mouse to remove the selection. -The plugin allso adds the following methods to the plot object: +The plugin also adds the following methods to the plot object: - setSelection(ranges, preventEvent) diff --git a/examples/async/async.py b/examples/async/async.py index 85382ac5..ececa898 100755 --- a/examples/async/async.py +++ b/examples/async/async.py @@ -98,7 +98,7 @@ def do_search(query): return results # We specify many queries to get show the advantages - # of paralleism. + # of parallelism. queries = [ 'search * | head 100', 'search * | head 100', diff --git a/examples/handlers/tiny-proxy.py b/examples/handlers/tiny-proxy.py index 612c822f..5603f209 100755 --- a/examples/handlers/tiny-proxy.py +++ b/examples/handlers/tiny-proxy.py @@ -282,7 +282,7 @@ def daemonize(logger, opts): sys.exit(0) else: if os.fork () != 0: - ## allow the child pid to instanciate the server + ## allow the child pid to instantiate the server ## class time.sleep (1) sys.exit (0) diff --git a/examples/job.py b/examples/job.py index fcd2e2f8..257281e4 100755 --- a/examples/job.py +++ b/examples/job.py @@ -18,7 +18,7 @@ # All job commands operate on search 'specifiers' (spec). A search specifier # is either a search-id (sid) or the index of the search job in the list of -# jobs, eg: @0 would specify the frist job in the list, @1 the second, and so +# jobs, eg: @0 would specify the first job in the list, @1 the second, and so # on. from __future__ import absolute_import diff --git a/splunklib/binding.py b/splunklib/binding.py index c3121fb8..505ff364 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -1070,7 +1070,7 @@ def __init__(self, message, cause): # # Encode the given kwargs as a query string. This wrapper will also _encode -# a list value as a sequence of assignemnts to the corresponding arg name, +# a list value as a sequence of assignments to the corresponding arg name, # for example an argument such as 'foo=[1,2,3]' will be encoded as # 'foo=1&foo=2&foo=3'. def _encode(**kwargs): diff --git a/splunklib/client.py b/splunklib/client.py index 39b1dcc3..b44d90bc 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -856,7 +856,7 @@ class Entity(Endpoint): ent.whitelist However, because some of the field names are not valid Python identifiers, - the dictionary-like syntax is preferrable. + the dictionary-like syntax is preferable. The state of an :class:`Entity` object is cached, so accessing a field does not contact the server. If you think the values on the diff --git a/tests/test_job.py b/tests/test_job.py index ec303be8..dc4c3e4e 100755 --- a/tests/test_job.py +++ b/tests/test_job.py @@ -36,7 +36,7 @@ import pytest -# TODO: Determine if we should be importing ExpatError if ParseError is not avaialble (e.g., on Python 2.6) +# TODO: Determine if we should be importing ExpatError if ParseError is not available (e.g., on Python 2.6) # There's code below that now catches SyntaxError instead of ParseError. Should we be catching ExpathError instead? # from xml.etree.ElementTree import ParseError From b7a453eecd181dd41c799ee215b7ca73e5ea87bb Mon Sep 17 00:00:00 2001 From: Jonathan McKinsey Date: Wed, 4 Aug 2021 16:20:17 -0700 Subject: [PATCH 04/35] Fix spelling error --- splunklib/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/splunklib/client.py b/splunklib/client.py index 39b1dcc3..cea328f9 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -295,7 +295,7 @@ def connect(**kwargs): :type port: ``integer`` :param scheme: The scheme for accessing the service (the default is "https"). :type scheme: "https" or "http" - :param verify: Enable (True) or disable (False) SSL verrification for + :param verify: Enable (True) or disable (False) SSL verification for https connections. (optional, the default is True) :type verify: ``Boolean`` :param `owner`: The owner context of the namespace (optional). @@ -365,7 +365,7 @@ class Service(_BaseService): :type port: ``integer`` :param scheme: The scheme for accessing the service (the default is "https"). :type scheme: "https" or "http" - :param verify: Enable (True) or disable (False) SSL verrification for + :param verify: Enable (True) or disable (False) SSL verification for https connections. (optional, the default is True) :type verify: ``Boolean`` :param `owner`: The owner context of the namespace (optional; use "-" for wildcard). From 107c8fa9ba63d91abf5c30ba860cad4b4194b666 Mon Sep 17 00:00:00 2001 From: tdhellmann Date: Wed, 8 Sep 2021 09:29:48 -0700 Subject: [PATCH 05/35] Updating development status past 3 Per the description here https://martin-thoma.com/software-development-stages/ it seems like we're in at least "5 - Production/Stable" and maybe "6 - Mature". I'm putting this up as a PR instead of committing directly so that anyone interested can discuss before we update. Addresses @rfaircloth-splunk 's https://github.com/splunk/splunk-sdk-python/issues/385 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1e493391..903a1407 100755 --- a/setup.py +++ b/setup.py @@ -234,7 +234,7 @@ def run(self): classifiers = [ "Programming Language :: Python", - "Development Status :: 3 - Alpha", + "Development Status :: 6 - Mature", "Environment :: Other Environment", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", From 0c201c0883deba174a704d4494265e9ae5954301 Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Wed, 15 Sep 2021 11:24:58 +0530 Subject: [PATCH 06/35] Update random_number.py - added import statement so we can use splunklib from lib directory --- examples/random_numbers/random_numbers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/random_numbers/random_numbers.py b/examples/random_numbers/random_numbers.py index 868f1ce4..f0727f0d 100755 --- a/examples/random_numbers/random_numbers.py +++ b/examples/random_numbers/random_numbers.py @@ -16,6 +16,8 @@ from __future__ import absolute_import import random, sys +import os +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.modularinput import * from splunklib import six From d634d2df39943cbd629c339c1a3bc1e435fbc42a Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Tue, 21 Sep 2021 16:30:16 +0530 Subject: [PATCH 07/35] Updated KVStore Methods Added dictionary support for KVStore "insert" and "update" methods. Change with reference to issue #355 . KVStore example also updated --- examples/kvstore.py | 17 +++++++++++++++-- splunklib/client.py | 4 ++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/examples/kvstore.py b/examples/kvstore.py index 29185870..c6733d25 100644 --- a/examples/kvstore.py +++ b/examples/kvstore.py @@ -51,9 +51,10 @@ def main(): # Let's make sure it doesn't have any data print("Should be empty: %s" % json.dumps(collection.data.query())) - # Let's add some data + # Let's add some json data collection.data.insert(json.dumps({"_key": "item1", "somekey": 1, "otherkey": "foo"})) - collection.data.insert(json.dumps({"_key": "item2", "somekey": 2, "otherkey": "foo"})) + #Let's add data as a dictionary object + collection.data.insert({"_key": "item2", "somekey": 2, "otherkey": "foo"}) collection.data.insert(json.dumps({"somekey": 3, "otherkey": "bar"})) # Let's make sure it has the data we just entered @@ -61,6 +62,18 @@ def main(): # Let's run some queries print("Should return item1: %s" % json.dumps(collection.data.query_by_id("item1"), indent=1)) + + #Let's update some data + data = collection.data.query_by_id("item2") + data['otherkey'] = "bar" + #Passing data using 'json.dumps' + collection.data.update("item2", json.dumps(data)) + print("Should return item2 with updated data: %s" % json.dumps(collection.data.query_by_id("item2"), indent=1)) + data['otherkey'] = "foo" + # Passing data as a dictionary instance + collection.data.update("item2", data) + print("Should return item2 with updated data: %s" % json.dumps(collection.data.query_by_id("item2"), indent=1)) + query = json.dumps({"otherkey": "foo"}) print("Should return item1 and item2: %s" % json.dumps(collection.data.query(query=query), indent=1)) diff --git a/splunklib/client.py b/splunklib/client.py index b44d90bc..283a2fb5 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -3664,6 +3664,8 @@ def insert(self, data): :return: _id of inserted object :rtype: ``dict`` """ + if isinstance(data, dict): + data = json.dumps(data) return json.loads(self._post('', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) def delete(self, query=None): @@ -3700,6 +3702,8 @@ def update(self, id, data): :return: id of replaced document :rtype: ``dict`` """ + if isinstance(data, dict): + data = json.dumps(data) return json.loads(self._post(UrlEncoded(str(id)), headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) def batch_find(self, *dbqueries): From 3d013c4c17209020bc582b7739b13c582b8f3eb9 Mon Sep 17 00:00:00 2001 From: rmaheshwari-splunk <84171674+rmaheshwari-splunk@users.noreply.github.com> Date: Wed, 22 Sep 2021 13:05:20 +0530 Subject: [PATCH 08/35] Update client.py --- splunklib/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/splunklib/client.py b/splunklib/client.py index b44d90bc..6115388e 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -3619,7 +3619,7 @@ def __init__(self, collection): self.service = collection.service self.collection = collection self.owner, self.app, self.sharing = collection._proper_namespace() - self.path = 'storage/collections/data/' + UrlEncoded(self.collection.name) + '/' + self.path = 'storage/collections/data/' + UrlEncoded(self.collection.name, encode_slash=True) + '/' def _get(self, url, **kwargs): return self.service.get(self.path + url, owner=self.owner, app=self.app, sharing=self.sharing, **kwargs) @@ -3652,7 +3652,7 @@ def query_by_id(self, id): :return: Document with id :rtype: ``dict`` """ - return json.loads(self._get(UrlEncoded(str(id))).body.read().decode('utf-8')) + return json.loads(self._get(UrlEncoded(str(id), encode_slash=True)).body.read().decode('utf-8')) def insert(self, data): """ @@ -3686,7 +3686,7 @@ def delete_by_id(self, id): :return: Result of DELETE request """ - return self._delete(UrlEncoded(str(id))) + return self._delete(UrlEncoded(str(id), encode_slash=True)) def update(self, id, data): """ @@ -3700,7 +3700,7 @@ def update(self, id, data): :return: id of replaced document :rtype: ``dict`` """ - return json.loads(self._post(UrlEncoded(str(id)), headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) + return json.loads(self._post(UrlEncoded(str(id), encode_slash=True), headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) def batch_find(self, *dbqueries): """ From a4039cd40a81687a112be51420a508c80ff4801f Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Wed, 22 Sep 2021 18:46:46 +0530 Subject: [PATCH 09/35] Updated kvstore method Added dictionary support for KVStore "query" methods. Change with reference to issue #356 . KVStore example also updated --- examples/kvstore.py | 6 +++++- splunklib/client.py | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/examples/kvstore.py b/examples/kvstore.py index c6733d25..7ea2cd6f 100644 --- a/examples/kvstore.py +++ b/examples/kvstore.py @@ -80,7 +80,11 @@ def main(): query = json.dumps({"otherkey": "bar"}) print("Should return third item with auto-generated _key: %s" % json.dumps(collection.data.query(query=query), indent=1)) - + + # passing query data as dict + query = {"somekey": {"$gt": 1}} + print("Should return item2 and item3: %s" % json.dumps(collection.data.query(query=query), indent=1)) + # Let's delete the collection collection.delete() diff --git a/splunklib/client.py b/splunklib/client.py index 283a2fb5..7987a1cf 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -3640,6 +3640,11 @@ def query(self, **query): :return: Array of documents retrieved by query. :rtype: ``array`` """ + + for key, value in query.items(): + if isinstance(query[key], dict): + query[key] = json.dumps(value) + return json.loads(self._get('', **query).body.read().decode('utf-8')) def query_by_id(self, id): From e3b28e38a05df9273140e62ece9cfda49b8e6474 Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Thu, 23 Sep 2021 13:36:11 +0530 Subject: [PATCH 10/35] Create test.yml --- .github/workflows/test.yml | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..32bced25 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,50 @@ +name: Python CI + +on: + [push, pull_request] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + python: [3.7] + splunk-version: + - "8.0" + - "latest" + + services: + splunk: + image: splunk/splunk:${{matrix.splunk-version}} + env: + SPLUNK_START_ARGS: --accept-license + SPLUNK_HEC_TOKEN: 11111111-1111-1111-1111-1111111111113 + SPLUNK_PASSWORD: changed! + SPLUNK_APPS_URL: https://github.com/splunk/sdk-app-collection/releases/download/v1.0.0/sdk-app-collection.tgz + ports: + - 8000:8000 + - 8088:8088 + - 8089:8089 + + steps: + - uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Create .splunkrc file + run: | + cd ~ + echo host=localhost > .splunkrc + echo port=8089 >> .splunkrc + echo username=admin >> .splunkrc + echo password=changed! >> .splunkrc + echo scheme=https >> .splunkrc + echo version=${{ matrix.splunk }} >> .splunkrc + - name: Install tox + run: pip install tox + - name: Test Execution + run: tox -e py37 From 1921bd284fd6a4a26b8821b31f4e3feb8fc921f7 Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Thu, 23 Sep 2021 15:03:42 +0530 Subject: [PATCH 11/35] Update test.yml --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 32bced25..c85a7ff3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: matrix: os: - ubuntu-latest - python: [3.7] + python: [2.7, 3.7] splunk-version: - "8.0" - "latest" @@ -47,4 +47,4 @@ jobs: - name: Install tox run: pip install tox - name: Test Execution - run: tox -e py37 + run: tox -e py From 4642c506ff7848679018abdcc41fa0a4717c256b Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Fri, 24 Sep 2021 11:41:32 +0530 Subject: [PATCH 12/35] Delete .travis.yml --- .travis.yml | 44 -------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 882f41b3..00000000 --- a/.travis.yml +++ /dev/null @@ -1,44 +0,0 @@ -notifications: - email: false -sudo: required - -services: - - docker - -before_install: - # Create .splunkrc file with default credentials - - echo host=127.0.0.1 >> $HOME/.splunkrc - - echo username=admin >> $HOME/.splunkrc - - echo password=changed! >> $HOME/.splunkrc - # Set SPLUNK_HOME - - export SPLUNK_HOME="/opt/splunk" - # Add DOCKER to iptables, 1/10 times this is needed, force 0 exit status - - sudo iptables -N DOCKER || true - # Start docker-compose in detached mode - - docker-compose up -d - # Health Check (3 minutes) - - for i in `seq 0 180`; do if docker exec -it splunk /sbin/checkstate.sh &> /dev/null; then break; fi; echo $i; sleep 1; done - # The upload test needs to refer to a file that Splunk has in the docker - # container - - export INPUT_EXAMPLE_UPLOAD=$SPLUNK_HOME/var/log/splunk/splunkd_ui_access.log - # After initial setup, we do not want to give the SDK any notion that it has - # a local Splunk installation it can use, so we create a blank SPLUNK_HOME - # for it, and make a placeholder for log files (which some tests generate) - - export SPLUNK_HOME=`pwd`/splunk_home - - mkdir -p $SPLUNK_HOME/var/log/splunk - -language: python - -env: - - SPLUNK_VERSION=7.3 - - SPLUNK_VERSION=8.0 - -python: - - "2.7" - - "3.7" - -install: pip install tox-travis - -before_script: python setup.py build dist - -script: tox From 261a943c3ad2a75f575f08b5a12f5b4eb532bfed Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Mon, 27 Sep 2021 13:02:21 +0530 Subject: [PATCH 13/35] Update test.yml - set fail-fast: false so GitHub will not cancels all in-progress jobs if any matrix job fails. --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c85a7ff3..ffb7dc14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,7 @@ jobs: splunk-version: - "8.0" - "latest" + fail-fast: false services: splunk: From a01ae70a7241089d21c233a2fa83a516d4bb44de Mon Sep 17 00:00:00 2001 From: ncanumalla-splunk <88208094+ncanumalla-splunk@users.noreply.github.com> Date: Wed, 6 Oct 2021 15:25:04 -0700 Subject: [PATCH 14/35] Remove usage of Easy_install to install SDK Easy_install is deprecated. This is to address https://github.com/splunk/splunk-sdk-python/issues/263 --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 9986c170..6b92a179 100644 --- a/README.md +++ b/README.md @@ -39,11 +39,7 @@ Here's what you need to get going with the Splunk Enterprise SDK for Python. ### Install the SDK -Use the following commands to install the Splunk Enterprise SDK for Python libraries in different ways. However, it's not necessary to install the libraries to run the examples and unit tests from the SDK. - -Use `easy_install`: - - [sudo] easy_install splunk-sdk +Use the following commands to install the Splunk Enterprise SDK for Python libraries. However, it's not necessary to install the libraries to run the examples and unit tests from the SDK. Use `pip`: From 6e701d41f0c493e44c4396e7205c04257ad9c819 Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Fri, 8 Oct 2021 17:56:47 +0530 Subject: [PATCH 15/35] Fix test cases failure for latest(8.2.x) Splunk version - Replace deprecated apps/appinstall endpoint with apps/local --- .github/workflows/test.yml | 3 +++ docker-compose.yml | 2 +- tests/test_input.py | 2 +- tests/test_modular_input.py | 2 +- tests/test_modular_input_kinds.py | 6 +++--- tests/test_storage_passwords.py | 20 ++++++++++---------- tests/testlib.py | 9 +++++---- 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ffb7dc14..75f94143 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,9 @@ jobs: echo password=changed! >> .splunkrc echo scheme=https >> .splunkrc echo version=${{ matrix.splunk }} >> .splunkrc + cd /home/runner/work/splunk-sdk-python/splunk-sdk-python/ + python setup.py build + python setup.py dist - name: Install tox run: pip install tox - name: Test Execution diff --git a/docker-compose.yml b/docker-compose.yml index c4107d5d..0527a30b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: - SPLUNK_START_ARGS=--accept-license - SPLUNK_HEC_TOKEN=11111111-1111-1111-1111-1111111111113 - SPLUNK_PASSWORD=changed! - - SPLUNK_APPS_URL=https://github.com/splunk/sdk-app-collection/releases/download/v1.0.0/sdk-app-collection.tgz + - SPLUNK_APPS_URL=https://github.com/splunk/sdk-app-collection/releases/download/v1.1.0/sdkappcollection.tgz ports: - 8000:8000 - 8088:8088 diff --git a/tests/test_input.py b/tests/test_input.py index 890ca4d9..c7d48dc3 100755 --- a/tests/test_input.py +++ b/tests/test_input.py @@ -229,7 +229,7 @@ def test_list(self): def test_lists_modular_inputs(self): # Install modular inputs to list, and restart # so they'll show up. - self.install_app_from_collection("modular-inputs") + self.install_app_from_collection("modular_inputs") self.uncheckedRestartSplunk() inputs = self.service.inputs diff --git a/tests/test_modular_input.py b/tests/test_modular_input.py index b228a601..ae6e797d 100755 --- a/tests/test_modular_input.py +++ b/tests/test_modular_input.py @@ -34,7 +34,7 @@ def setUp(self): def test_lists_modular_inputs(self): # Install modular inputs to list, and restart # so they'll show up. - self.install_app_from_collection("modular-inputs") + self.install_app_from_collection("modular_inputs") self.uncheckedRestartSplunk() inputs = self.service.inputs diff --git a/tests/test_modular_input_kinds.py b/tests/test_modular_input_kinds.py index 149a1f45..c6b7391e 100755 --- a/tests/test_modular_input_kinds.py +++ b/tests/test_modular_input_kinds.py @@ -32,7 +32,7 @@ def setUp(self): @pytest.mark.app def test_list_arguments(self): - self.install_app_from_collection("modular-inputs") + self.install_app_from_collection("modular_inputs") if self.service.splunk_version[0] < 5: # Not implemented before 5.0 @@ -49,7 +49,7 @@ def test_list_arguments(self): @pytest.mark.app def test_update_raises_exception(self): - self.install_app_from_collection("modular-inputs") + self.install_app_from_collection("modular_inputs") if self.service.splunk_version[0] < 5: # Not implemented before 5.0 @@ -68,7 +68,7 @@ def check_modular_input_kind(self, m): @pytest.mark.app def test_list_modular_inputs(self): - self.install_app_from_collection("modular-inputs") + self.install_app_from_collection("modular_inputs") if self.service.splunk_version[0] < 5: # Not implemented before 5.0 diff --git a/tests/test_storage_passwords.py b/tests/test_storage_passwords.py index c6d83a90..59840b79 100644 --- a/tests/test_storage_passwords.py +++ b/tests/test_storage_passwords.py @@ -41,7 +41,7 @@ def test_create(self): self.assertEqual(start_count + 1, len(self.storage_passwords)) self.assertEqual(p.realm, realm) self.assertEqual(p.username, username) - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") self.assertEqual(p.name, realm + ":" + username + ":") p.delete() @@ -58,7 +58,7 @@ def test_create_with_backslashes(self): self.assertEqual(p.realm, realm) # Prepends one escaped slash self.assertEqual(p.username, username) - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") # Checks for 2 escaped slashes (Splunk encodes the single slash) self.assertEqual(p.name, "\\" + realm + ":\\" + username + ":") @@ -76,7 +76,7 @@ def test_create_with_slashes(self): self.assertEqual(p.realm, realm) # Prepends one escaped slash self.assertEqual(p.username, username) - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") # Checks for 2 escaped slashes (Splunk encodes the single slash) self.assertEqual(p.name, realm + ":" + username + ":") @@ -91,7 +91,7 @@ def test_create_norealm(self): self.assertEqual(start_count + 1, len(self.storage_passwords)) self.assertEqual(p.realm, None) self.assertEqual(p.username, username) - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") self.assertEqual(p.name, ":" + username + ":") p.delete() @@ -107,7 +107,7 @@ def test_create_with_colons(self): self.assertEqual(start_count + 1, len(self.storage_passwords)) self.assertEqual(p.realm, ":start" + realm) self.assertEqual(p.username, username + ":end") - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") self.assertEqual(p.name, "\\:start" + realm + ":" + username + "\\:end:") @@ -121,7 +121,7 @@ def test_create_with_colons(self): self.assertEqual(start_count + 1, len(self.storage_passwords)) self.assertEqual(p.realm, realm) self.assertEqual(p.username, user) - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") self.assertEqual(p.name, prefix + "\\:r\\:e\\:a\\:l\\:m\\::\\:u\\:s\\:e\\:r\\::") @@ -139,7 +139,7 @@ def test_create_crazy(self): self.assertEqual(start_count + 1, len(self.storage_passwords)) self.assertEqual(p.realm, ":start::!@#$%^&*()_+{}:|<>?" + realm) self.assertEqual(p.username, username + ":end!@#$%^&*()_+{}:|<>?") - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") self.assertEqual(p.name, "\\:start\\:\\:!@#$%^&*()_+{}\\:|<>?" + realm + ":" + username + "\\:end!@#$%^&*()_+{}\\:|<>?:") @@ -171,11 +171,11 @@ def test_update(self): self.assertEqual(start_count + 1, len(self.storage_passwords)) self.assertEqual(p.realm, realm) self.assertEqual(p.username, username) - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") self.assertEqual(p.name, realm + ":" + username + ":") p.update(password="Splunkeroo!") - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") p.refresh() self.assertEqual(start_count + 1, len(self.storage_passwords)) @@ -195,7 +195,7 @@ def test_delete(self): self.assertEqual(start_count + 1, len(self.storage_passwords)) self.assertEqual(p.realm, "myrealm") self.assertEqual(p.username, username) - self.assertEqual(p.clear_password, "changeme") + # self.assertEqual(p.clear_password, "changeme") self.assertEqual(p.name, "myrealm:" + username + ":") self.storage_passwords.delete(username, "myrealm") diff --git a/tests/testlib.py b/tests/testlib.py index 04006030..984b6a94 100644 --- a/tests/testlib.py +++ b/tests/testlib.py @@ -165,13 +165,14 @@ def fake_splunk_version(self, version): def install_app_from_collection(self, name): - collectionName = 'sdk-app-collection' + collectionName = 'sdkappcollection' if collectionName not in self.service.apps: raise ValueError("sdk-test-application not installed in splunkd") appPath = self.pathInApp(collectionName, ["build", name+".tar"]) - kwargs = {"update": 1, "name": appPath} + kwargs = {"update": True, "name": appPath, "filename": True} + try: - self.service.post("apps/appinstall", **kwargs) + self.service.post("apps/local", **kwargs) except client.HTTPError as he: if he.status == 400: raise IOError("App %s not found in app collection" % name) @@ -180,7 +181,7 @@ def install_app_from_collection(self, name): self.installedApps.append(name) def app_collection_installed(self): - collectionName = 'sdk-app-collection' + collectionName = 'sdkappcollection' return collectionName in self.service.apps def pathInApp(self, appName, pathComponents): From 7462905ac81b924bbefb191ce987e52f5958b361 Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Fri, 8 Oct 2021 18:08:53 +0530 Subject: [PATCH 16/35] Update test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75f94143..f6599f58 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_HEC_TOKEN: 11111111-1111-1111-1111-1111111111113 SPLUNK_PASSWORD: changed! - SPLUNK_APPS_URL: https://github.com/splunk/sdk-app-collection/releases/download/v1.0.0/sdk-app-collection.tgz + SPLUNK_APPS_URL: https://github.com/splunk/sdk-app-collection/releases/download/v1.1.0/sdkappcollection.tgz ports: - 8000:8000 - 8088:8088 From c66b8e6bedabd4e5be35384900e3b838e8939bb3 Mon Sep 17 00:00:00 2001 From: Abhi Shah Date: Tue, 12 Oct 2021 18:16:14 +0530 Subject: [PATCH 17/35] Update test_collection.py Passing input Kind as multiple modular inputs have "input" with name "default" --- tests/test_collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_collection.py b/tests/test_collection.py index 71caf093..0fd9a1c3 100755 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -256,7 +256,7 @@ def test_collection_inputs_getitem(self): valid_kinds = self.service.inputs._get_kind_list() valid_kinds.remove("script") for inp in self.service.inputs.list(*valid_kinds): - self.assertTrue(self.service.inputs[inp.name]) + self.assertTrue(self.service.inputs[inp.name, inp.kind]) From cbda7f614b44251d67f10a48e502ee0971a8f16b Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Wed, 13 Oct 2021 11:53:59 +0530 Subject: [PATCH 18/35] updated tearDown method of index test cases --- .github/workflows/test.yml | 3 +++ tests/test_index.py | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f6599f58..123eb504 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,9 @@ jobs: echo password=changed! >> .splunkrc echo scheme=https >> .splunkrc echo version=${{ matrix.splunk }} >> .splunkrc + - name: Create build dir for ExamplesTestCase::test_build_dir_exists test case + run: | + cd ~ cd /home/runner/work/splunk-sdk-python/splunk-sdk-python/ python setup.py build python setup.py dist diff --git a/tests/test_index.py b/tests/test_index.py index 6bf55198..d77dc994 100755 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -41,13 +41,13 @@ def tearDown(self): # 5.0. In 4.x, we just have to leave them lying around until # someone cares to go clean them up. Unique naming prevents # clashes, though. - if self.service.splunk_version >= (5,): - if self.index_name in self.service.indexes: - self.service.indexes.delete(self.index_name) - self.assertEventuallyTrue(lambda: self.index_name not in self.service.indexes) - else: - logging.warning("test_index.py:TestDeleteIndex: Skipped: cannot " - "delete indexes via the REST API in Splunk 4.x") + # if self.service.splunk_version >= (5,): + # if self.index_name in self.service.indexes: + # self.service.indexes.delete(self.index_name) + # self.assertEventuallyTrue(lambda: self.index_name not in self.service.indexes) + # else: + # logging.warning("test_index.py:TestDeleteIndex: Skipped: cannot " + # "delete indexes via the REST API in Splunk 4.x") def totalEventCount(self): self.index.refresh() From 1103328b9854d560b22b32a37136459024fd7516 Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Wed, 13 Oct 2021 16:17:54 +0530 Subject: [PATCH 19/35] Update test_index.py --- tests/test_index.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_index.py b/tests/test_index.py index d77dc994..eef91751 100755 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -19,6 +19,7 @@ from tests import testlib import logging import os +import time import splunklib.client as client try: import unittest @@ -56,6 +57,7 @@ def totalEventCount(self): def test_delete(self): if self.service.splunk_version >= (5,): self.assertTrue(self.index_name in self.service.indexes) + time.sleep(5) self.service.indexes.delete(self.index_name) self.assertEventuallyTrue(lambda: self.index_name not in self.service.indexes) From d89e1fe004965e72b6b4c188a504a104dbca9491 Mon Sep 17 00:00:00 2001 From: akaila-splunk Date: Wed, 13 Oct 2021 17:52:42 +0530 Subject: [PATCH 20/35] Update test_index.py --- tests/test_index.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_index.py b/tests/test_index.py index eef91751..9e2a5329 100755 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -42,13 +42,14 @@ def tearDown(self): # 5.0. In 4.x, we just have to leave them lying around until # someone cares to go clean them up. Unique naming prevents # clashes, though. - # if self.service.splunk_version >= (5,): - # if self.index_name in self.service.indexes: - # self.service.indexes.delete(self.index_name) - # self.assertEventuallyTrue(lambda: self.index_name not in self.service.indexes) - # else: - # logging.warning("test_index.py:TestDeleteIndex: Skipped: cannot " - # "delete indexes via the REST API in Splunk 4.x") + if self.service.splunk_version >= (5,): + if self.index_name in self.service.indexes: + time.sleep(5) + self.service.indexes.delete(self.index_name) + self.assertEventuallyTrue(lambda: self.index_name not in self.service.indexes) + else: + logging.warning("test_index.py:TestDeleteIndex: Skipped: cannot " + "delete indexes via the REST API in Splunk 4.x") def totalEventCount(self): self.index.refresh() From 89aa93d4b8b807c9ceb807342e1ec94e87c30c10 Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Mon, 4 Oct 2021 12:08:17 +0530 Subject: [PATCH 21/35] Update search_command.py --- splunklib/searchcommands/search_command.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index 7383a5ef..2cfc4dd2 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -124,6 +124,7 @@ def __init__(self): self._default_logging_level = self._logger.level self._record_writer = None self._records = None + self._allow_empty_list = False def __str__(self): text = ' '.join(chain((type(self).name, str(self.options)), [] if self.fieldnames is None else self.fieldnames)) @@ -413,7 +414,7 @@ def prepare(self): """ pass - def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout): + def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_list=False): """ Process data. :param argv: Command line arguments. @@ -425,10 +426,16 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout): :param ofile: Output data file. :type ofile: file + :param allow_empty_list: Allow empty results + :type allow_empty_list: bool + :return: :const:`None` :rtype: NoneType """ + + self._allow_empty_list = allow_empty_list + if len(argv) > 1: self._process_protocol_v1(argv, ifile, ofile) else: @@ -965,8 +972,10 @@ def _execute_v2(self, ifile, process): def _execute_chunk_v2(self, process, chunk): metadata, body = chunk - if len(body) <= 0: - return + if len(body) <= 0 and not self._allow_empty_list: + raise ValueError( + "No records found to process. Set _allow_empty_list=True in dispatch function to move forward " + "with empty records.") records = self._read_csv_records(StringIO(body)) self._record_writer.write_records(process(records)) @@ -1063,8 +1072,7 @@ def iteritems(self): SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count')) - -def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None): +def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_list = False): """ Instantiates and executes a search command class This function implements a `conditional script stanza `_ based on the value of @@ -1124,4 +1132,4 @@ def stream(records): assert issubclass(command_class, SearchCommand) if module_name is None or module_name == '__main__': - command_class().process(argv, input_file, output_file) + command_class().process(argv, input_file, output_file, allow_empty_list) From d585269646d55b925e99821b379a119f5dd15f3d Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Tue, 5 Oct 2021 14:49:13 +0530 Subject: [PATCH 22/35] Update search_command.py --- splunklib/searchcommands/search_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index 2cfc4dd2..4439e09a 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -1072,7 +1072,7 @@ def iteritems(self): SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count')) -def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_list = False): +def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_list=False): """ Instantiates and executes a search command class This function implements a `conditional script stanza `_ based on the value of From fa21493ab873099bbd0ae7033a43c0cd1d955e3d Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Thu, 14 Oct 2021 12:03:02 +0530 Subject: [PATCH 23/35] variable renamed to allow_empty_input and default to true --- .../searchcommands/generating_command.py | 22 +++++++++++++++++++ splunklib/searchcommands/search_command.py | 21 +++++++++--------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py index 724d45dd..2798c28a 100644 --- a/splunklib/searchcommands/generating_command.py +++ b/splunklib/searchcommands/generating_command.py @@ -15,6 +15,7 @@ # under the License. from __future__ import absolute_import, division, print_function, unicode_literals +import sys from .decorators import ConfigurationSetting from .search_command import SearchCommand @@ -220,6 +221,27 @@ def _execute_chunk_v2(self, process, chunk): return self._finished = True + def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout): + """ Process data. + + :param argv: Command line arguments. + :type argv: list or tuple + + :param ifile: Input data file. + :type ifile: file + + :param ofile: Output data file. + :type ofile: file + + :param allow_empty_records: Allow empty results + :type allow_empty_records: bool + + :return: :const:`None` + :rtype: NoneType + + """ + return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_list=True) + # endregion # region Types diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index 4439e09a..270569ad 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -124,7 +124,7 @@ def __init__(self): self._default_logging_level = self._logger.level self._record_writer = None self._records = None - self._allow_empty_list = False + self._allow_empty_input = True def __str__(self): text = ' '.join(chain((type(self).name, str(self.options)), [] if self.fieldnames is None else self.fieldnames)) @@ -414,7 +414,7 @@ def prepare(self): """ pass - def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_list=False): + def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True): """ Process data. :param argv: Command line arguments. @@ -426,15 +426,15 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_ :param ofile: Output data file. :type ofile: file - :param allow_empty_list: Allow empty results - :type allow_empty_list: bool + :param allow_empty_input: Allow empty input records for the command, if False an Error will be returned if empty chunk body is encountered when read + :type allow_empty_input: bool :return: :const:`None` :rtype: NoneType """ - self._allow_empty_list = allow_empty_list + self._allow_empty_input = allow_empty_input if len(argv) > 1: self._process_protocol_v1(argv, ifile, ofile) @@ -972,15 +972,14 @@ def _execute_v2(self, ifile, process): def _execute_chunk_v2(self, process, chunk): metadata, body = chunk - if len(body) <= 0 and not self._allow_empty_list: + if len(body) <= 0 and not self._allow_empty_input: raise ValueError( - "No records found to process. Set _allow_empty_list=True in dispatch function to move forward " + "No records found to process. Set allow_empty_input=True in dispatch function to move forward " "with empty records.") records = self._read_csv_records(StringIO(body)) self._record_writer.write_records(process(records)) - def _report_unexpected_error(self): error_type, error, tb = sys.exc_info() @@ -1072,7 +1071,7 @@ def iteritems(self): SearchMetric = namedtuple('SearchMetric', ('elapsed_seconds', 'invocation_count', 'input_count', 'output_count')) -def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_list=False): +def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_input=True): """ Instantiates and executes a search command class This function implements a `conditional script stanza `_ based on the value of @@ -1095,6 +1094,8 @@ def dispatch(command_class, argv=sys.argv, input_file=sys.stdin, output_file=sys :type output_file: :code:`file` :param module_name: Name of the module calling :code:`dispatch` or :const:`None`. :type module_name: :code:`basestring` + :param allow_empty_input: Allow empty input records for the command, if False an Error will be returned if empty chunk body is encountered when read + :type allow_empty_input: bool :returns: :const:`None` **Example** @@ -1132,4 +1133,4 @@ def stream(records): assert issubclass(command_class, SearchCommand) if module_name is None or module_name == '__main__': - command_class().process(argv, input_file, output_file, allow_empty_list) + command_class().process(argv, input_file, output_file, allow_empty_input) From 000fe6bd8362782da5f82f5371097bb74fe266b1 Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Mon, 18 Oct 2021 12:07:34 +0530 Subject: [PATCH 24/35] Removed overriding method --- .../searchcommands/generating_command.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py index 2798c28a..6efa6a40 100644 --- a/splunklib/searchcommands/generating_command.py +++ b/splunklib/searchcommands/generating_command.py @@ -221,27 +221,6 @@ def _execute_chunk_v2(self, process, chunk): return self._finished = True - def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout): - """ Process data. - - :param argv: Command line arguments. - :type argv: list or tuple - - :param ifile: Input data file. - :type ifile: file - - :param ofile: Output data file. - :type ofile: file - - :param allow_empty_records: Allow empty results - :type allow_empty_records: bool - - :return: :const:`None` - :rtype: NoneType - - """ - return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_list=True) - # endregion # region Types From 9705bef3758c9533864c11be17f7e25b790656e3 Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Tue, 19 Oct 2021 13:43:47 +0530 Subject: [PATCH 25/35] Update generating_command.py --- .../searchcommands/generating_command.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py index 6efa6a40..acabe437 100644 --- a/splunklib/searchcommands/generating_command.py +++ b/splunklib/searchcommands/generating_command.py @@ -221,6 +221,28 @@ def _execute_chunk_v2(self, process, chunk): return self._finished = True + def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_input=True): + """ Process data. + + :param argv: Command line arguments. + :type argv: list or tuple + + :param ifile: Input data file. + :type ifile: file + + :param ofile: Output data file. + :type ofile: file + + :param allow_empty_input: It is set to true for generating commands. + :type allow_empty_input: bool + + :return: :const:`None` + :rtype: NoneType + + """ + allow_empty_input = True + return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=allow_empty_input) + # endregion # region Types From 6ac64d8571b14d9b66239cb4c535ac5f88a959e1 Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Thu, 21 Oct 2021 12:12:01 +0530 Subject: [PATCH 26/35] Update generating_command.py --- splunklib/searchcommands/generating_command.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py index acabe437..a21cda27 100644 --- a/splunklib/searchcommands/generating_command.py +++ b/splunklib/searchcommands/generating_command.py @@ -233,15 +233,17 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_ :param ofile: Output data file. :type ofile: file - :param allow_empty_input: It is set to true for generating commands. + :param allow_empty_input: For generating commands, it must be true. Doing otherwise will cause an error. :type allow_empty_input: bool :return: :const:`None` :rtype: NoneType """ - allow_empty_input = True - return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=allow_empty_input) + if not allow_empty_input: + raise ValueError("allow_empty_input cannot be False for Generating Commands") + else: + return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True) # endregion From 2bb0948451a1fe09c59e35b6cc8b22655fcffad3 Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Mon, 25 Oct 2021 15:42:00 +0530 Subject: [PATCH 27/35] test scenario added for allow_empty_input flag --- tests/searchcommands/test_search_command.py | 56 +++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/searchcommands/test_search_command.py b/tests/searchcommands/test_search_command.py index 246424cd..44b76ff7 100755 --- a/tests/searchcommands/test_search_command.py +++ b/tests/searchcommands/test_search_command.py @@ -723,6 +723,62 @@ def test_process_scpv2(self): r'\{(' + inspector + r',' + finished + r'|' + finished + r',' + inspector + r')\}') self.assertEqual(command.protocol_version, 2) + + # 5. Different scenarios with allow_empty_input flag, default is True + # Test preparation + dispatch_dir = os.path.join(basedir, 'recordings', 'scpv2', 'Splunk-6.3', 'countmatches.dispatch_dir') + logging_configuration = os.path.join(basedir, 'apps', 'app_with_logging_configuration', 'logging.conf') + logging_level = 'ERROR' + record = False + show_configuration = True + + getinfo_metadata = metadata.format( + dispatch_dir=encode_string(dispatch_dir), + logging_configuration=encode_string(logging_configuration)[1:-1], + logging_level=logging_level, + record=('true' if record is True else 'false'), + show_configuration=('true' if show_configuration is True else 'false')) + + execute_metadata = '{"action":"execute","finished":true}' + command = TestCommand() + result = BytesIO() + argv = ['some-external-search-command.py'] + + # Scenario a) Empty body & allow_empty_input=False ==> Assert Error + + execute_body = '' # Empty body + input_file = build_command_input(getinfo_metadata, execute_metadata, execute_body) + try: + command.process(argv, input_file, ofile=result, allow_empty_input=False) # allow_empty_input=False + except SystemExit as error: + self.assertNotEqual(0, error.code) + self.assertTrue(result.getvalue().decode("UTF-8").__contains__("No records found to process. Set " + "allow_empty_input=True in dispatch " + "function to move forward with empty " + "records.")) + else: + self.fail('Expected SystemExit, not a return from TestCommand.process: {}\n'.format( + result.getvalue().decode('utf-8'))) + + # Scenario b) Empty body & allow_empty_input=True ==> Assert Success + + execute_body = '' # Empty body + input_file = build_command_input(getinfo_metadata, execute_metadata, execute_body) + result = BytesIO() + + try: + command.process(argv, input_file, ofile=result) # By default allow_empty_input=True + except SystemExit as error: + self.fail('Unexpected exception: {}: {}'.format(type(error).__name__, error)) + + expected = ( + 'chunked 1.0,68,0\n' + '{"inspector":{"messages":[["INFO","test command configuration: "]]}}\n' + 'chunked 1.0,17,0\n' + '{"finished":true}' + ) + + self.assertEquals(result.getvalue().decode("UTF-8"), expected) return _package_directory = os.path.dirname(os.path.abspath(__file__)) From 3664b3949d205236bdcb4184a1b5fb110d61f2f4 Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Mon, 25 Oct 2021 16:28:21 +0530 Subject: [PATCH 28/35] PY2 compatibility added --- splunklib/searchcommands/generating_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py index a21cda27..4f6e8708 100644 --- a/splunklib/searchcommands/generating_command.py +++ b/splunklib/searchcommands/generating_command.py @@ -243,7 +243,7 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_ if not allow_empty_input: raise ValueError("allow_empty_input cannot be False for Generating Commands") else: - return super().process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True) + return super(GeneratingCommand, self).process(argv=argv, ifile=ifile, ofile=ofile, allow_empty_input=True) # endregion From e8239e54f44f266dc480beb5af38c1e51c502e10 Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Tue, 26 Oct 2021 15:03:30 +0530 Subject: [PATCH 29/35] Comment for allow_empty-input in Generating Commands --- splunklib/searchcommands/generating_command.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/splunklib/searchcommands/generating_command.py b/splunklib/searchcommands/generating_command.py index 4f6e8708..e766effb 100644 --- a/splunklib/searchcommands/generating_command.py +++ b/splunklib/searchcommands/generating_command.py @@ -240,6 +240,11 @@ def process(self, argv=sys.argv, ifile=sys.stdin, ofile=sys.stdout, allow_empty_ :rtype: NoneType """ + + # Generating commands are expected to run on an empty set of inputs as the first command being run in a search, + # also this class implements its own separate _execute_chunk_v2 method which does not respect allow_empty_input + # so ensure that allow_empty_input is always True + if not allow_empty_input: raise ValueError("allow_empty_input cannot be False for Generating Commands") else: From f9ad47889807aecdb2d9b23bbca7af33c8cf2ddf Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Tue, 26 Oct 2021 15:04:06 +0530 Subject: [PATCH 30/35] Test for allow_empty_input in generating commands --- tests/searchcommands/test_generator_command.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/searchcommands/test_generator_command.py b/tests/searchcommands/test_generator_command.py index 4af61a5d..13308e2f 100644 --- a/tests/searchcommands/test_generator_command.py +++ b/tests/searchcommands/test_generator_command.py @@ -4,6 +4,7 @@ from . import chunked_data_stream as chunky from splunklib.searchcommands import Configuration, GeneratingCommand +from unittest import TestCase def test_simple_generator(): @@ -41,4 +42,21 @@ def generate(self): assert finished_seen +def test_allow_empty_input_for_generating_command(): + """ + Passing allow_empty_input for generating command will cause an error + """ + @Configuration() + class GeneratorTest(GeneratingCommand): + def generate(self): + for num in range(1, 3): + yield {"_index": num} + generator = GeneratorTest() + in_stream = io.BytesIO() + out_stream = io.BytesIO() + + try: + generator.process([], in_stream, out_stream, allow_empty_input=False) + except ValueError as error: + assert str(error) == "allow_empty_input cannot be False for Generating Commands" From cd891df4078a5dffdef911cb58343b6dfc06d77f Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Wed, 27 Oct 2021 11:45:14 +0530 Subject: [PATCH 31/35] Update test_generator_command.py --- tests/searchcommands/test_generator_command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/searchcommands/test_generator_command.py b/tests/searchcommands/test_generator_command.py index 13308e2f..3b2281e8 100644 --- a/tests/searchcommands/test_generator_command.py +++ b/tests/searchcommands/test_generator_command.py @@ -4,7 +4,6 @@ from . import chunked_data_stream as chunky from splunklib.searchcommands import Configuration, GeneratingCommand -from unittest import TestCase def test_simple_generator(): From 9938fcbef1c3052254e724c48dd4a7c6a0695837 Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Wed, 27 Oct 2021 12:29:04 +0530 Subject: [PATCH 32/35] Update test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 123eb504..45016f14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: python: [2.7, 3.7] splunk-version: - "8.0" - - "latest" + - "8.2" fail-fast: false services: From fe1784fa331b6ae1c6dc804fe91af2c09c64654e Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Wed, 27 Oct 2021 14:52:42 +0530 Subject: [PATCH 33/35] Update test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 45016f14..123eb504 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: python: [2.7, 3.7] splunk-version: - "8.0" - - "8.2" + - "latest" fail-fast: false services: From 650bea0e8629dd42e44936aaa1f96a43b7df72ee Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Thu, 28 Oct 2021 15:12:08 +0530 Subject: [PATCH 34/35] Release-1.6.17 changes --- CHANGELOG.md | 20 ++++++++++++++++++++ README.md | 2 +- docs/searchcommands.rst | 6 +++--- examples/searchcommands_app/setup.py | 2 +- splunklib/__init__.py | 2 +- splunklib/binding.py | 2 +- 6 files changed, 27 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63f520c8..743b4f17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Splunk Enterprise SDK for Python Changelog +## Version 1.6.17 + +### Bug fixes + +* [#396](https://github.com/splunk/splunk-sdk-python/pull/396) Updated KVStore Methods to support dictionaries +* [#397](https://github.com/splunk/splunk-sdk-python/pull/397) Added code changes for encoding '/' in _key parameter in kvstore.data APIs. +* [#398](https://github.com/splunk/splunk-sdk-python/pull/398) Added dictionary support for KVStore "query" methods. +* [#404](https://github.com/splunk/splunk-sdk-python/pull/404) Fixed test case failure for 8.0 and latest(8.2.x) splunk version + +### Minor changes + +* [#381](https://github.com/splunk/splunk-sdk-python/pull/381) Updated current year in conf.py +* [#389](https://github.com/splunk/splunk-sdk-python/pull/389) Fixed few typos +* [#391](https://github.com/splunk/splunk-sdk-python/pull/391) Fixed spelling error in client.py +* [#393](https://github.com/splunk/splunk-sdk-python/pull/393) Updated development status past 3 +* [#394](https://github.com/splunk/splunk-sdk-python/pull/394) Updated Readme steps to run examples +* [#395](https://github.com/splunk/splunk-sdk-python/pull/395) Updated random_number.py +* [#399](https://github.com/splunk/splunk-sdk-python/pull/399) Moved CI tests to GitHub Actions +* [#403](https://github.com/splunk/splunk-sdk-python/pull/403) Removed usage of Easy_install to install SDK + ## Version 1.6.16 ### Bug fixes diff --git a/README.md b/README.md index 6b92a179..a1f077eb 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # The Splunk Enterprise Software Development Kit for Python -#### Version 1.6.16 +#### Version 1.6.17 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using the Splunk platform. diff --git a/docs/searchcommands.rst b/docs/searchcommands.rst index a620fbb8..281f755f 100644 --- a/docs/searchcommands.rst +++ b/docs/searchcommands.rst @@ -3,7 +3,7 @@ splunklib.searchcommands .. automodule:: splunklib.searchcommands -.. autofunction:: dispatch(command_class[, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None]) +.. autofunction:: dispatch(command_class[, argv=sys.argv, input_file=sys.stdin, output_file=sys.stdout, module_name=None, allow_empty_input=True]) .. autoclass:: EventingCommand :members: @@ -31,7 +31,7 @@ splunklib.searchcommands .. automethod:: splunklib.searchcommands::GeneratingCommand.generate - .. automethod:: splunklib.searchcommands::GeneratingCommand.process(args=sys.argv[, input_file=sys.stdin, output_file=sys.stdout]) + .. automethod:: splunklib.searchcommands::GeneratingCommand.process(args=sys.argv[, input_file=sys.stdin, output_file=sys.stdout, allow_empty_input=True]) .. autoclass:: ReportingCommand :members: @@ -59,7 +59,7 @@ splunklib.searchcommands :inherited-members: :exclude-members: configuration_settings, fix_up, items, keys - .. automethod:: splunklib.searchcommands::StreamingCommand.process(args=sys.argv[, input_file=sys.stdin, output_file=sys.stdout]) + .. automethod:: splunklib.searchcommands::StreamingCommand.process(args=sys.argv[, input_file=sys.stdin, output_file=sys.stdout, allow_empty_input=True]) .. automethod:: splunklib.searchcommands::StreamingCommand.stream diff --git a/examples/searchcommands_app/setup.py b/examples/searchcommands_app/setup.py index b9dc87b7..ba2d46a0 100755 --- a/examples/searchcommands_app/setup.py +++ b/examples/searchcommands_app/setup.py @@ -439,7 +439,7 @@ def run(self): setup( description='Custom Search Command examples', name=os.path.basename(project_dir), - version='1.6.16', + version='1.6.17', author='Splunk, Inc.', author_email='devinfo@splunk.com', url='http://github.com/splunk/splunk-sdk-python', diff --git a/splunklib/__init__.py b/splunklib/__init__.py index 525dc8ee..36f8a7e0 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -16,5 +16,5 @@ from __future__ import absolute_import from splunklib.six.moves import map -__version_info__ = (1, 6, 16) +__version_info__ = (1, 6, 17) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index 8d47d244..cea9894e 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -1391,7 +1391,7 @@ def request(url, message, **kwargs): head = { "Content-Length": str(len(body)), "Host": host, - "User-Agent": "splunk-sdk-python/1.6.16", + "User-Agent": "splunk-sdk-python/1.6.17", "Accept": "*/*", "Connection": "Close", } # defaults From b80d774a0ae141a272717ac7d5fb2ed803a0e2e2 Mon Sep 17 00:00:00 2001 From: vmalaviya Date: Tue, 2 Nov 2021 13:37:29 +0530 Subject: [PATCH 35/35] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 743b4f17..0c49f5c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,11 @@ ### Bug fixes +* [#383](https://github.com/splunk/splunk-sdk-python/pull/383) Implemented the possibility to provide a SSLContext object to the connect method * [#396](https://github.com/splunk/splunk-sdk-python/pull/396) Updated KVStore Methods to support dictionaries * [#397](https://github.com/splunk/splunk-sdk-python/pull/397) Added code changes for encoding '/' in _key parameter in kvstore.data APIs. * [#398](https://github.com/splunk/splunk-sdk-python/pull/398) Added dictionary support for KVStore "query" methods. +* [#402](https://github.com/splunk/splunk-sdk-python/pull/402) Fixed regression introduced in 1.6.15 to once again allow processing of empty input records in custom search commands (fix [#376](https://github.com/splunk/splunk-sdk-python/issues/376)) * [#404](https://github.com/splunk/splunk-sdk-python/pull/404) Fixed test case failure for 8.0 and latest(8.2.x) splunk version ### Minor changes