diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e97c7630..dc8bd208 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,7 @@ name: Python CI on: - [ push, pull_request ] + [ push, pull_request, workflow_dispatch ] jobs: build: @@ -11,7 +11,7 @@ jobs: matrix: os: - ubuntu-latest - python: [ 3.7, 3.9] + python: [ 3.7, 3.9, 3.13] splunk-version: - "8.1" - "8.2" @@ -22,8 +22,8 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Run docker-compose - run: SPLUNK_VERSION=${{matrix.splunk-version}} docker-compose up -d + - name: Run docker compose + run: SPLUNK_VERSION=${{matrix.splunk-version}} docker compose up -d - name: Setup Python uses: actions/setup-python@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e2137a6..288543c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Splunk Enterprise SDK for Python Changelog +## Version 2.1.0 + +### Changes +* [#516](https://github.com/splunk/splunk-sdk-python/pull/516) Added support for macros +* Remove deprecated `wrap_socket` in `Contex` class. +* Added explicit support for self signed certificates in https +* Enforce minimal required tls version in https connection +* Add support for python 3.13 +* [#559](https://github.com/splunk/splunk-sdk-python/pull/559/) Add exception logging + ## Version 2.0.2 ### Minor changes diff --git a/README.md b/README.md index e7481836..7cd66cc2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # The Splunk Enterprise Software Development Kit for Python -#### Version 2.0.2 +#### Version 2.1.0 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code designed to enable developers to build applications using the Splunk platform. @@ -25,9 +25,9 @@ The Splunk Enterprise SDK for Python contains library code, and its examples are Here's what you need to get going with the Splunk Enterprise SDK for Python. -* Python 3.7 or Python 3.9 +* Python 3.7, Python 3.9 and Python 3.13 - The Splunk Enterprise SDK for Python is compatible with python3 and has been tested with Python v3.7 and v3.9. + The Splunk Enterprise SDK for Python is compatible with python3 and has been tested with Python v3.7, v3.9 and v3.13. * Splunk Enterprise 9.2 or 8.2 diff --git a/setup.py b/setup.py index c80ddf37..d19a09eb 100755 --- a/setup.py +++ b/setup.py @@ -14,124 +14,15 @@ # License for the specific language governing permissions and limitations # under the License. -from setuptools import setup, Command - -import os -import sys +from setuptools import setup import splunklib -failed = False - -def run_test_suite(): - import unittest - - def mark_failed(): - global failed - failed = True - - class _TrackingTextTestResult(unittest._TextTestResult): - def addError(self, test, err): - unittest._TextTestResult.addError(self, test, err) - mark_failed() - - def addFailure(self, test, err): - unittest._TextTestResult.addFailure(self, test, err) - mark_failed() - - class TrackingTextTestRunner(unittest.TextTestRunner): - def _makeResult(self): - return _TrackingTextTestResult( - self.stream, self.descriptions, self.verbosity) - - original_cwd = os.path.abspath(os.getcwd()) - os.chdir('tests') - suite = unittest.defaultTestLoader.discover('.') - runner = TrackingTextTestRunner(verbosity=2) - runner.run(suite) - os.chdir(original_cwd) - - return failed - - -def run_test_suite_with_junit_output(): - try: - import unittest2 as unittest - except ImportError: - import unittest - import xmlrunner - original_cwd = os.path.abspath(os.getcwd()) - os.chdir('tests') - suite = unittest.defaultTestLoader.discover('.') - xmlrunner.XMLTestRunner(output='../test-reports').run(suite) - os.chdir(original_cwd) - - -class CoverageCommand(Command): - """setup.py command to run code coverage of the test suite.""" - description = "Create an HTML coverage report from running the full test suite." - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - try: - import coverage - except ImportError: - print("Could not import coverage. Please install it and try again.") - exit(1) - cov = coverage.coverage(source=['splunklib']) - cov.start() - run_test_suite() - cov.stop() - cov.html_report(directory='coverage_report') - - -class TestCommand(Command): - """setup.py command to run the whole test suite.""" - description = "Run test full test suite." - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - failed = run_test_suite() - if failed: - sys.exit(1) - - -class JunitXmlTestCommand(Command): - """setup.py command to run the whole test suite.""" - description = "Run test full test suite with JUnit-formatted output." - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - run_test_suite_with_junit_output() - - setup( author="Splunk, Inc.", author_email="devinfo@splunk.com", - cmdclass={'coverage': CoverageCommand, - 'test': TestCommand, - 'testjunit': JunitXmlTestCommand}, - description="The Splunk Software Development Kit for Python.", license="http://www.apache.org/licenses/LICENSE-2.0", diff --git a/splunklib/__init__.py b/splunklib/__init__.py index f999a52a..5f83b2ac 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -30,5 +30,5 @@ def setup_logging(level, log_format=DEFAULT_LOG_FORMAT, date_format=DEFAULT_DATE datefmt=date_format) -__version_info__ = (2, 0, 2) +__version_info__ = (2, 1, 0) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index 958be96e..25a09948 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -465,6 +465,8 @@ class Context: :type scheme: "https" or "http" :param verify: Enable (True) or disable (False) SSL verification for https connections. :type verify: ``Boolean`` + :param self_signed_certificate: Specifies if self signed certificate is used + :type self_signed_certificate: ``Boolean`` :param sharing: The sharing mode for the namespace (the default is "user"). :type sharing: "global", "system", "app", or "user" :param owner: The owner context of the namespace (optional, the default is "None"). @@ -526,6 +528,7 @@ def __init__(self, handler=None, **kwargs): self.bearerToken = kwargs.get("splunkToken", "") self.autologin = kwargs.get("autologin", False) self.additional_headers = kwargs.get("headers", []) + self._self_signed_certificate = kwargs.get("self_signed_certificate", True) # Store any cookies in the self.http._cookies dict if "cookie" in kwargs and kwargs['cookie'] not in [None, _NoAuthenticationToken]: @@ -604,7 +607,11 @@ def connect(self): """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.scheme == "https": - sock = ssl.wrap_socket(sock) + context = ssl.create_default_context() + context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 + context.check_hostname = not self._self_signed_certificate + context.verify_mode = ssl.CERT_NONE if self._self_signed_certificate else ssl.CERT_REQUIRED + sock = context.wrap_socket(sock, server_hostname=self.host) sock.connect((socket.gethostbyname(self.host), self.port)) return sock diff --git a/splunklib/client.py b/splunklib/client.py index 48861880..ee390c9e 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -101,6 +101,7 @@ PATH_JOBS = "search/jobs/" PATH_JOBS_V2 = "search/v2/jobs/" PATH_LOGGER = "/services/server/logger/" +PATH_MACROS = "configs/conf-macros/" PATH_MESSAGES = "messages/" PATH_MODULAR_INPUTS = "data/modular-inputs" PATH_ROLES = "authorization/roles/" @@ -667,6 +668,15 @@ def saved_searches(self): """ return SavedSearches(self) + @property + def macros(self): + """Returns the collection of macros. + + :return: A :class:`Macros` collection of :class:`Macro` + entities. + """ + return Macros(self) + @property def settings(self): """Returns the configuration settings for this instance of Splunk. @@ -3440,6 +3450,90 @@ def create(self, name, search, **kwargs): return Collection.create(self, name, search=search, **kwargs) +class Macro(Entity): + """This class represents a search macro.""" + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + @property + def args(self): + """Returns the macro arguments. + :return: The macro arguments. + :rtype: ``string`` + """ + return self._state.content.get('args', '') + + @property + def definition(self): + """Returns the macro definition. + :return: The macro definition. + :rtype: ``string`` + """ + return self._state.content.get('definition', '') + + @property + def errormsg(self): + """Returns the validation error message for the macro. + :return: The validation error message for the macro. + :rtype: ``string`` + """ + return self._state.content.get('errormsg', '') + + @property + def iseval(self): + """Returns the eval-based definition status of the macro. + :return: The iseval value for the macro. + :rtype: ``string`` + """ + return self._state.content.get('iseval', '0') + + def update(self, definition=None, **kwargs): + """Updates the server with any changes you've made to the current macro + along with any additional arguments you specify. + :param `definition`: The macro definition (optional). + :type definition: ``string`` + :param `kwargs`: Additional arguments (optional). Available parameters are: + 'disabled', 'iseval', 'validation', and 'errormsg'. + :type kwargs: ``dict`` + :return: The :class:`Macro`. + """ + # Updates to a macro *require* that the definition be + # passed, so we pass the current definition if a value wasn't + # provided by the caller. + if definition is None: definition = self.content.definition + Entity.update(self, definition=definition, **kwargs) + return self + + @property + def validation(self): + """Returns the validation expression for the macro. + :return: The validation expression for the macro. + :rtype: ``string`` + """ + return self._state.content.get('validation', '') + + +class Macros(Collection): + """This class represents a collection of macros. Retrieve this + collection using :meth:`Service.macros`.""" + def __init__(self, service): + Collection.__init__( + self, service, PATH_MACROS, item=Macro) + + def create(self, name, definition, **kwargs): + """ Creates a macro. + :param name: The name for the macro. + :type name: ``string`` + :param definition: The macro definition. + :type definition: ``string`` + :param kwargs: Additional arguments (optional). Available parameters are: + 'disabled', 'iseval', 'validation', and 'errormsg'. + :type kwargs: ``dict`` + :return: The :class:`Macros` collection. + """ + return Collection.create(self, name, definition=definition, **kwargs) + + class Settings(Entity): """This class represents configuration settings for a Splunk service. Retrieve this collection using :meth:`Service.settings`.""" @@ -3905,4 +3999,4 @@ def batch_save(self, *documents): data = json.dumps(documents) return json.loads( - self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) + self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) \ No newline at end of file diff --git a/splunklib/modularinput/event_writer.py b/splunklib/modularinput/event_writer.py index c048a5b5..7be3845a 100644 --- a/splunklib/modularinput/event_writer.py +++ b/splunklib/modularinput/event_writer.py @@ -13,6 +13,7 @@ # under the License. import sys +import traceback from splunklib.utils import ensure_str from .event import ET @@ -66,6 +67,25 @@ def log(self, severity, message): self._err.write(f"{severity} {message}\n") self._err.flush() + def log_exception(self, message, exception=None, severity=None): + """Logs messages about the exception thrown by this modular input to Splunk. + These messages will show up in Splunk's internal logs. + + :param message: ``string``, message to log. + :param exception: ``Exception``, exception thrown by this modular input; if none, sys.exc_info() is used + :param severity: ``string``, severity of message, see severities defined as class constants. Default severity: ERROR + """ + if exception is not None: + tb_str = traceback.format_exception(type(exception), exception, exception.__traceback__) + else: + tb_str = traceback.format_exc() + + if severity is None: + severity = EventWriter.ERROR + + self._err.write(("%s %s - %s" % (severity, message, tb_str)).replace("\n", " ")) + self._err.flush() + def write_xml_document(self, document): """Writes a string representation of an ``ElementTree`` object to the output stream. diff --git a/splunklib/modularinput/script.py b/splunklib/modularinput/script.py index e912d737..a6c96193 100644 --- a/splunklib/modularinput/script.py +++ b/splunklib/modularinput/script.py @@ -91,13 +91,12 @@ def run_script(self, args, event_writer, input_stream): event_writer.write_xml_document(root) return 1 - err_string = "ERROR Invalid arguments to modular input script:" + ' '.join( - args) - event_writer._err.write(err_string) + event_writer.log(EventWriter.ERROR, "Invalid arguments to modular input script:" + ' '.join( + args)) return 1 except Exception as e: - event_writer.log(EventWriter.ERROR, str(e)) + event_writer.log_exception(str(e)) return 1 @property diff --git a/tests/modularinput/test_event.py b/tests/modularinput/test_event.py index 4039d8e2..35e9c09c 100644 --- a/tests/modularinput/test_event.py +++ b/tests/modularinput/test_event.py @@ -15,7 +15,9 @@ # under the License. +import re import sys +from io import StringIO import pytest @@ -150,3 +152,29 @@ def test_write_xml_is_sane(capsys): found_xml = ET.fromstring(captured.out) assert xml_compare(expected_xml, found_xml) + + +def test_log_exception(): + out, err = StringIO(), StringIO() + ew = EventWriter(out, err) + + exc = Exception("Something happened!") + + try: + raise exc + except Exception: + ew.log_exception("ex1") + + assert out.getvalue() == "" + + # Remove paths and line + err = re.sub(r'File "[^"]+', 'File "...', err.getvalue()) + err = re.sub(r'line \d+', 'line 123', err) + + # One line + assert err == ( + 'ERROR ex1 - Traceback (most recent call last): ' + ' File "...", line 123, in test_log_exception ' + ' raise exc ' + 'Exception: Something happened! ' + ) diff --git a/tests/modularinput/test_script.py b/tests/modularinput/test_script.py index 48be8826..49c25972 100644 --- a/tests/modularinput/test_script.py +++ b/tests/modularinput/test_script.py @@ -1,6 +1,7 @@ import sys import io +import re import xml.etree.ElementTree as ET from splunklib.client import Service from splunklib.modularinput import Script, EventWriter, Scheme, Argument, Event @@ -228,3 +229,38 @@ def stream_events(self, inputs, ew): assert output.err == "" assert isinstance(script.service, Service) assert script.service.authority == script.authority_uri + + +def test_log_script_exception(monkeypatch): + out, err = io.StringIO(), io.StringIO() + + # Override abstract methods + class NewScript(Script): + def get_scheme(self): + return None + + def stream_events(self, inputs, ew): + raise RuntimeError("Some error") + + script = NewScript() + input_configuration = data_open("data/conf_with_2_inputs.xml") + + ew = EventWriter(out, err) + + assert script.run_script([TEST_SCRIPT_PATH], ew, input_configuration) == 1 + + # Remove paths and line numbers + err = re.sub(r'File "[^"]+', 'File "...', err.getvalue()) + err = re.sub(r'line \d+', 'line 123', err) + err = re.sub(r' +~+\^+', '', err) + + assert out.getvalue() == "" + assert err == ( + 'ERROR Some error - ' + 'Traceback (most recent call last): ' + ' File "...", line 123, in run_script ' + ' self.stream_events(self._input_definition, event_writer) ' + ' File "...", line 123, in stream_events ' + ' raise RuntimeError("Some error") ' + 'RuntimeError: Some error ' + ) diff --git a/tests/test_macro.py b/tests/test_macro.py new file mode 100755 index 00000000..25b72e4d --- /dev/null +++ b/tests/test_macro.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# +# Copyright 2011-2015 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import +from tests import testlib +import logging + +import splunklib.client as client + +import pytest + +@pytest.mark.smoke +class TestMacro(testlib.SDKTestCase): + def setUp(self): + super(TestMacro, self).setUp() + macros = self.service.macros + logging.debug("Macros namespace: %s", macros.service.namespace) + self.macro_name = testlib.tmpname() + definition = '| eval test="123"' + self.macro = macros.create(self.macro_name, definition) + + def tearDown(self): + super(TestMacro, self).setUp() + for macro in self.service.macros: + if macro.name.startswith('delete-me'): + self.service.macros.delete(macro.name) + + def check_macro(self, macro): + self.check_entity(macro) + expected_fields = ['definition', + 'iseval', + 'args', + 'validation', + 'errormsg'] + for f in expected_fields: + macro[f] + is_eval = macro.iseval + self.assertTrue(is_eval == '1' or is_eval == '0') + + def test_create(self): + self.assertTrue(self.macro_name in self.service.macros) + self.check_macro(self.macro) + + def test_create_with_args(self): + macro_name = testlib.tmpname() + '(1)' + definition = '| eval value="$value$"' + kwargs = { + 'args': 'value', + 'validation': '$value$ > 10', + 'errormsg': 'value must be greater than 10' + } + macro = self.service.macros.create(macro_name, definition=definition, **kwargs) + self.assertTrue(macro_name in self.service.macros) + self.check_macro(macro) + self.assertEqual(macro.iseval, '0') + self.assertEqual(macro.args, kwargs.get('args')) + self.assertEqual(macro.validation, kwargs.get('validation')) + self.assertEqual(macro.errormsg, kwargs.get('errormsg')) + self.service.macros.delete(macro_name) + + def test_delete(self): + self.assertTrue(self.macro_name in self.service.macros) + self.service.macros.delete(self.macro_name) + self.assertFalse(self.macro_name in self.service.macros) + self.assertRaises(client.HTTPError, + self.macro.refresh) + + def test_update(self): + new_definition = '| eval updated="true"' + self.macro.update(definition=new_definition) + self.macro.refresh() + self.assertEqual(self.macro['definition'], new_definition) + + is_eval = testlib.to_bool(self.macro['iseval']) + self.macro.update(iseval=not is_eval) + self.macro.refresh() + self.assertEqual(testlib.to_bool(self.macro['iseval']), not is_eval) + + def test_cannot_update_name(self): + new_name = self.macro_name + '-alteration' + self.assertRaises(client.IllegalOperationException, + self.macro.update, name=new_name) + + def test_name_collision(self): + opts = self.opts.kwargs.copy() + opts['owner'] = '-' + opts['app'] = '-' + opts['sharing'] = 'user' + service = client.connect(**opts) + logging.debug("Namespace for collision testing: %s", service.namespace) + macros = service.macros + name = testlib.tmpname() + + dispatch1 = '| eval macro_one="1"' + dispatch2 = '| eval macro_two="2"' + namespace1 = client.namespace(app='search', sharing='app') + namespace2 = client.namespace(owner='admin', app='search', sharing='user') + new_macro2 = macros.create( + name, dispatch2, + namespace=namespace1) + new_macro1 = macros.create( + name, dispatch1, + namespace=namespace2) + + self.assertRaises(client.AmbiguousReferenceException, + macros.__getitem__, name) + macro1 = macros[name, namespace1] + self.check_macro(macro1) + macro1.update(**{'definition': '| eval number=1'}) + macro1.refresh() + self.assertEqual(macro1['definition'], '| eval number=1') + macro2 = macros[name, namespace2] + macro2.update(**{'definition': '| eval number=2'}) + macro2.refresh() + self.assertEqual(macro2['definition'], '| eval number=2') + self.check_macro(macro2) + + def test_no_equality(self): + self.assertRaises(client.IncomparableException, + self.macro.__eq__, self.macro) + + def test_acl(self): + self.assertEqual(self.macro.access["perms"], None) + self.macro.acl_update(sharing="app", owner="admin", **{"perms.read": "admin, nobody"}) + self.assertEqual(self.macro.access["owner"], "admin") + self.assertEqual(self.macro.access["sharing"], "app") + self.assertEqual(self.macro.access["perms"]["read"], ['admin', 'nobody']) + + def test_acl_fails_without_sharing(self): + self.assertRaisesRegex( + ValueError, + "Required argument 'sharing' is missing.", + self.macro.acl_update, + owner="admin", app="search", **{"perms.read": "admin, nobody"} + ) + + def test_acl_fails_without_owner(self): + self.assertRaisesRegex( + ValueError, + "Required argument 'owner' is missing.", + self.macro.acl_update, + sharing="app", app="search", **{"perms.read": "admin, nobody"} + ) + + +if __name__ == "__main__": + try: + import unittest2 as unittest + except ImportError: + import unittest + unittest.main() diff --git a/tox.ini b/tox.ini index e45dbfb9..e69c2e6a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = clean,docs,py37,py39 +envlist = clean,docs,py37,py39,313 skipsdist = {env:TOXBUILD:false} [testenv:pep8] @@ -28,8 +28,6 @@ setenv = SPLUNK_HOME=/opt/splunk allowlist_externals = make deps = pytest pytest-cov - xmlrunner - unittest-xml-reporting python-dotenv distdir = build