From ca3d9eb01204652ebf9b9e6abd6265201b600edb Mon Sep 17 00:00:00 2001 From: kkedziak Date: Thu, 22 Feb 2024 13:05:43 +0100 Subject: [PATCH 01/22] Add exception logging in Script --- splunklib/modularinput/event_writer.py | 21 ++++++++++++++++ splunklib/modularinput/script.py | 7 +++--- tests/modularinput/test_event.py | 28 +++++++++++++++++++++ tests/modularinput/test_script.py | 35 ++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) diff --git a/splunklib/modularinput/event_writer.py b/splunklib/modularinput/event_writer.py index 38a110c12..07347725d 100644 --- a/splunklib/modularinput/event_writer.py +++ b/splunklib/modularinput/event_writer.py @@ -14,6 +14,7 @@ from __future__ import absolute_import import sys +import traceback from splunklib.six import ensure_str from .event import ET @@ -23,6 +24,7 @@ except ImportError: from splunklib.six import StringIO + class EventWriter(object): """``EventWriter`` writes events and error messages to Splunk from a modular input. Its two important methods are ``writeEvent``, which takes an ``Event`` object, @@ -71,6 +73,25 @@ def log(self, severity, message): self._err.write("%s %s\n" % (severity, message)) self._err.flush() + def log_exception(self, message, exception=None, severity=None): + """Logs messages about the exception thrown by this modular input to Splunk. + These messages will show up in Splunk's internal logs. + + :param message: ``string``, message to log. + :param exception: ``Exception``, exception thrown by this modular input; if none, sys.exc_info() is used + :param severity: ``string``, severity of message, see severities defined as class constants. Default: ERROR + """ + if exception is not None: + tb_str = traceback.format_exception(type(exception), exception, exception.__traceback__) + else: + tb_str = traceback.format_exc() + + if severity is None: + severity = EventWriter.ERROR + + self._err.write(("%s %s - %s" % (severity, message, tb_str)).replace("\n", " ")) + self._err.flush() + def write_xml_document(self, document): """Writes a string representation of an ``ElementTree`` object to the output stream. diff --git a/splunklib/modularinput/script.py b/splunklib/modularinput/script.py index 8595dc4bd..fe9ebbe7f 100644 --- a/splunklib/modularinput/script.py +++ b/splunklib/modularinput/script.py @@ -99,13 +99,12 @@ def run_script(self, args, event_writer, input_stream): return 1 else: - err_string = "ERROR Invalid arguments to modular input script:" + ' '.join( - args) - event_writer._err.write(err_string) + event_writer.log(EventWriter.ERROR, "Invalid arguments to modular input script:" + ' '.join( + args)) return 1 except Exception as e: - event_writer.log(EventWriter.ERROR, str(e)) + event_writer.log_exception(str(e)) return 1 @property diff --git a/tests/modularinput/test_event.py b/tests/modularinput/test_event.py index 865656031..03bf2de94 100644 --- a/tests/modularinput/test_event.py +++ b/tests/modularinput/test_event.py @@ -16,7 +16,9 @@ from __future__ import absolute_import +import re import sys +from io import StringIO import pytest @@ -151,3 +153,29 @@ def test_write_xml_is_sane(capsys): found_xml = ET.fromstring(captured.out) assert xml_compare(expected_xml, found_xml) + + +def test_log_exception(): + out, err = StringIO(), StringIO() + ew = EventWriter(out, err) + + exc = Exception("Something happened!") + + try: + raise exc + except: + ew.log_exception("ex1") + + assert out.getvalue() == "" + + # Remove paths and line + err = re.sub(r'File "[^"]+', 'File "...', err.getvalue()) + err = re.sub(r'line \d+', 'line 123', err) + + # One line + assert err == ( + 'ERROR ex1 - Traceback (most recent call last): ' + ' File "...", line 123, in test_log_exception ' + ' raise exc ' + 'Exception: Something happened! ' + ) diff --git a/tests/modularinput/test_script.py b/tests/modularinput/test_script.py index b15885dc7..9bdcb562f 100644 --- a/tests/modularinput/test_script.py +++ b/tests/modularinput/test_script.py @@ -3,6 +3,7 @@ from splunklib.client import Service from splunklib.modularinput import Script, EventWriter, Scheme, Argument, Event import io +import re from splunklib.modularinput.utils import xml_compare from tests.modularinput.modularinput_testlib import data_open @@ -231,3 +232,37 @@ def stream_events(self, inputs, ew): assert output.err == "" assert isinstance(script.service, Service) assert script.service.authority == script.authority_uri + + +def test_log_script_exception(monkeypatch): + out, err = io.StringIO(), io.StringIO() + + # Override abstract methods + class NewScript(Script): + def get_scheme(self): + return None + + def stream_events(self, inputs, ew): + raise RuntimeError("Some error") + + script = NewScript() + input_configuration = data_open("data/conf_with_2_inputs.xml") + + ew = EventWriter(out, err) + + assert script.run_script([TEST_SCRIPT_PATH], ew, input_configuration) == 1 + + # Remove paths and line numbers + err = re.sub(r'File "[^"]+', 'File "...', err.getvalue()) + err = re.sub(r'line \d+', 'line 123', err) + + assert out.getvalue() == "" + assert err == ( + 'ERROR Some error - ' + 'Traceback (most recent call last): ' + ' File "...", line 123, in run_script ' + ' self.stream_events(self._input_definition, event_writer) ' + ' File "...", line 123, in stream_events ' + ' raise RuntimeError("Some error") ' + 'RuntimeError: Some error ' + ) From 2869237ec343d0cf0fe5a1cec6ee03f61bb7ac38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20K=C4=99dziak?= Date: Fri, 5 Apr 2024 14:38:45 +0200 Subject: [PATCH 02/22] Revert CHANGELOG.md --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87d27a229..0e423e67d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,5 @@ # Splunk Enterprise SDK for Python Changelog -## Version 2.0.1 - -### Bug fixes -* [#567](https://github.com/splunk/splunk-sdk-python/issues/567) Moved "deprecation" dependency - - ## Version 2.0.0 ### Feature updates From e5f7af0f64afe582c5893726e6cb77e78e5356a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20K=C4=99dziak?= Date: Fri, 5 Apr 2024 14:39:15 +0200 Subject: [PATCH 03/22] Revert README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0969bbc2b..2bbae4de9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # The Splunk Enterprise Software Development Kit for Python -#### Version 2.0.1 +#### Version 2.0.0 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code designed to enable developers to build applications using the Splunk platform. From 3dab0b528bb429f5b8c795f86294f0f4a6853f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20K=C4=99dziak?= Date: Fri, 5 Apr 2024 14:39:36 +0200 Subject: [PATCH 04/22] Update __init__.py --- splunklib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/splunklib/__init__.py b/splunklib/__init__.py index c86dfdb8a..2613d2849 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -30,5 +30,5 @@ def setup_logging(level, log_format=DEFAULT_LOG_FORMAT, date_format=DEFAULT_DATE datefmt=date_format) -__version_info__ = (2, 0, 1) +__version_info__ = (2, 0, 0) __version__ = ".".join(map(str, __version_info__)) From be7dfe04ead6d1f6e5912afa646ecb78abae4856 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20K=C4=99dziak?= Date: Wed, 8 May 2024 12:33:38 +0000 Subject: [PATCH 05/22] Refactor comment --- splunklib/modularinput/event_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/splunklib/modularinput/event_writer.py b/splunklib/modularinput/event_writer.py index cdf694ee9..7be3845ab 100644 --- a/splunklib/modularinput/event_writer.py +++ b/splunklib/modularinput/event_writer.py @@ -73,7 +73,7 @@ def log_exception(self, message, exception=None, severity=None): :param message: ``string``, message to log. :param exception: ``Exception``, exception thrown by this modular input; if none, sys.exc_info() is used - :param severity: ``string``, severity of message, see severities defined as class constants. Default: ERROR + :param severity: ``string``, severity of message, see severities defined as class constants. Default severity: ERROR """ if exception is not None: tb_str = traceback.format_exception(type(exception), exception, exception.__traceback__) From 40e7b2a305d8e67034c13711c41c678e3a8d9fc5 Mon Sep 17 00:00:00 2001 From: maszyk99 Date: Thu, 16 May 2024 17:07:35 +0200 Subject: [PATCH 06/22] DVPL-0: don't use latest splunk version because of HEC token issue --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e97c7630f..dbc71e1b9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,8 @@ jobs: splunk-version: - "8.1" - "8.2" - - "latest" + - "9.0.8" + - "9.1.3" fail-fast: false steps: From 3ef0519242ebf7733aae41c965801242427bd7b6 Mon Sep 17 00:00:00 2001 From: kkedziak Date: Tue, 21 May 2024 15:01:24 +0200 Subject: [PATCH 07/22] Change --- tests/modularinput/test_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modularinput/test_event.py b/tests/modularinput/test_event.py index 3180feb62..35e9c09cd 100644 --- a/tests/modularinput/test_event.py +++ b/tests/modularinput/test_event.py @@ -162,7 +162,7 @@ def test_log_exception(): try: raise exc - except: + except Exception: ew.log_exception("ex1") assert out.getvalue() == "" From 1a2d17fa72cbd11139bc00973c9558f2dcd7b9c5 Mon Sep 17 00:00:00 2001 From: maszyk99 Date: Tue, 25 Jun 2024 16:52:42 +0200 Subject: [PATCH 08/22] Add python2/3 utils file back --- splunklib/six.py | 993 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 993 insertions(+) create mode 100644 splunklib/six.py diff --git a/splunklib/six.py b/splunklib/six.py new file mode 100644 index 000000000..d13e50c93 --- /dev/null +++ b/splunklib/six.py @@ -0,0 +1,993 @@ +# Copyright (c) 2010-2020 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""Utilities for writing code that runs on Python 2 and 3""" + +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.14.0" + + +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) + +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + get_source = get_code # same as get_code + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), + MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule("tkinter_colorchooser", "tkColorChooser", + "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", + "tkinter.commondialog"), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", + "tkinter.simpledialog"), + MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), + MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), + MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [ + MovedModule("winreg", "_winreg"), + ] + +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr + +_MovedItems._moved_attributes = _moved_attributes + +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr + +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes + +_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", "moves.urllib.parse") + + +class Module_six_moves_urllib_error(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr + +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes + +_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", "moves.urllib.error") + + +class Module_six_moves_urllib_request(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr + +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes + +_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", "moves.urllib.request") + + +class Module_six_moves_urllib_response(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr + +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes + +_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", "moves.urllib.response") + + +class Module_six_moves_urllib_robotparser(_LazyModule): + + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr + +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes + +_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", "moves.urllib.robotparser") + + +class Module_six_moves_urllib(types.ModuleType): + + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + +_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), + "moves.urllib") + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" + + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() +next = advance_iterator + + +try: + callable = callable +except NameError: + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc(get_unbound_function, + """Get the function out of a possibly unbound function""") + + +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) + + +if PY3: + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + + viewvalues = operator.methodcaller("values") + + viewitems = operator.methodcaller("items") +else: + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + + viewvalues = operator.methodcaller("viewvalues") + + viewitems = operator.methodcaller("viewitems") + +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc(iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc(iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.") + + +if PY3: + def b(s): + return s.encode("latin-1") + + def u(s): + return s + unichr = chr + import struct + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + StringIO = io.StringIO + BytesIO = io.BytesIO + del io + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" +else: + def b(s): + return s + # Workaround for standalone backslash + + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + +else: + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec("""exec _code_ in _globs_, _locs_""") + + exec_("""def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""") + + +if sys.version_info[:2] > (3,): + exec_("""def raise_from(value, from_value): + try: + raise value from from_value + finally: + value = None +""") +else: + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if (isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + if kwargs: + raise TypeError("invalid keyword arguments to print()") + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + +_add_doc(reraise, """Reraise an exception.""") + +if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper(wrapper, wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ + + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + return functools.partial(_update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + wraps.__doc__ = functools.wraps.__doc__ + +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(type): + + def __new__(cls, name, this_bases, d): + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ + return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper + + +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def python_2_unicode_compatible(klass): + """ + A class decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError("@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % + klass.__name__) + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if (type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__): + del sys.meta_path[i] + break + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) + +import warnings + +def deprecated(message): + def deprecated_decorator(func): + def deprecated_func(*args, **kwargs): + warnings.warn("{} is a deprecated function. {}".format(func.__name__, message), + category=DeprecationWarning, + stacklevel=2) + warnings.simplefilter('default', DeprecationWarning) + return func(*args, **kwargs) + return deprecated_func + return deprecated_decorator \ No newline at end of file From a02b2ce263734e0a395801f6b44b3e1997539366 Mon Sep 17 00:00:00 2001 From: Szymon Date: Tue, 15 Oct 2024 09:56:24 +0200 Subject: [PATCH 09/22] update pipelines so they would be testing python3.13 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f660e8ec..0024c7830 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: matrix: os: - ubuntu-latest - python: [ 3.7, 3.9] + python: [ 3.7, 3.9, 3.13] splunk-version: - "8.1" - "8.2" From a4085e0240a52ac299f59e8685008e8ebbb9df26 Mon Sep 17 00:00:00 2001 From: maszyk99 Date: Wed, 16 Oct 2024 13:22:37 +0200 Subject: [PATCH 10/22] DVPL-0: enable manual workflow trigger --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dbc71e1b9..237eafab9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,7 @@ name: Python CI on: - [ push, pull_request ] + [ push, pull_request, workflow_dispatch ] jobs: build: From 6b55844e3a4a7261b4177ad9217de2bff1792d3c Mon Sep 17 00:00:00 2001 From: maszyk99 Date: Sun, 20 Oct 2024 18:55:08 +0200 Subject: [PATCH 11/22] DVPL-0: fix test pipeline --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 237eafab9..99ee7c350 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,8 +23,8 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Run docker-compose - run: SPLUNK_VERSION=${{matrix.splunk-version}} docker-compose up -d + - name: Run docker compose + run: SPLUNK_VERSION=${{matrix.splunk-version}} docker compose up -d - name: Setup Python uses: actions/setup-python@v4 From 93f51c4c7ba2fb0966e256d4ed370375833a39c3 Mon Sep 17 00:00:00 2001 From: Matt Anderson Date: Mon, 13 Feb 2023 20:42:27 +0000 Subject: [PATCH 12/22] Add Service.macros --- splunklib/client.py | 96 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/splunklib/client.py b/splunklib/client.py index 48861880b..ee390c9ed 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -101,6 +101,7 @@ PATH_JOBS = "search/jobs/" PATH_JOBS_V2 = "search/v2/jobs/" PATH_LOGGER = "/services/server/logger/" +PATH_MACROS = "configs/conf-macros/" PATH_MESSAGES = "messages/" PATH_MODULAR_INPUTS = "data/modular-inputs" PATH_ROLES = "authorization/roles/" @@ -667,6 +668,15 @@ def saved_searches(self): """ return SavedSearches(self) + @property + def macros(self): + """Returns the collection of macros. + + :return: A :class:`Macros` collection of :class:`Macro` + entities. + """ + return Macros(self) + @property def settings(self): """Returns the configuration settings for this instance of Splunk. @@ -3440,6 +3450,90 @@ def create(self, name, search, **kwargs): return Collection.create(self, name, search=search, **kwargs) +class Macro(Entity): + """This class represents a search macro.""" + def __init__(self, service, path, **kwargs): + Entity.__init__(self, service, path, **kwargs) + + @property + def args(self): + """Returns the macro arguments. + :return: The macro arguments. + :rtype: ``string`` + """ + return self._state.content.get('args', '') + + @property + def definition(self): + """Returns the macro definition. + :return: The macro definition. + :rtype: ``string`` + """ + return self._state.content.get('definition', '') + + @property + def errormsg(self): + """Returns the validation error message for the macro. + :return: The validation error message for the macro. + :rtype: ``string`` + """ + return self._state.content.get('errormsg', '') + + @property + def iseval(self): + """Returns the eval-based definition status of the macro. + :return: The iseval value for the macro. + :rtype: ``string`` + """ + return self._state.content.get('iseval', '0') + + def update(self, definition=None, **kwargs): + """Updates the server with any changes you've made to the current macro + along with any additional arguments you specify. + :param `definition`: The macro definition (optional). + :type definition: ``string`` + :param `kwargs`: Additional arguments (optional). Available parameters are: + 'disabled', 'iseval', 'validation', and 'errormsg'. + :type kwargs: ``dict`` + :return: The :class:`Macro`. + """ + # Updates to a macro *require* that the definition be + # passed, so we pass the current definition if a value wasn't + # provided by the caller. + if definition is None: definition = self.content.definition + Entity.update(self, definition=definition, **kwargs) + return self + + @property + def validation(self): + """Returns the validation expression for the macro. + :return: The validation expression for the macro. + :rtype: ``string`` + """ + return self._state.content.get('validation', '') + + +class Macros(Collection): + """This class represents a collection of macros. Retrieve this + collection using :meth:`Service.macros`.""" + def __init__(self, service): + Collection.__init__( + self, service, PATH_MACROS, item=Macro) + + def create(self, name, definition, **kwargs): + """ Creates a macro. + :param name: The name for the macro. + :type name: ``string`` + :param definition: The macro definition. + :type definition: ``string`` + :param kwargs: Additional arguments (optional). Available parameters are: + 'disabled', 'iseval', 'validation', and 'errormsg'. + :type kwargs: ``dict`` + :return: The :class:`Macros` collection. + """ + return Collection.create(self, name, definition=definition, **kwargs) + + class Settings(Entity): """This class represents configuration settings for a Splunk service. Retrieve this collection using :meth:`Service.settings`.""" @@ -3905,4 +3999,4 @@ def batch_save(self, *documents): data = json.dumps(documents) return json.loads( - self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) + self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) \ No newline at end of file From 6de12b169003622f4715d4dd7b342cb69c6d53b3 Mon Sep 17 00:00:00 2001 From: Matt Anderson Date: Mon, 13 Feb 2023 20:44:14 +0000 Subject: [PATCH 13/22] Add tests for macros --- tests/test_macro.py | 164 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100755 tests/test_macro.py diff --git a/tests/test_macro.py b/tests/test_macro.py new file mode 100755 index 000000000..25b72e4d9 --- /dev/null +++ b/tests/test_macro.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# +# Copyright 2011-2015 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import +from tests import testlib +import logging + +import splunklib.client as client + +import pytest + +@pytest.mark.smoke +class TestMacro(testlib.SDKTestCase): + def setUp(self): + super(TestMacro, self).setUp() + macros = self.service.macros + logging.debug("Macros namespace: %s", macros.service.namespace) + self.macro_name = testlib.tmpname() + definition = '| eval test="123"' + self.macro = macros.create(self.macro_name, definition) + + def tearDown(self): + super(TestMacro, self).setUp() + for macro in self.service.macros: + if macro.name.startswith('delete-me'): + self.service.macros.delete(macro.name) + + def check_macro(self, macro): + self.check_entity(macro) + expected_fields = ['definition', + 'iseval', + 'args', + 'validation', + 'errormsg'] + for f in expected_fields: + macro[f] + is_eval = macro.iseval + self.assertTrue(is_eval == '1' or is_eval == '0') + + def test_create(self): + self.assertTrue(self.macro_name in self.service.macros) + self.check_macro(self.macro) + + def test_create_with_args(self): + macro_name = testlib.tmpname() + '(1)' + definition = '| eval value="$value$"' + kwargs = { + 'args': 'value', + 'validation': '$value$ > 10', + 'errormsg': 'value must be greater than 10' + } + macro = self.service.macros.create(macro_name, definition=definition, **kwargs) + self.assertTrue(macro_name in self.service.macros) + self.check_macro(macro) + self.assertEqual(macro.iseval, '0') + self.assertEqual(macro.args, kwargs.get('args')) + self.assertEqual(macro.validation, kwargs.get('validation')) + self.assertEqual(macro.errormsg, kwargs.get('errormsg')) + self.service.macros.delete(macro_name) + + def test_delete(self): + self.assertTrue(self.macro_name in self.service.macros) + self.service.macros.delete(self.macro_name) + self.assertFalse(self.macro_name in self.service.macros) + self.assertRaises(client.HTTPError, + self.macro.refresh) + + def test_update(self): + new_definition = '| eval updated="true"' + self.macro.update(definition=new_definition) + self.macro.refresh() + self.assertEqual(self.macro['definition'], new_definition) + + is_eval = testlib.to_bool(self.macro['iseval']) + self.macro.update(iseval=not is_eval) + self.macro.refresh() + self.assertEqual(testlib.to_bool(self.macro['iseval']), not is_eval) + + def test_cannot_update_name(self): + new_name = self.macro_name + '-alteration' + self.assertRaises(client.IllegalOperationException, + self.macro.update, name=new_name) + + def test_name_collision(self): + opts = self.opts.kwargs.copy() + opts['owner'] = '-' + opts['app'] = '-' + opts['sharing'] = 'user' + service = client.connect(**opts) + logging.debug("Namespace for collision testing: %s", service.namespace) + macros = service.macros + name = testlib.tmpname() + + dispatch1 = '| eval macro_one="1"' + dispatch2 = '| eval macro_two="2"' + namespace1 = client.namespace(app='search', sharing='app') + namespace2 = client.namespace(owner='admin', app='search', sharing='user') + new_macro2 = macros.create( + name, dispatch2, + namespace=namespace1) + new_macro1 = macros.create( + name, dispatch1, + namespace=namespace2) + + self.assertRaises(client.AmbiguousReferenceException, + macros.__getitem__, name) + macro1 = macros[name, namespace1] + self.check_macro(macro1) + macro1.update(**{'definition': '| eval number=1'}) + macro1.refresh() + self.assertEqual(macro1['definition'], '| eval number=1') + macro2 = macros[name, namespace2] + macro2.update(**{'definition': '| eval number=2'}) + macro2.refresh() + self.assertEqual(macro2['definition'], '| eval number=2') + self.check_macro(macro2) + + def test_no_equality(self): + self.assertRaises(client.IncomparableException, + self.macro.__eq__, self.macro) + + def test_acl(self): + self.assertEqual(self.macro.access["perms"], None) + self.macro.acl_update(sharing="app", owner="admin", **{"perms.read": "admin, nobody"}) + self.assertEqual(self.macro.access["owner"], "admin") + self.assertEqual(self.macro.access["sharing"], "app") + self.assertEqual(self.macro.access["perms"]["read"], ['admin', 'nobody']) + + def test_acl_fails_without_sharing(self): + self.assertRaisesRegex( + ValueError, + "Required argument 'sharing' is missing.", + self.macro.acl_update, + owner="admin", app="search", **{"perms.read": "admin, nobody"} + ) + + def test_acl_fails_without_owner(self): + self.assertRaisesRegex( + ValueError, + "Required argument 'owner' is missing.", + self.macro.acl_update, + sharing="app", app="search", **{"perms.read": "admin, nobody"} + ) + + +if __name__ == "__main__": + try: + import unittest2 as unittest + except ImportError: + import unittest + unittest.main() From 6ec2559b79cccb4f550c675ae3749fa4c66e48a2 Mon Sep 17 00:00:00 2001 From: Szymon Date: Thu, 24 Oct 2024 09:20:34 +0200 Subject: [PATCH 14/22] remove commands used to run test directly from setup.py --- setup.py | 111 +------------------------------------------------------ 1 file changed, 1 insertion(+), 110 deletions(-) diff --git a/setup.py b/setup.py index c80ddf37e..d19a09eb1 100755 --- a/setup.py +++ b/setup.py @@ -14,124 +14,15 @@ # License for the specific language governing permissions and limitations # under the License. -from setuptools import setup, Command - -import os -import sys +from setuptools import setup import splunklib -failed = False - -def run_test_suite(): - import unittest - - def mark_failed(): - global failed - failed = True - - class _TrackingTextTestResult(unittest._TextTestResult): - def addError(self, test, err): - unittest._TextTestResult.addError(self, test, err) - mark_failed() - - def addFailure(self, test, err): - unittest._TextTestResult.addFailure(self, test, err) - mark_failed() - - class TrackingTextTestRunner(unittest.TextTestRunner): - def _makeResult(self): - return _TrackingTextTestResult( - self.stream, self.descriptions, self.verbosity) - - original_cwd = os.path.abspath(os.getcwd()) - os.chdir('tests') - suite = unittest.defaultTestLoader.discover('.') - runner = TrackingTextTestRunner(verbosity=2) - runner.run(suite) - os.chdir(original_cwd) - - return failed - - -def run_test_suite_with_junit_output(): - try: - import unittest2 as unittest - except ImportError: - import unittest - import xmlrunner - original_cwd = os.path.abspath(os.getcwd()) - os.chdir('tests') - suite = unittest.defaultTestLoader.discover('.') - xmlrunner.XMLTestRunner(output='../test-reports').run(suite) - os.chdir(original_cwd) - - -class CoverageCommand(Command): - """setup.py command to run code coverage of the test suite.""" - description = "Create an HTML coverage report from running the full test suite." - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - try: - import coverage - except ImportError: - print("Could not import coverage. Please install it and try again.") - exit(1) - cov = coverage.coverage(source=['splunklib']) - cov.start() - run_test_suite() - cov.stop() - cov.html_report(directory='coverage_report') - - -class TestCommand(Command): - """setup.py command to run the whole test suite.""" - description = "Run test full test suite." - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - failed = run_test_suite() - if failed: - sys.exit(1) - - -class JunitXmlTestCommand(Command): - """setup.py command to run the whole test suite.""" - description = "Run test full test suite with JUnit-formatted output." - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - run_test_suite_with_junit_output() - - setup( author="Splunk, Inc.", author_email="devinfo@splunk.com", - cmdclass={'coverage': CoverageCommand, - 'test': TestCommand, - 'testjunit': JunitXmlTestCommand}, - description="The Splunk Software Development Kit for Python.", license="http://www.apache.org/licenses/LICENSE-2.0", From 1b441352b332dcd50960f5818723b84b731c903a Mon Sep 17 00:00:00 2001 From: Szymon Date: Thu, 24 Oct 2024 09:28:32 +0200 Subject: [PATCH 15/22] modify tox ini not to use deprecated libraries for unittests and add python 3.13 to tests --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index e45dbfb9d..e69c2e6ad 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = clean,docs,py37,py39 +envlist = clean,docs,py37,py39,313 skipsdist = {env:TOXBUILD:false} [testenv:pep8] @@ -28,8 +28,6 @@ setenv = SPLUNK_HOME=/opt/splunk allowlist_externals = make deps = pytest pytest-cov - xmlrunner - unittest-xml-reporting python-dotenv distdir = build From 2f2e17465656b647d5e2bf1fc039ba67eb0b9588 Mon Sep 17 00:00:00 2001 From: Szymon Date: Thu, 24 Oct 2024 10:13:43 +0200 Subject: [PATCH 16/22] modify testing traceback so it would filter out additions to tracback coming from python 3.13 --- tests/modularinput/test_script.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/modularinput/test_script.py b/tests/modularinput/test_script.py index bb4717107..49c259725 100644 --- a/tests/modularinput/test_script.py +++ b/tests/modularinput/test_script.py @@ -252,6 +252,7 @@ def stream_events(self, inputs, ew): # Remove paths and line numbers err = re.sub(r'File "[^"]+', 'File "...', err.getvalue()) err = re.sub(r'line \d+', 'line 123', err) + err = re.sub(r' +~+\^+', '', err) assert out.getvalue() == "" assert err == ( From e3f3968da91ed1d8b67fb20e2334e2cad43b6e3d Mon Sep 17 00:00:00 2001 From: maszyk99 Date: Fri, 25 Oct 2024 20:11:45 +0200 Subject: [PATCH 17/22] Revert "DVPL-0: don't use latest splunk version because of HEC token issue" This reverts commit 40e7b2a305d8e67034c13711c41c678e3a8d9fc5. --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99ee7c350..c9396b079 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,8 +15,7 @@ jobs: splunk-version: - "8.1" - "8.2" - - "9.0.8" - - "9.1.3" + - "latest" fail-fast: false steps: From a8948f8f7baff673a043797a6ec085dcf2a89534 Mon Sep 17 00:00:00 2001 From: Szymon Date: Tue, 29 Oct 2024 08:28:05 +0100 Subject: [PATCH 18/22] resolve conflicts --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e4fcd4ae..c8e67c155 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Run docker-compose + - name: Run docker compose run: SPLUNK_VERSION=${{matrix.splunk-version}} docker compose up -d - name: Setup Python From f637ade7eeba88ed17074ae261f273b774e5168c Mon Sep 17 00:00:00 2001 From: Szymon Date: Wed, 25 Sep 2024 13:48:08 +0200 Subject: [PATCH 19/22] use SSL DEFAULT CONTEX, use tls version min 1.2, allow for usage of Self-signed certificate (cherry picked from commit 43041d14eba7423227bef7fc7548927bfb282756) --- splunklib/binding.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/splunklib/binding.py b/splunklib/binding.py index 958be96eb..25a099489 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -465,6 +465,8 @@ class Context: :type scheme: "https" or "http" :param verify: Enable (True) or disable (False) SSL verification for https connections. :type verify: ``Boolean`` + :param self_signed_certificate: Specifies if self signed certificate is used + :type self_signed_certificate: ``Boolean`` :param sharing: The sharing mode for the namespace (the default is "user"). :type sharing: "global", "system", "app", or "user" :param owner: The owner context of the namespace (optional, the default is "None"). @@ -526,6 +528,7 @@ def __init__(self, handler=None, **kwargs): self.bearerToken = kwargs.get("splunkToken", "") self.autologin = kwargs.get("autologin", False) self.additional_headers = kwargs.get("headers", []) + self._self_signed_certificate = kwargs.get("self_signed_certificate", True) # Store any cookies in the self.http._cookies dict if "cookie" in kwargs and kwargs['cookie'] not in [None, _NoAuthenticationToken]: @@ -604,7 +607,11 @@ def connect(self): """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.scheme == "https": - sock = ssl.wrap_socket(sock) + context = ssl.create_default_context() + context.options |= ssl.OP_NO_TLSv1 | ssl.OP_NO_TLSv1_1 + context.check_hostname = not self._self_signed_certificate + context.verify_mode = ssl.CERT_NONE if self._self_signed_certificate else ssl.CERT_REQUIRED + sock = context.wrap_socket(sock, server_hostname=self.host) sock.connect((socket.gethostbyname(self.host), self.port)) return sock From 180cdd23057b51bd5c0b003c44ada96493db3255 Mon Sep 17 00:00:00 2001 From: Szymon Date: Tue, 29 Oct 2024 16:57:52 +0100 Subject: [PATCH 20/22] Add changelog --- CHANGELOG.md | 21 +++++++++++++++++++++ README.md | 2 +- splunklib/__init__.py | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e423e67d..f37733cbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Splunk Enterprise SDK for Python Changelog +## Version 2.1.0 + +### Changes +* [#516](https://github.com/splunk/splunk-sdk-python/pull/516) Added support for macros +* Remove deprecated `wrap_socket` in `Contex` class. +* Added explicit support for self signed certificates in https +* Enforce minimal required tls version in https connection +* Add support for python 3.13 + +## Version 2.0.2 + +### Minor changes +* Added six.py file back + + +## Version 2.0.1 + +### Bug fixes +* [#567](https://github.com/splunk/splunk-sdk-python/issues/567) Moved "deprecation" dependency + + ## Version 2.0.0 ### Feature updates diff --git a/README.md b/README.md index 2bbae4de9..e200fc9bd 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # The Splunk Enterprise Software Development Kit for Python -#### Version 2.0.0 +#### Version 2.1.0 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code designed to enable developers to build applications using the Splunk platform. diff --git a/splunklib/__init__.py b/splunklib/__init__.py index 2613d2849..5f83b2ac4 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -30,5 +30,5 @@ def setup_logging(level, log_format=DEFAULT_LOG_FORMAT, date_format=DEFAULT_DATE datefmt=date_format) -__version_info__ = (2, 0, 0) +__version_info__ = (2, 1, 0) __version__ = ".".join(map(str, __version_info__)) From 25c44e9098e8c48df13e080b2d0202dcf585e748 Mon Sep 17 00:00:00 2001 From: Szymon Date: Tue, 29 Oct 2024 17:04:15 +0100 Subject: [PATCH 21/22] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f37733cbe..288543c6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * Added explicit support for self signed certificates in https * Enforce minimal required tls version in https connection * Add support for python 3.13 +* [#559](https://github.com/splunk/splunk-sdk-python/pull/559/) Add exception logging ## Version 2.0.2 From d36db5ee76b3ba1837ac0157e7386fd17a63ff7f Mon Sep 17 00:00:00 2001 From: Szymon Date: Wed, 30 Oct 2024 10:35:38 +0100 Subject: [PATCH 22/22] update ReadMe --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e200fc9bd..7cd66cc23 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ The Splunk Enterprise SDK for Python contains library code, and its examples are Here's what you need to get going with the Splunk Enterprise SDK for Python. -* Python 3.7 or Python 3.9 +* Python 3.7, Python 3.9 and Python 3.13 - The Splunk Enterprise SDK for Python is compatible with python3 and has been tested with Python v3.7 and v3.9. + The Splunk Enterprise SDK for Python is compatible with python3 and has been tested with Python v3.7, v3.9 and v3.13. * Splunk Enterprise 9.2 or 8.2