Merge pull request #533 from splunk/release/1.7.4

Release/1.7.4

.github/workflows/release.yml

@@ -9,17 +9,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout source
-        uses: actions/checkout@v2.3.2
+        uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.7
       - name: Install dependencies
         run: pip install twine
       - name: Build package
         run: python setup.py sdist
       - name: Publish package to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.3.1
+        uses: pypa/gh-action-pypi-publish@v1.8.8
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-        python: [ 2.7, 3.7 ]
+        python: [3.7]
         splunk-version:
           - "8.1"
           - "8.2"
@@ -20,13 +20,13 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Run docker-compose
         run: SPLUNK_VERSION=${{matrix.splunk-version}} docker-compose up -d
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
 

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Splunk Enterprise SDK for Python Changelog
 
+## Version 1.7.4
+
+### Bug fixes
+* [#532](https://github.com/splunk/splunk-sdk-python/pull/532) update encoding errors mode to 'replace' [[issue#505](https://github.com/splunk/splunk-sdk-python/issues/505)]
+* [#507](https://github.com/splunk/splunk-sdk-python/pull/507) masked sensitive data in logs [[issue#506](https://github.com/splunk/splunk-sdk-python/issues/506)]
+
+### Minor changes
+* [#530](https://github.com/splunk/splunk-sdk-python/pull/530) Update GitHub CI build status in README and removed RTD(Read The Docs) reference
+
 ## Version 1.7.3
 
 ### Bug fixes

README.md

@@ -1,9 +1,10 @@
-[![Build Status](https://travis-ci.org/splunk/splunk-sdk-python.svg?branch=master)](https://travis-ci.org/splunk/splunk-sdk-python)
-[![Documentation Status](https://readthedocs.org/projects/splunk-python-sdk/badge/?version=latest)](https://splunk-python-sdk.readthedocs.io/en/latest/?badge=latest)
+[![Build Status](https://github.com/splunk/splunk-sdk-python/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/splunk/splunk-sdk-python/actions/workflows/test.yml)
+
+[Reference Docs](https://dev.splunk.com/enterprise/reference)
 
 # The Splunk Enterprise Software Development Kit for Python
 
-#### Version 1.7.3
+#### Version 1.7.4
 
 The Splunk Enterprise Software Development Kit (SDK) for Python contains library code designed to enable developers to build applications using the Splunk platform.
 
@@ -127,7 +128,7 @@ The Splunk Enterprise SDK for Python contains a collection of unit tests. To run
 
 You can also run individual test files, which are located in **/splunk-sdk-python/tests**. To run a specific test, enter:
 
-    make specific_test_name
+    make test_specific
 
 The test suite uses Python's standard library, the built-in `unittest` library, `pytest`, and `tox`.
 

scripts/test_specific.sh

@@ -1,2 +1,4 @@
 echo "To run a specific test:"
-echo "  tox -e py27,py37 [test_file_path]::[test_name]"
+echo "  tox -e py27,py37 [test_file_path]::[TestClassName]::[test_method]"
+echo "For Example, To run 'test_autologin' testcase from 'test_service.py' file run"
+echo "  tox -e py37 -- tests/test_service.py::ServiceTestCase::test_autologin"

splunklib/__init__.py

@@ -31,5 +31,5 @@ def setup_logging(level, log_format=DEFAULT_LOG_FORMAT, date_format=DEFAULT_DATE
                         format=log_format,
                         datefmt=date_format)
 
-__version_info__ = (1, 7, 3)
+__version_info__ = (1, 7, 4)
 __version__ = ".".join(map(str, __version_info__))

splunklib/binding.py

@@ -27,6 +27,7 @@
 from __future__ import absolute_import
 
 import io
+import json
 import logging
 import socket
 import ssl
@@ -60,12 +61,17 @@
     "HTTPError"
 ]
 
+SENSITIVE_KEYS = ['Authorization', 'Cookie', 'action.email.auth_password', 'auth', 'auth_password', 'clear_password', 'clientId',
+                  'crc-salt', 'encr_password', 'oldpassword', 'passAuth', 'password', 'session', 'suppressionKey',
+                  'token']
+
 # If you change these, update the docstring
 # on _authority as well.
 DEFAULT_HOST = "localhost"
 DEFAULT_PORT = "8089"
 DEFAULT_SCHEME = "https"
 
+
 def _log_duration(f):
     @wraps(f)
     def new_f(*args, **kwargs):
@@ -77,6 +83,28 @@ def new_f(*args, **kwargs):
     return new_f
 
 
+def mask_sensitive_data(data):
+    '''
+    Masked sensitive fields data for logging purpose
+    '''
+    if not isinstance(data, dict):
+        try:
+            data = json.loads(data)
+        except Exception as ex:
+            return data
+
+    # json.loads will return "123"(str) as 123(int), so return the data if it's not 'dict' type
+    if not isinstance(data, dict):
+        return data
+    mdata = {}
+    for k, v in data.items():
+        if k in SENSITIVE_KEYS:
+            mdata[k] = "******"
+        else:
+            mdata[k] = mask_sensitive_data(v)
+    return mdata
+
+
 def _parse_cookies(cookie_str, dictionary):
     """Tries to parse any key-value pairs of cookies in a string,
     then updates the the dictionary with any key-value pairs found.
@@ -631,7 +659,7 @@ def delete(self, path_segment, owner=None, app=None, sharing=None, **query):
         """
         path = self.authority + self._abspath(path_segment, owner=owner,
                                               app=app, sharing=sharing)
-        logger.debug("DELETE request to %s (body: %s)", path, repr(query))
+        logger.debug("DELETE request to %s (body: %s)", path, mask_sensitive_data(query))
         response = self.http.delete(path, self._auth_headers, **query)
         return response
 
@@ -694,7 +722,7 @@ def get(self, path_segment, owner=None, app=None, headers=None, sharing=None, **
 
         path = self.authority + self._abspath(path_segment, owner=owner,
                                               app=app, sharing=sharing)
-        logger.debug("GET request to %s (body: %s)", path, repr(query))
+        logger.debug("GET request to %s (body: %s)", path, mask_sensitive_data(query))
         all_headers = headers + self.additional_headers + self._auth_headers
         response = self.http.get(path, all_headers, **query)
         return response
@@ -773,12 +801,7 @@ def post(self, path_segment, owner=None, app=None, sharing=None, headers=None, *
 
         path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing)
 
-        # To avoid writing sensitive data in debug logs
-        endpoint_having_sensitive_data = ["/storage/passwords"]
-        if any(endpoint in path for endpoint in endpoint_having_sensitive_data):
-            logger.debug("POST request to %s ", path)
-        else:
-            logger.debug("POST request to %s (body: %s)", path, repr(query))
+        logger.debug("POST request to %s (body: %s)", path, mask_sensitive_data(query))
         all_headers = headers + self.additional_headers + self._auth_headers
         response = self.http.post(path, all_headers, **query)
         return response
@@ -845,8 +868,7 @@ def request(self, path_segment, method="GET", headers=None, body={},
 
         all_headers = headers + self.additional_headers + self._auth_headers
         logger.debug("%s request to %s (headers: %s, body: %s)",
-                      method, path, str(all_headers), repr(body))
-
+                     method, path, str(mask_sensitive_data(dict(all_headers))), mask_sensitive_data(body))
         if body:
             body = _encode(**body)
 

splunklib/modularinput/event_writer.py

@@ -77,7 +77,7 @@ def write_xml_document(self, document):
 
         :param document: An ``ElementTree`` object.
         """
-        self._out.write(ensure_str(ET.tostring(document)))
+        self._out.write(ensure_str(ET.tostring(document), errors="replace"))
         self._out.flush()
 
     def close(self):

splunklib/searchcommands/search_command.py

@@ -934,7 +934,7 @@ def _read_chunk(istream):
         except Exception as error:
             raise RuntimeError('Failed to read body of length {}: {}'.format(body_length, error))
 
-        return metadata, six.ensure_str(body)
+        return metadata, six.ensure_str(body, errors="replace")
 
     _header = re.compile(r'chunked\s+1.0\s*,\s*(\d+)\s*,\s*(\d+)\s*\n')
 

tests/searchcommands/test_csc_apps.py

@@ -15,10 +15,12 @@
 # under the License.
 
 import unittest
+import pytest
+
 from tests import testlib
 from splunklib import results
 
-
+@pytest.mark.smoke
 class TestCSC(testlib.SDKTestCase):
 
     def test_eventing_app(self):

tests/test_binding.py

@@ -641,6 +641,7 @@ def test_got_updated_cookie_with_get(self):
                 self.assertEqual(list(new_cookies.values())[0], list(old_cookies.values())[0])
         self.assertTrue(found)
 
+    @pytest.mark.smoke
     def test_login_fails_with_bad_cookie(self):
         # We should get an error if using a bad cookie
         try:
@@ -649,6 +650,7 @@ def test_login_fails_with_bad_cookie(self):
         except AuthenticationError as ae:
             self.assertEqual(str(ae), "Login failed.")
 
+    @pytest.mark.smoke
     def test_login_with_multiple_cookies(self):
         # We should get an error if using a bad cookie
         new_context = binding.Context()

tests/test_event_type.py

@@ -63,10 +63,10 @@ def tearDown(self):
         except KeyError:
             pass
 
-    def test_delete(self):
-        self.assertTrue(self.event_type_name in self.service.event_types)
-        self.service.event_types.delete(self.event_type_name)
-        self.assertFalse(self.event_type_name in self.service.event_types)
+    # def test_delete(self):
+    #     self.assertTrue(self.event_type_name in self.service.event_types)
+    #     self.service.event_types.delete(self.event_type_name)
+    #     self.assertFalse(self.event_type_name in self.service.event_types)
 
     def test_update(self):
         kwargs = {}

tests/test_job.py

@@ -57,6 +57,7 @@ def test_oneshot_with_garbage_fails(self):
         jobs = self.service.jobs
         self.assertRaises(TypeError, jobs.create, "abcd", exec_mode="oneshot")
 
+    @pytest.mark.smoke
     def test_oneshot(self):
         jobs = self.service.jobs
         stream = jobs.oneshot("search index=_internal earliest=-1m | head 3", output_mode='json')
@@ -382,6 +383,7 @@ def test_search_invalid_query_as_json(self):
         except Exception as e:
             self.fail("Got some unexpected error. %s" % e.message)
 
+    @pytest.mark.smoke
     def test_v1_job_fallback(self):
         self.assertEventuallyTrue(self.job.is_done)
         self.assertLessEqual(int(self.job['eventCount']), 3)

tests/test_service.py

@@ -15,6 +15,8 @@
 # under the License.
 
 from __future__ import absolute_import
+import pytest
+
 from tests import testlib
 import unittest
 
@@ -168,6 +170,7 @@ def _create_unauthenticated_service(self):
         })
 
     # To check the HEC event endpoint using Endpoint instance
+    @pytest.mark.smoke
     def test_hec_event(self):
         import json
         service_hec = client.connect(host='localhost', scheme='https', port=8088,