Skip to content

Commit 181cbfe

Browse files
authored
Merge pull request #341 from splunk/csc-multibyte
Custom search command support for multibyte characters in Python 3
2 parents d1553e5 + 37077d6 commit 181cbfe

File tree

11 files changed

+108
-39
lines changed

11 files changed

+108
-39
lines changed

CHANGELOG.md

+11-6
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
# Splunk SDK for Python Changelog
22

3+
## Version 1.6.14
4+
5+
### Bug fix
6+
* `SearchCommand` now correctly supports multibyte characters in Python 3.
7+
38
## Version 1.6.13
49

510
### Bug fix
6-
* Fixed regression in mod inputs which resulted in error ’file' object has no attribute 'readable’, by not forcing to text/bytes in mod inputs event writer any longer.
11+
* Fixed regression in mod inputs which resulted in error ’file' object has no attribute 'readable’, by not forcing to text/bytes in mod inputs event writer any longer.
712

813
### Minor changes
9-
* Minor updates to the splunklib search commands to support Python3
14+
* Minor updates to the splunklib search commands to support Python3
1015

1116
## Version 1.6.12
1217

@@ -22,25 +27,25 @@
2227

2328
### Bug Fix
2429

25-
* Fix custom search command V2 failures on Windows for Python3
30+
* Fix custom search command V2 failures on Windows for Python3
2631

2732
## Version 1.6.10
2833

2934
### Bug Fix
3035

31-
* Fix long type gets wrong values on windows for python 2
36+
* Fix long type gets wrong values on windows for python 2
3237

3338
## Version 1.6.9
3439

3540
### Bug Fix
3641

37-
* Fix buffered input in python 3
42+
* Fix buffered input in python 3
3843

3944
## Version 1.6.8
4045

4146
### Bug Fix
4247

43-
* Fix custom search command on python 3 on windows
48+
* Fix custom search command on python 3 on windows
4449

4550
## Version 1.6.7
4651

README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
# The Splunk Software Development Kit for Python
55

6-
#### Version 1.6.13
6+
#### Version 1.6.14
77

88
The Splunk Software Development Kit (SDK) for Python contains library code and
99
examples designed to enable developers to build applications using Splunk.

examples/searchcommands_app/setup.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ def splunk_restart(uri, auth):
111111

112112

113113
class AnalyzeCommand(Command):
114-
"""
115-
setup.py command to run code coverage of the test suite.
114+
"""
115+
setup.py command to run code coverage of the test suite.
116116
117117
"""
118118
description = 'Create an HTML coverage report from running the full test suite.'
@@ -367,8 +367,8 @@ def _link_debug_client(self):
367367

368368

369369
class TestCommand(Command):
370-
"""
371-
setup.py command to run the whole test suite.
370+
"""
371+
setup.py command to run the whole test suite.
372372
373373
"""
374374
description = 'Run full test suite.'
@@ -439,7 +439,7 @@ def run(self):
439439
setup(
440440
description='Custom Search Command examples',
441441
name=os.path.basename(project_dir),
442-
version='1.6.13',
442+
version='1.6.14',
443443
author='Splunk, Inc.',
444444
author_email='[email protected]',
445445
url='http://github.com/splunk/splunk-sdk-python',

splunklib/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,5 @@
1616

1717
from __future__ import absolute_import
1818
from splunklib.six.moves import map
19-
__version_info__ = (1, 6, 13)
19+
__version_info__ = (1, 6, 14)
2020
__version__ = ".".join(map(str, __version_info__))

splunklib/binding.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1378,7 +1378,7 @@ def request(url, message, **kwargs):
13781378
head = {
13791379
"Content-Length": str(len(body)),
13801380
"Host": host,
1381-
"User-Agent": "splunk-sdk-python/1.6.13",
1381+
"User-Agent": "splunk-sdk-python/1.6.14",
13821382
"Accept": "*/*",
13831383
"Connection": "Close",
13841384
} # defaults

splunklib/searchcommands/generating_command.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ def _execute(self, ifile, process):
204204
205205
"""
206206
if self._protocol_version == 2:
207-
result = self._read_chunk(ifile)
207+
result = self._read_chunk(self._as_binary_stream(ifile))
208208

209209
if not result:
210210
return

splunklib/searchcommands/search_command.py

+22-9
Original file line numberDiff line numberDiff line change
@@ -656,7 +656,7 @@ def _process_protocol_v2(self, argv, ifile, ofile):
656656
# noinspection PyBroadException
657657
try:
658658
debug('Reading metadata')
659-
metadata, body = self._read_chunk(ifile)
659+
metadata, body = self._read_chunk(self._as_binary_stream(ifile))
660660

661661
action = getattr(metadata, 'action', None)
662662

@@ -850,17 +850,29 @@ def _execute(self, ifile, process):
850850
self.finish()
851851

852852
@staticmethod
853-
def _read_chunk(ifile):
853+
def _as_binary_stream(ifile):
854+
if six.PY2:
855+
return ifile
856+
857+
try:
858+
return ifile.buffer
859+
except AttributeError as error:
860+
raise RuntimeError('Failed to get underlying buffer: {}'.format(error))
861+
862+
@staticmethod
863+
def _read_chunk(istream):
854864
# noinspection PyBroadException
865+
assert isinstance(istream.read(0), six.binary_type), 'Stream must be binary'
866+
855867
try:
856-
header = ifile.readline()
868+
header = istream.readline()
857869
except Exception as error:
858870
raise RuntimeError('Failed to read transport header: {}'.format(error))
859871

860872
if not header:
861873
return None
862874

863-
match = SearchCommand._header.match(header)
875+
match = SearchCommand._header.match(six.ensure_str(header))
864876

865877
if match is None:
866878
raise RuntimeError('Failed to parse transport header: {}'.format(header))
@@ -870,14 +882,14 @@ def _read_chunk(ifile):
870882
body_length = int(body_length)
871883

872884
try:
873-
metadata = ifile.read(metadata_length)
885+
metadata = istream.read(metadata_length)
874886
except Exception as error:
875887
raise RuntimeError('Failed to read metadata of length {}: {}'.format(metadata_length, error))
876888

877889
decoder = MetadataDecoder()
878890

879891
try:
880-
metadata = decoder.decode(metadata)
892+
metadata = decoder.decode(six.ensure_str(metadata))
881893
except Exception as error:
882894
raise RuntimeError('Failed to parse metadata of length {}: {}'.format(metadata_length, error))
883895

@@ -887,11 +899,11 @@ def _read_chunk(ifile):
887899
body = ""
888900
try:
889901
if body_length > 0:
890-
body = ifile.read(body_length)
902+
body = istream.read(body_length)
891903
except Exception as error:
892904
raise RuntimeError('Failed to read body of length {}: {}'.format(body_length, error))
893905

894-
return metadata, body
906+
return metadata, six.ensure_str(body)
895907

896908
_header = re.compile(r'chunked\s+1.0\s*,\s*(\d+)\s*,\s*(\d+)\s*\n')
897909

@@ -922,9 +934,10 @@ def _records_protocol_v1(self, ifile):
922934
yield record
923935

924936
def _records_protocol_v2(self, ifile):
937+
istream = self._as_binary_stream(ifile)
925938

926939
while True:
927-
result = self._read_chunk(ifile)
940+
result = self._read_chunk(istream)
928941

929942
if not result:
930943
return
25.7 KB
Binary file not shown.
61.7 KB
Binary file not shown.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import io
2+
import gzip
3+
import sys
4+
5+
from os import path
6+
7+
from splunklib import six
8+
from splunklib.searchcommands import StreamingCommand, Configuration
9+
10+
11+
def build_test_command():
12+
@Configuration()
13+
class TestSearchCommand(StreamingCommand):
14+
def stream(self, records):
15+
for record in records:
16+
yield record
17+
18+
return TestSearchCommand()
19+
20+
21+
def get_input_file(name):
22+
return path.join(
23+
path.dirname(path.dirname(__file__)), 'data', 'custom_search', name + '.gz')
24+
25+
26+
def test_multibyte_chunked():
27+
data = gzip.open(get_input_file("multibyte_input"))
28+
if not six.PY2:
29+
data = io.TextIOWrapper(data)
30+
cmd = build_test_command()
31+
cmd._process_protocol_v2(sys.argv, data, sys.stdout)
32+
33+
34+
def test_v1_searchcommand():
35+
data = gzip.open(get_input_file("v1_search_input"))
36+
if not six.PY2:
37+
data = io.TextIOWrapper(data)
38+
cmd = build_test_command()
39+
cmd._process_protocol_v1(["test_script.py", "__EXECUTE__"], data, sys.stdout)

tests/searchcommands/test_search_command.py

+27-15
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,21 @@
3333
import os
3434
import re
3535

36+
from io import TextIOWrapper
37+
3638
import pytest
3739

40+
def build_command_input(getinfo_metadata, execute_metadata, execute_body):
41+
input = ('chunked 1.0,{},0\n{}'.format(len(six.ensure_binary(getinfo_metadata)), getinfo_metadata) +
42+
'chunked 1.0,{},{}\n{}{}'.format(len(six.ensure_binary(execute_metadata)), len(six.ensure_binary(execute_body)), execute_metadata, execute_body))
43+
44+
ifile = BytesIO(six.ensure_binary(input))
45+
46+
if not six.PY2:
47+
ifile = TextIOWrapper(ifile)
48+
49+
return ifile
50+
3851
@Configuration()
3952
class TestCommand(SearchCommand):
4053

@@ -428,11 +441,9 @@ def test_process_scpv2(self):
428441
show_configuration=('true' if show_configuration is True else 'false'))
429442

430443
execute_metadata = '{"action":"execute","finished":true}'
431-
execute_body = 'test\r\ndata\r\n'
444+
execute_body = 'test\r\ndata\r\n测试\r\n'
432445

433-
ifile = StringIO(
434-
'chunked 1.0,{},0\n{}'.format(len(getinfo_metadata), getinfo_metadata) +
435-
'chunked 1.0,{},{}\n{}{}'.format(len(execute_metadata), len(execute_body), execute_metadata, execute_body))
446+
ifile = build_command_input(getinfo_metadata, execute_metadata, execute_body)
436447

437448
command = TestCommand()
438449
result = BytesIO()
@@ -455,12 +466,17 @@ def test_process_scpv2(self):
455466
self.assertEqual(command.required_option_1, 'value_1')
456467
self.assertEqual(command.required_option_2, 'value_2')
457468

458-
self.assertEqual(
469+
expected = (
459470
'chunked 1.0,68,0\n'
460471
'{"inspector":{"messages":[["INFO","test command configuration: "]]}}\n'
461-
'chunked 1.0,17,23\n'
472+
'chunked 1.0,17,32\n'
462473
'{"finished":true}test,__mv_test\r\n'
463-
'data,\r\n',
474+
'data,\r\n'
475+
'测试,\r\n'
476+
)
477+
478+
self.assertEqual(
479+
expected,
464480
result.getvalue().decode('utf-8'))
465481

466482
self.assertEqual(command.protocol_version, 2)
@@ -620,11 +636,9 @@ def test_process_scpv2(self):
620636
show_configuration=show_configuration)
621637

622638
execute_metadata = '{"action":"execute","finished":true}'
623-
execute_body = 'test\r\ndata\r\n'
639+
execute_body = 'test\r\ndata\r\n测试\r\n'
624640

625-
ifile = StringIO(
626-
'chunked 1.0,{},0\n{}'.format(len(getinfo_metadata), getinfo_metadata) +
627-
'chunked 1.0,{},{}\n{}{}'.format(len(execute_metadata), len(execute_body), execute_metadata, execute_body))
641+
ifile = build_command_input(getinfo_metadata, execute_metadata, execute_body)
628642

629643
command = TestCommand()
630644
result = BytesIO()
@@ -666,11 +680,9 @@ def test_process_scpv2(self):
666680
show_configuration=('true' if show_configuration is True else 'false'))
667681

668682
execute_metadata = '{"action":"execute","finished":true}'
669-
execute_body = 'action\r\nraise_exception\r\n'
683+
execute_body = 'action\r\nraise_exception\r\n测试\r\n'
670684

671-
ifile = StringIO(
672-
'chunked 1.0,{},0\n{}'.format(len(getinfo_metadata), getinfo_metadata) +
673-
'chunked 1.0,{},{}\n{}{}'.format(len(execute_metadata), len(execute_body), execute_metadata, execute_body))
685+
ifile = build_command_input(getinfo_metadata, execute_metadata, execute_body)
674686

675687
command = TestCommand()
676688
result = BytesIO()

0 commit comments

Comments
 (0)