[WIP] Telemetry

splunklib/modularinput/script.py

@@ -13,15 +13,19 @@
 # under the License.
 
 from __future__ import absolute_import
+
+import sys
+
 from abc import ABCMeta, abstractmethod
+import splunklib
+from splunklib import six
 from splunklib.six.moves.urllib.parse import urlsplit
-import sys
 
 from ..client import Service
 from .event_writer import EventWriter
 from .input_definition import InputDefinition
 from .validation_definition import ValidationDefinition
-from splunklib import six
+from ..wire._internal import Telemetry, TelemetryMetric
 
 try:
     import xml.etree.cElementTree as ET
@@ -70,6 +74,20 @@ def run_script(self, args, event_writer, input_stream):
                 # passed on stdin as XML, and the script will write events on
                 # stdout and log entries on stderr.
                 self._input_definition = InputDefinition.parse(input_stream)
+
+                # create a telemetry metric
+                metric = TelemetryMetric(**{
+                    'metric_type': 'event',
+                    'component': 'splunk-sdk-python',
+                    'data': {
+                        'version': splunklib.__version__
+                    }
+                })
+
+                # call out to telemetry
+                telemetry = Telemetry(self.service)
+                telemetry.submit(metric.to_wire())
+
                 self.stream_events(self._input_definition, event_writer)
                 event_writer.close()
                 return 0

splunklib/wire/__init__.py

@@ -0,0 +1,17 @@
+# coding=utf-8
+#
+# Copyright © 2011-2020 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ._internal import *

splunklib/wire/_internal/__init__.py

@@ -0,0 +1,18 @@
+# coding=utf-8
+#
+# Copyright © 2011-2020 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from .telemetry import *
+from .telemetry_metric import *

splunklib/wire/_internal/json_sink.py

@@ -0,0 +1,50 @@
+# coding=utf-8
+#
+# Copyright © 2011-2020 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+
+from splunklib.client import Entity
+
+class JsonSink(Entity):
+    """This class represents a JSON-based write-only sink of entities in the Splunk
+    instance, notably telemetry-metric.
+    """
+    JSON_HEADER = [('Content-Type', 'application/json')]
+
+    def __init__(self, service, path, **kwargs):
+        super(JsonSink, self).__init__(service, path, skip_refresh=True, **kwargs)
+
+    def _post(self, url, **kwargs):
+        owner, app, sharing = self._proper_namespace()
+
+        return self.service.post(self.path + url, owner=owner, app=app, sharing=sharing, **kwargs)
+
+    def submit(self, data):
+        """
+        Submits an item to the sink.
+
+        :param data: data to submit
+        :type data: ``dict``
+
+        :return: return data
+        :rtype: ``dict``
+        """
+
+        response = self._post('', headers=self.__class__.JSON_HEADER, body=json.dumps(data))
+
+        body = json.loads(response.body.read().decode('utf-8'))
+
+        return response, body

splunklib/wire/_internal/telemetry.py

@@ -0,0 +1,23 @@
+# coding=utf-8
+#
+# Copyright © 2011-2020 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from splunklib.wire._internal.json_sink import JsonSink
+
+PATH_TELEMETRY = "telemetry-metric"
+
+class Telemetry(JsonSink):
+    def __init__(self, service, **kwargs):
+        super(Telemetry, self).__init__(service, PATH_TELEMETRY, **kwargs)

splunklib/wire/_internal/telemetry_metric.py

@@ -0,0 +1,62 @@
+# coding=utf-8
+#
+# Copyright © 2011-2020 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+class TelemetryMetric:
+    def __init__(self, metric_type, component, data, opt_in_required=2):
+        self.metric_type = metric_type
+        self.component = component
+        self.data = data
+        self.opt_in_required = opt_in_required
+
+    @property
+    def metric_type(self):
+        return self._metric_type
+
+    @metric_type.setter
+    def metric_type(self, value):
+        self._metric_type = value
+
+    @property
+    def component(self):
+        return self._component
+
+    @component.setter
+    def component(self, value):
+        self._component = value
+
+    @property
+    def data(self):
+        return self._data
+
+    @data.setter
+    def data(self, value):
+        self._data = value
+
+    @property
+    def opt_in_required(self):
+        return self._opt_in_required
+
+    @opt_in_required.setter
+    def opt_in_required(self, value):
+        self._opt_in_required = value
+
+    def to_wire(self):
+        return {
+            'type': self.metric_type,
+            'component': self.component,
+            'data': self.data,
+            'optInRequired': self.opt_in_required,
+        }

tests/modularinput/test_script.py

@@ -1,8 +1,12 @@
+import io
+import json
 import sys
 
+from mock import Mock, patch
+
+from splunklib import six
 from splunklib.client import Service
 from splunklib.modularinput import Script, EventWriter, Scheme, Argument, Event
-import io
 
 from splunklib.modularinput.utils import xml_compare
 from tests.modularinput.modularinput_testlib import data_open
@@ -14,6 +18,17 @@
 
 TEST_SCRIPT_PATH = "__IGNORED_SCRIPT_PATH__"
 
+PATCHED_TELEMETRY_RESPONSE = {
+    'status': 201,
+    'reason': 'Created',
+    'body.read.return_value': six.ensure_binary(json.dumps({
+        'message': 'Data submitted successfully',
+        'metricValueID': '26844DB9-7806-40E0-96C0-1BD554930BA8'
+    })),
+    'headers': [
+        ('content-type', 'application/json; charset=UTF-8')
+    ]
+}
 
 def test_error_on_script_with_null_scheme(capsys):
     """A script that returns a null scheme should generate no output on
@@ -184,9 +199,12 @@ def stream_events(self, inputs, ew):
     script = NewScript()
     input_configuration = data_open("data/conf_with_2_inputs.xml")
 
-    ew = EventWriter(sys.stdout, sys.stderr)
+    event_writer = EventWriter(sys.stdout, sys.stderr)
 
-    return_value = script.run_script([TEST_SCRIPT_PATH], ew, input_configuration)
+    with patch.object(Service, 'post') as patched_telemetry_post:
+        patched_telemetry_post.return_value = Mock(**PATCHED_TELEMETRY_RESPONSE)
+
+        return_value = script.run_script([TEST_SCRIPT_PATH], event_writer, input_configuration)
 
     output = capsys.readouterr()
     assert output.err == ""
@@ -218,7 +236,12 @@ def stream_events(self, inputs, ew):
             self.authority_uri = inputs.metadata['server_uri']
 
     script = NewScript()
-    with data_open("data/conf_with_2_inputs.xml") as input_configuration:
+
+    with data_open("data/conf_with_2_inputs.xml") as input_configuration, \
+            patch.object(Service, 'post') as patched_telemetry_post:
+
+        patched_telemetry_post.return_value = Mock(**PATCHED_TELEMETRY_RESPONSE)
+
         ew = EventWriter(sys.stdout, sys.stderr)
 
         assert script.service is None

tests/test_telemetry.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+#
+# Copyright 2011-2014 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import
+import pytest
+
+from tests import testlib
+from splunklib.wire._internal.telemetry import Telemetry
+from splunklib.wire._internal.telemetry_metric import TelemetryMetric
+
+@pytest.mark.app
+class TestTelemetry(testlib.SDKTestCase):
+    def setUp(self):
+        super(TestTelemetry, self).setUp()
+
+        self.service.namespace['owner'] = 'nobody'
+        self.service.namespace['app'] = 'sdk-app-collection'
+
+        self.telemetry = Telemetry(self.service)
+
+    def test_submit(self):
+        # create a telemetry metric
+        metric = TelemetryMetric(**{
+            'metric_type': 'event',
+            'component': 'telemetry_test_case',
+            'data': {
+                'testValue': 32
+            }
+        })
+
+        # call out to telemetry
+        response, _body = self.telemetry.submit(metric.to_wire())
+
+        # it should return a 201
+        self.assertEqual(response.status, 201)

tox.ini

@@ -27,7 +27,8 @@ passenv = LANG
 setenv = SPLUNK_HOME=/opt/splunk
          INPUT_EXAMPLE_UPLOAD=/opt/splunk/var/log/splunk/splunkd_ui_access.log
 whitelist_externals = true
-deps = pytest
+deps = mock
+       pytest
        pytest-cov
        xmlrunner
        unittest2