Merge pull request #147 from stuart-warren/jolokia

initial jolokia collector rewrite

THANKS

@@ -15,3 +15,4 @@ Chad Rhyner <[email protected]>
 Jacek Masiulaniec <[email protected]>
 Manuel Amador <[email protected]>
 Tim Douglas <[email protected]>
+Stuart Warren <[email protected]>

collectors/0/jolokia.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python
+# This file is part of tcollector.
+# Copyright (C) 2013 The tcollector Authors.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version. This program is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
+# General Public License for more details. You should have received a copy
+# of the GNU Lesser General Public License along with this program. If not,
+# see <http://www.gnu.org/licenses/>.
+
+"""
+# Script to use with tcollector and OpenTSDB
+# to grab metrics from a java process using
+# jolokia.
+#
+# Requires pyjolokia > 0.3.1
+"""
+
+import time
+import sys
+import copy
+try:
+    import simplejson as json
+except ImportError:
+    import json
+
+from collectors.lib import utils
+
+try:
+    from pyjolokia import Jolokia, JolokiaError
+except ImportError:
+    utils.err('requires pyjolokia python module > 0.3.1')
+    sys.exit(13)
+
+try:
+    from collectors.etc import jolokia_conf
+except ImportError:
+    jolokia_conf = None
+
+
+class JolokiaCollector():
+    """
+    Create an instance of JolokiaCollector for each instance of Jolokia you
+    will connect to.
+    """
+
+    url = ''
+    monitors = []
+    j4p = None
+
+    def __init__(self, url, auth, tags, monitors):
+        """
+        url - HTTP location of Jolokia
+        auth - Dict of username and password
+        tags - Dict of key value pairs
+        monitors - List of Dicts.
+           - mbean - String, eg 'java.lang:type=*'
+           - metric - String, prefix to generated metric name, eg 'java.lang'
+           - tags - (optional) Same as above, but specific for this mbean only
+           - not_tags - (optional) List of autogenerated tags to remove, eg ['type']
+        """
+        self.url = url
+        if 'username' not in auth:
+            auth['username'] = ''
+        if 'password' not in auth:
+            auth['password'] = ''
+        self.auth = auth
+        self._set_monitors(monitors, tags)
+
+    def _set_monitors(self, monitors, tags):
+        self.j4p = Jolokia(self.url)
+        self.j4p.auth(httpusername=self.auth['username'], httppassword=self.auth['password'])
+        self.j4p.config(ignoreErrors=True)
+
+        self.monitors = copy.deepcopy(monitors)
+        for m in self.monitors:
+            if 'tags' in m:
+                m['tags'].update(tags)
+            else:
+                m['tags'] = tags
+
+            if 'tags' in m and len(m['tags']) > 0:
+                m['taglist'] = ["%s=%s" % (k, v) for k, v in m['tags'].items()]
+            else:
+                m['taglist'] = []
+
+            # set a default not_tags to '' to catch empty tag keys and append
+            # the value to metric name instead
+            m['not_tags'] += ['']
+
+            self.j4p.add_request(type='read', mbean=m['mbean'])
+
+    def print_metrics(self, d, metric_prefix, timestamp, tags, not_tags=[]):
+        """ Take a dict of attributes and print out numerical metric strings
+        Recurse if necessary
+        """
+        for k, v in d.iteritems():
+            # Tack on the name of the attribute
+            attribute, more_tags = self.parse_attribute(k.lower(), not_tags)
+            metric_name = '.'.join([metric_prefix, attribute])
+            my_tags = tags + more_tags
+            # If numerical
+            if utils.is_numeric(v):
+                print "%s %d %s %s" % (metric_name, timestamp, str(v),
+                                        ' '.join(my_tags))
+            # If a bool, True=1, False=0
+            elif type(v) is bool:
+                print "%s %d %s %s" % (metric_name, timestamp, str(int(v)),
+                                        ' '.join(my_tags))
+            # Or a dict of more attributes, call ourselves again
+            elif type(v) is dict:
+                self.print_metrics(v, metric_name, timestamp, my_tags, not_tags)
+            else:
+                #lists, strings, etc
+                #print '# ', type(v), metric_name, str(v)
+                pass
+
+    def process_data(self):
+        """ Make request to Jolokia, make sure we have valid data, print out
+        the metrics for each mbean.
+        """
+        data = []
+        try:
+            data = self.j4p.getRequests()
+        except JolokiaError:
+            utils.err('error: issue connecting to Jolokia ' + self.url)
+        if len(data) >= 1:
+            for mbean in data:
+                if 'error' in mbean:
+                    utils.err("error: " + mbean['error'])
+                for monitor in self.monitors:
+                    if monitor['mbean'] == mbean['request']['mbean']:
+                        if mbean['status'] == 200:
+                            self.print_metrics(mbean['value'], monitor['metric'], mbean['timestamp'],
+                                                   monitor['taglist'], monitor['not_tags'])
+                            break
+                        else:
+                            utils.err("error: mbean not found - " + monitor['mbean'])
+
+    def parse_attribute(self, attr, not_tags=[]):
+        """ Parse and order attribute text
+        eg from:
+            org.apache.cassandra.metrics:name=CurrentlyBlockedTasks,path=request,
+               scope=RequestResponseStage,type=ThreadPools
+        to: cassandra.metrics.threadpools.currentlyblockedtasks.count,
+               [path=request, scope=requestresponsestage]
+        """
+        pruned = {}
+        parts = attr.split(',')
+        for p in parts:
+            # Take anything to the right of a =
+            tag_name, _, attrname = p.rpartition('=')
+            tag_name = tag_name.split(':')[-1]
+            # Swap out bad chars
+            attrname = attrname.replace('.', '_').replace('/', '').replace(' ', '_')
+            pruned[tag_name] = attrname
+
+        attr_list = []
+        for t in not_tags:
+            if t in pruned:
+                attr_list.append(pruned[t])
+
+        return '.'.join(attr_list), ["%s=%s" % (k, v) for k, v in pruned.items()
+                                     if k not in not_tags]
+
+
+def main():
+    if not (jolokia_conf and jolokia_conf.enabled()):
+        utils.err("Jolokia collector disable by config")
+        sys.exit(13)
+    utils.drop_privileges()
+
+    CONFIG = jolokia_conf.get_config()
+    instances = []
+
+    for instance in CONFIG['instances']:
+        if 'common_tags' in CONFIG:
+            if 'tags' in instance:
+                instance['tags'].update(CONFIG['common_tags'])
+            else:
+                instance['tags'] = CONFIG['common_tags']
+        if 'common_monitors' in CONFIG:
+            if 'monitors' in instance:
+                instance['monitors'] += CONFIG['common_monitors']
+            else:
+                instance['monitors'] = CONFIG['common_monitors']
+
+        if not 'monitors' in instance:
+            utils.err("error: no monitors configured")
+            sys.exit(13)
+        if not 'tags' in instance:
+            instance['tags'] = []
+
+        if not 'auth' in instance:
+            instance['auth'] = {'username': '', 'password': ''}
+
+        jc = JolokiaCollector(instance['url'], instance['auth'], instance['tags'], instance['monitors'])
+        instances.append(jc)
+
+    # LOOP!!
+    while True:
+        for i in instances:
+            i.process_data()
+
+        try:
+            time.sleep(CONFIG['interval'])
+        except KeyboardInterrupt:
+            break
+    # End while True
+
+if __name__ == "__main__":
+    main()
+
+sys.exit(0)

collectors/etc/jolokia_conf.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+
+
+def enabled():
+    return False
+
+
+def get_config():
+    # config = {
+    #     ## interval in seconds to fetch metrics
+    #     'interval': 20,
+    #     ## List of tags to be added to all metrics
+    #     'common_tags': {
+    #         'env': 'prd'
+    #     },
+    #     ## For each instance of Jolokia, fetch this list of mbeans
+    #     ## mbean parameter keys MUST be in alphanumeric order
+    #     ##      eg: name, type
+    #     'common_monitors': [{
+    #         'mbean': 'java.lang:type=*',
+    #         'metric': 'java.lang',
+    #         'not_tags': ['type']
+    #         }, {
+    #         'mbean': 'java.lang:type=Runtime',
+    #         'metric': 'java.lang',
+    #         'not_tags': ['type']
+    #         }, {
+    #         'mbean': 'java.lang:name=*,type=GarbageCollector',
+    #         'metric': 'java.lang',
+    #         'not_tags': ['type']
+    #         }, {
+    #         'mbean': 'java.lang:name=*,type=MemoryPool',
+    #         'metric': 'java.lang',
+    #         'not_tags': ['type']
+    #     }],
+    #     ## List of instances of Jolokia to query for metrics
+    #     'instances': [{
+    #         ## url: required
+    #         'url': 'http://localhost:8778/jolokia/',
+    #         ## optional basic auth credentials
+    #         'auth': {
+    #             'username': 'adminRole',
+    #             'password': 'passwordhere'
+    #         }
+    #         ## list of additional tags for this instance
+    #         'tags': {
+    #             'cluster': 'cluster01'
+    #         },
+    #         ## list of additional mbeans to monitor for this instance
+    #         'monitors': [{
+    #             'mbean': 'org.apache.cassandra.*:*',
+    #             'metric': 'cassandra.metrics',
+    #             'not_tags': ['type', 'name']
+    #         }]
+    #     }, {
+    #         'url': 'http://localhost:8998/jolokia/',
+    #         'tags': {
+    #             'cluster': 'cluster02'
+    #         },
+    #         'monitors': [{
+    #             'mbean': 'org.apache.zookeeper.*:*',
+    #             'metric': 'zookeeper.metrics',
+    #             'not_tags': ['type', 'name']
+    #         }]
+    #     }]
+    # }
+
+    config = {
+        'interval': 20,
+        'common_monitors': [{
+            'mbean': 'java.lang:type=*',
+            'metric': 'java.lang',
+            'not_tags': ['type']
+            }, {
+            'mbean': 'java.lang:type=Runtime',
+            'metric': 'java.lang',
+            'not_tags': ['type']
+            }, {
+            'mbean': 'java.lang:name=*,type=GarbageCollector',
+            'metric': 'java.lang',
+            'not_tags': ['type']
+            }, {
+            'mbean': 'java.lang:name=*,type=MemoryPool',
+            'metric': 'java.lang',
+            'not_tags': ['type']
+        }],
+        'instances': [{
+            'url': 'http://localhost:8778/jolokia/'
+        }]
+    }
+
+    return config