[kubernetes] gather pods.running metric using kubelet api locally on …

…hosts

added tests and fixtures for pods.running metric

fixed typo

cleaned up code

python2.6 compat fix

added checks in test_historate

fixed reviewed items

allow caller to avoid strings escaping while reading fixture files

refactoring: call kubelet api only once for labels and running pods

remove node_name tag, we already have host tag carrying same info

checks.d/kubernetes.py

@@ -5,14 +5,16 @@
 import numbers
 from fnmatch import fnmatch
 import re
+import simplejson as json
+from collections import defaultdict
 
 # 3rd party
 import requests
 
 # project
 from checks import AgentCheck
 from config import _is_affirmative
-from utils.kubeutil import set_kube_settings, get_kube_settings, get_kube_labels
+from utils.kubeutil import set_kube_settings, get_kube_settings, extract_kube_labels
 from utils.http import retrieve_json
 
 NAMESPACE = "kubernetes"
@@ -40,6 +42,7 @@
     RATE: {True: HISTORATE, False: RATE}
 }
 
+
 class Kubernetes(AgentCheck):
     """ Collect metrics and events from kubelet """
 
@@ -105,7 +108,6 @@ def _perform_master_checks(self, url):
             self.log.warning('master checks url=%s exception=%s' % (url, str(e)))
             raise
 
-
     def check(self, instance):
         kube_settings = get_kube_settings()
         if not kube_settings.get("host"):
@@ -217,13 +219,15 @@ def _update_container_metrics(self, instance, subcontainer, kube_labels):
     def _retrieve_metrics(self, url):
         return retrieve_json(url)
 
-    def _retrieve_kube_labels(self):
-        return get_kube_labels()
-
+    def _retrieve_pods_list(self):
+        kube_settings = get_kube_settings()
+        return retrieve_json(kube_settings["pods_list_url"])
 
     def _update_metrics(self, instance, kube_settings):
+        pods_list = self._retrieve_pods_list()
         metrics = self._retrieve_metrics(kube_settings["metrics_url"])
-        kube_labels = self._retrieve_kube_labels()
+        kube_labels = extract_kube_labels(pods_list)
+
         if not metrics:
             raise Exception('No metrics retrieved cmd=%s' % self.metrics_cmd)
 
@@ -233,3 +237,22 @@ def _update_metrics(self, instance, kube_settings):
             except Exception, e:
                 self.log.error("Unable to collect metrics for container: {0} ({1}".format(
                     subcontainer.get('name'), e))
+
+        self._update_pods_metrics(instance, pods_list)
+
+    def _update_pods_metrics(self, instance, pods):
+        controllers_map = defaultdict(list)
+        for pod in pods['items']:
+            node_name = pod['spec']['nodeName']
+            try:
+                created_by = json.loads(pod['metadata']['annotations']['kubernetes.io/created-by'])
+                if created_by['reference']['kind'] == 'ReplicationController':
+                    controllers_map[created_by['reference']['name']].append(node_name)
+            except KeyError:
+                continue
+
+        tags = instance.get('tags', [])
+        for ctrl, pods in controllers_map.iteritems():
+            _tags = tags[:]  # copy base tags
+            _tags.append('kube_replication_controller:{0}'.format(ctrl))
+            self.publish_gauge(self, NAMESPACE + '.pods.running', len(pods), _tags)

tests/checks/common.py

@@ -128,9 +128,12 @@ def file(file_name):
         return os.path.join(Fixtures.directory(), file_name)
 
     @staticmethod
-    def read_file(file_name):
+    def read_file(file_name, string_escape=True):
         with open(Fixtures.file(file_name)) as f:
-            return f.read().decode('string-escape').decode("utf-8")
+            contents = f.read()
+            if string_escape:
+                contents = contents.decode('string-escape')
+            return contents.decode("utf-8")
 
 
 class AgentCheckTest(unittest.TestCase):