[kubernetes] Add ability to specify service discovery via kubernetes …

…annotations.

This change makes the docker service discovery read the kubernetes annotations
to discover how to monitor a pod. This behavior is only triggered when a
service discovery backend isn't set. The 3 annotations looked for are:
 - `com.datadoghq.sd/check_names`
 - `com.datadoghq.sd/init_configs`
 - `com.datadoghq.sd/instances`

The semantics are exactly the same as that of a KV store.

Fixes DataDog#2794

tests/core/test_service_discovery.py

@@ -40,7 +40,7 @@ def _get_container_inspect(c_id):
         return None
 
 
-def _get_conf_tpls(image_name, trace_config=False):
+def _get_conf_tpls(image_name, trace_config=False, kube_annotations=None):
     """Return a mocked configuration template from self.mock_templates."""
     return copy.deepcopy(TestServiceDiscovery.mock_templates.get(image_name)[0])
 
@@ -524,6 +524,28 @@ def test_get_check_tpls(self, mock_client_read):
             tpl = self.mock_tpls.get(image)[1]
             self.assertEquals(tpl, config_store.get_check_tpls(image))
 
+    @mock.patch.object(AbstractConfigStore, 'client_read', side_effect=client_read)
+    def test_get_check_tpls_kube(self, mock_client_read):
+        """Test get_check_tpls"""
+        valid_config = ['image_0', 'image_1', 'image_2']
+        invalid_config = ['bad_image_0']
+        config_store = get_config_store(self.auto_conf_agentConfig)
+        for image in valid_config + invalid_config:
+            tpl = self.mock_tpls.get(image)[1]
+            if tpl:
+                self.assertNotEquals(
+                    tpl,
+                    config_store.get_check_tpls('k8s-' + image, auto_conf=True))
+            self.assertEquals(
+                tpl,
+                config_store.get_check_tpls(
+                    'k8s-' + image, auto_conf=True,
+                    kube_annotations=dict(zip(
+                        ['com.datadoghq.sd/check_names',
+                         'com.datadoghq.sd/init_configs',
+                         'com.datadoghq.sd/instances'],
+                        self.mock_tpls[image][0]))))
+
     def test_get_config_id(self):
         """Test get_config_id"""
         with mock.patch('utils.dockerutil.DockerUtil.client', return_value=None):

utils/service_discovery/abstract_config_store.py

@@ -20,10 +20,13 @@
 CONFIG_FROM_AUTOCONF = 'auto-configuration'
 CONFIG_FROM_FILE = 'YAML file'
 CONFIG_FROM_TEMPLATE = 'template'
+CONFIG_FROM_KUBE = 'Kubernetes Pod Annotation'
 TRACE_CONFIG = 'trace_config'  # used for tracing config load by service discovery
 CHECK_NAMES = 'check_names'
 INIT_CONFIGS = 'init_configs'
 INSTANCES = 'instances'
+KUBE_ANNOTATIONS = 'kube_annotations'
+KUBE_ANNOTATION_PREFIX = 'com.datadoghq.sd/'
 
 
 class KeyNotFound(Exception):
@@ -95,6 +98,19 @@ def _populate_identifier_to_checks(self):
 
         return identifier_to_checks
 
+    def _get_kube_config(self, identifier, kube_annotations):
+        try:
+            check_names = json.loads(kube_annotations[KUBE_ANNOTATION_PREFIX + CHECK_NAMES])
+            init_config_tpls = json.loads(kube_annotations[KUBE_ANNOTATION_PREFIX + INIT_CONFIGS])
+            instance_tpls = json.loads(kube_annotations[KUBE_ANNOTATION_PREFIX + INSTANCES])
+            return [check_names, init_config_tpls, instance_tpls]
+        except KeyError:
+            return None
+        except json.JSONDecodeError:
+            log.exception('Could not decode the JSON configuration template '
+                          'for the kubernetes pod with ident %s...' % identifier)
+            return None
+
     def _get_auto_config(self, image_name):
         ident = self._get_image_ident(image_name)
         if ident in self.auto_conf_images:
@@ -115,12 +131,22 @@ def _get_auto_config(self, image_name):
 
     def get_check_tpls(self, identifier, **kwargs):
         """Retrieve template configs for an identifier from the config_store or auto configuration."""
-        templates = []
         trace_config = kwargs.get(TRACE_CONFIG, False)
 
         # this flag is used when no valid configuration store was provided
         # it makes the method skip directly to the auto_conf
         if kwargs.get('auto_conf') is True:
+            # When not using a configuration store on kubernetes, check the pod
+            # annotations for configs before falling back to autoconf.
+            kube_annotations = kwargs.get(KUBE_ANNOTATIONS)
+            if kube_annotations:
+                kube_config = self._get_kube_config(identifier, kube_annotations)
+                if kube_config is not None:
+                    check_names, init_config_tpls, instance_tpls = kube_config
+                    source = CONFIG_FROM_KUBE
+                    return [(source, vs) if trace_config else vs
+                            for vs in zip(check_names, init_config_tpls, instance_tpls)]
+
             # in auto config mode, identifier is the image name
             auto_config = self._get_auto_config(identifier)
             if auto_config is not None:
@@ -147,12 +173,8 @@ def get_check_tpls(self, identifier, **kwargs):
         # Try to update the identifier_to_checks cache
         self._update_identifier_to_checks(identifier, check_names)
 
-        for idx, c_name in enumerate(check_names):
-            if trace_config:
-                templates.append((source, (c_name, init_config_tpls[idx], instance_tpls[idx])))
-            else:
-                templates.append((c_name, init_config_tpls[idx], instance_tpls[idx]))
-        return templates
+        return [(source, values) if trace_config else values
+                for values in zip(check_names, init_config_tpls, instance_tpls)]
 
     def read_config_from_store(self, identifier):
         """Try to read from the config store, falls back to auto-config in case of failure."""

utils/service_discovery/sd_docker_backend.py

@@ -14,6 +14,9 @@
 from utils.service_discovery.config_stores import get_config_store, TRACE_CONFIG
 
 DATADOG_ID = 'com.datadoghq.sd.check.id'
+K8S_ANNOTATION_CHECK_NAMES = 'com.datadoghq.sd/check_names'
+K8S_ANNOTATION_INIT_CONFIGS = 'com.datadoghq.sd/init_configs'
+K8S_ANNOTATION_INSTANCES = 'com.datadoghq.sd/instances'
 log = logging.getLogger(__name__)
 
 
@@ -255,7 +258,8 @@ def get_config_id(self, image, labels):
     def _get_check_configs(self, c_id, identifier, trace_config=False):
         """Retrieve configuration templates and fill them with data pulled from docker and tags."""
         inspect = self.docker_client.inspect_container(c_id)
-        config_templates = self._get_config_templates(identifier, trace_config=trace_config)
+        annotations = self._get_kube_config(inspect.get('Id'), 'annotations') if Platform.is_k8s() else None
+        config_templates = self._get_config_templates(identifier, trace_config=trace_config, kube_annotations=annotations)
         if not config_templates:
             log.debug('No config template for container %s with identifier %s. '
                       'It will be left unconfigured.' % (c_id[:12], identifier))
@@ -281,7 +285,7 @@ def _get_check_configs(self, c_id, identifier, trace_config=False):
 
         return check_configs
 
-    def _get_config_templates(self, identifier, trace_config=False):
+    def _get_config_templates(self, identifier, trace_config=False, kube_annotations=None):
         """Extract config templates for an identifier from a K/V store and returns it as a dict object."""
         config_backend = self.agentConfig.get('sd_config_backend')
         templates = []
@@ -294,7 +298,7 @@ def _get_config_templates(self, identifier, trace_config=False):
         # format: [('ident', {init_tpl}, {instance_tpl})] without trace_config
         # or      [(source, ('ident', {init_tpl}, {instance_tpl}))] with trace_config
         raw_tpls = self.config_store.get_check_tpls(
-            identifier, auto_conf=auto_conf, trace_config=trace_config)
+            identifier, auto_conf=auto_conf, trace_config=trace_config, kube_annotations=kube_annotations)
         for tpl in raw_tpls:
             if trace_config and tpl is not None:
                 # each template can come from either auto configuration or user-supplied templates