Skip to content

Commit

Permalink
[SD] create a template cache to reduce calls to the KV store
Browse files Browse the repository at this point in the history
This cache is invalidated when a template change is detected.
It doesn't detect changes to the auto_conf folder or JMX checks,
only to the KV store.
Refreshing configs from files still requires a SIGHUP or a restart.
  • Loading branch information
hkaj committed Dec 5, 2016
1 parent 74f6746 commit 9122643
Show file tree
Hide file tree
Showing 7 changed files with 294 additions and 161 deletions.
2 changes: 1 addition & 1 deletion config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1088,7 +1088,7 @@ def load_check(agentConfig, hostname, checkname):

# try to load the check and return the result
load_success, load_failure = load_check_from_places(check_config, check_name, checks_places, agentConfig)
return load_success.values()[0] or load_failure
return load_success.values()[0] if load_success else load_failure

return None

Expand Down
115 changes: 98 additions & 17 deletions tests/core/test_service_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import copy
import mock
import unittest
from collections import defaultdict

# 3p
from nose.plugins.attrib import attr
Expand All @@ -10,7 +11,8 @@
from utils.service_discovery.config_stores import get_config_store
from utils.service_discovery.consul_config_store import ConsulStore
from utils.service_discovery.etcd_config_store import EtcdStore
from utils.service_discovery.abstract_config_store import AbstractConfigStore, CONFIG_FROM_KUBE
from utils.service_discovery.abstract_config_store import AbstractConfigStore, \
_TemplateCache, CONFIG_FROM_KUBE, CONFIG_FROM_TEMPLATE, CONFIG_FROM_AUTOCONF
from utils.service_discovery.sd_backend import get_sd_backend
from utils.service_discovery.sd_docker_backend import SDDockerBackend, _SDDockerBackendConfigFetchState

Expand Down Expand Up @@ -65,11 +67,11 @@ def client_read(path, **kwargs):
if 'all' in kwargs:
return {}
else:
return TestServiceDiscovery.mock_tpls.get(image)[0][config_parts.index(config_part)]
return TestServiceDiscovery.mock_raw_templates.get(image)[0][config_parts.index(config_part)]


def issue_read(identifier):
return TestServiceDiscovery.mock_tpls.get(identifier)
return TestServiceDiscovery.mock_raw_templates.get(identifier)


@attr('unix')
Expand Down Expand Up @@ -122,7 +124,7 @@ class TestServiceDiscovery(unittest.TestCase):
}

# raw templates coming straight from the config store
mock_tpls = {
mock_raw_templates = {
# image_name: ('[check_name]', '[init_tpl]', '[instance_tpl]', expected_python_tpl_list)
'image_0': (
('["check_0"]', '[{}]', '[{"host": "%%host%%"}]'),
Expand Down Expand Up @@ -501,12 +503,12 @@ def test_fill_tpl(self, *args):
def test_get_auto_config(self):
"""Test _get_auto_config"""
expected_tpl = {
'redis': ('redisdb', None, {"host": "%%host%%", "port": "%%port%%"}),
'consul': ('consul', None, {
"url": "http://%%host%%:%%port%%", "catalog_checks": True, "new_leader_checks": True
}),
'redis:v1': ('redisdb', None, {"host": "%%host%%", "port": "%%port%%"}),
'foobar': None
'redis': [('redisdb', None, {"host": "%%host%%", "port": "%%port%%"})],
'consul': [('consul', None, {
"url": "http://%%host%%:%%port%%", "catalog_checks": True, "new_leader_checks": True
})],
'redis:v1': [('redisdb', None, {"host": "%%host%%", "port": "%%port%%"})],
'foobar': []
}

config_store = get_config_store(self.auto_conf_agentConfig)
Expand All @@ -521,10 +523,10 @@ def test_get_check_tpls(self, mock_client_read):
invalid_config = ['bad_image_0', 'bad_image_1']
config_store = get_config_store(self.auto_conf_agentConfig)
for image in valid_config:
tpl = self.mock_tpls.get(image)[1]
tpl = self.mock_raw_templates.get(image)[1]
self.assertEquals(tpl, config_store.get_check_tpls(image))
for image in invalid_config:
tpl = self.mock_tpls.get(image)[1]
tpl = self.mock_raw_templates.get(image)[1]
self.assertEquals(tpl, config_store.get_check_tpls(image))

@mock.patch.object(AbstractConfigStore, 'client_read', side_effect=client_read)
Expand All @@ -534,7 +536,7 @@ def test_get_check_tpls_kube(self, mock_client_read):
invalid_config = ['bad_image_0']
config_store = get_config_store(self.auto_conf_agentConfig)
for image in valid_config + invalid_config:
tpl = self.mock_tpls.get(image)[1]
tpl = self.mock_raw_templates.get(image)[1]
tpl = [(CONFIG_FROM_KUBE, t[1]) for t in tpl]
if tpl:
self.assertNotEquals(
Expand All @@ -550,7 +552,7 @@ def test_get_check_tpls_kube(self, mock_client_read):
['service-discovery.datadoghq.com/foo.check_names',
'service-discovery.datadoghq.com/foo.init_configs',
'service-discovery.datadoghq.com/foo.instances'],
self.mock_tpls[image][0]))))
self.mock_raw_templates[image][0]))))

def test_get_config_id(self):
"""Test get_config_id"""
Expand All @@ -562,8 +564,8 @@ def test_get_config_id(self):
expected_ident)
clear_singletons(self.auto_conf_agentConfig)

@mock.patch.object(AbstractConfigStore, '_issue_read', side_effect=issue_read)
def test_read_config_from_store(self, issue_read):
@mock.patch.object(_TemplateCache, '_issue_read', side_effect=issue_read)
def test_read_config_from_store(self, args):
"""Test read_config_from_store"""
valid_idents = [('nginx', 'nginx'), ('nginx:latest', 'nginx:latest'),
('custom-nginx', 'custom-nginx'), ('custom-nginx:latest', 'custom-nginx'),
Expand All @@ -574,6 +576,85 @@ def test_read_config_from_store(self, issue_read):
for ident, expected_key in valid_idents:
tpl = config_store.read_config_from_store(ident)
# source is added after reading from the store
self.assertEquals(tpl, ('template',) + self.mock_tpls.get(expected_key))
self.assertEquals(
tpl,
{
CONFIG_FROM_AUTOCONF: None,
CONFIG_FROM_TEMPLATE: self.mock_raw_templates.get(expected_key)
}
)
for ident in invalid_idents:
self.assertEquals(config_store.read_config_from_store(ident), [])

# Template cache
@mock.patch('utils.service_discovery.abstract_config_store.get_auto_conf_images')
def test_populate_auto_conf(self, mock_get_auto_conf_images):
"""test _populate_auto_conf"""
auto_tpls = {
'foo': [['check0', 'check1'], [{}, {}], [{}, {}]],
'bar': [['check2', 'check3', 'check3'], [{}, {}, {}], [{}, {'foo': 'bar'}, {'bar': 'foo'}]],
}
cache = _TemplateCache(issue_read, '')
cache.auto_conf_templates = defaultdict(lambda: [[]] * 3)
mock_get_auto_conf_images.return_value = auto_tpls

cache._populate_auto_conf()
self.assertEquals(cache.auto_conf_templates['foo'], auto_tpls['foo'])
self.assertEquals(cache.auto_conf_templates['bar'],
[['check2', 'check3'], [{}, {}], [{}, {'foo': 'bar'}]])

@mock.patch.object(_TemplateCache, '_issue_read', return_value=None)
def test_get_templates(self, args):
"""test get_templates"""
kv_tpls = {
'foo': [['check0', 'check1'], [{}, {}], [{}, {}]],
'bar': [['check2', 'check3'], [{}, {}], [{}, {}]],
}
auto_tpls = {
'foo': [['check3', 'check5'], [{}, {}], [{}, {}]],
'bar': [['check2', 'check6'], [{}, {}], [{}, {}]],
'foobar': [['check4'], [{}], [{}]],
}
cache = _TemplateCache(issue_read, '')
cache.kv_templates = kv_tpls
cache.auto_conf_templates = auto_tpls
self.assertEquals(cache.get_templates('foo'),
{
CONFIG_FROM_TEMPLATE: [['check0', 'check1'], [{}, {}], [{}, {}]],
CONFIG_FROM_AUTOCONF: [['check3', 'check5'], [{}, {}], [{}, {}]]
}
)

self.assertEquals(cache.get_templates('bar'),
{
# check3 must come from template not autoconf
CONFIG_FROM_TEMPLATE: [['check2', 'check3'], [{}, {}], [{}, {}]],
CONFIG_FROM_AUTOCONF: [['check6'], [{}], [{}]]
}
)

self.assertEquals(cache.get_templates('foobar'),
{
CONFIG_FROM_TEMPLATE: None,
CONFIG_FROM_AUTOCONF: [['check4'], [{}], [{}]]
}
)

self.assertEquals(cache.get_templates('baz'), None)

def test_get_check_names(self):
"""Test get_check_names"""
kv_tpls = {
'foo': [['check0', 'check1'], [{}, {}], [{}, {}]],
'bar': [['check2', 'check3'], [{}, {}], [{}, {}]],
}
auto_tpls = {
'foo': [['check4', 'check5'], [{}, {}], [{}, {}]],
'bar': [['check2', 'check6'], [{}, {}], [{}, {}]],
}
cache = _TemplateCache(issue_read, '')
cache.kv_templates = kv_tpls
cache.auto_conf_templates = auto_tpls
self.assertEquals(cache.get_check_names('foo'), set(['check0', 'check1', 'check4', 'check5']))
self.assertEquals(cache.get_check_names('bar'), set(['check2', 'check3', 'check6']))
self.assertEquals(cache.get_check_names('baz'), set())
22 changes: 18 additions & 4 deletions utils/checkfiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# std
import logging
import os
from collections import defaultdict
from urlparse import urljoin

# project
Expand Down Expand Up @@ -50,7 +51,7 @@ def get_check_class(agentConfig, check_name):
return None


def get_auto_conf(agentConfig, check_name):
def get_auto_conf(check_name):
"""Return the yaml auto_config dict for a check name (None if it doesn't exist)."""
from config import PathNotFound, get_auto_confd_path

Expand All @@ -75,11 +76,11 @@ def get_auto_conf(agentConfig, check_name):
return auto_conf


def get_auto_conf_images(agentConfig):
def get_auto_conf_images(full_tpl=False):
"""Walk through the auto_config folder and build a dict of auto-configurable images."""
from config import PathNotFound, get_auto_confd_path
auto_conf_images = {
# image_name: check_name
# image_name: [check_names] or [[check_names], [init_tpls], [instance_tpls]]
}

try:
Expand All @@ -100,5 +101,18 @@ def get_auto_conf_images(agentConfig):
# extract the supported image list
images = auto_conf.get('docker_images', [])
for image in images:
auto_conf_images[image] = check_name
if full_tpl:
init_tpl = auto_conf.get('init_config') or {}
instance_tpl = auto_conf.get('instances', [])
if image not in auto_conf_images:
auto_conf_images[image] = [[check_name], [init_tpl], [instance_tpl]]
else:
for idx, item in enumerate([check_name, init_tpl, instance_tpl]):
auto_conf_images[image][idx].append(item)
else:
if image in auto_conf_images:
auto_conf_images[image].append(check_name)
else:
auto_conf_images[image] = [check_name]

return auto_conf_images
6 changes: 3 additions & 3 deletions utils/configcheck.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,10 @@ def print_templates(agentConfig):
except Exception as ex:
print("Failed to extract configuration templates from the backend:\n%s" % str(ex))

for img, tpl in templates.iteritems():
for ident, tpl in templates.iteritems():
print(
"- Image %s:\n\tcheck names: %s\n\tinit_configs: %s\n\tinstances: %s" % (
img,
"- Identifier %s:\n\tcheck names: %s\n\tinit_configs: %s\n\tinstances: %s" % (
ident,
tpl.get('check_names'),
tpl.get('init_configs'),
tpl.get('instances'),
Expand Down
Loading

0 comments on commit 9122643

Please sign in to comment.