Skip to content

Commit

Permalink
Add config spec data model consumer
Browse files Browse the repository at this point in the history
  • Loading branch information
ofek committed Feb 23, 2021
1 parent e6a3e0c commit 5de72b1
Show file tree
Hide file tree
Showing 49 changed files with 2,593 additions and 15 deletions.
5 changes: 5 additions & 0 deletions .azure-pipelines/templates/run-validations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ steps:
ddev validate config
displayName: 'Validate default configuration files'

- script: |
echo "ddev validate models"
ddev validate models
displayName: 'Validate configuration data models'

- script: |
echo "ddev validate dashboards"
ddev validate dashboards
Expand Down
8 changes: 8 additions & 0 deletions .flake8
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,11 @@ ignore = E203,E722,E741,W503,G200
exclude = .eggs,.tox,build,compat.py,__init__.py,datadog_checks_dev/datadog_checks/dev/tooling/templates/*,*/datadog_checks/*/vendor/*
max-line-length = 120
enable-extensions=G
per-file-ignores =
# potentially long literal strings
datadog_checks/*/config_models/deprecations.py: E501
# https://pydantic-docs.helpmanual.io/usage/validators/
# > validators are "class methods", so the first argument value they
# receive is the UserModel class, not an instance of UserModel
datadog_checks/*/config_models/instance.py: B902
datadog_checks/*/config_models/shared.py: B902
79 changes: 78 additions & 1 deletion datadog_checks_base/datadog_checks/base/checks/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,11 @@
from typing import TYPE_CHECKING, Any, AnyStr, Callable, Deque, Dict, List, Optional, Sequence, Tuple, Union

import yaml
from six import binary_type, iteritems, text_type
from six import PY2, binary_type, iteritems, raise_from, text_type

from ..config import is_affirmative
from ..constants import ServiceCheck
from ..errors import ConfigurationError
from ..types import (
AgentConfigType,
Event,
Expand Down Expand Up @@ -63,6 +64,9 @@

monkey_patch_pyyaml()

if not PY2:
from pydantic import ValidationError

if TYPE_CHECKING:
import ssl

Expand Down Expand Up @@ -241,9 +245,16 @@ def __init__(self, *args, **kwargs):
# Setup metric limits
self.metric_limiter = self._get_metric_limiter(self.name, instance=self.instance)

# Lazily load and validate config
self.__config = None
self.__shared_config = None

# Functions that will be called exactly once (if successful) before the first check run
self.check_initializations = deque([self.send_config_metadata]) # type: Deque[Callable[[], None]]

if not PY2:
self.check_initializations.append(self.load_configuration_models)

def _get_metric_limiter(self, name, instance=None):
# type: (str, InstanceType) -> Optional[Limiter]
limit = self._get_metric_limit(instance=instance)
Expand Down Expand Up @@ -364,6 +375,72 @@ def in_developer_mode(self):
self._log_deprecation('in_developer_mode')
return False

@property
def config(self):
return self.__config

@property
def shared_config(self):
return self.__shared_config

def load_configuration_models(self):
# 'datadog_checks.<PACKAGE>.<MODULE>...'
module_parts = self.__module__.split('.')
package_path = '{}.config_models'.format('.'.join(module_parts[:2]))

if self.__shared_config is None:
self.__shared_config = self.load_configuration_model(
package_path, 'SharedConfig', self._get_shared_config()
)

if self.__config is None:
self.__config = self.load_configuration_model(package_path, 'InstanceConfig', self._get_instance_config())

@staticmethod
def load_configuration_model(import_path, model_name, config):
try:
package = importlib.import_module(import_path)
except ModuleNotFoundError as e: # type: ignore
# Don't fail if there are no models
if str(e) == 'No module named {!r}'.format(import_path):
return

raise

model = getattr(package, model_name, None)
if model is not None:
try:
config_model = model(**copy.deepcopy(config))
except ValidationError as e:
errors = e.errors()
num_errors = len(errors)
message_lines = [
'Detected {} error{} while loading configuration model `{}`:'.format(
num_errors, 's' if num_errors > 1 else '', model_name
)
]

for error in e.errors():
message_lines.append(
' -> '.join(
# Start array indexes at one for user-friendliness
str(loc + 1) if isinstance(loc, int) else str(loc) for loc in error['loc']
)
)
message_lines.append(' {}'.format(error['msg']))

raise_from(ConfigurationError('\n'.join(message_lines)), None)
else:
return config_model

def _get_instance_config(self):
# Any extra fields will be available during a config model's initial validation stage
return self.instance

def _get_shared_config(self):
# Any extra fields will be available during a config model's initial validation stage
return self.init_config

def register_secret(self, secret):
# type: (str) -> None
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ flup-py3==1.0.3; python_version > "3.0"
flup==1.0.3.dev-20110405; python_version < "3.0"
futures==3.3.0; python_version < "3.0"
gearman==2.0.2; sys_platform != "win32" and python_version < "3.0"
immutables==0.15; python_version > "3.0"
in-toto==0.5.0
ipaddress==1.0.22; python_version < "3.0"
jaydebeapi==1.2.3
Expand All @@ -42,6 +43,7 @@ psutil==5.7.2
psycopg2-binary==2.8.4
pyasn1==0.4.6
pycryptodomex==3.9.4
pydantic==1.7.3; python_version > "3.0"
pyhdb==0.3.4
pyjwt==1.7.1
pymongo==3.8.0
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# (C) Datadog, Inc. 2021-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
17 changes: 17 additions & 0 deletions datadog_checks_base/datadog_checks/base/utils/models/fields.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# (C) Datadog, Inc. 2021-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
from pydantic.fields import SHAPE_MAPPING, SHAPE_SEQUENCE, SHAPE_SINGLETON


def get_default_field_value(field, value):
if field.shape == SHAPE_MAPPING:
return {}
elif field.shape == SHAPE_SEQUENCE:
return []
elif field.shape == SHAPE_SINGLETON:
field_type = field.type_
if field_type in (float, int, str):
return field_type()

return value
15 changes: 15 additions & 0 deletions datadog_checks_base/datadog_checks/base/utils/models/types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# (C) Datadog, Inc. 2021-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
from immutables import Map


def make_immutable_check_config(obj):
# Only consider container types that can be de-serialized from YAML/JSON
if isinstance(obj, list):
return tuple(make_immutable_check_config(item) for item in obj)
elif isinstance(obj, dict):
# There are no ordering guarantees, see https://github.com/MagicStack/immutables/issues/57
return Map((k, make_immutable_check_config(v)) for k, v in obj.items())

return obj
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# (C) Datadog, Inc. 2021-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
from . import core
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# (C) Datadog, Inc. 2021-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
from ..types import make_immutable_check_config


def initialize_config(values):
# This is what is returned by the initial root validator of each config model.
return values


def finalize_config(values):
# This is what is returned by the final root validator of each config model. Note:
#
# 1. the final object must be a dict
# 2. we maintain the original order of keys
return {field: make_immutable_check_config(value) for field, value in values.items()}
2 changes: 2 additions & 0 deletions datadog_checks_base/requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ contextlib2==0.6.0; python_version < '3.0'
cryptography==3.3.2
ddtrace==0.32.2
enum34==1.1.6; python_version < '3.0'
immutables==0.15; python_version > '3.0'
ipaddress==1.0.22; python_version < '3.0'
kubernetes==8.0.1
mmh3==2.5.1
orjson==2.6.1; python_version > '3.0'
prometheus-client==0.9.0
protobuf==3.7.0
pydantic==1.7.3; python_version > '3.0'
pyjwt==1.7.1
pysocks==1.7.0
python-dateutil==2.8.0
Expand Down
3 changes: 3 additions & 0 deletions datadog_checks_base/tests/models/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# (C) Datadog, Inc. 2021-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
44 changes: 44 additions & 0 deletions datadog_checks_base/tests/models/test_types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# (C) Datadog, Inc. 2021-present
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
from ..utils import requires_py3

pytestmark = [requires_py3]


def test_make_immutable_check_config():
# TODO: move imports up top when we drop Python 2
from immutables import Map

from datadog_checks.base.utils.models.types import make_immutable_check_config

obj = make_immutable_check_config(
{
'string': 'foo',
'integer': 9000,
'float': 3.14,
'boolean': True,
'array': [{'key': 'foo'}, {'key': 'bar'}],
'mapping': {'foo': 'bar'},
}
)

assert isinstance(obj, Map)
assert len(obj) == 6
assert isinstance(obj['string'], str)
assert obj['string'] == 'foo'
assert isinstance(obj['integer'], int)
assert obj['integer'] == 9000
assert isinstance(obj['float'], float)
assert obj['float'] == 3.14
assert isinstance(obj['boolean'], bool)
assert obj['boolean'] is True
assert isinstance(obj['array'], tuple)
assert len(obj['array']) == 2
assert isinstance(obj['array'][0], Map)
assert obj['array'][0]['key'] == 'foo'
assert isinstance(obj['array'][1], Map)
assert obj['array'][1]['key'] == 'bar'
assert isinstance(obj['mapping'], Map)
assert len(obj['mapping']) == 1
assert obj['mapping']['foo'] == 'bar'
31 changes: 31 additions & 0 deletions datadog_checks_base/tests/test_agent_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,16 @@

import mock
import pytest
from immutables import Map
from six import PY3

from datadog_checks.base import AgentCheck
from datadog_checks.base import __version__ as base_package_version
from datadog_checks.base import to_native_string
from datadog_checks.base.checks.base import datadog_agent

from .utils import requires_py3


def test_instance():
"""
Expand Down Expand Up @@ -840,3 +843,31 @@ def check(self, _):
check.run()

assert check.initialize.call_count == 2


@requires_py3
def test_load_configuration_models(dd_run_check, mocker):
instance = {'endpoint': 'url', 'tags': ['foo:bar'], 'proxy': {'http': 'http://1.2.3.4:9000'}}
init_config = {'proxy': {'https': 'https://1.2.3.4:4242'}}
check = AgentCheck('test', init_config, [instance])
check.check_id = 'test:123'
check.check = lambda _: None

assert check.config is None
assert check.shared_config is None

config_model = {'endpoint': 'url', 'tags': ('foo:bar',), 'proxy': Map(http='http://1.2.3.4:9000')}
shared_config_model = {'proxy': Map(https='https://1.2.3.4:4242')}
package = mocker.MagicMock()
package.InstanceConfig = mocker.MagicMock(return_value=config_model)
package.SharedConfig = mocker.MagicMock(return_value=shared_config_model)
import_module = mocker.patch('importlib.import_module', return_value=package)

dd_run_check(check)

import_module.assert_called_once_with('datadog_checks.base.models')
package.InstanceConfig.assert_called_once_with(**config_model)
package.SharedConfig.assert_called_once_with(**shared_config_model)

assert check.config is config_model
assert check.shared_config is shared_config_model
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from .jmx_metrics import jmx_metrics
from .manifest import manifest
from .metadata import metadata
from .models import models
from .package import package
from .readmes import readmes
from .recommended_monitors import recommended_monitors
Expand All @@ -37,6 +38,7 @@
imports,
manifest,
metadata,
models,
package,
readmes,
recommended_monitors,
Expand Down
Loading

0 comments on commit 5de72b1

Please sign in to comment.