[mongo] Emit amazon documentdb cloud metadata (#18921)

* emit docdb internal resources * fix instance * submit cloud_metadata * add documentdb prefix * sync config * remove documentdb prefix for consistency * add changelog * add unit test * fix lint * fix tests * fix tests
DataDog · Oct 29, 2024 · 37caf92 · 37caf92
1 parent 1fce6b8
commit 37caf92
Show file tree

Hide file tree

Showing 21 changed files with 142 additions and 3 deletions.
diff --git a/mongo/assets/configuration/spec.yaml b/mongo/assets/configuration/spec.yaml
@@ -425,6 +425,36 @@ files:
                 type: <METRIC_TYPE>
             tags:
               - test:mongodb
+    - name: aws
+      description: |
+        This block defines the configuration for Amazon DocumentDB instances.
+
+        Complete this section if you have installed the Datadog AWS Integration
+        (https://docs.datadoghq.com/integrations/amazon_web_services) to enrich instances
+        with DocumentDB integration telemetry.
+
+        These values are only applied when `dbm: true` option is set.
+      options:
+        - name: instance_endpoint
+          description: |
+            Equal to the Endpoint.Address of the instance the agent is connecting to.
+            This value is optional if the value of `host` is already configured to the instance endpoint.
+
+            For more information on instance endpoints,
+            see the AWS docs https://docs.aws.amazon.com/documentdb/latest/developerguide/API_Endpoint.html
+          value:
+            type: string
+            example: mydocdb.c0xa1xvklawc.us-east-1.docdb.amazonaws.com
+        - name: cluster_identifier
+          description: |
+            Equal to the Cluster identifier of the instance the agent is connecting to.
+            This value is optional if the value of `cluster_name` is already configured to the cluster identifier.
+
+            For more information on cluster identifiers,
+            see the AWS docs https://docs.aws.amazon.com/documentdb/latest/developerguide/API_DBCluster.html
+          value:
+            type: string
+            example: mydocdb
     - name: server
       deprecation:
         Agent version: "8.0.0"

diff --git a/mongo/changelog.d/18921.added b/mongo/changelog.d/18921.added
@@ -0,0 +1 @@
+Add `aws` to the instance configuration to allow cloud resource linking with Amazon DocumentDB.
diff --git a/mongo/datadog_checks/mongo/config.py b/mongo/datadog_checks/mongo/config.py
@@ -102,6 +102,7 @@ def __init__(self, instance, log, init_config):
         self.dbm_enabled = is_affirmative(instance.get('dbm', False))
         self.database_instance_collection_interval = instance.get('database_instance_collection_interval', 300)
         self.cluster_name = instance.get('cluster_name', None)
+        self.cloud_metadata = self._compute_cloud_metadata(instance)
         self._operation_samples_config = instance.get('operation_samples', {})
         self._slow_operations_config = instance.get('slow_operations', {})
         self._schemas_config = instance.get('schemas', {})
@@ -159,6 +160,15 @@ def _compute_metric_tags(self):
             tags.append('clustername:%s' % self.cluster_name)
         return tags
 
+    def _compute_cloud_metadata(self, instance):
+        cloud_metadata = {}
+        if aws := instance.get('aws'):
+            cloud_metadata['aws'] = {
+                'instance_endpoint': aws.get('instance_endpoint'),
+                'cluster_identifier': aws.get('cluster_identifier') or self.cluster_name,
+            }
+        return cloud_metadata
+
     @property
     def operation_samples(self):
         enabled = False

diff --git a/mongo/datadog_checks/mongo/config_models/instance.py b/mongo/datadog_checks/mongo/config_models/instance.py
@@ -20,6 +20,15 @@
 from . import defaults, deprecations, validators
 
 
+class Aws(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    cluster_identifier: Optional[str] = None
+    instance_endpoint: Optional[str] = None
+
+
 class Field(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
@@ -103,6 +112,7 @@ class InstanceConfig(BaseModel):
     )
     add_node_tag_to_events: Optional[bool] = None
     additional_metrics: Optional[tuple[str, ...]] = None
+    aws: Optional[Aws] = None
     cluster_name: Optional[str] = None
     collections: Optional[tuple[str, ...]] = None
     collections_indexes_stats: Optional[bool] = None

diff --git a/mongo/datadog_checks/mongo/data/conf.yaml.example b/mongo/datadog_checks/mongo/data/conf.yaml.example
@@ -346,6 +346,34 @@ instances:
     #     tags:
     #     - test:mongodb
 
+    ## This block defines the configuration for Amazon DocumentDB instances.
+    ##
+    ## Complete this section if you have installed the Datadog AWS Integration
+    ## (https://docs.datadoghq.com/integrations/amazon_web_services) to enrich instances
+    ## with DocumentDB integration telemetry.
+    ##
+    ## These values are only applied when `dbm: true` option is set.
+    #
+    # aws:
+
+        ## @param instance_endpoint - string - optional - default: mydocdb.c0xa1xvklawc.us-east-1.docdb.amazonaws.com
+        ## Equal to the Endpoint.Address of the instance the agent is connecting to.
+        ## This value is optional if the value of `host` is already configured to the instance endpoint.
+        ##
+        ## For more information on instance endpoints,
+        ## see the AWS docs https://docs.aws.amazon.com/documentdb/latest/developerguide/API_Endpoint.html
+        #
+        # instance_endpoint: mydocdb.c0xa1xvklawc.us-east-1.docdb.amazonaws.com
+
+        ## @param cluster_identifier - string - optional - default: mydocdb
+        ## Equal to the Cluster identifier of the instance the agent is connecting to.
+        ## This value is optional if the value of `cluster_name` is already configured to the cluster identifier.
+        ##
+        ## For more information on cluster identifiers,
+        ## see the AWS docs https://docs.aws.amazon.com/documentdb/latest/developerguide/API_DBCluster.html
+        #
+        # cluster_identifier: mydocdb
+
     ## @param server - string - optional
     ## Specify the MongoDB URI, with database to use for reporting (defaults to "admin")
     ## E.g. mongodb://datadog:LnCbkX4uhpuLHSUrcayEoAZA@localhost:27016/admin

diff --git a/mongo/datadog_checks/mongo/dbm/operation_samples.py b/mongo/datadog_checks/mongo/dbm/operation_samples.py
@@ -309,6 +309,7 @@ def _create_operation_sample_payload(
             "network": {
                 "client": operation_metadata["client"],
             },
+            "cloud_metadata": self._check._config.cloud_metadata,
             "db": {
                 "instance": operation_metadata["dbname"],
                 "plan": explain_plan,
@@ -360,6 +361,7 @@ def _create_activities_payload(
             "dbm_type": "activity",
             "collection_interval": self._collection_interval,
             "ddtags": self._check._get_tags(),
+            "cloud_metadata": self._check._config.cloud_metadata,
             "timestamp": now * 1000,
             "service": self._check._config.service,
             "mongodb_activity": activities,

diff --git a/mongo/datadog_checks/mongo/dbm/schemas.py b/mongo/datadog_checks/mongo/dbm/schemas.py
@@ -63,6 +63,7 @@ def collect_schemas(self):
             "collection_interval": self._collection_interval,
             "dbms_version": self._check._mongo_version,
             "tags": self._check._get_tags(),
+            "cloud_metadata": self._check._config.cloud_metadata,
         }
 
         collected_collections = 0

diff --git a/mongo/datadog_checks/mongo/dbm/slow_operations.py b/mongo/datadog_checks/mongo/dbm/slow_operations.py
@@ -309,6 +309,7 @@ def _create_slow_operation_explain_plan_payload(self, slow_operation: dict, expl
             "ddagentversion": datadog_agent.get_version(),
             "ddsource": "mongo",
             "ddtags": ",".join(self._check._get_tags()),
+            "cloud_metadata": self._check._config.cloud_metadata,
             "timestamp": slow_operation["ts"] * 1000,
             "service": self._check._config.service,
             "network": {
@@ -405,6 +406,7 @@ def _submit_slow_operation_payload(self, slow_operation_events):
             "dbm_type": "slow_query",
             "collection_interval": self._collection_interval,
             "ddtags": self._check._get_tags(),
+            "cloud_metadata": self._check._config.cloud_metadata,
             "timestamp": time.time() * 1000,
             "service": self._check._config.service,
             "mongodb_slow_queries": slow_operation_events,

diff --git a/mongo/datadog_checks/mongo/mongo.py b/mongo/datadog_checks/mongo/mongo.py
@@ -246,9 +246,16 @@ def internal_resource_tags(self):
         '''
         Return the internal resource tags for the database instance.
         '''
-        if not self._resolved_hostname:
-            return []
-        return [f"dd.internal.resource:database_instance:{self._resolved_hostname}"]
+        tags = []
+        if self._resolved_hostname:
+            tags.append(f"dd.internal.resource:database_instance:{self._resolved_hostname}")
+        if self._config.cloud_metadata:
+            aws = self._config.cloud_metadata.get('aws')
+            if instance_endpoint := aws.get('instance_endpoint'):
+                tags.append(f"dd.internal.resource:aws_docdb_instance:{instance_endpoint}")
+            if cluster_identifier := aws.get('cluster_identifier'):
+                tags.append(f"dd.internal.resource:aws_docdb_cluster:{cluster_identifier}")
+        return tags
 
     def _get_tags(self, include_internal_resource_tags=False):
         tags = deepcopy(self._config.metric_tags)

diff --git a/mongo/tests/results/operation-activities-mongos.json b/mongo/tests/results/operation-activities-mongos.json
@@ -5,6 +5,7 @@
         "ddsource": "mongo",
         "dbm_type": "activity",
         "collection_interval": 10,
+        "cloud_metadata": {},
         "ddtags": [
             "server:mongodb://testUser2:*****@localhost:27017/test",
             "clustername:my_cluster",

diff --git a/mongo/tests/results/operation-activities-standalone.json b/mongo/tests/results/operation-activities-standalone.json
@@ -5,6 +5,7 @@
         "ddsource": "mongo",
         "dbm_type": "activity",
         "collection_interval": 10,
+        "cloud_metadata": {},
         "ddtags": [
             "server:mongodb://testUser2:*****@localhost:27017/test",
             "clustername:my_cluster",

diff --git a/mongo/tests/results/operation-samples-mongos.json b/mongo/tests/results/operation-samples-mongos.json
@@ -5,6 +5,7 @@
         "ddagentversion": "0.0.0",
         "ddsource": "mongo",
         "ddtags": "server:mongodb://testUser2:*****@localhost:27017/test,clustername:my_cluster,hosting_type:self-hosted,sharding_cluster_role:mongos",
+        "cloud_metadata": {},
         "timestamp": 1715911398111.2722,
         "service": "my_service",
         "network": {

diff --git a/mongo/tests/results/operation-samples-standalone.json b/mongo/tests/results/operation-samples-standalone.json
@@ -5,6 +5,7 @@
         "ddagentversion": "0.0.0",
         "ddsource": "mongo",
         "ddtags": "server:mongodb://testUser2:*****@localhost:27017/test,clustername:my_cluster,hosting_type:self-hosted",
+        "cloud_metadata": {},
         "timestamp": 1715911398111.2722,
         "service": "my_service",
         "network": {

diff --git a/mongo/tests/results/schemas-mongos.json b/mongo/tests/results/schemas-mongos.json
@@ -6,6 +6,7 @@
         "kind": "mongodb_databases",
         "collection_interval": 600,
         "dbms_version": "4.1.13",
+        "cloud_metadata": {},
         "tags": [
             "server:mongodb://testUser2:*****@localhost:27017/test",
             "clustername:my_cluster",
@@ -596,6 +597,7 @@
         "kind": "mongodb_databases",
         "collection_interval": 600,
         "dbms_version": "4.1.13",
+        "cloud_metadata": {},
         "tags": [
             "server:mongodb://testUser2:*****@localhost:27017/test",
             "clustername:my_cluster",

diff --git a/mongo/tests/results/schemas-standalone-atlas.json b/mongo/tests/results/schemas-standalone-atlas.json
@@ -6,6 +6,7 @@
         "kind": "mongodb_databases",
         "collection_interval": 600,
         "dbms_version": "4.1.13",
+        "cloud_metadata": {},
         "tags": [
             "server:mongodb://testUser2:*****@localhost:27017/test",
             "clustername:my_cluster",
@@ -629,6 +630,7 @@
         "kind": "mongodb_databases",
         "collection_interval": 600,
         "dbms_version": "4.1.13",
+        "cloud_metadata": {},
         "tags": [
             "server:mongodb://testUser2:*****@localhost:27017/test",
             "clustername:my_cluster",

diff --git a/mongo/tests/results/schemas-standalone.json b/mongo/tests/results/schemas-standalone.json
@@ -6,6 +6,7 @@
         "kind": "mongodb_databases",
         "collection_interval": 600,
         "dbms_version": "4.1.13",
+        "cloud_metadata": {},
         "tags": [
             "server:mongodb://testUser2:*****@localhost:27017/test",
             "clustername:my_cluster",
@@ -595,6 +596,7 @@
         "kind": "mongodb_databases",
         "collection_interval": 600,
         "dbms_version": "4.1.13",
+        "cloud_metadata": {},
         "tags": [
             "server:mongodb://testUser2:*****@localhost:27017/test",
             "clustername:my_cluster",

diff --git a/mongo/tests/results/slow-operations-explain-plans-mongos.json b/mongo/tests/results/slow-operations-explain-plans-mongos.json
@@ -5,6 +5,7 @@
         "ddagentversion": "0.0.0",
         "ddsource": "mongo",
         "ddtags": "server:mongodb://testUser2:*****@localhost:27017/test,clustername:my_cluster,hosting_type:self-hosted,sharding_cluster_role:mongos",
+        "cloud_metadata": {},
         "timestamp": 32479987545338.0,
         "service": "my_service",
         "network": {
@@ -730,6 +731,7 @@
         "ddagentversion": "0.0.0",
         "ddsource": "mongo",
         "ddtags": "server:mongodb://testUser2:*****@localhost:27017/test,clustername:my_cluster,hosting_type:self-hosted,sharding_cluster_role:mongos",
+        "cloud_metadata": {},
         "timestamp": 32479987545521.0,
         "service": "my_service",
         "network": {
@@ -1450,6 +1452,7 @@
         "ddagentversion": "0.0.0",
         "ddsource": "mongo",
         "ddtags": "server:mongodb://testUser2:*****@localhost:27017/test,clustername:my_cluster,hosting_type:self-hosted,sharding_cluster_role:mongos",
+        "cloud_metadata": {},
         "timestamp": 32479987545338.0,
         "service": "my_service",
         "network": {
@@ -2175,6 +2178,7 @@
         "ddagentversion": "0.0.0",
         "ddsource": "mongo",
         "ddtags": "server:mongodb://testUser2:*****@localhost:27017/test,clustername:my_cluster,hosting_type:self-hosted,sharding_cluster_role:mongos",
+        "cloud_metadata": {},
         "timestamp": 32479987545521.0,
         "service": "my_service",
         "network": {

diff --git a/mongo/tests/results/slow-operations-explain-plans-standalone.json b/mongo/tests/results/slow-operations-explain-plans-standalone.json
@@ -5,6 +5,7 @@
         "ddagentversion": "0.0.0",
         "ddsource": "mongo",
         "ddtags": "server:mongodb://testUser2:*****@localhost:27017/test,clustername:my_cluster,hosting_type:self-hosted",
+        "cloud_metadata": {},
         "timestamp": 1720550476223.0,
         "service": "my_service",
         "network": {
@@ -136,6 +137,7 @@
         "ddagentversion": "0.0.0",
         "ddsource": "mongo",
         "ddtags": "server:mongodb://testUser2:*****@localhost:27017/test,clustername:my_cluster,hosting_type:self-hosted",
+        "cloud_metadata": {},
         "timestamp": 32479987545338.0,
         "service": "my_service",
         "network": {
@@ -282,6 +284,7 @@
         "ddagentversion": "0.0.0",
         "ddsource": "mongo",
         "ddtags": "server:mongodb://testUser2:*****@localhost:27017/test,clustername:my_cluster,hosting_type:self-hosted",
+        "cloud_metadata": {},
         "timestamp": 32479987545521.0,
         "service": "my_service",
         "network": {

diff --git a/mongo/tests/results/slow-operations-mongos.json b/mongo/tests/results/slow-operations-mongos.json
@@ -11,6 +11,7 @@
             "hosting_type:self-hosted",
             "sharding_cluster_role:mongos"
         ],
+        "cloud_metadata": {},
         "timestamp": 1715911398111.2722,
         "service": "my_service",
         "mongodb_slow_queries": [

diff --git a/mongo/tests/results/slow-operations-standalone.json b/mongo/tests/results/slow-operations-standalone.json
@@ -10,6 +10,7 @@
             "clustername:my_cluster",
             "hosting_type:self-hosted"
         ],
+        "cloud_metadata": {},
         "timestamp": 1715911398111.2722,
         "service": "my_service",
         "mongodb_slow_queries": [

diff --git a/mongo/tests/test_unit_config.py b/mongo/tests/test_unit_config.py
@@ -171,3 +171,32 @@ def test_database_autodiscovery_dbnames_deprecation(instance_user):
     assert config.database_autodiscovery_config is not None
     assert config.database_autodiscovery_config['enabled'] is True
     assert config.database_autodiscovery_config['include'] == ['test$', 'integration$']
+
+
+@pytest.mark.parametrize(
+    'aws_cloud_metadata',
+    [
+        pytest.param(
+            {
+                'instance_endpoint': 'mycluster.cluster-123456789012.us-east-1.docdb.amazonaws.com',
+                'cluster_identifier': 'mydocdbcluster',
+            },
+            id='aws_cloud_metadata',
+        ),
+        pytest.param(
+            {'instance_endpoint': 'mycluster.cluster-123456789012.us-east-1.docdb.amazonaws.com'},
+            id='aws_cloud_metadata_no_cluster_identifier',
+        ),
+    ],
+)
+def test_amazon_docdb_cloud_metadata(instance_integration_cluster, aws_cloud_metadata):
+    instance_integration_cluster['aws'] = aws_cloud_metadata
+    config = MongoConfig(instance_integration_cluster, mock.Mock(), {})
+    assert config.cloud_metadata is not None
+    aws = config.cloud_metadata['aws']
+    assert aws['instance_endpoint'] == aws_cloud_metadata['instance_endpoint']
+    assert aws['cluster_identifier'] is not None
+    if 'cluster_identifier' in aws_cloud_metadata:
+        assert aws['cluster_identifier'] == aws_cloud_metadata['cluster_identifier']
+    else:
+        assert aws['cluster_identifier'] == instance_integration_cluster['cluster_name']
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Add `aws` to the instance configuration to allow cloud resource linking with Amazon DocumentDB.