[Azure Metrics] Move database_account metrics from beats to integrations

packages/azure_metrics/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.0.9"
+  changes:
+    - description: Move database_account metrics config from beats to integrations
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3631
 - version: "1.0.8"
   changes:
     - description: Move container_registry metrics config from beats to integrations

packages/azure_metrics/data_stream/database_account/agent/stream/stream.yml.hbs

@@ -1,4 +1,4 @@
-metricsets: ["database_account"]
+metricsets: ["monitor"]
 period: {{period}}
 {{#if client_id}}
 client_id: {{client_id}}
@@ -26,10 +26,136 @@ resources:
 {{#if resource_groups}}
     {{#each resource_groups}}
         - resource_group: "{{this}}"
+          resource_type: "Microsoft.DocumentDb/databaseAccounts"
+          metrics:
+          - name: ["AddRegion", "RemoveRegion", "UpdateAccountReplicationSettings", "UpdateAccountNetworkSettings", "UpdateAccountKeys", "ServiceAvailability", "ReplicationLatency",
+                   "RegionFailover", "DeleteAccount", "CreateAccount", "CassandraConnectionClosures", "UpdateDiagnosticsSettings"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+          - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount",
+                   "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            timegrain: "PT5M"
+            dimensions:
+            - name: "DatabaseName"
+              value: "*"
+          - name: ["TotalRequestUnits", "TotalRequests"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            timegrain: "PT5M"
+            dimensions:
+            - name: "DatabaseName"
+              value: "*"
+            - name: "StatusCode"
+              value: "*"
+          - name: ["CassandraRequestCharges", "CassandraRequests"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            timegrain: "PT1M"
+            dimensions:
+            - name: "DatabaseName"
+              value: "*"
+          - name: [ "GremlinDatabaseDelete", "GremlinDatabaseThroughputUpdate", "GremlinDatabaseUpdate", "GremlinGraphDelete","GremlinGraphThroughputUpdate", "GremlinGraphUpdate",
+            "MongoCollectionDelete", "MongoCollectionThroughputUpdate", "MongoCollectionUpdate", "MongoDBDatabaseUpdate", "MongoDatabaseDelete", "MongoDatabaseThroughputUpdate",
+            "CassandraKeyspaceDelete", "CassandraKeyspaceThroughputUpdate", "CassandraKeyspaceUpdate","CassandraTableDelete", "CassandraTableThroughputUpdate", "CassandraTableUpdate",
+            "SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete",
+                    "TableTableThroughputUpdate","TableTableUpdate"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            dimensions:
+            - name: "ResourceName"
+              value: "*"
     {{/each}}
 {{/if}}
 {{#if resource_ids}}
     {{#each resource_ids}}
         - resource_id: "{{this}}"
+          metrics:
+          - name: ["AddRegion", "RemoveRegion", "UpdateAccountReplicationSettings", "UpdateAccountNetworkSettings", "UpdateAccountKeys", "ServiceAvailability", "ReplicationLatency",
+                   "RegionFailover", "DeleteAccount", "CreateAccount", "CassandraConnectionClosures", "UpdateDiagnosticsSettings"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+          - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount",
+                   "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            timegrain: "PT5M"
+            dimensions:
+            - name: "DatabaseName"
+              value: "*"
+          - name: ["TotalRequestUnits", "TotalRequests"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            timegrain: "PT5M"
+            dimensions:
+            - name: "DatabaseName"
+              value: "*"
+            - name: "StatusCode"
+              value: "*"
+          - name: ["CassandraRequestCharges", "CassandraRequests"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            timegrain: "PT1M"
+            dimensions:
+            - name: "DatabaseName"
+              value: "*"
+          - name: [ "GremlinDatabaseDelete", "GremlinDatabaseThroughputUpdate", "GremlinDatabaseUpdate", "GremlinGraphDelete","GremlinGraphThroughputUpdate", "GremlinGraphUpdate",
+                    "MongoCollectionDelete", "MongoCollectionThroughputUpdate", "MongoCollectionUpdate", "MongoDBDatabaseUpdate", "MongoDatabaseDelete", "MongoDatabaseThroughputUpdate",
+                    "CassandraKeyspaceDelete", "CassandraKeyspaceThroughputUpdate", "CassandraKeyspaceUpdate","CassandraTableDelete", "CassandraTableThroughputUpdate", "CassandraTableUpdate",
+                    "SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete",
+                    "TableTableThroughputUpdate","TableTableUpdate"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            dimensions:
+            - name: "ResourceName"
+              value: "*"
     {{/each}}
 {{/if}}
+
+{{!
+    When no resource group and resource ID are specified by the user, we want to 
+    collect metrics for all the resource groups in the subscription.
+}}
+
+{{#unless resource_ids }}
+    {{#unless resource_groups }}
+        - resource_query: "resourceType eq 'Microsoft.DocumentDb/databaseAccounts'"
+          metrics:
+          - name: ["AddRegion", "RemoveRegion", "UpdateAccountReplicationSettings", "UpdateAccountNetworkSettings", "UpdateAccountKeys", "ServiceAvailability", "ReplicationLatency",
+                   "RegionFailover", "DeleteAccount", "CreateAccount", "CassandraConnectionClosures", "UpdateDiagnosticsSettings"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+          - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount",
+                   "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            timegrain: "PT5M"
+            dimensions:
+            - name: "DatabaseName"
+              value: "*"
+          - name: ["TotalRequestUnits", "TotalRequests"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            timegrain: "PT5M"
+            dimensions:
+            - name: "DatabaseName"
+              value: "*"
+            - name: "StatusCode"
+              value: "*"
+          - name: ["CassandraRequestCharges", "CassandraRequests"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            timegrain: "PT1M"
+            dimensions:
+            - name: "DatabaseName"
+              value: "*"
+          - name: [ "GremlinDatabaseDelete", "GremlinDatabaseThroughputUpdate", "GremlinDatabaseUpdate", "GremlinGraphDelete","GremlinGraphThroughputUpdate", "GremlinGraphUpdate",
+            "MongoCollectionDelete", "MongoCollectionThroughputUpdate", "MongoCollectionUpdate", "MongoDBDatabaseUpdate", "MongoDatabaseDelete", "MongoDatabaseThroughputUpdate",
+            "CassandraKeyspaceDelete", "CassandraKeyspaceThroughputUpdate", "CassandraKeyspaceUpdate","CassandraTableDelete", "CassandraTableThroughputUpdate", "CassandraTableUpdate",
+            "SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete",
+                    "TableTableThroughputUpdate","TableTableUpdate"]
+            namespace: "Microsoft.DocumentDb/databaseAccounts"
+            ignore_unsupported: true
+            dimensions:
+            - name: "ResourceName"
+              value: "*"
+    {{/unless}}
+{{/unless}}

packages/azure_metrics/data_stream/database_account/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,14 @@
+---
+description: Pipeline for parsing azure database_account metrics.
+processors:
+  - set:
+      field: ecs.version
+      value: "8.0.0"
+  - rename:
+      field: azure.metrics
+      target_field: azure.database_account
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'

packages/azure_metrics/data_stream/database_account/sample_event.json

@@ -1,24 +1,23 @@
 {
     "agent": {
-        "hostname": "docker-fleet-agent",
-        "name": "docker-fleet-agent",
-        "id": "6493df64-791a-4b55-b2e9-c5b1dd347fe7",
+        "name": "azure-vm",
+        "id": "2f167534-86cd-4751-b39f-a9cc7a155584",
+        "ephemeral_id": "3e900895-9bbb-46c2-b651-8f0067dfacb8",
         "type": "metricbeat",
-        "ephemeral_id": "833a8afe-815e-4ed5-b7a1-bd0e30d626ed",
-        "version": "7.14.0"
+        "version": "8.3.0"
     },
     "elastic_agent": {
-        "id": "6493df64-791a-4b55-b2e9-c5b1dd347fe7",
-        "version": "7.14.0",
+        "id": "2f167533-86cd-5751-b39f-a9cc7a155584",
+        "version": "8.3.0",
         "snapshot": true
     },
     "cloud": {
         "provider": "azure",
-        "region": "westus"
+        "region": "ukwest"
     },
-    "@timestamp": "2021-07-26T09:52:00.000Z",
+    "@timestamp": "2022-07-06T15:27:00.000Z",
     "ecs": {
-        "version": "1.10.0"
+        "version": "8.0.0"
     },
     "data_stream": {
         "namespace": "default",
@@ -29,51 +28,51 @@
         "type": "azure"
     },
     "host": {
-        "hostname": "docker-fleet-agent",
+        "hostname": "azure-vm",
         "os": {
-            "kernel": "4.19.128-microsoft-standard",
+            "kernel": "3.10.0-1160.53.1.el7.x86_64",
             "codename": "Core",
             "name": "CentOS Linux",
-            "type": "linux",
             "family": "redhat",
+            "type": "linux",
             "version": "7 (Core)",
             "platform": "centos"
         },
-        "containerized": true,
+        "containerized": false,
         "ip": [
-            "172.20.0.7"
+            "10.1.0.9",
+            "fe80::222:48ff:fe3f:2d92"
         ],
-        "name": "docker-fleet-agent",
-        "id": "d4845dae196bc2de62b7b208c215d5bc",
+        "name": "azure-vm",
+        "id": "09bbcdbd0f374f0e9fb890d89dae3a37",
         "mac": [
-            "02:42:ac:14:00:07"
+            "00:22:48:3f:2d:97"
         ],
         "architecture": "x86_64"
     },
     "metricset": {
-        "period": 300000,
-        "name": "database_account"
+        "period": 60000,
+        "name": "monitor"
     },
     "event": {
-        "duration": 4437397900,
+        "duration": 3006143506,
         "agent_id_status": "verified",
-        "ingested": "2021-07-26T10:02:22.992720500Z",
+        "ingested": "2022-07-06T15:29:03Z",
         "module": "azure",
         "dataset": "azure.database_account"
     },
     "azure": {
-        "subscription_id": "70bd6e77-4b1e-4835-8896-db77b8eef364",
+        "subscription_id": "0e073ec1-c22f-4488-add5-da35ed609cce",
         "timegrain": "PT1H",
         "resource": {
-            "name": "obscosmosmongodb",
-            "id": "/subscriptions/70bd6e77-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.DocumentDb/databaseAccounts/obscosmosmongodb",
+            "name": "db-account",
+            "id": "/subscriptions/0e073ec1-c22f-4488-adde-da25ed609ccd/resourceGroups/rg/providers/Microsoft.DocumentDb/databaseAccounts/db-account",
             "type": "Microsoft.DocumentDb/databaseAccounts",
+            "group": "rg",
             "tags": {
-                "defaultExperience": "Azure Cosmos DB for MongoDB API",
-                "hidden-cosmos-mmspecial": "",
-                "CosmosAccountType": "Non-Production"
-            },
-            "group": "obs-infrastructure"
+                "defaultExperience": "Core (SQL)",
+                "hidden-cosmos-mmspecial": ""
+            }
         },
         "database_account": {
             "service_availability": {

packages/azure_metrics/manifest.yml

@@ -1,6 +1,6 @@
 name: azure_metrics
 title: Azure Resource Metrics
-version: 1.0.8
+version: 1.0.9
 release: ga
 description: Collect metrics from Azure resources with Elastic Agent.
 type: integration