From d89675fc3bc4f6a8e72671ed887bf96ad8ca159b Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Tue, 3 Mar 2020 11:59:00 +0100 Subject: [PATCH] Add container azure metricset (#16421) * add container * update changelog * fix test * separate container * separate * regenerate * fixes * tests * fix test --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 30 +++++ metricbeat/docs/modules/azure.asciidoc | 43 ++++++ .../modules/azure/container_instance.asciidoc | 24 ++++ .../modules/azure/container_registry.asciidoc | 24 ++++ .../modules/azure/container_service.asciidoc | 24 ++++ metricbeat/docs/modules_list.asciidoc | 5 +- x-pack/metricbeat/metricbeat.reference.yml | 31 +++++ .../module/azure/_meta/config.reference.yml | 31 +++++ .../metricbeat/module/azure/_meta/config.yml | 33 +++++ x-pack/metricbeat/module/azure/azure.go | 23 +++- x-pack/metricbeat/module/azure/client.go | 6 +- .../metricbeat/module/azure/client_utils.go | 10 ++ .../module/azure/client_utils_test.go | 26 ++++ .../compute_vm_scaleset/client_helper.go | 17 +-- .../compute_vm_scaleset/client_helper_test.go | 26 ---- .../azure/container_instance/_meta/data.json | 46 +++++++ .../container_instance/_meta/docs.asciidoc | 19 +++ .../azure/container_instance/_meta/fields.yml | 7 + .../container_instance_integration_test.go | 51 +++++++ .../container_instance_test.go | 17 +++ .../azure/container_instance/manifest.yml | 30 +++++ .../azure/container_registry/_meta/data.json | 37 ++++++ .../container_registry/_meta/docs.asciidoc | 19 +++ .../azure/container_registry/_meta/fields.yml | 7 + .../container_registry_integration_test.go | 51 +++++++ .../container_registry_test.go | 17 +++ .../azure/container_registry/manifest.yml | 19 +++ .../azure/container_service/_meta/data.json | 46 +++++++ .../container_service/_meta/docs.asciidoc | 19 +++ .../azure/container_service/_meta/fields.yml | 7 + .../container_service_integration_test.go | 51 +++++++ .../container_service_test.go | 17 +++ .../azure/container_service/manifest.yml | 30 +++++ x-pack/metricbeat/module/azure/data.go | 27 +++- x-pack/metricbeat/module/azure/data_test.go | 125 ++++++++++++++++++ x-pack/metricbeat/module/azure/fields.go | 2 +- x-pack/metricbeat/module/azure/module.yml | 5 +- .../metricbeat/modules.d/azure.yml.disabled | 33 +++++ 39 files changed, 984 insertions(+), 52 deletions(-) create mode 100644 metricbeat/docs/modules/azure/container_instance.asciidoc create mode 100644 metricbeat/docs/modules/azure/container_registry.asciidoc create mode 100644 metricbeat/docs/modules/azure/container_service.asciidoc create mode 100644 x-pack/metricbeat/module/azure/container_instance/_meta/data.json create mode 100644 x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/container_instance/container_instance_test.go create mode 100644 x-pack/metricbeat/module/azure/container_instance/manifest.yml create mode 100644 x-pack/metricbeat/module/azure/container_registry/_meta/data.json create mode 100644 x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/container_registry/container_registry_test.go create mode 100644 x-pack/metricbeat/module/azure/container_registry/manifest.yml create mode 100644 x-pack/metricbeat/module/azure/container_service/_meta/data.json create mode 100644 x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/azure/container_service/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/container_service/container_service_test.go create mode 100644 x-pack/metricbeat/module/azure/container_service/manifest.yml create mode 100644 x-pack/metricbeat/module/azure/data_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e5fff6f9f94..95ad20ea4d3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -191,6 +191,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for NATS 2.1. {pull}16317[16317] - Add Load Balancing metricset to GCP {pull}15559[15559] - Add support for Dropwizard metrics 4.1. {pull}16332[16332] +- Add azure container metricset in order to monitor containers. {issue}15751[15751] {pull}16421[16421] - Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333] - Add filtering option for prometheus collector. {pull}16420[16420] - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7c339b1020c..4cb142e1ee7 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3209,6 +3209,36 @@ type: object compute_vm_scaleset +type: object + +-- + +*`azure.container_instance.*.*`*:: ++ +-- +container instance + + +type: object + +-- + +*`azure.container_registry.*.*`*:: ++ +-- +container registry + + +type: object + +-- + +*`azure.container_service.*.*`*:: ++ +-- +container service + + type: object -- diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index ed5a266e3d8..a19996d7c2a 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -150,6 +150,37 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - container_instance + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_service + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_registry + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + + - module: azure metricsets: - database_account @@ -171,6 +202,12 @@ The following metricsets are available: * <> +* <> + +* <> + +* <> + * <> * <> @@ -181,6 +218,12 @@ include::azure/compute_vm.asciidoc[] include::azure/compute_vm_scaleset.asciidoc[] +include::azure/container_instance.asciidoc[] + +include::azure/container_registry.asciidoc[] + +include::azure/container_service.asciidoc[] + include::azure/database_account.asciidoc[] include::azure/monitor.asciidoc[] diff --git a/metricbeat/docs/modules/azure/container_instance.asciidoc b/metricbeat/docs/modules/azure/container_instance.asciidoc new file mode 100644 index 00000000000..88f1959637d --- /dev/null +++ b/metricbeat/docs/modules/azure/container_instance.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-container_instance]] +=== azure container_instance metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/container_instance/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/azure/container_registry.asciidoc b/metricbeat/docs/modules/azure/container_registry.asciidoc new file mode 100644 index 00000000000..67ff888d807 --- /dev/null +++ b/metricbeat/docs/modules/azure/container_registry.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-container_registry]] +=== azure container_registry metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/container_registry/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/azure/container_service.asciidoc b/metricbeat/docs/modules/azure/container_service.asciidoc new file mode 100644 index 00000000000..2aed94c7749 --- /dev/null +++ b/metricbeat/docs/modules/azure/container_service.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-container_service]] +=== azure container_service metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/container_service/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 1ca316b1c95..e3773152861 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -31,8 +31,11 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> beta[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.5+| .5+| |<> beta[] +.8+| .8+| |<> beta[] |<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] |<> beta[] |<> beta[] |<> beta[] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 710e4556a47..71922b4b590 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -265,6 +265,37 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - container_instance + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_service + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_registry + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + + - module: azure metricsets: - database_account diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml index 7dfd3068070..5823b66c671 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.reference.yml @@ -43,6 +43,37 @@ tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - container_instance + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_service + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_registry + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + + - module: azure metricsets: - database_account diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml index 525e421369f..3f3b29713e9 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.yml @@ -47,6 +47,39 @@ # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s +#- module: azure +# metricsets: +# - container_instance +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container_service +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container_registry +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + #- module: azure # metricsets: # - database_account diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index dc1ea3a86ed..675e5c35d3c 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -106,14 +106,16 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // check for lightweight resources if no groups or ids have been entered, if not a new resource is created to check the entire subscription var resources []ResourceConfig for _, resource := range config.Resources { - if len(resource.Group) != 0 || len(resource.ID) != 0 { + if hasConfigOptions(resource.Group) || hasConfigOptions(resource.ID) { resources = append(resources, resource) } } + // check if this is a light metricset or not and no resources have been configured if len(resources) == 0 && len(config.Resources) != 0 { - res := config.Resources[0] - res.Query = fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType) - resources = append(resources, res) + resources = append(resources, ResourceConfig{ + Query: fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType), + Metrics: config.Resources[0].Metrics, + }) } config.Resources = resources } @@ -152,3 +154,16 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } return nil } + +// hasConfigOptions func will check if any resource id or resource group options have been entered in the light metricsets +func hasConfigOptions(config []string) bool { + if config == nil { + return false + } + for _, group := range config { + if group == "" { + return false + } + } + return true +} diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index ffed01497d4..1d0064785ee 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -114,13 +114,13 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] client.Log.Error(err) report.Error(err) } else { - if metric.TimeGrain == "" { - metric.TimeGrain = timegrain - } for i, currentMetric := range client.Resources.Metrics { if matchMetrics(currentMetric, metric) { current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute)) client.Resources.Metrics[i].Values = current + if client.Resources.Metrics[i].TimeGrain == "" { + client.Resources.Metrics[i].TimeGrain = timegrain + } resultedMetrics = append(resultedMetrics, client.Resources.Metrics[i]) } } diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index b03964044ea..ce55a7a1459 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -188,3 +188,13 @@ func groupMetricsByResource(metrics []Metric) map[string][]Metric { } return grouped } + +// ContainsDimension will check if the dimension value is found in the list +func ContainsDimension(dimension string, dimensions []insights.LocalizableString) bool { + for _, dim := range dimensions { + if *dim.Value == dimension { + return true + } + } + return false +} diff --git a/x-pack/metricbeat/module/azure/client_utils_test.go b/x-pack/metricbeat/module/azure/client_utils_test.go index a63034ff77d..a21ebf026e4 100644 --- a/x-pack/metricbeat/module/azure/client_utils_test.go +++ b/x-pack/metricbeat/module/azure/client_utils_test.go @@ -142,3 +142,29 @@ func TestCompareMetricValues(t *testing.T) { result = compareMetricValues(val1, val2) assert.True(t, result) } + +func TestContainsDimension(t *testing.T) { + dimension := "VMName" + dim1 := "SlotID" + dim2 := "VNU" + dim3 := "VMName" + dimensionList := []insights.LocalizableString{ + { + Value: &dim1, + LocalizedValue: &dim1, + }, + { + Value: &dim2, + LocalizedValue: &dim2, + }, + { + Value: &dim3, + LocalizedValue: &dim3, + }, + } + result := ContainsDimension(dimension, dimensionList) + assert.True(t, result) + dimension = "VirtualMachine" + result = ContainsDimension(dimension, dimensionList) + assert.False(t, result) +} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go index 526ed99ea68..e39c591bdc2 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go @@ -18,7 +18,6 @@ const ( defaultVMDimension = "VMName" customVMDimension = "VirtualMachine" defaultSlotIDDimension = "SlotId" - defaultTimeGrain = "PT5M" ) // mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters @@ -61,9 +60,9 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res for _, metricName := range supportedMetricNames { if metricName.Dimensions == nil || len(*metricName.Dimensions) == 0 { groupedMetrics[azure.NoDimension] = append(groupedMetrics[azure.NoDimension], metricName) - } else if containsDimension(vmdim, *metricName.Dimensions) { + } else if azure.ContainsDimension(vmdim, *metricName.Dimensions) { groupedMetrics[vmdim] = append(groupedMetrics[vmdim], metricName) - } else if containsDimension(defaultSlotIDDimension, *metricName.Dimensions) { + } else if azure.ContainsDimension(defaultSlotIDDimension, *metricName.Dimensions) { groupedMetrics[defaultSlotIDDimension] = append(groupedMetrics[defaultSlotIDDimension], metricName) } } @@ -76,19 +75,9 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res if key != azure.NoDimension { dimensions = []azure.Dimension{{Name: key, Value: "*"}} } - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", metric.Namespace, dimensions, defaultTimeGrain)...) + metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", metric.Namespace, dimensions, azure.DefaultTimeGrain)...) } } } return metrics, nil } - -// containsDimension will check if the dimension value is found in the list -func containsDimension(dimension string, dimensions []insights.LocalizableString) bool { - for _, dim := range dimensions { - if *dim.Value == dimension { - return true - } - } - return false -} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go index f3ac49198bb..e81582d48b3 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go @@ -116,29 +116,3 @@ func TestMapMetric(t *testing.T) { m.AssertExpectations(t) }) } - -func TestContainsDimension(t *testing.T) { - dimension := "VMName" - dim1 := "SlotID" - dim2 := "VNU" - dim3 := "VMName" - dimensionList := []insights.LocalizableString{ - { - Value: &dim1, - LocalizedValue: &dim1, - }, - { - Value: &dim2, - LocalizedValue: &dim2, - }, - { - Value: &dim3, - LocalizedValue: &dim3, - }, - } - result := containsDimension(dimension, dimensionList) - assert.True(t, result) - dimension = "VirtualMachine" - result = containsDimension(dimension, dimensionList) - assert.False(t, result) -} diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json new file mode 100644 index 00000000000..163b491021e --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json @@ -0,0 +1,46 @@ +{ + "@timestamp" : "2020-02-28T14:20:00.000Z", + "cloud" : { + "provider" : "azure", + "region" : "westeurope" + }, + "ecs" : { + "version" : "1.4.0" + }, + "metricset" : { + "period" : 300000, + "name" : "container_instance" + }, + "event" : { + "duration" : 2383878000, + "dataset" : "azure.container_instance", + "module" : "azure" + }, + "azure" : { + "dimensions" : { + "container_name" : "testcontainergroup" + }, + "container_instance" : { + "cpu_usage" : { + "avg" : 0 + }, + "memory_usage" : { + "avg" : 0 + } + }, + "timegrain" : "PT5M", + "resource" : { + "group" : "obs-infrastructure", + "tags" : { + "tag1" : "value1" + }, + "name" : "testcontainergroup", + "type" : "Microsoft.ContainerInstance/containerGroups" + }, + "subscription_id" : "123456-qwer-1234-5678-12345678", + "namespace" : "Microsoft.ContainerInstance/containerGroups" + }, + "service" : { + "type" : "azure" + } +} diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc new file mode 100644 index 00000000000..a144105cf98 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc @@ -0,0 +1,19 @@ +This is the container_instance metricset of the module azure. + +This metricset allows users to retrieve all metrics from specified container groups. + +include::../../_meta/shared-azure.asciidoc[] + +[float] +==== Config options to identify resources + +`resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. + Should return a list of resources. + +`resource_group`:: (_[]string_) This option should return a list of container groups we want to apply our metric configuration options on. + +If none of the options are entered then we will select all the container groups from the entire subscription +For each metric the primary aggregation assigned will be retrieved. +A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. + + diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml b/x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml new file mode 100644 index 00000000000..d79ddd14999 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: container_instance.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" + description: > + container instance diff --git a/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go new file mode 100644 index 00000000000..37b814b070f --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_instance + +import ( + "errors" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + config, err := getConfig() + if err != nil { + t.Skip("Skipping TestData: " + err.Error()) + } + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} + +func getConfig() (map[string]interface{}, error) { + clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_ID key") + } + clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_SECRET key") + } + tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") + if !ok { + return nil, errors.New("missing AZURE_TENANT_ID key") + } + subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") + if !ok { + return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") + } + config := map[string]interface{}{ + "module": "azure", + "metricsets": []string{"container_instance"}, + "client_id": clientId, + "client_secret": clientSecret, + "tenant_id": tenantId, + "subscription_id": subscriptionId, + } + return config, nil +} diff --git a/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go b/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go new file mode 100644 index 00000000000..a92237aea4b --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_instance + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/azure/container_instance/manifest.yml b/x-pack/metricbeat/module/azure/container_instance/manifest.yml new file mode 100644 index 00000000000..a0e6cd5ec27 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/manifest.yml @@ -0,0 +1,30 @@ +default: true +input: + module: azure + metricset: monitor + defaults: + default_resource_type: "Microsoft.ContainerInstance/containerGroups" + resources: + - resource_group: "" + resource_type: "Microsoft.ContainerInstance/containerGroups" + metrics: + - name: ["CpuUsage", "MemoryUsage"] + namespace: "Microsoft.ContainerInstance/containerGroups" + timegrain: "PT5M" + dimensions: + - name: "containerName" + value: "*" + - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] + namespace: "Microsoft.ContainerInstance/containerGroups" + timegrain: "PT5M" + - resource_id: "" + metrics: + - name: ["CpuUsage", "MemoryUsage"] + namespace: "Microsoft.ContainerInstance/containerGroups" + timegrain: "PT5M" + dimensions: + - name: "containerName" + value: "*" + - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] + namespace: "Microsoft.ContainerInstance/containerGroups" + timegrain: "PT5M" diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json new file mode 100644 index 00000000000..0167132328b --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json @@ -0,0 +1,37 @@ +{ + "@timestamp" : "2020-02-28T14:47:00.000Z", + "ecs" : { + "version" : "1.4.0" + }, + "cloud" : { + "region" : "westeurope", + "provider" : "azure" + }, + "event" : { + "dataset" : "azure.container_registry", + "module" : "azure", + "duration" : 2730896000 + }, + "metricset" : { + "name" : "container_registry", + "period" : 300000 + }, + "azure" : { + "subscription_id" : "123456-qwer-1234-5678-12345678", + "namespace" : "Microsoft.ContainerRegistry/registries", + "container_registry" : { + "successful_push_count" : { + "avg" : 1 + } + }, + "timegrain" : "PT5M", + "resource" : { + "type" : "Microsoft.ContainerRegistry/registries", + "group" : "obs-infrastructure", + "name" : "obstest" + } + }, + "service" : { + "type" : "azure" + } +} diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc new file mode 100644 index 00000000000..fa742cad052 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc @@ -0,0 +1,19 @@ +This is the container_registry metricset of the module azure. + +This metricset allows users to retrieve all metrics from specified virtual machines. + +include::../../_meta/shared-azure.asciidoc[] + +[float] +==== Config options to identify resources + +`resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. + Should return a list of resources. + +`resource_group`:: (_[]string_) This option should return a list virtual machines we want to apply our metric configuration options on. + +If none of the options are entered then we will select all virtual machine from the entire subscription +For each metric the primary aggregation assigned will be retrieved. +A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. + + diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml b/x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml new file mode 100644 index 00000000000..c8296603a77 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: container_registry.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" + description: > + container registry diff --git a/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go new file mode 100644 index 00000000000..41414529997 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_registry + +import ( + "errors" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + config, err := getConfig() + if err != nil { + t.Skip("Skipping TestData: " + err.Error()) + } + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} + +func getConfig() (map[string]interface{}, error) { + clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_ID key") + } + clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_SECRET key") + } + tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") + if !ok { + return nil, errors.New("missing AZURE_TENANT_ID key") + } + subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") + if !ok { + return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") + } + config := map[string]interface{}{ + "module": "azure", + "metricsets": []string{"container_registry"}, + "client_id": clientId, + "client_secret": clientSecret, + "tenant_id": tenantId, + "subscription_id": subscriptionId, + } + return config, nil +} diff --git a/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go b/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go new file mode 100644 index 00000000000..7b65c68056f --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_registry + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/azure/container_registry/manifest.yml b/x-pack/metricbeat/module/azure/container_registry/manifest.yml new file mode 100644 index 00000000000..77993952390 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/manifest.yml @@ -0,0 +1,19 @@ +default: true +input: + module: azure + metricset: monitor + defaults: + default_resource_type: "Microsoft.ContainerRegistry/registries" + resources: + - resource_group: "" + resource_type: "Microsoft.ContainerRegistry/registries" + metrics: + - name: "*" + namespace: "Microsoft.ContainerRegistry/registries" + timegrain: "PT5M" + - resource_id: "" + timegrain: "PT5M" + metrics: + - name: "*" + namespace: "Microsoft.ContainerRegistry/registries" + timegrain: "PT5M" diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/data.json b/x-pack/metricbeat/module/azure/container_service/_meta/data.json new file mode 100644 index 00000000000..fe6623057a3 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/_meta/data.json @@ -0,0 +1,46 @@ +{ + "@timestamp" : "2020-02-28T14:47:00.000Z", + "ecs" : { + "version" : "1.4.0" + }, + "cloud" : { + "region" : "westeurope", + "provider" : "azure" + }, + "event" : { + "dataset" : "azure.container_service", + "module" : "azure", + "duration" : 2730896000 + }, + "metricset" : { + "name" : "container_service", + "period" : 300000 + }, + "azure" : { + "subscription_id" : "123456-qwer-1234-5678-12345678", + "namespace" : "Microsoft.ContainerService/managedClusters", + "container_service" : { + "kube_pod_status_ready" : { + "avg" : 9 + }, + "kube_pod_status_phase" : { + "avg" : 9 + }, + "kube_node_status_allocatable_cpu_cores" : { + "avg" : 2 + }, + "kube_node_status_allocatable_memory_bytes" : { + "avg" : 2.25349632E9 + } + }, + "timegrain" : "PT5M", + "resource" : { + "type" : "Microsoft.ContainerService/managedClusters", + "group" : "obs-infrastructure", + "name" : "obskube" + } + }, + "service" : { + "type" : "azure" + } +} diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc new file mode 100644 index 00000000000..db015dd1513 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc @@ -0,0 +1,19 @@ +This is the container_service metricset of the module azure. + +This metricset allows users to retrieve all metrics from specified virtual machines. + +include::../../_meta/shared-azure.asciidoc[] + +[float] +==== Config options to identify resources + +`resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. + Should return a list of resources. + +`resource_group`:: (_[]string_) This option should return a list virtual machines we want to apply our metric configuration options on. + +If none of the options are entered then we will select all virtual machine from the entire subscription +For each metric the primary aggregation assigned will be retrieved. +A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. + + diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/fields.yml b/x-pack/metricbeat/module/azure/container_service/_meta/fields.yml new file mode 100644 index 00000000000..9cccb41591d --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: container_service.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" + description: > + container service diff --git a/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go new file mode 100644 index 00000000000..01c9f5a053e --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_service + +import ( + "errors" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + config, err := getConfig() + if err != nil { + t.Skip("Skipping TestData: " + err.Error()) + } + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} + +func getConfig() (map[string]interface{}, error) { + clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_ID key") + } + clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_SECRET key") + } + tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") + if !ok { + return nil, errors.New("missing AZURE_TENANT_ID key") + } + subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") + if !ok { + return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") + } + config := map[string]interface{}{ + "module": "azure", + "metricsets": []string{"container_registry"}, + "client_id": clientId, + "client_secret": clientSecret, + "tenant_id": tenantId, + "subscription_id": subscriptionId, + } + return config, nil +} diff --git a/x-pack/metricbeat/module/azure/container_service/container_service_test.go b/x-pack/metricbeat/module/azure/container_service/container_service_test.go new file mode 100644 index 00000000000..0fb0b53347e --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/container_service_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_service + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/azure/container_service/manifest.yml b/x-pack/metricbeat/module/azure/container_service/manifest.yml new file mode 100644 index 00000000000..1580c5797f4 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/manifest.yml @@ -0,0 +1,30 @@ +default: true +input: + module: azure + metricset: monitor + defaults: + default_resource_type: "Microsoft.ContainerService/managedClusters" + resources: + - resource_group: "" + resource_type: "Microsoft.ContainerService/managedClusters" + metrics: + - name: ["kube_node_status_condition"] + namespace: "Microsoft.ContainerService/managedClusters" + timegrain: "PT5M" + dimensions: + - name: "status" + value: "*" + - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes", "kube_pod_status_ready", "kube_pod_status_phase"] + namespace: "Microsoft.ContainerService/managedClusters" + timegrain: "PT5M" + - resource_id: "" + metrics: + - name: ["kube_node_status_condition"] + namespace: "Microsoft.ContainerService/managedClusters" + timegrain: "PT5M" + dimensions: + - name: "status" + value: "*" + - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes", "kube_pod_status_ready", "kube_pod_status_phase"] + namespace: "Microsoft.ContainerService/managedClusters" + timegrain: "PT5M" diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index 2c3612a3866..64920f1eb64 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -6,8 +6,10 @@ package azure import ( "fmt" + "regexp" "strings" "time" + "unicode" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" @@ -15,8 +17,9 @@ import ( const ( // NoDimension is used to group metrics in separate api calls in order to reduce the number of executions - NoDimension = "none" - nativeMetricset = "monitor" + NoDimension = "none" + nativeMetricset = "monitor" + replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]` ) // EventsMapping will map metric values to beats events @@ -106,6 +109,9 @@ func managePropertyName(metric string) string { resultMetricName = strings.Replace(resultMetricName, ":", "_", -1) // create an object in case of ":" resultMetricName = strings.Replace(resultMetricName, "_-_", "_", -1) + // replace uppercases with underscores + resultMetricName = replaceUpperCase(resultMetricName) + // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" obj := strings.Split(resultMetricName, ".") for index := range obj { @@ -118,6 +124,23 @@ func managePropertyName(metric string) string { return resultMetricName } +// replaceUpperCase func will replace upper case with '_' +func replaceUpperCase(src string) string { + replaceUpperCaseRegexp := regexp.MustCompile(replaceUpperCaseRegex) + return replaceUpperCaseRegexp.ReplaceAllStringFunc(src, func(str string) string { + var newStr string + for _, r := range str { + // split into fields based on class of unicode character + if unicode.IsUpper(r) { + newStr += "_" + strings.ToLower(string(r)) + } else { + newStr += string(r) + } + } + return newStr + }) +} + // createEvent will create a new base event func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) (mb.Event, common.MapStr) { event := mb.Event{ diff --git a/x-pack/metricbeat/module/azure/data_test.go b/x-pack/metricbeat/module/azure/data_test.go new file mode 100644 index 00000000000..2498d92a558 --- /dev/null +++ b/x-pack/metricbeat/module/azure/data_test.go @@ -0,0 +1,125 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azure + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +func TestReturnAllDimensions(t *testing.T) { + dimensionList := []Dimension{ + { + Value: "vm1", + Name: "VMName", + }, + { + Value: "*", + Name: "SlotID", + }, + } + result, dims := returnAllDimensions(dimensionList) + assert.True(t, result) + assert.Equal(t, len(dims), 1) + assert.Equal(t, dims[0].Name, "SlotID") + assert.Equal(t, dims[0].Value, "*") +} + +func TestGetDimensionValue(t *testing.T) { + dimensionList := []Dimension{ + { + Value: "vm1", + Name: "VMName", + }, + { + Value: "*", + Name: "SlotID", + }, + } + result := getDimensionValue("VMName", dimensionList) + assert.Equal(t, result, "vm1") +} + +func TestReplaceUpperCase(t *testing.T) { + result := replaceUpperCase("TestReplaceUpper_Case") + assert.Equal(t, result, "Test_replace_upper_Case") + // should not split on acronyms + result = replaceUpperCase("CPU_Percentage") + assert.Equal(t, result, "CPU_Percentage") +} + +func TestManagePropertyName(t *testing.T) { + result := managePropertyName("TestManageProperty_Name") + assert.Equal(t, result, "test_manage_property_name") + + result = managePropertyName("Test ManageProperty_Name/sec") + assert.Equal(t, result, "test_manage_property_name_per_sec") + + result = managePropertyName("Test_-_Manage:Property.Name") + assert.Equal(t, result, "test_manage_property_name") + + result = managePropertyName("Percentage CPU") + assert.Equal(t, result, "percentage_cpu") +} + +func TestCreateEvent(t *testing.T) { + createTime, err := time.Parse(time.RFC3339, "2020-02-28T20:53:03Z") + if !assert.NoError(t, err) { + t.Fatal(err) + } + metric := Metric{ + Resource: Resource{ + ID: "resId", + Name: "res", + Location: "west_europe", + Type: "resType", + Group: "resGroup", + Tags: nil, + Subscription: "subId", + }, + Namespace: "namespace1", + Names: []string{"Percentage CPU"}, + Aggregations: "", + Dimensions: nil, + Values: nil, + TimeGrain: "", + } + var total float64 = 23 + metricValues := []MetricValue{ + { + name: "Percentage CPU", + avg: nil, + min: nil, + max: nil, + total: &total, + count: nil, + timestamp: time.Time{}, + dimensions: nil, + }, + } + event, list := createEvent(createTime, metric, metricValues) + assert.NotNil(t, event) + assert.NotNil(t, list) + assert.Equal(t, event.Timestamp, createTime) + sub, err := event.ModuleFields.GetValue("subscription_id") + if !assert.NoError(t, err) { + t.Fatal(err) + } + assert.Equal(t, sub, metric.Resource.Subscription) + namespace, err := event.ModuleFields.GetValue("namespace") + if !assert.NoError(t, err) { + t.Fatal(err) + } + assert.Equal(t, namespace, metric.Namespace) + val, err := list.GetValue("percentage_cpu") + if !assert.NoError(t, err) { + t.Fatal(err) + } + assert.Equal(t, val.(common.MapStr), common.MapStr{"total": total}) +} diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index e470f6e4b2d..25f8d1b6fcc 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUVb2OnDAQ7vcpRluudPsAFJEipUmRLj0yZiDO4R/Z40Sbp4+8gM9gfpbkij0XSHjM9zPzAS/wircC2B9v8QRAgjos4Hy/P58AanTcCkNCqwI+nQCgPwtS174Lj1jskDksoEJiJ4BGYFe74n70BRST+AYfFt0MFtBa7c2ws8AxhUmhSEhsLRMqVkbIV7z91rZO9heB+/X9B8Ln3giSFXwBd2S06LS3HDPC1MMDdCMOOINcNAJTqXO7qYBwnRTWHe/IGKUERNANUCJrkTrwvC91ePwh6nl334E7TiDHjoZZ666XRVpd/UROs1K/WW4JS46UkhkjVDucP1/Ox0z0iY027mKzvIarM2whsIffkAgFDjvklER2ZHO+ihClqP+fMwWEr18ywlpIVE5oNZ3Tyox25vPobDY0T74iibhMONfSeMLyl7xeJtLnX9AjlppOs5Xivxp607lhoXScdeiQPoqXKDjW82QxYhVzWDLOtVdPbm1UC4PazI7USpC2u7+tNV8b3Dn01v+rfzfcrJ0brdtt35EW7lgJ61svECyStwrr63pGHGnLWnzuaAwiYzL+BgAA///JVKQ8" + return "eJzUlr+O2zAMxvc8BZExwOUBPBQo0KVDt+6GIjMue7ZkkPQV6dMXjm2d/D/p3ZBoCBBJ/r7fJ9KCX+AVLwmYvzXjDkBJC0xgf/2/3wFkKJapUvIugS87AGj3QumzumgeYSzQCCZwQjU7gDNhkUly3foCzpT4Lt8MvVSYQM6+rrqZGY+hTCylVGLOhlxY6SVf8fLHcxbNzwq34+cvhK9tEFQmO6PbOzKKr9nixDDOcINdrwNSoaUzYYw6jhsDNL+DheXEGxg9SqMI/gwaYc1aNz6fa908fpP1+HQ/wTtUYKodAptcjodZW3/6jVZHS+1kugYWbUlLU1Xk8m7//rC/L0TbsSHGFXbSr82vVGamYe9+Q4IUCBZoNWrZ3k3qU5BIKfu4ZywI379NDDMq0Ql5N6zTQo026nNrbVaYB7dIBDcBt76sasX0rTweBujjG/SeSOfCm4XF/w30zrkSIRVrChTUZ8kSgMP6TDqnhhxySk7UOIuPHq7jhZ53JRJjTqJ8eZZIPe9KJEF+o+cpUoc7vdCMmpMRTI21vnYP/kb1tNDRTuKU3pF63vxaWsq14j2VXvtsaq9kGR3nytFtHt89R7gRpRk/WkBg1JodZsflq0nUs8kfvNU7yNAZ/wIAAP//ilZVQA==" } diff --git a/x-pack/metricbeat/module/azure/module.yml b/x-pack/metricbeat/module/azure/module.yml index 9505846e81c..cc106a3c59b 100644 --- a/x-pack/metricbeat/module/azure/module.yml +++ b/x-pack/metricbeat/module/azure/module.yml @@ -1,3 +1,6 @@ name: azure metricsets: - - database_account + - container_registry + - container_instance + - container_service + - database_account diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml.disabled index 6e715512bed..a0ca31b5bc3 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -50,6 +50,39 @@ # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s +#- module: azure +# metricsets: +# - container_instance +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container_service +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container_registry +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + #- module: azure # metricsets: # - database_account