From db567d6a2a2347c7f4d449b1479dbecb7223a2e0 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 19 Feb 2020 15:59:28 +0100 Subject: [PATCH 1/9] add container --- metricbeat/docs/fields.asciidoc | 10 ++ metricbeat/docs/modules/azure.asciidoc | 14 ++ .../docs/modules/azure/container.asciidoc | 23 +++ metricbeat/docs/modules_list.asciidoc | 3 +- x-pack/metricbeat/include/list.go | 1 + x-pack/metricbeat/metricbeat.reference.yml | 10 ++ .../module/azure/_meta/config.reference.yml | 10 ++ .../metricbeat/module/azure/_meta/config.yml | 11 ++ .../metricbeat/module/azure/client_utils.go | 10 ++ .../module/azure/client_utils_test.go | 26 ++++ .../compute_vm_scaleset/client_helper.go | 17 +-- .../compute_vm_scaleset/client_helper_test.go | 26 ---- .../module/azure/container/_meta/data.json | 45 ++++++ .../azure/container/_meta/docs.asciidoc | 19 +++ .../module/azure/container/_meta/fields.yml | 7 + .../module/azure/container/client_helper.go | 66 +++++++++ .../azure/container/client_helper_test.go | 135 ++++++++++++++++++ .../module/azure/container/container.go | 79 ++++++++++ .../module/azure/container/container_test.go | 75 ++++++++++ x-pack/metricbeat/module/azure/data.go | 19 +++ x-pack/metricbeat/module/azure/fields.go | 2 +- .../metricbeat/modules.d/azure.yml.disabled | 11 ++ 22 files changed, 577 insertions(+), 42 deletions(-) create mode 100644 metricbeat/docs/modules/azure/container.asciidoc create mode 100644 x-pack/metricbeat/module/azure/container/_meta/data.json create mode 100644 x-pack/metricbeat/module/azure/container/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/azure/container/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/azure/container/client_helper.go create mode 100644 x-pack/metricbeat/module/azure/container/client_helper_test.go create mode 100644 x-pack/metricbeat/module/azure/container/container.go create mode 100644 x-pack/metricbeat/module/azure/container/container_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 754041ae8ba..650236d6247 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3190,6 +3190,16 @@ type: object compute_vm_scaleset +type: object + +-- + +*`azure.container.*.*`*:: ++ +-- +container + + type: object -- diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index b442bf294dd..b284cd3f134 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -149,6 +149,16 @@ metricbeat.modules: client_secret: '${AZURE_CLIENT_SECRET:""}' tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' ---- [float] @@ -160,6 +170,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -168,6 +180,8 @@ include::azure/compute_vm.asciidoc[] include::azure/compute_vm_scaleset.asciidoc[] +include::azure/container.asciidoc[] + include::azure/monitor.asciidoc[] include::azure/storage.asciidoc[] diff --git a/metricbeat/docs/modules/azure/container.asciidoc b/metricbeat/docs/modules/azure/container.asciidoc new file mode 100644 index 00000000000..b8ff22b283c --- /dev/null +++ b/metricbeat/docs/modules/azure/container.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-container]] +=== azure container metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/container/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/container/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index e35f54b7351..abfbfb1f6b7 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -30,8 +30,9 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> beta[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.4+| .4+| |<> beta[] +.5+| .5+| |<> beta[] |<> beta[] +|<> beta[] |<> beta[] |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 590accfd862..3ae0a19d523 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -21,6 +21,7 @@ import ( _ "github.com/elastic/beats/x-pack/metricbeat/module/azure" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/compute_vm" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/compute_vm_scaleset" + _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/container" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/monitor" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/storage" _ "github.com/elastic/beats/x-pack/metricbeat/module/cockroachdb" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 28ce31cc400..4c181b59e1a 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -265,6 +265,16 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - container + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + #--------------------------------- Beat Module --------------------------------- - module: beat metricsets: diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml index 8b3a6e64ec6..3e10e869002 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.reference.yml @@ -42,3 +42,13 @@ client_secret: '${AZURE_CLIENT_SECRET:""}' tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml index 009159d21b3..b058dc77118 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.yml @@ -46,3 +46,14 @@ # tenant_id: '${AZURE_TENANT_ID:""}' # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index b03964044ea..ce55a7a1459 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -188,3 +188,13 @@ func groupMetricsByResource(metrics []Metric) map[string][]Metric { } return grouped } + +// ContainsDimension will check if the dimension value is found in the list +func ContainsDimension(dimension string, dimensions []insights.LocalizableString) bool { + for _, dim := range dimensions { + if *dim.Value == dimension { + return true + } + } + return false +} diff --git a/x-pack/metricbeat/module/azure/client_utils_test.go b/x-pack/metricbeat/module/azure/client_utils_test.go index a63034ff77d..a21ebf026e4 100644 --- a/x-pack/metricbeat/module/azure/client_utils_test.go +++ b/x-pack/metricbeat/module/azure/client_utils_test.go @@ -142,3 +142,29 @@ func TestCompareMetricValues(t *testing.T) { result = compareMetricValues(val1, val2) assert.True(t, result) } + +func TestContainsDimension(t *testing.T) { + dimension := "VMName" + dim1 := "SlotID" + dim2 := "VNU" + dim3 := "VMName" + dimensionList := []insights.LocalizableString{ + { + Value: &dim1, + LocalizedValue: &dim1, + }, + { + Value: &dim2, + LocalizedValue: &dim2, + }, + { + Value: &dim3, + LocalizedValue: &dim3, + }, + } + result := ContainsDimension(dimension, dimensionList) + assert.True(t, result) + dimension = "VirtualMachine" + result = ContainsDimension(dimension, dimensionList) + assert.False(t, result) +} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go index aa5e699109f..382b99b40dd 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go @@ -18,7 +18,6 @@ const ( defaultVMDimension = "VMName" customVMDimension = "VirtualMachine" defaultSlotIDDimension = "SlotId" - defaultTimeGrain = "PT5M" ) // mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters @@ -61,9 +60,9 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res for _, metricName := range supportedMetricNames { if metricName.Dimensions == nil || len(*metricName.Dimensions) == 0 { groupedMetrics[azure.NoDimension] = append(groupedMetrics[azure.NoDimension], metricName) - } else if containsDimension(vmdim, *metricName.Dimensions) { + } else if azure.ContainsDimension(vmdim, *metricName.Dimensions) { groupedMetrics[vmdim] = append(groupedMetrics[vmdim], metricName) - } else if containsDimension(defaultSlotIDDimension, *metricName.Dimensions) { + } else if azure.ContainsDimension(defaultSlotIDDimension, *metricName.Dimensions) { groupedMetrics[defaultSlotIDDimension] = append(groupedMetrics[defaultSlotIDDimension], metricName) } } @@ -76,19 +75,9 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res if key != azure.NoDimension { dimensions = []azure.Dimension{{Name: key, Value: "*"}} } - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", metric.Namespace, dimensions, defaultTimeGrain)...) + metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", metric.Namespace, dimensions, azure.DefaultTimeGrain)...) } } } return metrics, nil } - -// containsDimension will check if the dimension value is found in the list -func containsDimension(dimension string, dimensions []insights.LocalizableString) bool { - for _, dim := range dimensions { - if *dim.Value == dimension { - return true - } - } - return false -} diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go index 889c029a0e9..a79afb25e5c 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go @@ -116,29 +116,3 @@ func TestMapMetric(t *testing.T) { m.AssertExpectations(t) }) } - -func TestContainsDimension(t *testing.T) { - dimension := "VMName" - dim1 := "SlotID" - dim2 := "VNU" - dim3 := "VMName" - dimensionList := []insights.LocalizableString{ - { - Value: &dim1, - LocalizedValue: &dim1, - }, - { - Value: &dim2, - LocalizedValue: &dim2, - }, - { - Value: &dim3, - LocalizedValue: &dim3, - }, - } - result := containsDimension(dimension, dimensionList) - assert.True(t, result) - dimension = "VirtualMachine" - result = containsDimension(dimension, dimensionList) - assert.False(t, result) -} diff --git a/x-pack/metricbeat/module/azure/container/_meta/data.json b/x-pack/metricbeat/module/azure/container/_meta/data.json new file mode 100644 index 00000000000..0dacb4b32e3 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container/_meta/data.json @@ -0,0 +1,45 @@ +{ + "@timestamp" : "2020-02-19T14:41:00.000Z", + "event" : { + "duration" : 52412509300, + "dataset" : "azure.container", + "module" : "azure" + }, + "metricset" : { + "name" : "container", + "period" : 300000 + }, + "azure" : { + "subscription_id" : "123456-qwer-1234-5678-12345678", + "namespace" : "Microsoft.ContainerService/managedClusters", + "container" : { + "kube_pod_status_phase" : { + "avg" : 9 + }, + "kube_node_status_allocatable_cpu_cores" : { + "avg" : 2 + }, + "kube_node_status_allocatable_memory_bytes" : { + "avg" : 2.25349632E9 + }, + "kube_pod_status_ready" : { + "avg" : 9 + } + }, + "resource" : { + "group" : "obs-infrastructure", + "name" : "obskube", + "type" : "Microsoft.ContainerService/managedClusters" + } + }, + "service" : { + "type" : "azure" + }, + "ecs" : { + "version" : "1.4.0" + }, + "cloud":{ + "provider":"azure", + "region":"westeurope" + } +} diff --git a/x-pack/metricbeat/module/azure/container/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/container/_meta/docs.asciidoc new file mode 100644 index 00000000000..719742a52a0 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container/_meta/docs.asciidoc @@ -0,0 +1,19 @@ +This is the container metricset of the module azure. + +This metricset allows users to retrieve all metrics from specified containers, container registries and container groups. + +include::../../_meta/shared-azure.asciidoc[] + +[float] +==== Config options to identify resources + +`resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. + Should return a list of resources. + +`resource_group`:: (_[]string_) This option should return a list of containers we want to apply our metric configuration options on. + +If none of the options are entered then we will select all containers from the entire subscription +For each metric the primary aggregation assigned will be retrieved. +A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. + + diff --git a/x-pack/metricbeat/module/azure/container/_meta/fields.yml b/x-pack/metricbeat/module/azure/container/_meta/fields.yml new file mode 100644 index 00000000000..0a331319901 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: container.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" + description: > + container diff --git a/x-pack/metricbeat/module/azure/container/client_helper.go b/x-pack/metricbeat/module/azure/container/client_helper.go new file mode 100644 index 00000000000..ba75bd824f8 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container/client_helper.go @@ -0,0 +1,66 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container + +import ( + "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" + "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" + "github.com/pkg/errors" + + "github.com/elastic/beats/x-pack/metricbeat/module/azure" +) + +var retquiredDimension = "status" + +// mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters +func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { + var metrics []azure.Metric + if len(resourceConfig.Metrics) == 0 { + return nil, nil + } + for _, resource := range resources { + // return all namespaces supported for this resource + namespaces, err := client.AzureMonitorService.GetMetricNamespaces(*resource.ID) + if err != nil { + return nil, errors.Wrapf(err, "no metric namespaces were found for resource %s", *resource.ID) + } + for _, namespace := range *namespaces.Value { + // get all metric definitions supported by the namespace provided + metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, *namespace.Properties.MetricNamespaceName) + if err != nil { + return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) + } + if len(*metricDefinitions.Value) == 0 { + return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) + } + var filteredMetricDefinitions []insights.MetricDefinition + for _, metricDefinition := range *metricDefinitions.Value { + filteredMetricDefinitions = append(filteredMetricDefinitions, metricDefinition) + } + + groupedMetrics := make(map[string][]insights.MetricDefinition) + for _, metricName := range filteredMetricDefinitions { + if *metricName.IsDimensionRequired == false { + groupedMetrics[azure.NoDimension] = append(groupedMetrics[azure.NoDimension], metricName) + } else if azure.ContainsDimension(retquiredDimension, *metricName.Dimensions) { + groupedMetrics[retquiredDimension] = append(groupedMetrics[retquiredDimension], metricName) + } + } + for key, metricGroup := range groupedMetrics { + var metricNameList []string + for _, metricName := range metricGroup { + metricNameList = append(metricNameList, *metricName.Name.Value) + } + var dimensions []azure.Dimension + if key != azure.NoDimension { + dimensions = []azure.Dimension{{Name: key, Value: "*"}} + } + // map azure metric definitions to client metrics + metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", *namespace.Properties.MetricNamespaceName, dimensions, azure.DefaultTimeGrain)...) + } + } + } + return metrics, nil +} diff --git a/x-pack/metricbeat/module/azure/container/client_helper_test.go b/x-pack/metricbeat/module/azure/container/client_helper_test.go new file mode 100644 index 00000000000..006b3ef1d58 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container/client_helper_test.go @@ -0,0 +1,135 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" + "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "github.com/elastic/beats/x-pack/metricbeat/module/azure" +) + +func MockResource() resources.GenericResource { + id := "123" + name := "resourceName" + location := "resourceLocation" + rType := "resourceType" + return resources.GenericResource{ + ID: &id, + Name: &name, + Location: &location, + Type: &rType, + } +} + +func MockNamespace() insights.MetricNamespaceCollection { + name := "namespace" + property := insights.MetricNamespaceName{ + MetricNamespaceName: &name, + } + namespace := insights.MetricNamespace{ + Name: &name, + Properties: &property, + } + list := []insights.MetricNamespace{namespace} + return insights.MetricNamespaceCollection{ + Value: &list, + } +} + +func MockMetricDefinitions() *[]insights.MetricDefinition { + metric1 := "TotalRequests" + metric2 := "Capacity" + metric3 := "BytesRead" + defs := []insights.MetricDefinition{ + { + Name: &insights.LocalizableString{Value: &metric1}, + PrimaryAggregationType: insights.Average, + SupportedAggregationTypes: &[]insights.AggregationType{insights.Maximum, insights.Count, insights.Total, insights.Average}, + }, + { + Name: &insights.LocalizableString{Value: &metric2}, + PrimaryAggregationType: insights.Average, + SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, + }, + { + Name: &insights.LocalizableString{Value: &metric3}, + PrimaryAggregationType: insights.Minimum, + SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, + }, + } + return &defs +} + +func TestMapMetric(t *testing.T) { + resource := MockResource() + namespace := MockNamespace() + metricDefinitions := insights.MetricDefinitionCollection{ + Value: MockMetricDefinitions(), + } + emptyList := []insights.MetricDefinition{} + emptyMetricDefinitions := insights.MetricDefinitionCollection{ + Value: &emptyList, + } + metricConfig := azure.MetricConfig{Name: []string{"*"}} + var resourceConfig = azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}} + client := azure.NewMockClient() + t.Run("return error when the metric namespaces api call returns an error", func(t *testing.T) { + m := &azure.MockService{} + m.On("GetMetricNamespaces", mock.Anything).Return(insights.MetricNamespaceCollection{}, errors.New("invalid resource ID")) + client.AzureMonitorService = m + metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) + assert.NotNil(t, err) + assert.Equal(t, err.Error(), "no metric namespaces were found for resource 123: invalid resource ID") + assert.Equal(t, metric, []azure.Metric(nil)) + m.AssertExpectations(t) + }) + t.Run("return error when no metric definitions were found", func(t *testing.T) { + m := &azure.MockService{} + m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) + m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) + client.AzureMonitorService = m + metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) + assert.NotNil(t, err) + assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace namespace.") + assert.Equal(t, metric, []azure.Metric(nil)) + m.AssertExpectations(t) + }) + t.Run("return mapped metrics correctly", func(t *testing.T) { + m := &azure.MockService{} + m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) + m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) + client.AzureMonitorService = m + metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) + assert.Nil(t, err) + assert.Equal(t, metrics[0].Resource.ID, "123") + assert.Equal(t, metrics[0].Resource.Name, "resourceName") + assert.Equal(t, metrics[0].Resource.Type, "resourceType") + assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[0].Namespace, "namespace") + assert.Equal(t, metrics[1].Resource.ID, "123") + assert.Equal(t, metrics[1].Resource.Name, "resourceName") + assert.Equal(t, metrics[1].Resource.Type, "resourceType") + assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[1].Namespace, "namespace") + assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) + assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) + + //order of elements can be different when running the test + if metrics[0].Aggregations == "Average" { + assert.Equal(t, metrics[0].Names, []string{"TotalRequests", "Capacity"}) + } else { + assert.Equal(t, metrics[0].Names, []string{"BytesRead"}) + assert.Equal(t, metrics[0].Aggregations, "Minimum") + } + + m.AssertExpectations(t) + }) +} diff --git a/x-pack/metricbeat/module/azure/container/container.go b/x-pack/metricbeat/module/azure/container/container.go new file mode 100644 index 00000000000..724d4274d8d --- /dev/null +++ b/x-pack/metricbeat/module/azure/container/container.go @@ -0,0 +1,79 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container + +import ( + "fmt" + + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/x-pack/metricbeat/module/azure" +) + +const defaultContainerNamespace = "Microsoft.ContainerService/managedClusters" +const registryNamespace = "Microsoft.ContainerRegistry/registries" +const groupNamespace = "Microsoft.ContainerInstance/containerGroups" + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("azure", "container", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *azure.MetricSet +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + ms, err := azure.NewMetricSet(base) + if err != nil { + return nil, err + } + // if no options are entered we will retrieve all the vm's from the entire subscription + if len(ms.Client.Config.Resources) == 0 { + ms.Client.Config.Resources = []azure.ResourceConfig{ + { + Query: fmt.Sprintf("resourceType eq '%s' OR resourceType eq '%s' or resourceType eq '%s' ", defaultContainerNamespace, registryNamespace, groupNamespace), + }, + } + } + var filledResources []azure.ResourceConfig + for _, res := range ms.Client.Config.Resources { + // if any resource groups were configured the resource type should be added + if len(res.Group) > 0 { + containerRes := res + containerRes.Type = defaultContainerNamespace + groupRes := res + containerRes.Type = groupNamespace + registryRes := res + containerRes.Type = registryNamespace + filledResources = append(filledResources, containerRes, groupRes, registryRes) + + } else { + filledResources = append(filledResources, res) + } + + } + for index := range filledResources { + // one metric configuration will be added containing all metrics names + filledResources[index].Metrics = []azure.MetricConfig{ + { + Name: []string{"*"}, + }, + } + } + ms.Client.Config.Resources = filledResources + ms.MapMetrics = mapMetrics + return &MetricSet{ + MetricSet: ms, + }, nil +} diff --git a/x-pack/metricbeat/module/azure/container/container_test.go b/x-pack/metricbeat/module/azure/container/container_test.go new file mode 100644 index 00000000000..47cf3934dd3 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container/container_test.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" +) + +var ( + missingResourcesConfig = common.MapStr{ + "module": "azure", + "period": "60s", + "metricsets": []string{"container"}, + "client_secret": "unique identifier", + "client_id": "unique identifier", + "subscription_id": "unique identifier", + "tenant_id": "unique identifier", + } + + resourceConfig = common.MapStr{ + "module": "azure", + "period": "60s", + "metricsets": []string{"container"}, + "client_secret": "unique identifier", + "client_id": "unique identifier", + "subscription_id": "unique identifier", + "tenant_id": "unique identifier", + "resources": []common.MapStr{ + { + "resource_id": "test", + "metrics": []map[string]interface{}{ + { + "name": []string{"*"}, + }}, + }}, + } +) + +func TestFetch(t *testing.T) { + c, err := common.NewConfigFrom(missingResourcesConfig) + if err != nil { + t.Fatal(err) + } + module, metricsets, err := mb.NewModule(c, mb.Registry) + assert.NotNil(t, module) + assert.NotNil(t, metricsets) + assert.Nil(t, err) + ms, ok := metricsets[0].(*MetricSet) + assert.Equal(t, len(ms.Client.Config.Resources), 1) + assert.Equal(t, ms.Client.Config.Resources[0].Query, fmt.Sprintf("resourceType eq '%s' OR resourceType eq '%s' or resourceType eq '%s' ", + defaultContainerNamespace, registryNamespace, groupNamespace)) + + c, err = common.NewConfigFrom(resourceConfig) + if err != nil { + t.Fatal(err) + } + module, metricsets, err = mb.NewModule(c, mb.Registry) + if err != nil { + t.Fatal(err) + } + assert.NotNil(t, module) + assert.NotNil(t, metricsets) + ms, ok = metricsets[0].(*MetricSet) + require.True(t, ok, "metricset must be MetricSet") + assert.NotNil(t, ms) +} diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index 5b5ebc04eed..06174415708 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -8,6 +8,7 @@ import ( "fmt" "strings" "time" + "unicode" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" @@ -95,6 +96,7 @@ func EventsMapping(metrics []Metric, metricset string, report mb.ReporterV2) err // managePropertyName function will handle metric names, there are several formats the metric names are written func managePropertyName(metric string) string { + // replace spaces with underscores resultMetricName := strings.Replace(metric, " ", "_", -1) // replace backslashes with "per" @@ -106,6 +108,9 @@ func managePropertyName(metric string) string { resultMetricName = strings.Replace(resultMetricName, ":", "_", -1) // create an object in case of ":" resultMetricName = strings.Replace(resultMetricName, "_-_", "_", -1) + // replace uppercases with underscores + resultMetricName = replaceUpperCase(resultMetricName) + // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" obj := strings.Split(resultMetricName, ".") for index := range obj { @@ -118,6 +123,20 @@ func managePropertyName(metric string) string { return resultMetricName } +// replaceUpperCase func will replace upper case with '_' +func replaceUpperCase(src string) string { + var result string + // split into fields based on class of unicode character + for _, r := range src { + if unicode.IsUpper(r) { + result += "_" + strings.ToLower(string(r)) + } else { + result += string(r) + } + } + return result +} + // createEvent will create a new base event func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) (mb.Event, common.MapStr) { event := mb.Event{ diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index 8c678616d18..ddf5e52c143 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUVDtPwzAQ3vMrThkrtT8gAxISCwMbe+Tal2AaP+QHqPx65KR1nXcLHcCDpfjO3+Pu4i0c8FgA+fIGMwDHXYMF5O13ngEwtNRw7biSBTxkANDlglDMN+GKwQaJxQL26EgGUHFsmC3a1C1IIvACH5Y7aiygNsrr08kERx8mhTJolTcUY2AKcRa1W69vGHHAaqS84siSlCF5KiDsvcBZwAGPn8qwQWxBxllKQARVgUtkTVIHnvtSh+tXUQ+rewfu2IExdjRMarvbTNKq/TtSNwh1h+WSsCSlFERrLutTfr7JbzPx2P4I0UYrdjSvYbeaTAzsWN/KyEYosNggdcnIntms30eIkrPfc6aA8Pw0ImRcoLRcyX6fZnq00p9re7OgueuJQGc4TcWNhFMltHdYfojdpid9+J7dYqlqFJkJ/tTQReeChdJS0qBF91+8RMExPnInlOROmdV3fs7jgo4x9NKD3w2THZR2oYyrpbylnCtWwnrpBIJB541EtpsvqnXKkBr/9picRAKhVHnpsu8AAAD//wCRS6g=" + return "eJzUVb2O2zAM3v0UhMcAlwfwUKBAlw7duhuKRLvqWT+QqBbp0xeyHZ38f2kz5DQYiEh9PyQVvcArXitgf4LDAoAkdVhB2f8uCwCBnjtpSRpdwacCAIZcUEaELh5x2CHzWMEFiRUAjcRO+KpPfQHNFL7Bx0VXixW0zgQ77qxwTGFyKIfeBMcxBdYQN1GH9f0HJhzwFrlsJIosZU6eC4jfSeAm4BWvv40Ts9iOjJuUiAimAcpkrVJHnsdSx+Pvop5X9wHcqQNL7GSYtf58WqU1l5/IaRYaNus9YVlKrZi1Urdjfnkq7zPxub8IyUYvdjGv8estWxnYpb6DkU1Q4LFDTtnI3th8uCSIWor/58wB4euXBaGQCrWXRk/7tNGjg/68tzc7moeeKCQneS5uIZwbZQNh/UudTxPp8/+zeyw1nWEbwX819KZzx0LtOevQI30UL0lwiq+408SkRvfsnkaZCwPKaEnGHT5UW4Z2SJfQey/WcBv8rI47NTus2z21O7AS17dBIDik4DSK8/ZUeDKOtfjcMzGKBMa5CZqKvwEAAP//KrN/pg==" } diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml.disabled index 31cb3a18951..f470a0b900b 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -49,3 +49,14 @@ # tenant_id: '${AZURE_TENANT_ID:""}' # subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' # refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s From 0a382c676d47d8d32f539a3ddf39f5e3d25dbdea Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 19 Feb 2020 16:07:15 +0100 Subject: [PATCH 2/9] update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2676dc67ffd..4741bc7aad4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -161,6 +161,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for NATS 2.1. {pull}16317[16317] - Add Load Balancing metricset to GCP {pull}15559[15559] - Add support for Dropwizard metrics 4.1. {pull}16332[16332] +- Add azure container metricset in order to monitor containers. {issue}15751[15751] {pull}16421[16421] *Packetbeat* From fe681c484b5c6071fc43af3af5a7d239217bed41 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 20 Feb 2020 10:30:00 +0100 Subject: [PATCH 3/9] fix test --- .../module/azure/container/client_helper_test.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/metricbeat/module/azure/container/client_helper_test.go b/x-pack/metricbeat/module/azure/container/client_helper_test.go index 006b3ef1d58..cacfb5e301f 100644 --- a/x-pack/metricbeat/module/azure/container/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/container/client_helper_test.go @@ -47,22 +47,26 @@ func MockNamespace() insights.MetricNamespaceCollection { func MockMetricDefinitions() *[]insights.MetricDefinition { metric1 := "TotalRequests" metric2 := "Capacity" - metric3 := "BytesRead" + requird := true + notRequired := false + dimension := "status" + dimensionValue := "*" defs := []insights.MetricDefinition{ { Name: &insights.LocalizableString{Value: &metric1}, PrimaryAggregationType: insights.Average, SupportedAggregationTypes: &[]insights.AggregationType{insights.Maximum, insights.Count, insights.Total, insights.Average}, + IsDimensionRequired: ¬Required, }, { Name: &insights.LocalizableString{Value: &metric2}, PrimaryAggregationType: insights.Average, SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, - }, - { - Name: &insights.LocalizableString{Value: &metric3}, - PrimaryAggregationType: insights.Minimum, - SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, + IsDimensionRequired: &requird, + Dimensions: &[]insights.LocalizableString{{ + Value: &dimension, + LocalizedValue: &dimensionValue, + }}, }, } return &defs From abab5ea99c02a40f95efe3af60d9f5faea52c0b0 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 25 Feb 2020 19:54:34 +0100 Subject: [PATCH 4/9] separate container --- .../module/azure/_meta/config.reference.yml | 23 ++- .../metricbeat/module/azure/_meta/config.yml | 24 ++- .../module/azure/container/_meta/data.json | 45 ------ .../module/azure/container/client_helper.go | 66 -------- .../azure/container/client_helper_test.go | 139 ---------------- .../module/azure/container/container.go | 79 --------- .../module/azure/container/container_test.go | 75 --------- .../azure/container_instance/_meta/data.json | 153 ++++++++++++++++++ .../_meta/docs.asciidoc | 8 +- .../azure/container_instance/_meta/fields.yml | 7 + .../container_instance_integration_test.go | 51 ++++++ .../container_instance_test.go | 17 ++ .../azure/container_instance/manifest.yml | 16 ++ .../azure/container_registry/_meta/data.json | 153 ++++++++++++++++++ .../container_registry/_meta/docs.asciidoc | 19 +++ .../azure/container_registry/_meta/fields.yml | 7 + .../container_registry_integration_test.go | 51 ++++++ .../container_registry_test.go | 17 ++ .../azure/container_registry/manifest.yml | 16 ++ .../azure/container_service/_meta/data.json | 153 ++++++++++++++++++ .../container_service/_meta/docs.asciidoc | 19 +++ .../_meta/fields.yml | 4 +- .../container_service_integration_test.go | 51 ++++++ .../container_service_test.go | 17 ++ .../azure/container_service/manifest.yml | 16 ++ x-pack/metricbeat/module/azure/module.yml | 5 + 26 files changed, 819 insertions(+), 412 deletions(-) delete mode 100644 x-pack/metricbeat/module/azure/container/_meta/data.json delete mode 100644 x-pack/metricbeat/module/azure/container/client_helper.go delete mode 100644 x-pack/metricbeat/module/azure/container/client_helper_test.go delete mode 100644 x-pack/metricbeat/module/azure/container/container.go delete mode 100644 x-pack/metricbeat/module/azure/container/container_test.go create mode 100644 x-pack/metricbeat/module/azure/container_instance/_meta/data.json rename x-pack/metricbeat/module/azure/{container => container_instance}/_meta/docs.asciidoc (72%) create mode 100644 x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/container_instance/container_instance_test.go create mode 100644 x-pack/metricbeat/module/azure/container_instance/manifest.yml create mode 100644 x-pack/metricbeat/module/azure/container_registry/_meta/data.json create mode 100644 x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/container_registry/container_registry_test.go create mode 100644 x-pack/metricbeat/module/azure/container_registry/manifest.yml create mode 100644 x-pack/metricbeat/module/azure/container_service/_meta/data.json create mode 100644 x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc rename x-pack/metricbeat/module/azure/{container => container_service}/_meta/fields.yml (66%) create mode 100644 x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go create mode 100644 x-pack/metricbeat/module/azure/container_service/container_service_test.go create mode 100644 x-pack/metricbeat/module/azure/container_service/manifest.yml create mode 100644 x-pack/metricbeat/module/azure/module.yml diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml index 3e10e869002..b942eb6bd13 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.reference.yml @@ -45,10 +45,31 @@ - module: azure metricsets: - - container + - container_instance enabled: true period: 300s client_id: '${AZURE_CLIENT_ID:""}' client_secret: '${AZURE_CLIENT_SECRET:""}' tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_service + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_registry + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml index b058dc77118..38e1e0c4684 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.yml @@ -49,7 +49,29 @@ #- module: azure # metricsets: -# - container +# - container_instance +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container_service +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container_registry # enabled: true # period: 300s # client_id: '${AZURE_CLIENT_ID:""}' diff --git a/x-pack/metricbeat/module/azure/container/_meta/data.json b/x-pack/metricbeat/module/azure/container/_meta/data.json deleted file mode 100644 index 0dacb4b32e3..00000000000 --- a/x-pack/metricbeat/module/azure/container/_meta/data.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "@timestamp" : "2020-02-19T14:41:00.000Z", - "event" : { - "duration" : 52412509300, - "dataset" : "azure.container", - "module" : "azure" - }, - "metricset" : { - "name" : "container", - "period" : 300000 - }, - "azure" : { - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.ContainerService/managedClusters", - "container" : { - "kube_pod_status_phase" : { - "avg" : 9 - }, - "kube_node_status_allocatable_cpu_cores" : { - "avg" : 2 - }, - "kube_node_status_allocatable_memory_bytes" : { - "avg" : 2.25349632E9 - }, - "kube_pod_status_ready" : { - "avg" : 9 - } - }, - "resource" : { - "group" : "obs-infrastructure", - "name" : "obskube", - "type" : "Microsoft.ContainerService/managedClusters" - } - }, - "service" : { - "type" : "azure" - }, - "ecs" : { - "version" : "1.4.0" - }, - "cloud":{ - "provider":"azure", - "region":"westeurope" - } -} diff --git a/x-pack/metricbeat/module/azure/container/client_helper.go b/x-pack/metricbeat/module/azure/container/client_helper.go deleted file mode 100644 index ba75bd824f8..00000000000 --- a/x-pack/metricbeat/module/azure/container/client_helper.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package container - -import ( - "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" - "github.com/pkg/errors" - - "github.com/elastic/beats/x-pack/metricbeat/module/azure" -) - -var retquiredDimension = "status" - -// mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters -func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { - var metrics []azure.Metric - if len(resourceConfig.Metrics) == 0 { - return nil, nil - } - for _, resource := range resources { - // return all namespaces supported for this resource - namespaces, err := client.AzureMonitorService.GetMetricNamespaces(*resource.ID) - if err != nil { - return nil, errors.Wrapf(err, "no metric namespaces were found for resource %s", *resource.ID) - } - for _, namespace := range *namespaces.Value { - // get all metric definitions supported by the namespace provided - metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, *namespace.Properties.MetricNamespaceName) - if err != nil { - return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) - } - if len(*metricDefinitions.Value) == 0 { - return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) - } - var filteredMetricDefinitions []insights.MetricDefinition - for _, metricDefinition := range *metricDefinitions.Value { - filteredMetricDefinitions = append(filteredMetricDefinitions, metricDefinition) - } - - groupedMetrics := make(map[string][]insights.MetricDefinition) - for _, metricName := range filteredMetricDefinitions { - if *metricName.IsDimensionRequired == false { - groupedMetrics[azure.NoDimension] = append(groupedMetrics[azure.NoDimension], metricName) - } else if azure.ContainsDimension(retquiredDimension, *metricName.Dimensions) { - groupedMetrics[retquiredDimension] = append(groupedMetrics[retquiredDimension], metricName) - } - } - for key, metricGroup := range groupedMetrics { - var metricNameList []string - for _, metricName := range metricGroup { - metricNameList = append(metricNameList, *metricName.Name.Value) - } - var dimensions []azure.Dimension - if key != azure.NoDimension { - dimensions = []azure.Dimension{{Name: key, Value: "*"}} - } - // map azure metric definitions to client metrics - metrics = append(metrics, azure.MapMetricByPrimaryAggregation(client, metricGroup, resource, "", *namespace.Properties.MetricNamespaceName, dimensions, azure.DefaultTimeGrain)...) - } - } - } - return metrics, nil -} diff --git a/x-pack/metricbeat/module/azure/container/client_helper_test.go b/x-pack/metricbeat/module/azure/container/client_helper_test.go deleted file mode 100644 index cacfb5e301f..00000000000 --- a/x-pack/metricbeat/module/azure/container/client_helper_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package container - -import ( - "testing" - - "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - - "github.com/elastic/beats/x-pack/metricbeat/module/azure" -) - -func MockResource() resources.GenericResource { - id := "123" - name := "resourceName" - location := "resourceLocation" - rType := "resourceType" - return resources.GenericResource{ - ID: &id, - Name: &name, - Location: &location, - Type: &rType, - } -} - -func MockNamespace() insights.MetricNamespaceCollection { - name := "namespace" - property := insights.MetricNamespaceName{ - MetricNamespaceName: &name, - } - namespace := insights.MetricNamespace{ - Name: &name, - Properties: &property, - } - list := []insights.MetricNamespace{namespace} - return insights.MetricNamespaceCollection{ - Value: &list, - } -} - -func MockMetricDefinitions() *[]insights.MetricDefinition { - metric1 := "TotalRequests" - metric2 := "Capacity" - requird := true - notRequired := false - dimension := "status" - dimensionValue := "*" - defs := []insights.MetricDefinition{ - { - Name: &insights.LocalizableString{Value: &metric1}, - PrimaryAggregationType: insights.Average, - SupportedAggregationTypes: &[]insights.AggregationType{insights.Maximum, insights.Count, insights.Total, insights.Average}, - IsDimensionRequired: ¬Required, - }, - { - Name: &insights.LocalizableString{Value: &metric2}, - PrimaryAggregationType: insights.Average, - SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, - IsDimensionRequired: &requird, - Dimensions: &[]insights.LocalizableString{{ - Value: &dimension, - LocalizedValue: &dimensionValue, - }}, - }, - } - return &defs -} - -func TestMapMetric(t *testing.T) { - resource := MockResource() - namespace := MockNamespace() - metricDefinitions := insights.MetricDefinitionCollection{ - Value: MockMetricDefinitions(), - } - emptyList := []insights.MetricDefinition{} - emptyMetricDefinitions := insights.MetricDefinitionCollection{ - Value: &emptyList, - } - metricConfig := azure.MetricConfig{Name: []string{"*"}} - var resourceConfig = azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}} - client := azure.NewMockClient() - t.Run("return error when the metric namespaces api call returns an error", func(t *testing.T) { - m := &azure.MockService{} - m.On("GetMetricNamespaces", mock.Anything).Return(insights.MetricNamespaceCollection{}, errors.New("invalid resource ID")) - client.AzureMonitorService = m - metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) - assert.NotNil(t, err) - assert.Equal(t, err.Error(), "no metric namespaces were found for resource 123: invalid resource ID") - assert.Equal(t, metric, []azure.Metric(nil)) - m.AssertExpectations(t) - }) - t.Run("return error when no metric definitions were found", func(t *testing.T) { - m := &azure.MockService{} - m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) - client.AzureMonitorService = m - metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) - assert.NotNil(t, err) - assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace namespace.") - assert.Equal(t, metric, []azure.Metric(nil)) - m.AssertExpectations(t) - }) - t.Run("return mapped metrics correctly", func(t *testing.T) { - m := &azure.MockService{} - m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) - client.AzureMonitorService = m - metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) - assert.Nil(t, err) - assert.Equal(t, metrics[0].Resource.ID, "123") - assert.Equal(t, metrics[0].Resource.Name, "resourceName") - assert.Equal(t, metrics[0].Resource.Type, "resourceType") - assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") - assert.Equal(t, metrics[0].Namespace, "namespace") - assert.Equal(t, metrics[1].Resource.ID, "123") - assert.Equal(t, metrics[1].Resource.Name, "resourceName") - assert.Equal(t, metrics[1].Resource.Type, "resourceType") - assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") - assert.Equal(t, metrics[1].Namespace, "namespace") - assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) - assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) - - //order of elements can be different when running the test - if metrics[0].Aggregations == "Average" { - assert.Equal(t, metrics[0].Names, []string{"TotalRequests", "Capacity"}) - } else { - assert.Equal(t, metrics[0].Names, []string{"BytesRead"}) - assert.Equal(t, metrics[0].Aggregations, "Minimum") - } - - m.AssertExpectations(t) - }) -} diff --git a/x-pack/metricbeat/module/azure/container/container.go b/x-pack/metricbeat/module/azure/container/container.go deleted file mode 100644 index 724d4274d8d..00000000000 --- a/x-pack/metricbeat/module/azure/container/container.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package container - -import ( - "fmt" - - "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/x-pack/metricbeat/module/azure" -) - -const defaultContainerNamespace = "Microsoft.ContainerService/managedClusters" -const registryNamespace = "Microsoft.ContainerRegistry/registries" -const groupNamespace = "Microsoft.ContainerInstance/containerGroups" - -// init registers the MetricSet with the central registry as soon as the program -// starts. The New function will be called later to instantiate an instance of -// the MetricSet for each host defined in the module's configuration. After the -// MetricSet has been created then Fetch will begin to be called periodically. -func init() { - mb.Registry.MustAddMetricSet("azure", "container", New) -} - -// MetricSet holds any configuration or state information. It must implement -// the mb.MetricSet interface. And this is best achieved by embedding -// mb.BaseMetricSet because it implements all of the required mb.MetricSet -// interface methods except for Fetch. -type MetricSet struct { - *azure.MetricSet -} - -// New creates a new instance of the MetricSet. New is responsible for unpacking -// any MetricSet specific configuration options if there are any. -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - ms, err := azure.NewMetricSet(base) - if err != nil { - return nil, err - } - // if no options are entered we will retrieve all the vm's from the entire subscription - if len(ms.Client.Config.Resources) == 0 { - ms.Client.Config.Resources = []azure.ResourceConfig{ - { - Query: fmt.Sprintf("resourceType eq '%s' OR resourceType eq '%s' or resourceType eq '%s' ", defaultContainerNamespace, registryNamespace, groupNamespace), - }, - } - } - var filledResources []azure.ResourceConfig - for _, res := range ms.Client.Config.Resources { - // if any resource groups were configured the resource type should be added - if len(res.Group) > 0 { - containerRes := res - containerRes.Type = defaultContainerNamespace - groupRes := res - containerRes.Type = groupNamespace - registryRes := res - containerRes.Type = registryNamespace - filledResources = append(filledResources, containerRes, groupRes, registryRes) - - } else { - filledResources = append(filledResources, res) - } - - } - for index := range filledResources { - // one metric configuration will be added containing all metrics names - filledResources[index].Metrics = []azure.MetricConfig{ - { - Name: []string{"*"}, - }, - } - } - ms.Client.Config.Resources = filledResources - ms.MapMetrics = mapMetrics - return &MetricSet{ - MetricSet: ms, - }, nil -} diff --git a/x-pack/metricbeat/module/azure/container/container_test.go b/x-pack/metricbeat/module/azure/container/container_test.go deleted file mode 100644 index 47cf3934dd3..00000000000 --- a/x-pack/metricbeat/module/azure/container/container_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package container - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/metricbeat/mb" -) - -var ( - missingResourcesConfig = common.MapStr{ - "module": "azure", - "period": "60s", - "metricsets": []string{"container"}, - "client_secret": "unique identifier", - "client_id": "unique identifier", - "subscription_id": "unique identifier", - "tenant_id": "unique identifier", - } - - resourceConfig = common.MapStr{ - "module": "azure", - "period": "60s", - "metricsets": []string{"container"}, - "client_secret": "unique identifier", - "client_id": "unique identifier", - "subscription_id": "unique identifier", - "tenant_id": "unique identifier", - "resources": []common.MapStr{ - { - "resource_id": "test", - "metrics": []map[string]interface{}{ - { - "name": []string{"*"}, - }}, - }}, - } -) - -func TestFetch(t *testing.T) { - c, err := common.NewConfigFrom(missingResourcesConfig) - if err != nil { - t.Fatal(err) - } - module, metricsets, err := mb.NewModule(c, mb.Registry) - assert.NotNil(t, module) - assert.NotNil(t, metricsets) - assert.Nil(t, err) - ms, ok := metricsets[0].(*MetricSet) - assert.Equal(t, len(ms.Client.Config.Resources), 1) - assert.Equal(t, ms.Client.Config.Resources[0].Query, fmt.Sprintf("resourceType eq '%s' OR resourceType eq '%s' or resourceType eq '%s' ", - defaultContainerNamespace, registryNamespace, groupNamespace)) - - c, err = common.NewConfigFrom(resourceConfig) - if err != nil { - t.Fatal(err) - } - module, metricsets, err = mb.NewModule(c, mb.Registry) - if err != nil { - t.Fatal(err) - } - assert.NotNil(t, module) - assert.NotNil(t, metricsets) - ms, ok = metricsets[0].(*MetricSet) - require.True(t, ok, "metricset must be MetricSet") - assert.NotNil(t, ms) -} diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json new file mode 100644 index 00000000000..2751b9327db --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json @@ -0,0 +1,153 @@ +{ + "@timestamp":"2019-09-20T11:02:00.000Z", + "azure":{ + "resource":{ + "name":"obslinux", + "type":"Microsoft.Compute/virtualMachines", + "group":"resource-group" + }, + "subscription_id":"123456-qwer-1234-5678-12345678", + "namespace":"Microsoft.Compute/virtualMachines", + "compute_vm":{ + "inbound_flows_maximum_creation_rate":{ + "avg":14.6 + }, + "os_per_disk_write_bytes_per_sec":{ + "avg":15261.86 + }, + "outbound_flows_maximum_creation_rate":{ + "avg":18.6 + }, + "os_per_disk_read_operations_per_sec":{ + "avg":0 + }, + "data_disk_queue_depth":{ + "avg":0 + }, + "per_disk_read_operations_per_sec":{ + "avg":0 + }, + "inbound_flows":{ + "avg":60.4 + }, + "os_disk_write_operations_per_sec":{ + "avg":1.0175 + }, + "per_disk_read_bytes_per_sec":{ + "avg":0 + }, + "disk_write_operations_per_sec":{ + "avg":1.03 + }, + "outbound_flows":{ + "avg":60.4 + }, + "network_in":{ + "total":3624009.0 + }, + "os_per_disk_qd":{ + "avg":0 + }, + "data_disk_read_bytes_per_sec":{ + "avg":0 + }, + "disk_read_operations_per_sec":{ + "avg":0 + }, + "os_disk_read_operations_per_sec":{ + "avg":0 + }, + "per_disk_qd":{ + "avg":0 + }, + "os_disk_write_bytes_per_sec":{ + "avg":15261.86 + }, + "disk_write_bytes":{ + "total":2806782.57 + }, + "per_disk_write_operations_per_sec":{ + "avg":0 + }, + "network_out":{ + "total":704862 + }, + "os_disk_read_bytes_per_sec":{ + "avg":0 + }, + "os_per_disk_read_bytes_per_sec":{ + "avg":0 + }, + "os_per_disk_write_operations_per_sec":{ + "avg":1.0175 + }, + "percentage_cpu":{ + "avg":7.683333333333334 + }, + "data_disk_write_operations_per_sec":{ + "avg":0 + }, + "network_in_total":{ + "total":6787492.0 + }, + "disk_read_bytes":{ + "total":0 + }, + "data_disk_write_bytes_per_sec":{ + "avg":0 + }, + "per_disk_write_bytes_per_sec":{ + "avg":0 + }, + "data_disk_read_operations_per_sec":{ + "avg":0 + }, + "os_disk_queue_depth":{ + "avg":0 + }, + "network_out_total":{ + "total":1889767.0 + } + } + }, + "service":{ + "type":"azure" + }, + "cloud":{ + "provider":"azure", + "region":"westeurope" + }, + "ecs":{ + "version":"1.1.0" + }, + "host":{ + "architecture":"x86_64", + "os":{ + "build":"17134.1006", + "platform":"windows", + "version":"10.0", + "family":"windows", + "name":"Windows 10 Pro", + "kernel":"10.0.17134.1006 (WinBuild.160101.0800)" + }, + "id":"123456-qwer-1234-5678-12345678", + "name":"DESKTOP-RFOOE09", + "hostname":"DESKTOP-RFOOE09" + }, + "agent":{ + "hostname":"DESKTOP-RFOOE09", + "id":"123456-qwer-1234-5678-12345678", + "version":"8.0.0", + "type":"metricbeat", + "ephemeral_id":"123456-qwer-1234-5678-12345678" + }, + "event":{ + "duration":4611928900, + "dataset":"azure.compute_vm", + "module":"azure" + }, + "metricset":{ + "name":"compute_vm", + "period":300000 + } +} diff --git a/x-pack/metricbeat/module/azure/container/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc similarity index 72% rename from x-pack/metricbeat/module/azure/container/_meta/docs.asciidoc rename to x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc index 719742a52a0..a144105cf98 100644 --- a/x-pack/metricbeat/module/azure/container/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc @@ -1,6 +1,6 @@ -This is the container metricset of the module azure. +This is the container_instance metricset of the module azure. -This metricset allows users to retrieve all metrics from specified containers, container registries and container groups. +This metricset allows users to retrieve all metrics from specified container groups. include::../../_meta/shared-azure.asciidoc[] @@ -10,9 +10,9 @@ include::../../_meta/shared-azure.asciidoc[] `resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. Should return a list of resources. -`resource_group`:: (_[]string_) This option should return a list of containers we want to apply our metric configuration options on. +`resource_group`:: (_[]string_) This option should return a list of container groups we want to apply our metric configuration options on. -If none of the options are entered then we will select all containers from the entire subscription +If none of the options are entered then we will select all the container groups from the entire subscription For each metric the primary aggregation assigned will be retrieved. A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml b/x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml new file mode 100644 index 00000000000..d79ddd14999 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: container_instance.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" + description: > + container instance diff --git a/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go new file mode 100644 index 00000000000..37b814b070f --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/container_instance_integration_test.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_instance + +import ( + "errors" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + config, err := getConfig() + if err != nil { + t.Skip("Skipping TestData: " + err.Error()) + } + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} + +func getConfig() (map[string]interface{}, error) { + clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_ID key") + } + clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_SECRET key") + } + tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") + if !ok { + return nil, errors.New("missing AZURE_TENANT_ID key") + } + subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") + if !ok { + return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") + } + config := map[string]interface{}{ + "module": "azure", + "metricsets": []string{"container_instance"}, + "client_id": clientId, + "client_secret": clientSecret, + "tenant_id": tenantId, + "subscription_id": subscriptionId, + } + return config, nil +} diff --git a/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go b/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go new file mode 100644 index 00000000000..a92237aea4b --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/container_instance_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_instance + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/azure/container_instance/manifest.yml b/x-pack/metricbeat/module/azure/container_instance/manifest.yml new file mode 100644 index 00000000000..5e53428b13f --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_instance/manifest.yml @@ -0,0 +1,16 @@ +default: true +input: + module: azure + metricset: monitor + defaults: + default_resource_type: "Microsoft.ContainerInstance/containerGroups" + resources: + - resource_group: "" + resource_type: "Microsoft.ContainerInstance/containerGroups" + metrics: + - name: "*" + namespace: "Microsoft.ContainerInstance/containerGroups" + - resource_id: "" + metrics: + - name: "*" + namespace: "Microsoft.ContainerInstance/containerGroups" diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json new file mode 100644 index 00000000000..2751b9327db --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json @@ -0,0 +1,153 @@ +{ + "@timestamp":"2019-09-20T11:02:00.000Z", + "azure":{ + "resource":{ + "name":"obslinux", + "type":"Microsoft.Compute/virtualMachines", + "group":"resource-group" + }, + "subscription_id":"123456-qwer-1234-5678-12345678", + "namespace":"Microsoft.Compute/virtualMachines", + "compute_vm":{ + "inbound_flows_maximum_creation_rate":{ + "avg":14.6 + }, + "os_per_disk_write_bytes_per_sec":{ + "avg":15261.86 + }, + "outbound_flows_maximum_creation_rate":{ + "avg":18.6 + }, + "os_per_disk_read_operations_per_sec":{ + "avg":0 + }, + "data_disk_queue_depth":{ + "avg":0 + }, + "per_disk_read_operations_per_sec":{ + "avg":0 + }, + "inbound_flows":{ + "avg":60.4 + }, + "os_disk_write_operations_per_sec":{ + "avg":1.0175 + }, + "per_disk_read_bytes_per_sec":{ + "avg":0 + }, + "disk_write_operations_per_sec":{ + "avg":1.03 + }, + "outbound_flows":{ + "avg":60.4 + }, + "network_in":{ + "total":3624009.0 + }, + "os_per_disk_qd":{ + "avg":0 + }, + "data_disk_read_bytes_per_sec":{ + "avg":0 + }, + "disk_read_operations_per_sec":{ + "avg":0 + }, + "os_disk_read_operations_per_sec":{ + "avg":0 + }, + "per_disk_qd":{ + "avg":0 + }, + "os_disk_write_bytes_per_sec":{ + "avg":15261.86 + }, + "disk_write_bytes":{ + "total":2806782.57 + }, + "per_disk_write_operations_per_sec":{ + "avg":0 + }, + "network_out":{ + "total":704862 + }, + "os_disk_read_bytes_per_sec":{ + "avg":0 + }, + "os_per_disk_read_bytes_per_sec":{ + "avg":0 + }, + "os_per_disk_write_operations_per_sec":{ + "avg":1.0175 + }, + "percentage_cpu":{ + "avg":7.683333333333334 + }, + "data_disk_write_operations_per_sec":{ + "avg":0 + }, + "network_in_total":{ + "total":6787492.0 + }, + "disk_read_bytes":{ + "total":0 + }, + "data_disk_write_bytes_per_sec":{ + "avg":0 + }, + "per_disk_write_bytes_per_sec":{ + "avg":0 + }, + "data_disk_read_operations_per_sec":{ + "avg":0 + }, + "os_disk_queue_depth":{ + "avg":0 + }, + "network_out_total":{ + "total":1889767.0 + } + } + }, + "service":{ + "type":"azure" + }, + "cloud":{ + "provider":"azure", + "region":"westeurope" + }, + "ecs":{ + "version":"1.1.0" + }, + "host":{ + "architecture":"x86_64", + "os":{ + "build":"17134.1006", + "platform":"windows", + "version":"10.0", + "family":"windows", + "name":"Windows 10 Pro", + "kernel":"10.0.17134.1006 (WinBuild.160101.0800)" + }, + "id":"123456-qwer-1234-5678-12345678", + "name":"DESKTOP-RFOOE09", + "hostname":"DESKTOP-RFOOE09" + }, + "agent":{ + "hostname":"DESKTOP-RFOOE09", + "id":"123456-qwer-1234-5678-12345678", + "version":"8.0.0", + "type":"metricbeat", + "ephemeral_id":"123456-qwer-1234-5678-12345678" + }, + "event":{ + "duration":4611928900, + "dataset":"azure.compute_vm", + "module":"azure" + }, + "metricset":{ + "name":"compute_vm", + "period":300000 + } +} diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc new file mode 100644 index 00000000000..fa742cad052 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc @@ -0,0 +1,19 @@ +This is the container_registry metricset of the module azure. + +This metricset allows users to retrieve all metrics from specified virtual machines. + +include::../../_meta/shared-azure.asciidoc[] + +[float] +==== Config options to identify resources + +`resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. + Should return a list of resources. + +`resource_group`:: (_[]string_) This option should return a list virtual machines we want to apply our metric configuration options on. + +If none of the options are entered then we will select all virtual machine from the entire subscription +For each metric the primary aggregation assigned will be retrieved. +A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. + + diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml b/x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml new file mode 100644 index 00000000000..c8296603a77 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/fields.yml @@ -0,0 +1,7 @@ +- name: container_registry.*.* + release: beta + type: object + object_type: float + object_type_mapping_type: "*" + description: > + container registry diff --git a/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go new file mode 100644 index 00000000000..41414529997 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/container_registry_integration_test.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_registry + +import ( + "errors" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + config, err := getConfig() + if err != nil { + t.Skip("Skipping TestData: " + err.Error()) + } + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} + +func getConfig() (map[string]interface{}, error) { + clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_ID key") + } + clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_SECRET key") + } + tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") + if !ok { + return nil, errors.New("missing AZURE_TENANT_ID key") + } + subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") + if !ok { + return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") + } + config := map[string]interface{}{ + "module": "azure", + "metricsets": []string{"container_registry"}, + "client_id": clientId, + "client_secret": clientSecret, + "tenant_id": tenantId, + "subscription_id": subscriptionId, + } + return config, nil +} diff --git a/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go b/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go new file mode 100644 index 00000000000..7b65c68056f --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/container_registry_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_registry + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/azure/container_registry/manifest.yml b/x-pack/metricbeat/module/azure/container_registry/manifest.yml new file mode 100644 index 00000000000..787b46e71de --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_registry/manifest.yml @@ -0,0 +1,16 @@ +default: true +input: + module: azure + metricset: monitor + defaults: + default_resource_type: "Microsoft.ContainerRegistry/registries" + resources: + - resource_group: "" + resource_type: "Microsoft.ContainerRegistry/registries" + metrics: + - name: "*" + namespace: "Microsoft.ContainerRegistry/registries" + - resource_id: "" + metrics: + - name: "*" + namespace: "Microsoft.ContainerRegistry/registries" diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/data.json b/x-pack/metricbeat/module/azure/container_service/_meta/data.json new file mode 100644 index 00000000000..2751b9327db --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/_meta/data.json @@ -0,0 +1,153 @@ +{ + "@timestamp":"2019-09-20T11:02:00.000Z", + "azure":{ + "resource":{ + "name":"obslinux", + "type":"Microsoft.Compute/virtualMachines", + "group":"resource-group" + }, + "subscription_id":"123456-qwer-1234-5678-12345678", + "namespace":"Microsoft.Compute/virtualMachines", + "compute_vm":{ + "inbound_flows_maximum_creation_rate":{ + "avg":14.6 + }, + "os_per_disk_write_bytes_per_sec":{ + "avg":15261.86 + }, + "outbound_flows_maximum_creation_rate":{ + "avg":18.6 + }, + "os_per_disk_read_operations_per_sec":{ + "avg":0 + }, + "data_disk_queue_depth":{ + "avg":0 + }, + "per_disk_read_operations_per_sec":{ + "avg":0 + }, + "inbound_flows":{ + "avg":60.4 + }, + "os_disk_write_operations_per_sec":{ + "avg":1.0175 + }, + "per_disk_read_bytes_per_sec":{ + "avg":0 + }, + "disk_write_operations_per_sec":{ + "avg":1.03 + }, + "outbound_flows":{ + "avg":60.4 + }, + "network_in":{ + "total":3624009.0 + }, + "os_per_disk_qd":{ + "avg":0 + }, + "data_disk_read_bytes_per_sec":{ + "avg":0 + }, + "disk_read_operations_per_sec":{ + "avg":0 + }, + "os_disk_read_operations_per_sec":{ + "avg":0 + }, + "per_disk_qd":{ + "avg":0 + }, + "os_disk_write_bytes_per_sec":{ + "avg":15261.86 + }, + "disk_write_bytes":{ + "total":2806782.57 + }, + "per_disk_write_operations_per_sec":{ + "avg":0 + }, + "network_out":{ + "total":704862 + }, + "os_disk_read_bytes_per_sec":{ + "avg":0 + }, + "os_per_disk_read_bytes_per_sec":{ + "avg":0 + }, + "os_per_disk_write_operations_per_sec":{ + "avg":1.0175 + }, + "percentage_cpu":{ + "avg":7.683333333333334 + }, + "data_disk_write_operations_per_sec":{ + "avg":0 + }, + "network_in_total":{ + "total":6787492.0 + }, + "disk_read_bytes":{ + "total":0 + }, + "data_disk_write_bytes_per_sec":{ + "avg":0 + }, + "per_disk_write_bytes_per_sec":{ + "avg":0 + }, + "data_disk_read_operations_per_sec":{ + "avg":0 + }, + "os_disk_queue_depth":{ + "avg":0 + }, + "network_out_total":{ + "total":1889767.0 + } + } + }, + "service":{ + "type":"azure" + }, + "cloud":{ + "provider":"azure", + "region":"westeurope" + }, + "ecs":{ + "version":"1.1.0" + }, + "host":{ + "architecture":"x86_64", + "os":{ + "build":"17134.1006", + "platform":"windows", + "version":"10.0", + "family":"windows", + "name":"Windows 10 Pro", + "kernel":"10.0.17134.1006 (WinBuild.160101.0800)" + }, + "id":"123456-qwer-1234-5678-12345678", + "name":"DESKTOP-RFOOE09", + "hostname":"DESKTOP-RFOOE09" + }, + "agent":{ + "hostname":"DESKTOP-RFOOE09", + "id":"123456-qwer-1234-5678-12345678", + "version":"8.0.0", + "type":"metricbeat", + "ephemeral_id":"123456-qwer-1234-5678-12345678" + }, + "event":{ + "duration":4611928900, + "dataset":"azure.compute_vm", + "module":"azure" + }, + "metricset":{ + "name":"compute_vm", + "period":300000 + } +} diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc new file mode 100644 index 00000000000..db015dd1513 --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc @@ -0,0 +1,19 @@ +This is the container_service metricset of the module azure. + +This metricset allows users to retrieve all metrics from specified virtual machines. + +include::../../_meta/shared-azure.asciidoc[] + +[float] +==== Config options to identify resources + +`resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. + Should return a list of resources. + +`resource_group`:: (_[]string_) This option should return a list virtual machines we want to apply our metric configuration options on. + +If none of the options are entered then we will select all virtual machine from the entire subscription +For each metric the primary aggregation assigned will be retrieved. +A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. + + diff --git a/x-pack/metricbeat/module/azure/container/_meta/fields.yml b/x-pack/metricbeat/module/azure/container_service/_meta/fields.yml similarity index 66% rename from x-pack/metricbeat/module/azure/container/_meta/fields.yml rename to x-pack/metricbeat/module/azure/container_service/_meta/fields.yml index 0a331319901..9cccb41591d 100644 --- a/x-pack/metricbeat/module/azure/container/_meta/fields.yml +++ b/x-pack/metricbeat/module/azure/container_service/_meta/fields.yml @@ -1,7 +1,7 @@ -- name: container.*.* +- name: container_service.*.* release: beta type: object object_type: float object_type_mapping_type: "*" description: > - container + container service diff --git a/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go new file mode 100644 index 00000000000..01c9f5a053e --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/container_service_integration_test.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_service + +import ( + "errors" + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + config, err := getConfig() + if err != nil { + t.Skip("Skipping TestData: " + err.Error()) + } + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} + +func getConfig() (map[string]interface{}, error) { + clientId, ok := os.LookupEnv("AZURE_CLIENT_ID") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_ID key") + } + clientSecret, ok := os.LookupEnv("AZURE_CLIENT_SECRET") + if !ok { + return nil, errors.New("missing AZURE_CLIENT_SECRET key") + } + tenantId, ok := os.LookupEnv("AZURE_TENANT_ID") + if !ok { + return nil, errors.New("missing AZURE_TENANT_ID key") + } + subscriptionId, ok := os.LookupEnv("AZURE_SUBSCRIPTION_ID") + if !ok { + return nil, errors.New("missing AZURE_SUBSCRIPTION_ID key") + } + config := map[string]interface{}{ + "module": "azure", + "metricsets": []string{"container_registry"}, + "client_id": clientId, + "client_secret": clientSecret, + "tenant_id": tenantId, + "subscription_id": subscriptionId, + } + return config, nil +} diff --git a/x-pack/metricbeat/module/azure/container_service/container_service_test.go b/x-pack/metricbeat/module/azure/container_service/container_service_test.go new file mode 100644 index 00000000000..0fb0b53347e --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/container_service_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package container_service + +import ( + "os" + + "github.com/elastic/beats/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/azure/container_service/manifest.yml b/x-pack/metricbeat/module/azure/container_service/manifest.yml new file mode 100644 index 00000000000..9456f1eec8d --- /dev/null +++ b/x-pack/metricbeat/module/azure/container_service/manifest.yml @@ -0,0 +1,16 @@ +default: true +input: + module: azure + metricset: monitor + defaults: + default_resource_type: "Microsoft.ContainerService/managedClusters" + resources: + - resource_group: "" + resource_type: "Microsoft.ContainerService/managedClusters" + metrics: + - name: "*" + namespace: "Microsoft.ContainerService/managedClusters" + - resource_id: "" + metrics: + - name: "*" + namespace: "Microsoft.ContainerService/managedClusters" diff --git a/x-pack/metricbeat/module/azure/module.yml b/x-pack/metricbeat/module/azure/module.yml new file mode 100644 index 00000000000..78df52b8890 --- /dev/null +++ b/x-pack/metricbeat/module/azure/module.yml @@ -0,0 +1,5 @@ +name: azure +metricsets: + - container_registry + - container_instance + - container_service From eac4a39597d0703492e49763ed8dbd94e91116cb Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 25 Feb 2020 20:49:12 +0100 Subject: [PATCH 5/9] separate --- metricbeat/docs/fields.asciidoc | 24 +++++++++++-- metricbeat/docs/modules/azure.asciidoc | 35 +++++++++++++++++-- .../docs/modules/azure/container.asciidoc | 23 ------------ .../modules/azure/container_instance.asciidoc | 24 +++++++++++++ .../modules/azure/container_registry.asciidoc | 24 +++++++++++++ .../modules/azure/container_service.asciidoc | 24 +++++++++++++ metricbeat/docs/modules_list.asciidoc | 6 ++-- x-pack/metricbeat/include/list.go | 1 - x-pack/metricbeat/metricbeat.reference.yml | 23 +++++++++++- x-pack/metricbeat/module/azure/fields.go | 2 +- .../metricbeat/modules.d/azure.yml.disabled | 24 ++++++++++++- 11 files changed, 176 insertions(+), 34 deletions(-) delete mode 100644 metricbeat/docs/modules/azure/container.asciidoc create mode 100644 metricbeat/docs/modules/azure/container_instance.asciidoc create mode 100644 metricbeat/docs/modules/azure/container_registry.asciidoc create mode 100644 metricbeat/docs/modules/azure/container_service.asciidoc diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 69921fc4baf..d6e473b9074 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3194,10 +3194,30 @@ type: object -- -*`azure.container.*.*`*:: +*`azure.container_instance.*.*`*:: + -- -container +container instance + + +type: object + +-- + +*`azure.container_registry.*.*`*:: ++ +-- +container registry + + +type: object + +-- + +*`azure.container_service.*.*`*:: ++ +-- +container service type: object diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index b284cd3f134..6bdf571c1aa 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -152,13 +152,34 @@ metricbeat.modules: - module: azure metricsets: - - container + - container_instance enabled: true period: 300s client_id: '${AZURE_CLIENT_ID:""}' client_secret: '${AZURE_CLIENT_SECRET:""}' tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_service + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_registry + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + ---- [float] @@ -170,7 +191,11 @@ The following metricsets are available: * <> -* <> +* <> + +* <> + +* <> * <> @@ -180,7 +205,11 @@ include::azure/compute_vm.asciidoc[] include::azure/compute_vm_scaleset.asciidoc[] -include::azure/container.asciidoc[] +include::azure/container_instance.asciidoc[] + +include::azure/container_registry.asciidoc[] + +include::azure/container_service.asciidoc[] include::azure/monitor.asciidoc[] diff --git a/metricbeat/docs/modules/azure/container.asciidoc b/metricbeat/docs/modules/azure/container.asciidoc deleted file mode 100644 index b8ff22b283c..00000000000 --- a/metricbeat/docs/modules/azure/container.asciidoc +++ /dev/null @@ -1,23 +0,0 @@ -//// -This file is generated! See scripts/mage/docs_collector.go -//// - -[[metricbeat-metricset-azure-container]] -=== azure container metricset - -beta[] - -include::../../../../x-pack/metricbeat/module/azure/container/_meta/docs.asciidoc[] - - -==== Fields - -For a description of each field in the metricset, see the -<> section. - -Here is an example document generated by this metricset: - -[source,json] ----- -include::../../../../x-pack/metricbeat/module/azure/container/_meta/data.json[] ----- diff --git a/metricbeat/docs/modules/azure/container_instance.asciidoc b/metricbeat/docs/modules/azure/container_instance.asciidoc new file mode 100644 index 00000000000..88f1959637d --- /dev/null +++ b/metricbeat/docs/modules/azure/container_instance.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-container_instance]] +=== azure container_instance metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/container_instance/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/azure/container_registry.asciidoc b/metricbeat/docs/modules/azure/container_registry.asciidoc new file mode 100644 index 00000000000..67ff888d807 --- /dev/null +++ b/metricbeat/docs/modules/azure/container_registry.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-container_registry]] +=== azure container_registry metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/container_registry/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/azure/container_service.asciidoc b/metricbeat/docs/modules/azure/container_service.asciidoc new file mode 100644 index 00000000000..2aed94c7749 --- /dev/null +++ b/metricbeat/docs/modules/azure/container_service.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-container_service]] +=== azure container_service metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/container_service/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 14770b66a16..687b29301a7 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -30,9 +30,11 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> beta[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.5+| .5+| |<> beta[] +.7+| .7+| |<> beta[] |<> beta[] -|<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] |<> beta[] |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 3ae0a19d523..590accfd862 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -21,7 +21,6 @@ import ( _ "github.com/elastic/beats/x-pack/metricbeat/module/azure" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/compute_vm" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/compute_vm_scaleset" - _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/container" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/monitor" _ "github.com/elastic/beats/x-pack/metricbeat/module/azure/storage" _ "github.com/elastic/beats/x-pack/metricbeat/module/cockroachdb" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 00a3f88c2c6..237c0ffdedf 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -267,7 +267,7 @@ metricbeat.modules: - module: azure metricsets: - - container + - container_instance enabled: true period: 300s client_id: '${AZURE_CLIENT_ID:""}' @@ -275,6 +275,27 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - container_service + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_registry + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + + #--------------------------------- Beat Module --------------------------------- - module: beat metricsets: diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index ddf5e52c143..4393354aa8f 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUVb2O2zAM3v0UhMcAlwfwUKBAlw7duhuKRLvqWT+QqBbp0xeyHZ38f2kz5DQYiEh9PyQVvcArXitgf4LDAoAkdVhB2f8uCwCBnjtpSRpdwacCAIZcUEaELh5x2CHzWMEFiRUAjcRO+KpPfQHNFL7Bx0VXixW0zgQ77qxwTGFyKIfeBMcxBdYQN1GH9f0HJhzwFrlsJIosZU6eC4jfSeAm4BWvv40Ts9iOjJuUiAimAcpkrVJHnsdSx+Pvop5X9wHcqQNL7GSYtf58WqU1l5/IaRYaNus9YVlKrZi1Urdjfnkq7zPxub8IyUYvdjGv8estWxnYpb6DkU1Q4LFDTtnI3th8uCSIWor/58wB4euXBaGQCrWXRk/7tNGjg/68tzc7moeeKCQneS5uIZwbZQNh/UudTxPp8/+zeyw1nWEbwX819KZzx0LtOevQI30UL0lwiq+408SkRvfsnkaZCwPKaEnGHT5UW4Z2SJfQey/WcBv8rI47NTus2z21O7AS17dBIDik4DSK8/ZUeDKOtfjcMzGKBMa5CZqKvwEAAP//KrN/pg==" + return "eJzUlr+Om0AQxnueYuTS0vkBKCJFSpMiXXq0XsZkcrCLZoaLnKePMLDH3+Wcu+K8hSUzy/f9vp0B8QTPeE3B/G0YEwAlLTGFw+3/IQHIUSxTreRdCl8SAOj2QuXzpmxvYSzRCKZwRjUJwIWwzCW9bX0CZyp8lW+XXmtMoWDf1P2VFY+pzFiKUXzDFkNhTXFTtVs/f2HQAanR0oUwH22Zm48B2t9JYQB4xusfz/msFsEYUFpF8BfQEdaqdevzsdbt7W+ynp/uB3iHDiy1Q2BTyOm4auvPv9HqrNRdzGJgoy1ZZeqaXNHvPxwP94X4ensQQowb7GJe21+pzcrALvl2RjZIgWCJVkcjO7hJcw4SGeXv9xwLwvdvC8OcKnRC3k37tNGjnf68tTcR5q4nFSqTHcMtwK2v6kYxe6lOxwn6/H12T6RL6c1G8X8DvXJGImRiTYmC+ihZAnCor6RzasghZ+REjbP42cP1vDDwRiIxFiTK10eJNPBGIgnyCz1Ok3rcRaDKO1LPu58XW8Ei5kvp2HdG9w6T2XlGzm73/O45w50o7frRAQKjNuwwP20/y6KeTfHJZ6OHBGOtb5wm/wIAAP//Wib8nQ==" } diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml.disabled index f470a0b900b..bff59eebde1 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -52,7 +52,29 @@ #- module: azure # metricsets: -# - container +# - container_instance +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container_service +# enabled: true +# period: 300s +# client_id: '${AZURE_CLIENT_ID:""}' +# client_secret: '${AZURE_CLIENT_SECRET:""}' +# tenant_id: '${AZURE_TENANT_ID:""}' +# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +# refresh_list_interval: 600s + +#- module: azure +# metricsets: +# - container_registry # enabled: true # period: 300s # client_id: '${AZURE_CLIENT_ID:""}' From d588d4dfb46ab6e10ebc5f36aba507cee5364b61 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 27 Feb 2020 12:01:07 +0100 Subject: [PATCH 6/9] regenerate --- metricbeat/docs/fields.asciidoc | 30 +++++++++++++++++ metricbeat/docs/modules/azure.asciidoc | 43 ++++++++++++++++++++++++ metricbeat/docs/modules_list.asciidoc | 5 ++- x-pack/metricbeat/module/azure/fields.go | 2 +- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index cec790b06bf..56e98f80fa8 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3200,6 +3200,36 @@ type: object compute_vm_scaleset +type: object + +-- + +*`azure.container_instance.*.*`*:: ++ +-- +container instance + + +type: object + +-- + +*`azure.container_registry.*.*`*:: ++ +-- +container registry + + +type: object + +-- + +*`azure.container_service.*.*`*:: ++ +-- +container service + + type: object -- diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index ed5a266e3d8..a19996d7c2a 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -150,6 +150,37 @@ metricbeat.modules: tenant_id: '${AZURE_TENANT_ID:""}' subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' +- module: azure + metricsets: + - container_instance + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_service + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + +- module: azure + metricsets: + - container_registry + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + + - module: azure metricsets: - database_account @@ -171,6 +202,12 @@ The following metricsets are available: * <> +* <> + +* <> + +* <> + * <> * <> @@ -181,6 +218,12 @@ include::azure/compute_vm.asciidoc[] include::azure/compute_vm_scaleset.asciidoc[] +include::azure/container_instance.asciidoc[] + +include::azure/container_registry.asciidoc[] + +include::azure/container_service.asciidoc[] + include::azure/database_account.asciidoc[] include::azure/monitor.asciidoc[] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 5587b923345..0dc710ebb36 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -30,8 +30,11 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> beta[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.5+| .5+| |<> beta[] +.8+| .8+| |<> beta[] |<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] |<> beta[] |<> beta[] |<> beta[] diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index 735ee8e1020..1da96238fa0 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUVb2OnDAQ7vcpRluudPsAFJEipUmRLj0yZiDO4R/Z40Sbp4+8gM9gfpbkij0XSHjM9zPzAS/wircC2B9v8QRAgjos4Hy/P58AanTcCkNCqwI+nQCgPwtS174Lj1jskDksoEJiJ4BGYFe74n70BRST+AYfFt0MFtBa7c2ws8AxhUmhSEhsLRMqVkbIV7z91rZO9heB+/X9B8Ln3giSFXwBd2S06LS3HDPC1MMDdCMOOINcNAJTqXO7qYBwnRTWHe/IGKUERNANUCJrkTrwvC91ePwh6nl334E7TiDHjoZZ666XRVpd/UROs1K/WW4JS46UkhkjVDucP1/Ox0z0iY027mKzvIarM2whsIffkAgFDjvklER2ZHO+ihClqP+fMwWEr18ywlpIVE5oNZ3Tyox25vPobDY0T74iibhMONfSeMLyl7xeJtLnX9AjlppOs5Xivxp607lhoXScdeiQPoqXKDjW82QxYhVzWDLOtVdPbm1UC4PazI7USpC2u7+tNV8b3Dn01v+rfzfcrJ0brdtt35EW7lgJ61svECyStwrr63pGHGnLWnzuaAwiYzL+BgAA///JVKQ8" + return "eJzUlr+O2zAMxvc8BZExwOUBPBQo0KVDt+6GIjMue7ZkkPQV6dMXjm2d/D/p3ZBoCBBJ/r7fJ9KCX+AVLwmYvzXjDkBJC0xgf/2/3wFkKJapUvIugS87AGj3QumzumgeYSzQCCZwQjU7gDNhkUly3foCzpT4Lt8MvVSYQM6+rrqZGY+hTCylVGLOhlxY6SVf8fLHcxbNzwq34+cvhK9tEFQmO6PbOzKKr9nixDDOcINdrwNSoaUzYYw6jhsDNL+DheXEGxg9SqMI/gwaYc1aNz6fa908fpP1+HQ/wTtUYKodAptcjodZW3/6jVZHS+1kugYWbUlLU1Xk8m7//rC/L0TbsSHGFXbSr82vVGamYe9+Q4IUCBZoNWrZ3k3qU5BIKfu4ZywI379NDDMq0Ql5N6zTQo026nNrbVaYB7dIBDcBt76sasX0rTweBujjG/SeSOfCm4XF/w30zrkSIRVrChTUZ8kSgMP6TDqnhhxySk7UOIuPHq7jhZ53JRJjTqJ8eZZIPe9KJEF+o+cpUoc7vdCMmpMRTI21vnYP/kb1tNDRTuKU3pF63vxaWsq14j2VXvtsaq9kGR3nytFtHt89R7gRpRk/WkBg1JodZsflq0nUs8kfvNU7yNAZ/wIAAP//ilZVQA==" } From a854b08bfd72609b19e4c9e1ff768992de34d581 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 27 Feb 2020 17:34:07 +0100 Subject: [PATCH 7/9] fixes --- x-pack/metricbeat/module/azure/azure.go | 23 +++++++++++++++---- x-pack/metricbeat/module/azure/client.go | 6 ++--- .../azure/container_instance/manifest.yml | 16 +++++++++++-- .../azure/container_registry/manifest.yml | 2 ++ .../azure/container_service/manifest.yml | 18 +++++++++++++-- 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index 1d08b60951f..5eb63042587 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -106,14 +106,16 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // check for lightweight resources if no groups or ids have been entered, if not a new resource is created to check the entire subscription var resources []ResourceConfig for _, resource := range config.Resources { - if len(resource.Group) != 0 || len(resource.ID) != 0 { + if hasConfigOptions(resource.Group) || hasConfigOptions(resource.ID) { resources = append(resources, resource) } } + // check if this is a light metricset or not and no resources have been configured if len(resources) == 0 && len(config.Resources) != 0 { - res := config.Resources[0] - res.Query = fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType) - resources = append(resources, res) + resources = append(resources, ResourceConfig{ + Query: fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType), + Metrics: config.Resources[0].Metrics, + }) } config.Resources = resources } @@ -152,3 +154,16 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } return nil } + +// hasConfigOptions func will check if any resource id or resource group options have been entered in the light metricsets +func hasConfigOptions(config []string) bool { + if config == nil { + return false + } + for _, group := range config { + if group == "" { + return false + } + } + return true +} diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 5a3871f1046..e6b381f931e 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -114,11 +114,11 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] client.Log.Error(err) report.Error(err) } else { - if metric.TimeGrain == "" { - metric.TimeGrain = timegrain - } for i, currentMetric := range client.Resources.Metrics { if matchMetrics(currentMetric, metric) { + if metric.TimeGrain == "" { + metric.TimeGrain = timegrain + } current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute)) client.Resources.Metrics[i].Values = current resultedMetrics = append(resultedMetrics, client.Resources.Metrics[i]) diff --git a/x-pack/metricbeat/module/azure/container_instance/manifest.yml b/x-pack/metricbeat/module/azure/container_instance/manifest.yml index 5e53428b13f..90c0fd1f46f 100644 --- a/x-pack/metricbeat/module/azure/container_instance/manifest.yml +++ b/x-pack/metricbeat/module/azure/container_instance/manifest.yml @@ -7,10 +7,22 @@ input: resources: - resource_group: "" resource_type: "Microsoft.ContainerInstance/containerGroups" + timegrain: "PT5M" metrics: - - name: "*" + - name: ["CpuUsage", "MemoryUsage"] + namespace: "Microsoft.ContainerInstance/containerGroups" + dimensions: + - name: "containerName" + value: "*" + - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] namespace: "Microsoft.ContainerInstance/containerGroups" - resource_id: "" + timegrain: "PT5M" metrics: - - name: "*" + - name: ["CpuUsage", "MemoryUsage"] + namespace: "Microsoft.ContainerInstance/containerGroups" + dimensions: + - name: "containerName" + value: "*" + - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] namespace: "Microsoft.ContainerInstance/containerGroups" diff --git a/x-pack/metricbeat/module/azure/container_registry/manifest.yml b/x-pack/metricbeat/module/azure/container_registry/manifest.yml index 787b46e71de..47da4e625c7 100644 --- a/x-pack/metricbeat/module/azure/container_registry/manifest.yml +++ b/x-pack/metricbeat/module/azure/container_registry/manifest.yml @@ -7,10 +7,12 @@ input: resources: - resource_group: "" resource_type: "Microsoft.ContainerRegistry/registries" + timegrain: "PT5M" metrics: - name: "*" namespace: "Microsoft.ContainerRegistry/registries" - resource_id: "" + timegrain: "PT5M" metrics: - name: "*" namespace: "Microsoft.ContainerRegistry/registries" diff --git a/x-pack/metricbeat/module/azure/container_service/manifest.yml b/x-pack/metricbeat/module/azure/container_service/manifest.yml index 9456f1eec8d..1580c5797f4 100644 --- a/x-pack/metricbeat/module/azure/container_service/manifest.yml +++ b/x-pack/metricbeat/module/azure/container_service/manifest.yml @@ -8,9 +8,23 @@ input: - resource_group: "" resource_type: "Microsoft.ContainerService/managedClusters" metrics: - - name: "*" + - name: ["kube_node_status_condition"] namespace: "Microsoft.ContainerService/managedClusters" + timegrain: "PT5M" + dimensions: + - name: "status" + value: "*" + - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes", "kube_pod_status_ready", "kube_pod_status_phase"] + namespace: "Microsoft.ContainerService/managedClusters" + timegrain: "PT5M" - resource_id: "" metrics: - - name: "*" + - name: ["kube_node_status_condition"] + namespace: "Microsoft.ContainerService/managedClusters" + timegrain: "PT5M" + dimensions: + - name: "status" + value: "*" + - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes", "kube_pod_status_ready", "kube_pod_status_phase"] namespace: "Microsoft.ContainerService/managedClusters" + timegrain: "PT5M" From 0540d7fca1d8ed79bed49baf6449ad816c72e76a Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 28 Feb 2020 16:47:33 +0100 Subject: [PATCH 8/9] tests --- x-pack/metricbeat/module/azure/client.go | 6 +- .../azure/container_instance/_meta/data.json | 181 ++++-------------- .../azure/container_instance/manifest.yml | 6 +- .../azure/container_registry/_meta/data.json | 172 +++-------------- .../azure/container_registry/manifest.yml | 3 +- .../azure/container_service/_meta/data.json | 181 ++++-------------- x-pack/metricbeat/module/azure/data.go | 28 +-- x-pack/metricbeat/module/azure/data_test.go | 125 ++++++++++++ 8 files changed, 252 insertions(+), 450 deletions(-) create mode 100644 x-pack/metricbeat/module/azure/data_test.go diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index e6b381f931e..aeec2bec155 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -116,11 +116,11 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } else { for i, currentMetric := range client.Resources.Metrics { if matchMetrics(currentMetric, metric) { - if metric.TimeGrain == "" { - metric.TimeGrain = timegrain - } current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute)) client.Resources.Metrics[i].Values = current + if client.Resources.Metrics[i].TimeGrain == "" { + client.Resources.Metrics[i].TimeGrain = timegrain + } resultedMetrics = append(resultedMetrics, client.Resources.Metrics[i]) } } diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json index 2751b9327db..163b491021e 100644 --- a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json @@ -1,153 +1,46 @@ { - "@timestamp":"2019-09-20T11:02:00.000Z", - "azure":{ - "resource":{ - "name":"obslinux", - "type":"Microsoft.Compute/virtualMachines", - "group":"resource-group" - }, - "subscription_id":"123456-qwer-1234-5678-12345678", - "namespace":"Microsoft.Compute/virtualMachines", - "compute_vm":{ - "inbound_flows_maximum_creation_rate":{ - "avg":14.6 - }, - "os_per_disk_write_bytes_per_sec":{ - "avg":15261.86 - }, - "outbound_flows_maximum_creation_rate":{ - "avg":18.6 - }, - "os_per_disk_read_operations_per_sec":{ - "avg":0 - }, - "data_disk_queue_depth":{ - "avg":0 - }, - "per_disk_read_operations_per_sec":{ - "avg":0 - }, - "inbound_flows":{ - "avg":60.4 - }, - "os_disk_write_operations_per_sec":{ - "avg":1.0175 - }, - "per_disk_read_bytes_per_sec":{ - "avg":0 - }, - "disk_write_operations_per_sec":{ - "avg":1.03 - }, - "outbound_flows":{ - "avg":60.4 - }, - "network_in":{ - "total":3624009.0 - }, - "os_per_disk_qd":{ - "avg":0 - }, - "data_disk_read_bytes_per_sec":{ - "avg":0 - }, - "disk_read_operations_per_sec":{ - "avg":0 - }, - "os_disk_read_operations_per_sec":{ - "avg":0 - }, - "per_disk_qd":{ - "avg":0 - }, - "os_disk_write_bytes_per_sec":{ - "avg":15261.86 - }, - "disk_write_bytes":{ - "total":2806782.57 - }, - "per_disk_write_operations_per_sec":{ - "avg":0 - }, - "network_out":{ - "total":704862 - }, - "os_disk_read_bytes_per_sec":{ - "avg":0 - }, - "os_per_disk_read_bytes_per_sec":{ - "avg":0 - }, - "os_per_disk_write_operations_per_sec":{ - "avg":1.0175 - }, - "percentage_cpu":{ - "avg":7.683333333333334 - }, - "data_disk_write_operations_per_sec":{ - "avg":0 - }, - "network_in_total":{ - "total":6787492.0 - }, - "disk_read_bytes":{ - "total":0 - }, - "data_disk_write_bytes_per_sec":{ - "avg":0 - }, - "per_disk_write_bytes_per_sec":{ - "avg":0 - }, - "data_disk_read_operations_per_sec":{ - "avg":0 - }, - "os_disk_queue_depth":{ - "avg":0 - }, - "network_out_total":{ - "total":1889767.0 - } - } + "@timestamp" : "2020-02-28T14:20:00.000Z", + "cloud" : { + "provider" : "azure", + "region" : "westeurope" }, - "service":{ - "type":"azure" + "ecs" : { + "version" : "1.4.0" }, - "cloud":{ - "provider":"azure", - "region":"westeurope" + "metricset" : { + "period" : 300000, + "name" : "container_instance" }, - "ecs":{ - "version":"1.1.0" + "event" : { + "duration" : 2383878000, + "dataset" : "azure.container_instance", + "module" : "azure" }, - "host":{ - "architecture":"x86_64", - "os":{ - "build":"17134.1006", - "platform":"windows", - "version":"10.0", - "family":"windows", - "name":"Windows 10 Pro", - "kernel":"10.0.17134.1006 (WinBuild.160101.0800)" + "azure" : { + "dimensions" : { + "container_name" : "testcontainergroup" }, - "id":"123456-qwer-1234-5678-12345678", - "name":"DESKTOP-RFOOE09", - "hostname":"DESKTOP-RFOOE09" - }, - "agent":{ - "hostname":"DESKTOP-RFOOE09", - "id":"123456-qwer-1234-5678-12345678", - "version":"8.0.0", - "type":"metricbeat", - "ephemeral_id":"123456-qwer-1234-5678-12345678" - }, - "event":{ - "duration":4611928900, - "dataset":"azure.compute_vm", - "module":"azure" + "container_instance" : { + "cpu_usage" : { + "avg" : 0 + }, + "memory_usage" : { + "avg" : 0 + } + }, + "timegrain" : "PT5M", + "resource" : { + "group" : "obs-infrastructure", + "tags" : { + "tag1" : "value1" + }, + "name" : "testcontainergroup", + "type" : "Microsoft.ContainerInstance/containerGroups" + }, + "subscription_id" : "123456-qwer-1234-5678-12345678", + "namespace" : "Microsoft.ContainerInstance/containerGroups" }, - "metricset":{ - "name":"compute_vm", - "period":300000 + "service" : { + "type" : "azure" } } diff --git a/x-pack/metricbeat/module/azure/container_instance/manifest.yml b/x-pack/metricbeat/module/azure/container_instance/manifest.yml index 90c0fd1f46f..a0e6cd5ec27 100644 --- a/x-pack/metricbeat/module/azure/container_instance/manifest.yml +++ b/x-pack/metricbeat/module/azure/container_instance/manifest.yml @@ -7,22 +7,24 @@ input: resources: - resource_group: "" resource_type: "Microsoft.ContainerInstance/containerGroups" - timegrain: "PT5M" metrics: - name: ["CpuUsage", "MemoryUsage"] namespace: "Microsoft.ContainerInstance/containerGroups" + timegrain: "PT5M" dimensions: - name: "containerName" value: "*" - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] namespace: "Microsoft.ContainerInstance/containerGroups" + timegrain: "PT5M" - resource_id: "" - timegrain: "PT5M" metrics: - name: ["CpuUsage", "MemoryUsage"] namespace: "Microsoft.ContainerInstance/containerGroups" + timegrain: "PT5M" dimensions: - name: "containerName" value: "*" - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] namespace: "Microsoft.ContainerInstance/containerGroups" + timegrain: "PT5M" diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json index 2751b9327db..0167132328b 100644 --- a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json @@ -1,153 +1,37 @@ { - "@timestamp":"2019-09-20T11:02:00.000Z", - "azure":{ - "resource":{ - "name":"obslinux", - "type":"Microsoft.Compute/virtualMachines", - "group":"resource-group" - }, - "subscription_id":"123456-qwer-1234-5678-12345678", - "namespace":"Microsoft.Compute/virtualMachines", - "compute_vm":{ - "inbound_flows_maximum_creation_rate":{ - "avg":14.6 - }, - "os_per_disk_write_bytes_per_sec":{ - "avg":15261.86 - }, - "outbound_flows_maximum_creation_rate":{ - "avg":18.6 - }, - "os_per_disk_read_operations_per_sec":{ - "avg":0 - }, - "data_disk_queue_depth":{ - "avg":0 - }, - "per_disk_read_operations_per_sec":{ - "avg":0 - }, - "inbound_flows":{ - "avg":60.4 - }, - "os_disk_write_operations_per_sec":{ - "avg":1.0175 - }, - "per_disk_read_bytes_per_sec":{ - "avg":0 - }, - "disk_write_operations_per_sec":{ - "avg":1.03 - }, - "outbound_flows":{ - "avg":60.4 - }, - "network_in":{ - "total":3624009.0 - }, - "os_per_disk_qd":{ - "avg":0 - }, - "data_disk_read_bytes_per_sec":{ - "avg":0 - }, - "disk_read_operations_per_sec":{ - "avg":0 - }, - "os_disk_read_operations_per_sec":{ - "avg":0 - }, - "per_disk_qd":{ - "avg":0 - }, - "os_disk_write_bytes_per_sec":{ - "avg":15261.86 - }, - "disk_write_bytes":{ - "total":2806782.57 - }, - "per_disk_write_operations_per_sec":{ - "avg":0 - }, - "network_out":{ - "total":704862 - }, - "os_disk_read_bytes_per_sec":{ - "avg":0 - }, - "os_per_disk_read_bytes_per_sec":{ - "avg":0 - }, - "os_per_disk_write_operations_per_sec":{ - "avg":1.0175 - }, - "percentage_cpu":{ - "avg":7.683333333333334 - }, - "data_disk_write_operations_per_sec":{ - "avg":0 - }, - "network_in_total":{ - "total":6787492.0 - }, - "disk_read_bytes":{ - "total":0 - }, - "data_disk_write_bytes_per_sec":{ - "avg":0 - }, - "per_disk_write_bytes_per_sec":{ - "avg":0 - }, - "data_disk_read_operations_per_sec":{ - "avg":0 - }, - "os_disk_queue_depth":{ - "avg":0 - }, - "network_out_total":{ - "total":1889767.0 - } - } + "@timestamp" : "2020-02-28T14:47:00.000Z", + "ecs" : { + "version" : "1.4.0" }, - "service":{ - "type":"azure" + "cloud" : { + "region" : "westeurope", + "provider" : "azure" }, - "cloud":{ - "provider":"azure", - "region":"westeurope" + "event" : { + "dataset" : "azure.container_registry", + "module" : "azure", + "duration" : 2730896000 }, - "ecs":{ - "version":"1.1.0" + "metricset" : { + "name" : "container_registry", + "period" : 300000 }, - "host":{ - "architecture":"x86_64", - "os":{ - "build":"17134.1006", - "platform":"windows", - "version":"10.0", - "family":"windows", - "name":"Windows 10 Pro", - "kernel":"10.0.17134.1006 (WinBuild.160101.0800)" + "azure" : { + "subscription_id" : "123456-qwer-1234-5678-12345678", + "namespace" : "Microsoft.ContainerRegistry/registries", + "container_registry" : { + "successful_push_count" : { + "avg" : 1 + } }, - "id":"123456-qwer-1234-5678-12345678", - "name":"DESKTOP-RFOOE09", - "hostname":"DESKTOP-RFOOE09" - }, - "agent":{ - "hostname":"DESKTOP-RFOOE09", - "id":"123456-qwer-1234-5678-12345678", - "version":"8.0.0", - "type":"metricbeat", - "ephemeral_id":"123456-qwer-1234-5678-12345678" - }, - "event":{ - "duration":4611928900, - "dataset":"azure.compute_vm", - "module":"azure" + "timegrain" : "PT5M", + "resource" : { + "type" : "Microsoft.ContainerRegistry/registries", + "group" : "obs-infrastructure", + "name" : "obstest" + } }, - "metricset":{ - "name":"compute_vm", - "period":300000 + "service" : { + "type" : "azure" } } diff --git a/x-pack/metricbeat/module/azure/container_registry/manifest.yml b/x-pack/metricbeat/module/azure/container_registry/manifest.yml index 47da4e625c7..77993952390 100644 --- a/x-pack/metricbeat/module/azure/container_registry/manifest.yml +++ b/x-pack/metricbeat/module/azure/container_registry/manifest.yml @@ -7,12 +7,13 @@ input: resources: - resource_group: "" resource_type: "Microsoft.ContainerRegistry/registries" - timegrain: "PT5M" metrics: - name: "*" namespace: "Microsoft.ContainerRegistry/registries" + timegrain: "PT5M" - resource_id: "" timegrain: "PT5M" metrics: - name: "*" namespace: "Microsoft.ContainerRegistry/registries" + timegrain: "PT5M" diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/data.json b/x-pack/metricbeat/module/azure/container_service/_meta/data.json index 2751b9327db..fe6623057a3 100644 --- a/x-pack/metricbeat/module/azure/container_service/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_service/_meta/data.json @@ -1,153 +1,46 @@ { - "@timestamp":"2019-09-20T11:02:00.000Z", - "azure":{ - "resource":{ - "name":"obslinux", - "type":"Microsoft.Compute/virtualMachines", - "group":"resource-group" - }, - "subscription_id":"123456-qwer-1234-5678-12345678", - "namespace":"Microsoft.Compute/virtualMachines", - "compute_vm":{ - "inbound_flows_maximum_creation_rate":{ - "avg":14.6 - }, - "os_per_disk_write_bytes_per_sec":{ - "avg":15261.86 - }, - "outbound_flows_maximum_creation_rate":{ - "avg":18.6 - }, - "os_per_disk_read_operations_per_sec":{ - "avg":0 - }, - "data_disk_queue_depth":{ - "avg":0 - }, - "per_disk_read_operations_per_sec":{ - "avg":0 - }, - "inbound_flows":{ - "avg":60.4 - }, - "os_disk_write_operations_per_sec":{ - "avg":1.0175 - }, - "per_disk_read_bytes_per_sec":{ - "avg":0 - }, - "disk_write_operations_per_sec":{ - "avg":1.03 - }, - "outbound_flows":{ - "avg":60.4 - }, - "network_in":{ - "total":3624009.0 - }, - "os_per_disk_qd":{ - "avg":0 - }, - "data_disk_read_bytes_per_sec":{ - "avg":0 - }, - "disk_read_operations_per_sec":{ - "avg":0 - }, - "os_disk_read_operations_per_sec":{ - "avg":0 - }, - "per_disk_qd":{ - "avg":0 - }, - "os_disk_write_bytes_per_sec":{ - "avg":15261.86 - }, - "disk_write_bytes":{ - "total":2806782.57 - }, - "per_disk_write_operations_per_sec":{ - "avg":0 - }, - "network_out":{ - "total":704862 - }, - "os_disk_read_bytes_per_sec":{ - "avg":0 - }, - "os_per_disk_read_bytes_per_sec":{ - "avg":0 - }, - "os_per_disk_write_operations_per_sec":{ - "avg":1.0175 - }, - "percentage_cpu":{ - "avg":7.683333333333334 - }, - "data_disk_write_operations_per_sec":{ - "avg":0 - }, - "network_in_total":{ - "total":6787492.0 - }, - "disk_read_bytes":{ - "total":0 - }, - "data_disk_write_bytes_per_sec":{ - "avg":0 - }, - "per_disk_write_bytes_per_sec":{ - "avg":0 - }, - "data_disk_read_operations_per_sec":{ - "avg":0 - }, - "os_disk_queue_depth":{ - "avg":0 - }, - "network_out_total":{ - "total":1889767.0 - } - } + "@timestamp" : "2020-02-28T14:47:00.000Z", + "ecs" : { + "version" : "1.4.0" }, - "service":{ - "type":"azure" + "cloud" : { + "region" : "westeurope", + "provider" : "azure" }, - "cloud":{ - "provider":"azure", - "region":"westeurope" + "event" : { + "dataset" : "azure.container_service", + "module" : "azure", + "duration" : 2730896000 }, - "ecs":{ - "version":"1.1.0" + "metricset" : { + "name" : "container_service", + "period" : 300000 }, - "host":{ - "architecture":"x86_64", - "os":{ - "build":"17134.1006", - "platform":"windows", - "version":"10.0", - "family":"windows", - "name":"Windows 10 Pro", - "kernel":"10.0.17134.1006 (WinBuild.160101.0800)" + "azure" : { + "subscription_id" : "123456-qwer-1234-5678-12345678", + "namespace" : "Microsoft.ContainerService/managedClusters", + "container_service" : { + "kube_pod_status_ready" : { + "avg" : 9 + }, + "kube_pod_status_phase" : { + "avg" : 9 + }, + "kube_node_status_allocatable_cpu_cores" : { + "avg" : 2 + }, + "kube_node_status_allocatable_memory_bytes" : { + "avg" : 2.25349632E9 + } }, - "id":"123456-qwer-1234-5678-12345678", - "name":"DESKTOP-RFOOE09", - "hostname":"DESKTOP-RFOOE09" - }, - "agent":{ - "hostname":"DESKTOP-RFOOE09", - "id":"123456-qwer-1234-5678-12345678", - "version":"8.0.0", - "type":"metricbeat", - "ephemeral_id":"123456-qwer-1234-5678-12345678" - }, - "event":{ - "duration":4611928900, - "dataset":"azure.compute_vm", - "module":"azure" + "timegrain" : "PT5M", + "resource" : { + "type" : "Microsoft.ContainerService/managedClusters", + "group" : "obs-infrastructure", + "name" : "obskube" + } }, - "metricset":{ - "name":"compute_vm", - "period":300000 + "service" : { + "type" : "azure" } } diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index edbfcd6cbf1..80f36dd1156 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -6,6 +6,7 @@ package azure import ( "fmt" + "regexp" "strings" "time" "unicode" @@ -16,8 +17,9 @@ import ( const ( // NoDimension is used to group metrics in separate api calls in order to reduce the number of executions - NoDimension = "none" - nativeMetricset = "monitor" + NoDimension = "none" + nativeMetricset = "monitor" + replaceUpperCaseRegex = `(?:[^A-Z_\W])([A-Z])[^A-Z]` ) // EventsMapping will map metric values to beats events @@ -96,7 +98,6 @@ func EventsMapping(metrics []Metric, metricset string, report mb.ReporterV2) err // managePropertyName function will handle metric names, there are several formats the metric names are written func managePropertyName(metric string) string { - // replace spaces with underscores resultMetricName := strings.Replace(metric, " ", "_", -1) // replace backslashes with "per" @@ -125,16 +126,19 @@ func managePropertyName(metric string) string { // replaceUpperCase func will replace upper case with '_' func replaceUpperCase(src string) string { - var result string - // split into fields based on class of unicode character - for _, r := range src { - if unicode.IsUpper(r) { - result += "_" + strings.ToLower(string(r)) - } else { - result += string(r) + replaceUpperCaseRegexp := regexp.MustCompile(replaceUpperCaseRegex) + return replaceUpperCaseRegexp.ReplaceAllStringFunc(src, func(str string) string { + var newStr string + for _, r := range str { + // split into fields based on class of unicode character + if unicode.IsUpper(r) { + newStr += "_" + strings.ToLower(string(r)) + } else { + newStr += string(r) + } } - } - return result + return newStr + }) } // createEvent will create a new base event diff --git a/x-pack/metricbeat/module/azure/data_test.go b/x-pack/metricbeat/module/azure/data_test.go new file mode 100644 index 00000000000..7d63d94f92d --- /dev/null +++ b/x-pack/metricbeat/module/azure/data_test.go @@ -0,0 +1,125 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azure + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +func TestReturnAllDimensions(t *testing.T) { + dimensionList := []Dimension{ + { + Value: "vm1", + Name: "VMName", + }, + { + Value: "*", + Name: "SlotID", + }, + } + result, dims := returnAllDimensions(dimensionList) + assert.True(t, result) + assert.Equal(t, len(dims), 1) + assert.Equal(t, dims[0].Name, "SlotID") + assert.Equal(t, dims[0].Value, "*") +} + +func TestGetDimensionValue(t *testing.T) { + dimensionList := []Dimension{ + { + Value: "vm1", + Name: "VMName", + }, + { + Value: "*", + Name: "SlotID", + }, + } + result := getDimensionValue("VMName", dimensionList) + assert.Equal(t, result, "vm1") +} + +func TestReplaceUpperCase(t *testing.T) { + result := replaceUpperCase("TestReplaceUpper_Case") + assert.Equal(t, result, "test_replace_upper_Case") + // should not split on acronyms + result = replaceUpperCase("CPU_Percentage") + assert.Equal(t, result, "CPU_Percentage") +} + +func TestManagePropertyName(t *testing.T) { + result := managePropertyName("TestManageProperty_Name") + assert.Equal(t, result, "test_manage_property_name") + + result = managePropertyName("Test ManageProperty_Name/sec") + assert.Equal(t, result, "test_manage_property_name_per_sec") + + result = managePropertyName("Test_-_Manage:Property.Name") + assert.Equal(t, result, "test_manage_property_name") + + result = managePropertyName("Percentage CPU") + assert.Equal(t, result, "percentage_cpu") +} + +func TestCreateEvent(t *testing.T) { + createTime, err := time.Parse(time.RFC3339, "2020-02-28T20:53:03Z") + if !assert.NoError(t, err) { + t.Fatal(err) + } + metric := Metric{ + Resource: Resource{ + ID: "resId", + Name: "res", + Location: "west_europe", + Type: "resType", + Group: "resGroup", + Tags: nil, + Subscription: "subId", + }, + Namespace: "namespace1", + Names: []string{"Percentage CPU"}, + Aggregations: "", + Dimensions: nil, + Values: nil, + TimeGrain: "", + } + var total float64 = 23 + metricValues := []MetricValue{ + { + name: "Percentage CPU", + avg: nil, + min: nil, + max: nil, + total: &total, + count: nil, + timestamp: time.Time{}, + dimensions: nil, + }, + } + event, list := createEvent(createTime, metric, metricValues) + assert.NotNil(t, event) + assert.NotNil(t, list) + assert.Equal(t, event.Timestamp, createTime) + sub, err := event.ModuleFields.GetValue("subscription_id") + if !assert.NoError(t, err) { + t.Fatal(err) + } + assert.Equal(t, sub, metric.Resource.Subscription) + namespace, err := event.ModuleFields.GetValue("namespace") + if !assert.NoError(t, err) { + t.Fatal(err) + } + assert.Equal(t, namespace, metric.Namespace) + val, err := list.GetValue("percentage_cpu") + if !assert.NoError(t, err) { + t.Fatal(err) + } + assert.Equal(t, val.(common.MapStr), common.MapStr{"total": total}) +} From 8db3ba1ebca6ed74d944393af8db0e4079567d45 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 28 Feb 2020 18:31:28 +0100 Subject: [PATCH 9/9] fix test --- x-pack/metricbeat/module/azure/data_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/azure/data_test.go b/x-pack/metricbeat/module/azure/data_test.go index 7d63d94f92d..2498d92a558 100644 --- a/x-pack/metricbeat/module/azure/data_test.go +++ b/x-pack/metricbeat/module/azure/data_test.go @@ -48,7 +48,7 @@ func TestGetDimensionValue(t *testing.T) { func TestReplaceUpperCase(t *testing.T) { result := replaceUpperCase("TestReplaceUpper_Case") - assert.Equal(t, result, "test_replace_upper_Case") + assert.Equal(t, result, "Test_replace_upper_Case") // should not split on acronyms result = replaceUpperCase("CPU_Percentage") assert.Equal(t, result, "CPU_Percentage")