Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add container azure metricset #16421

Merged
merged 11 commits into from
Mar 3, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add support for NATS 2.1. {pull}16317[16317]
- Add Load Balancing metricset to GCP {pull}15559[15559]
- Add support for Dropwizard metrics 4.1. {pull}16332[16332]
- Add azure container metricset in order to monitor containers. {issue}15751[15751] {pull}16421[16421]
- Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333]
- Add filtering option for prometheus collector. {pull}16420[16420]
- Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254]
Expand Down
30 changes: 30 additions & 0 deletions metricbeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -3200,6 +3200,36 @@ type: object
compute_vm_scaleset


type: object

--

*`azure.container_instance.*.*`*::
+
--
container instance


type: object

--

*`azure.container_registry.*.*`*::
+
--
container registry


type: object

--

*`azure.container_service.*.*`*::
+
--
container service


type: object

--
Expand Down
43 changes: 43 additions & 0 deletions metricbeat/docs/modules/azure.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,37 @@ metricbeat.modules:
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'

- module: azure
metricsets:
- container_instance
enabled: true
period: 300s
client_id: '${AZURE_CLIENT_ID:""}'
client_secret: '${AZURE_CLIENT_SECRET:""}'
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'

- module: azure
metricsets:
- container_service
enabled: true
period: 300s
client_id: '${AZURE_CLIENT_ID:""}'
client_secret: '${AZURE_CLIENT_SECRET:""}'
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'

- module: azure
metricsets:
- container_registry
enabled: true
period: 300s
client_id: '${AZURE_CLIENT_ID:""}'
client_secret: '${AZURE_CLIENT_SECRET:""}'
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'


- module: azure
metricsets:
- database_account
Expand All @@ -171,6 +202,12 @@ The following metricsets are available:

* <<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>>

* <<metricbeat-metricset-azure-container_instance,container_instance>>

* <<metricbeat-metricset-azure-container_registry,container_registry>>

* <<metricbeat-metricset-azure-container_service,container_service>>

* <<metricbeat-metricset-azure-database_account,database_account>>

* <<metricbeat-metricset-azure-monitor,monitor>>
Expand All @@ -181,6 +218,12 @@ include::azure/compute_vm.asciidoc[]

include::azure/compute_vm_scaleset.asciidoc[]

include::azure/container_instance.asciidoc[]

include::azure/container_registry.asciidoc[]

include::azure/container_service.asciidoc[]

include::azure/database_account.asciidoc[]

include::azure/monitor.asciidoc[]
Expand Down
24 changes: 24 additions & 0 deletions metricbeat/docs/modules/azure/container_instance.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
////
This file is generated! See scripts/mage/docs_collector.go
////

[[metricbeat-metricset-azure-container_instance]]
=== azure container_instance metricset

beta[]

include::../../../../x-pack/metricbeat/module/azure/container_instance/_meta/docs.asciidoc[]

This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.

==== Fields

For a description of each field in the metricset, see the
<<exported-fields-azure,exported fields>> section.

Here is an example document generated by this metricset:

[source,json]
----
include::../../../../x-pack/metricbeat/module/azure/container_instance/_meta/data.json[]
----
24 changes: 24 additions & 0 deletions metricbeat/docs/modules/azure/container_registry.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
////
This file is generated! See scripts/mage/docs_collector.go
////

[[metricbeat-metricset-azure-container_registry]]
=== azure container_registry metricset

beta[]

include::../../../../x-pack/metricbeat/module/azure/container_registry/_meta/docs.asciidoc[]

This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.

==== Fields

For a description of each field in the metricset, see the
<<exported-fields-azure,exported fields>> section.

Here is an example document generated by this metricset:

[source,json]
----
include::../../../../x-pack/metricbeat/module/azure/container_registry/_meta/data.json[]
----
24 changes: 24 additions & 0 deletions metricbeat/docs/modules/azure/container_service.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
////
This file is generated! See scripts/mage/docs_collector.go
////

[[metricbeat-metricset-azure-container_service]]
=== azure container_service metricset

beta[]

include::../../../../x-pack/metricbeat/module/azure/container_service/_meta/docs.asciidoc[]

This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.

==== Fields

For a description of each field in the metricset, see the
<<exported-fields-azure,exported fields>> section.

Here is an example document generated by this metricset:

[source,json]
----
include::../../../../x-pack/metricbeat/module/azure/container_service/_meta/data.json[]
----
5 changes: 4 additions & 1 deletion metricbeat/docs/modules_list.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,11 @@ This file is generated! See scripts/mage/docs_collector.go
|<<metricbeat-metricset-aws-sqs,sqs>>
|<<metricbeat-metricset-aws-usage,usage>> beta[]
|<<metricbeat-module-azure,azure>> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] |
.5+| .5+| |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]
.8+| .8+| |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]
|<<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>> beta[]
|<<metricbeat-metricset-azure-container_instance,container_instance>> beta[]
|<<metricbeat-metricset-azure-container_registry,container_registry>> beta[]
|<<metricbeat-metricset-azure-container_service,container_service>> beta[]
|<<metricbeat-metricset-azure-database_account,database_account>> beta[]
|<<metricbeat-metricset-azure-monitor,monitor>> beta[]
|<<metricbeat-metricset-azure-storage,storage>> beta[]
Expand Down
31 changes: 31 additions & 0 deletions x-pack/metricbeat/metricbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,37 @@ metricbeat.modules:
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'

- module: azure
metricsets:
- container_instance
enabled: true
period: 300s
client_id: '${AZURE_CLIENT_ID:""}'
client_secret: '${AZURE_CLIENT_SECRET:""}'
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'

- module: azure
metricsets:
- container_service
enabled: true
period: 300s
client_id: '${AZURE_CLIENT_ID:""}'
client_secret: '${AZURE_CLIENT_SECRET:""}'
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'

- module: azure
metricsets:
- container_registry
enabled: true
period: 300s
client_id: '${AZURE_CLIENT_ID:""}'
client_secret: '${AZURE_CLIENT_SECRET:""}'
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'


- module: azure
metricsets:
- database_account
Expand Down
31 changes: 31 additions & 0 deletions x-pack/metricbeat/module/azure/_meta/config.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,37 @@
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'

- module: azure
metricsets:
- container_instance
enabled: true
period: 300s
client_id: '${AZURE_CLIENT_ID:""}'
client_secret: '${AZURE_CLIENT_SECRET:""}'
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'

- module: azure
metricsets:
- container_service
enabled: true
period: 300s
client_id: '${AZURE_CLIENT_ID:""}'
client_secret: '${AZURE_CLIENT_SECRET:""}'
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'

- module: azure
metricsets:
- container_registry
enabled: true
period: 300s
client_id: '${AZURE_CLIENT_ID:""}'
client_secret: '${AZURE_CLIENT_SECRET:""}'
tenant_id: '${AZURE_TENANT_ID:""}'
subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'


- module: azure
metricsets:
- database_account
Expand Down
33 changes: 33 additions & 0 deletions x-pack/metricbeat/module/azure/_meta/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,39 @@
# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
# refresh_list_interval: 600s

#- module: azure
# metricsets:
# - container_instance
# enabled: true
# period: 300s
# client_id: '${AZURE_CLIENT_ID:""}'
# client_secret: '${AZURE_CLIENT_SECRET:""}'
# tenant_id: '${AZURE_TENANT_ID:""}'
# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
# refresh_list_interval: 600s

#- module: azure
# metricsets:
# - container_service
# enabled: true
# period: 300s
# client_id: '${AZURE_CLIENT_ID:""}'
# client_secret: '${AZURE_CLIENT_SECRET:""}'
# tenant_id: '${AZURE_TENANT_ID:""}'
# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
# refresh_list_interval: 600s

#- module: azure
# metricsets:
# - container_registry
# enabled: true
# period: 300s
# client_id: '${AZURE_CLIENT_ID:""}'
# client_secret: '${AZURE_CLIENT_SECRET:""}'
# tenant_id: '${AZURE_TENANT_ID:""}'
# subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
# refresh_list_interval: 600s
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file is starting to grow a lot. Would it make sense to group all metricsets that share the same params? This can happen on a different PR

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could do, will add this to the new PR


#- module: azure
# metricsets:
# - database_account
Expand Down
23 changes: 19 additions & 4 deletions x-pack/metricbeat/module/azure/azure.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,14 +106,16 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
// check for lightweight resources if no groups or ids have been entered, if not a new resource is created to check the entire subscription
var resources []ResourceConfig
for _, resource := range config.Resources {
if len(resource.Group) != 0 || len(resource.ID) != 0 {
if hasConfigOptions(resource.Group) || hasConfigOptions(resource.ID) {
resources = append(resources, resource)
}
}
// check if this is a light metricset or not and no resources have been configured
if len(resources) == 0 && len(config.Resources) != 0 {
res := config.Resources[0]
res.Query = fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType)
resources = append(resources, res)
resources = append(resources, ResourceConfig{
Query: fmt.Sprintf("resourceType eq '%s'", config.DefaultResourceType),
Metrics: config.Resources[0].Metrics,
})
}
config.Resources = resources
}
Expand Down Expand Up @@ -152,3 +154,16 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
}
return nil
}

// hasConfigOptions func will check if any resource id or resource group options have been entered in the light metricsets
func hasConfigOptions(config []string) bool {
if config == nil {
return false
}
for _, group := range config {
if group == "" {
return false
}
}
return true
}
6 changes: 3 additions & 3 deletions x-pack/metricbeat/module/azure/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,13 +114,13 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) []
client.Log.Error(err)
report.Error(err)
} else {
if metric.TimeGrain == "" {
metric.TimeGrain = timegrain
}
for i, currentMetric := range client.Resources.Metrics {
if matchMetrics(currentMetric, metric) {
current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute))
client.Resources.Metrics[i].Values = current
if client.Resources.Metrics[i].TimeGrain == "" {
client.Resources.Metrics[i].TimeGrain = timegrain
}
resultedMetrics = append(resultedMetrics, client.Resources.Metrics[i])
}
}
Expand Down
10 changes: 10 additions & 0 deletions x-pack/metricbeat/module/azure/client_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,3 +188,13 @@ func groupMetricsByResource(metrics []Metric) map[string][]Metric {
}
return grouped
}

// ContainsDimension will check if the dimension value is found in the list
func ContainsDimension(dimension string, dimensions []insights.LocalizableString) bool {
for _, dim := range dimensions {
if *dim.Value == dimension {
return true
}
}
return false
}
Loading