From 6f79ab716c0f0e0ec19071bd673d40f302c6549d Mon Sep 17 00:00:00 2001 From: Tetiana Kravchenko Date: Mon, 12 Sep 2022 11:09:15 +0200 Subject: [PATCH] Add enricher to storageclass metricset (#33032) * add enricher for storageclass Signed-off-by: Tetiana Kravchenko * add pr number Signed-off-by: Tetiana Kravchenko * fix check-no-changes] Error 1 Signed-off-by: Tetiana Kravchenko * generate notice file Signed-off-by: Tetiana Kravchenko * readd removed metric from test file Signed-off-by: Tetiana Kravchenko Signed-off-by: Tetiana Kravchenko --- CHANGELOG.next.asciidoc | 3 +- NOTICE.txt | 4 +-- deploy/kubernetes/metricbeat-kubernetes.yaml | 7 ++++ .../metricbeat-daemonset-configmap.yaml | 3 ++ .../metricbeat/metricbeat-role.yaml | 4 +++ go.mod | 2 +- go.sum | 5 ++- .../test/docs/01_playground/metricbeat.yaml | 7 ++++ .../_meta/test/ksm.unit.v1.8.0 | 2 -- .../_meta/test/ksm.unit.v1.8.0.expected | 2 +- .../_meta/test/ksm.v1.8.0.expected | 2 +- .../_meta/test/ksm.v2.0.0.expected | 6 ++-- .../state_storageclass/state_storageclass.go | 34 +++++++++++++++---- .../module/kubernetes/util/kubernetes.go | 2 ++ 14 files changed, 62 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c1dac84aedef..c658bb61d165 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -83,8 +83,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Fix GCP storage field naming {pull}32806[32806] - in module/windows/perfmon, changed collection method of the second counter value required to create a displayable value {pull}32305[32305] - Fix and improve AWS metric period calculation to avoid zero-length intervals {pull}32724[32724] -- Add network name to data.json {pull}32986[32986] -- Add missing cluster metadata to k8s module metricsets {pull}32979[32979] +- Add missing cluster metadata to k8s module metricsets {pull}32979[32979] {pull}33032[33032] *Packetbeat* diff --git a/NOTICE.txt b/NOTICE.txt index a0463ecd5e95..27e1f73a4d35 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9444,11 +9444,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.2.1 +Version: v0.3.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.2.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.3.0/LICENSE: Apache License Version 2.0, January 2004 diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 56785cdbf96f..921fccd85d8e 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -41,6 +41,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass # If `https` is used to access `kube-state-metrics`, uncomment following settings: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: @@ -303,6 +306,10 @@ rules: - jobs - cronjobs verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml index 4dc934ccb267..9ddab21360cc 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml @@ -41,6 +41,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass # If `https` is used to access `kube-state-metrics`, uncomment following settings: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml index a9157a7e41d5..73ce911439b7 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-role.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-role.yaml @@ -36,6 +36,10 @@ rules: - jobs - cronjobs verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: diff --git a/go.mod b/go.mod index fc87b2812f1d..45b481496374 100644 --- a/go.mod +++ b/go.mod @@ -202,7 +202,7 @@ require ( github.com/aws/smithy-go v1.12.0 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.2.1 + github.com/elastic/elastic-agent-autodiscover v0.3.0 github.com/elastic/elastic-agent-libs v0.2.11 github.com/elastic/elastic-agent-shipper-client v0.4.0 github.com/elastic/elastic-agent-system-metrics v0.4.4 diff --git a/go.sum b/go.sum index b8befe58e1b8..ba7b7ab13e03 100644 --- a/go.sum +++ b/go.sum @@ -602,11 +602,10 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUdOu0EVlpj53CqCsbU0E4= -github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+iG9eyyXaQXBbAMHa+Y6Z8hXfcGY= +github.com/elastic/elastic-agent-autodiscover v0.3.0 h1:kdpNnIDnVk7gvQxxR6PzZY7aM8LyMTRkwI/p+FNS17s= +github.com/elastic/elastic-agent-autodiscover v0.3.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= -github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4= github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-shipper-client v0.4.0 h1:nsTJF9oo4RHLl+zxFUZqNHaE86C6Ba5aImfegcEf6Sk= diff --git a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml index e17b1c3a6870..204b4313e54f 100644 --- a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml +++ b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml @@ -37,6 +37,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass - module: kubernetes metricsets: - apiserver @@ -293,6 +296,10 @@ rules: - jobs - cronjobs verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 index 0a7712e74b7a..0efb02acbee6 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 @@ -7,5 +7,3 @@ kube_storageclass_created{storageclass="test_storageclass"} 1.501569018e+09 # HELP kube_storageclass_labels Kubernetes labels converted to Prometheus labels. # TYPE kube_storageclass_labels gauge kube_storageclass_labels{storageclass="test_storageclass",label_foo="bar"} 1 - - diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected index cd0595a14f4a..6fcfce949769 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected @@ -1,6 +1,6 @@ [ { - "RootFields": {}, + "RootFields": null, "ModuleFields": { "labels": { "foo": "bar" diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected index 346cc2cf1045..deedb1e2fea4 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected @@ -1,6 +1,6 @@ [ { - "RootFields": {}, + "RootFields": null, "ModuleFields": { "labels": { "addonmanager_kubernetes_io_mode": "EnsureExists" diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected index ace72a2ccd9b..7e3ff74968ec 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected @@ -1,11 +1,13 @@ [ { - "RootFields": {}, + "RootFields": null, "ModuleFields": null, "MetricSetFields": { "created": "2021-08-12T14:24:52.000Z", "name": "standard", - "provisioner": "rancher.io/local-path" + "provisioner": "rancher.io/local-path", + "reclaim_policy": "Delete", + "volume_binding_mode": "WaitForFirstConsumer" }, "Index": "", "ID": "", diff --git a/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go b/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go index 5a0ce3d01092..927468a676f4 100644 --- a/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go +++ b/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go @@ -23,6 +23,8 @@ import ( p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" + "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) func init() { @@ -38,6 +40,7 @@ type StorageClassMetricSet struct { prometheus p.Prometheus mapping *p.MetricsMapping mod k8smod.Module + enricher util.Enricher } // NewStorageClassMetricSet returns a prometheus based metricset for Storage classes @@ -54,6 +57,7 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { BaseMetricSet: base, prometheus: prometheus, mod: mod, + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.StorageClass{}, mod.GetMetricsRepo(), false), mapping: &p.MetricsMapping{ Metrics: map[string]p.MetricMap{ "kube_storageclass_info": p.InfoMetric(), @@ -66,10 +70,12 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { "kube_storageclass_created": p.Metric("created", p.OpUnixTimestampValue()), }, Labels: map[string]p.LabelMap{ - "storageclass": p.KeyLabel("name"), - "provisioner": p.Label("provisioner"), - "reclaimPolicy": p.Label("reclaim_policy"), - "volumeBindingMode": p.Label("volume_binding_mode"), + "storageclass": p.KeyLabel("name"), + "provisioner": p.Label("provisioner"), + "reclaimPolicy": p.Label("reclaim_policy"), + "reclaim_policy": p.Label("reclaim_policy"), + "volumeBindingMode": p.Label("volume_binding_mode"), + "volume_binding_mode": p.Label("volume_binding_mode"), }, }, }, nil @@ -78,6 +84,8 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch prometheus metrics and treats those prefixed by mb.ModuleDataKey as // module rooted fields at the event that gets reported func (m *StorageClassMetricSet) Fetch(reporter mb.ReporterV2) { + m.enricher.Start() + families, err := m.mod.GetStateMetricsFamilies(m.prometheus) if err != nil { m.Logger().Error(err) @@ -91,12 +99,24 @@ func (m *StorageClassMetricSet) Fetch(reporter mb.ReporterV2) { return } + m.enricher.Enrich(events) + for _, event := range events { - event[mb.NamespaceKey] = "storageclass" - reported := reporter.Event(mb.TransformMapStrToEvent("kubernetes", event, nil)) - if !reported { + + e, err := util.CreateEvent(event, "kubernetes.storageclass") + if err != nil { + m.Logger().Error(err) + } + + if reported := reporter.Event(e); !reported { m.Logger().Debug("error trying to emit event") return } } } + +// Close stops this metricset +func (m *StorageClassMetricSet) Close() error { + m.enricher.Stop() + return nil +} diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index c52f4bf85718..e6addbba7912 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -158,6 +158,8 @@ func NewResourceMetadataEnricher( m[id] = metaGen.Generate("persistentvolume", r) case *kubernetes.PersistentVolumeClaim: m[id] = metaGen.Generate("persistentvolumeclaim", r) + case *kubernetes.StorageClass: + m[id] = metaGen.Generate("storageclass", r) default: m[id] = metaGen.Generate(r.GetObjectKind().GroupVersionKind().Kind, r) }