diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e4120e0c0d2..9775e4a72fd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -50,6 +50,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Fix and improve AWS metric period calculation to avoid zero-length intervals {pull}32724[32724] - Add missing cluster metadata to k8s module metricsets {pull}32979[32979] +- Add missing cluster metadata to k8s module metricsets {pull}32979[32979] {pull}33032[33032] *Packetbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 1c5942ca796..62ffded5514 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -9414,11 +9414,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.2.1 +Version: v0.3.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.2.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.3.0/LICENSE: Apache License Version 2.0, January 2004 @@ -9858,11 +9858,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.2.9 +Version: v0.2.11 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.11/LICENSE: Apache License Version 2.0, January 2004 diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml index 17b6c3c4d59..f278fe403e7 100644 --- a/deploy/kubernetes/metricbeat-kubernetes.yaml +++ b/deploy/kubernetes/metricbeat-kubernetes.yaml @@ -41,6 +41,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass # If `https` is used to access `kube-state-metrics`, uncomment following settings: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: @@ -303,6 +306,10 @@ rules: - jobs - cronjobs verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml index 4dc934ccb26..9ddab21360c 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml @@ -41,6 +41,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass # If `https` is used to access `kube-state-metrics`, uncomment following settings: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml index a9157a7e41d..73ce911439b 100644 --- a/deploy/kubernetes/metricbeat/metricbeat-role.yaml +++ b/deploy/kubernetes/metricbeat/metricbeat-role.yaml @@ -36,6 +36,10 @@ rules: - jobs - cronjobs verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: diff --git a/go.mod b/go.mod index 89080d5829e..5b8ce85fe6f 100644 --- a/go.mod +++ b/go.mod @@ -201,8 +201,8 @@ require ( github.com/aws/smithy-go v1.12.0 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.2.1 - github.com/elastic/elastic-agent-libs v0.2.9 + github.com/elastic/elastic-agent-autodiscover v0.3.0 + github.com/elastic/elastic-agent-libs v0.2.11 github.com/elastic/elastic-agent-shipper-client v0.2.0 github.com/elastic/elastic-agent-system-metrics v0.4.4 github.com/elastic/go-elasticsearch/v8 v8.2.0 diff --git a/go.sum b/go.sum index 32f215ce130..624020bf325 100644 --- a/go.sum +++ b/go.sum @@ -593,15 +593,14 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUdOu0EVlpj53CqCsbU0E4= -github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+iG9eyyXaQXBbAMHa+Y6Z8hXfcGY= +github.com/elastic/elastic-agent-autodiscover v0.3.0 h1:kdpNnIDnVk7gvQxxR6PzZY7aM8LyMTRkwI/p+FNS17s= +github.com/elastic/elastic-agent-autodiscover v0.3.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo= github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc= github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s= -github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-libs v0.2.7/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= -github.com/elastic/elastic-agent-libs v0.2.9 h1:7jOCqNqEWG0kJb3fa8/SC6beSiys1TmAylH9+hWTnrM= -github.com/elastic/elastic-agent-libs v0.2.9/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= +github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4= +github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE= github.com/elastic/elastic-agent-shipper-client v0.2.0 h1:p+5ep48YCOe+3nICeWmiLwQV11yDLad2n4NunI66Shg= github.com/elastic/elastic-agent-shipper-client v0.2.0/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q= github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0= diff --git a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml index e17b1c3a687..204b4313e54 100644 --- a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml +++ b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml @@ -37,6 +37,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass - module: kubernetes metricsets: - apiserver @@ -293,6 +296,10 @@ rules: - jobs - cronjobs verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] - apiGroups: - "" resources: diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 index 0a7712e74b7..0efb02acbee 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 @@ -7,5 +7,3 @@ kube_storageclass_created{storageclass="test_storageclass"} 1.501569018e+09 # HELP kube_storageclass_labels Kubernetes labels converted to Prometheus labels. # TYPE kube_storageclass_labels gauge kube_storageclass_labels{storageclass="test_storageclass",label_foo="bar"} 1 - - diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected index cd0595a14f4..6fcfce94976 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected @@ -1,6 +1,6 @@ [ { - "RootFields": {}, + "RootFields": null, "ModuleFields": { "labels": { "foo": "bar" diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected index 346cc2cf104..deedb1e2fea 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected @@ -1,6 +1,6 @@ [ { - "RootFields": {}, + "RootFields": null, "ModuleFields": { "labels": { "addonmanager_kubernetes_io_mode": "EnsureExists" diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected index ace72a2ccd9..7e3ff74968e 100644 --- a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected +++ b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected @@ -1,11 +1,13 @@ [ { - "RootFields": {}, + "RootFields": null, "ModuleFields": null, "MetricSetFields": { "created": "2021-08-12T14:24:52.000Z", "name": "standard", - "provisioner": "rancher.io/local-path" + "provisioner": "rancher.io/local-path", + "reclaim_policy": "Delete", + "volume_binding_mode": "WaitForFirstConsumer" }, "Index": "", "ID": "", diff --git a/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go b/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go index 5a0ce3d0109..927468a676f 100644 --- a/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go +++ b/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go @@ -23,6 +23,8 @@ import ( p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes" + "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" + "github.com/elastic/elastic-agent-autodiscover/kubernetes" ) func init() { @@ -38,6 +40,7 @@ type StorageClassMetricSet struct { prometheus p.Prometheus mapping *p.MetricsMapping mod k8smod.Module + enricher util.Enricher } // NewStorageClassMetricSet returns a prometheus based metricset for Storage classes @@ -54,6 +57,7 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { BaseMetricSet: base, prometheus: prometheus, mod: mod, + enricher: util.NewResourceMetadataEnricher(base, &kubernetes.StorageClass{}, mod.GetMetricsRepo(), false), mapping: &p.MetricsMapping{ Metrics: map[string]p.MetricMap{ "kube_storageclass_info": p.InfoMetric(), @@ -66,10 +70,12 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { "kube_storageclass_created": p.Metric("created", p.OpUnixTimestampValue()), }, Labels: map[string]p.LabelMap{ - "storageclass": p.KeyLabel("name"), - "provisioner": p.Label("provisioner"), - "reclaimPolicy": p.Label("reclaim_policy"), - "volumeBindingMode": p.Label("volume_binding_mode"), + "storageclass": p.KeyLabel("name"), + "provisioner": p.Label("provisioner"), + "reclaimPolicy": p.Label("reclaim_policy"), + "reclaim_policy": p.Label("reclaim_policy"), + "volumeBindingMode": p.Label("volume_binding_mode"), + "volume_binding_mode": p.Label("volume_binding_mode"), }, }, }, nil @@ -78,6 +84,8 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch prometheus metrics and treats those prefixed by mb.ModuleDataKey as // module rooted fields at the event that gets reported func (m *StorageClassMetricSet) Fetch(reporter mb.ReporterV2) { + m.enricher.Start() + families, err := m.mod.GetStateMetricsFamilies(m.prometheus) if err != nil { m.Logger().Error(err) @@ -91,12 +99,24 @@ func (m *StorageClassMetricSet) Fetch(reporter mb.ReporterV2) { return } + m.enricher.Enrich(events) + for _, event := range events { - event[mb.NamespaceKey] = "storageclass" - reported := reporter.Event(mb.TransformMapStrToEvent("kubernetes", event, nil)) - if !reported { + + e, err := util.CreateEvent(event, "kubernetes.storageclass") + if err != nil { + m.Logger().Error(err) + } + + if reported := reporter.Event(e); !reported { m.Logger().Debug("error trying to emit event") return } } } + +// Close stops this metricset +func (m *StorageClassMetricSet) Close() error { + m.enricher.Stop() + return nil +} diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index c52f4bf8571..e6addbba791 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -158,6 +158,8 @@ func NewResourceMetadataEnricher( m[id] = metaGen.Generate("persistentvolume", r) case *kubernetes.PersistentVolumeClaim: m[id] = metaGen.Generate("persistentvolumeclaim", r) + case *kubernetes.StorageClass: + m[id] = metaGen.Generate("storageclass", r) default: m[id] = metaGen.Generate(r.GetObjectKind().GroupVersionKind().Kind, r) }