Add enricher to storageclass metricset (#33032)

* add enricher for storageclass Signed-off-by: Tetiana Kravchenko <[email protected]> * add pr number Signed-off-by: Tetiana Kravchenko <[email protected]> * fix check-no-changes] Error 1 Signed-off-by: Tetiana Kravchenko <[email protected]> * generate notice file Signed-off-by: Tetiana Kravchenko <[email protected]> * readd removed metric from test file Signed-off-by: Tetiana Kravchenko <[email protected]> Signed-off-by: Tetiana Kravchenko <[email protected]> (cherry picked from commit d20fff4) # Conflicts: # NOTICE.txt # go.mod # go.sum
elastic · Sep 12, 2022 · 7abc62a · 7abc62a
1 parent 0520908
commit 7abc62a
Show file tree

Hide file tree

Showing 14 changed files with 84 additions and 15 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -55,6 +55,10 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 
 - Check for nil metadata in GCP. {pull}32281[32281]
 - Add missing cluster metadata to k8s module metricsets {pull}32979[32979]
+- Fix GCP storage field naming {pull}32806[32806]
+- in module/windows/perfmon, changed collection method of the second counter value required to create a displayable value {pull}32305[32305]
+- Fix and improve AWS metric period calculation to avoid zero-length intervals {pull}32724[32724]
+- Add missing cluster metadata to k8s module metricsets {pull}32979[32979] {pull}33032[33032]
 
 *Packetbeat*
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -9607,6 +9607,7 @@ SOFTWARE.
 
 
 --------------------------------------------------------------------------------
+<<<<<<< HEAD
 Dependency : github.com/gocarina/gocsv
 Version: v0.0.0-20170324095351-ffef3ffc77be
 Licence type (autodetected): MIT
@@ -10031,6 +10032,14 @@ Licence type (autodetected): Apache-2.0
 
 Contents of probable licence file $GOMODCACHE/github.com/golang/[email protected]/LICENSE:
 
+=======
+Dependency : github.com/elastic/elastic-agent-autodiscover
+Version: v0.3.0
+Licence type (autodetected): Apache-2.0
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/LICENSE:
+>>>>>>> d20fff40fa (Add enricher to storageclass metricset (#33032))
 
                                  Apache License
                            Version 2.0, January 2004

diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml
@@ -41,6 +41,9 @@ data:
                     - state_resourcequota
                     - state_statefulset
                     - state_service
+                    - state_persistentvolume
+                    - state_persistentvolumeclaim
+                    - state_storageclass
                   # If `https` is used to access `kube-state-metrics`, uncomment following settings:
                   # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
                   # ssl.certificate_authorities:
@@ -303,6 +306,10 @@ rules:
   - jobs
   - cronjobs
   verbs: ["get", "list", "watch"]
+- apiGroups: ["storage.k8s.io"]
+  resources:
+    - storageclasses
+  verbs: ["get", "list", "watch"]
 - apiGroups:
   - ""
   resources:

diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml
@@ -41,6 +41,9 @@ data:
                     - state_resourcequota
                     - state_statefulset
                     - state_service
+                    - state_persistentvolume
+                    - state_persistentvolumeclaim
+                    - state_storageclass
                   # If `https` is used to access `kube-state-metrics`, uncomment following settings:
                   # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
                   # ssl.certificate_authorities:

diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml
@@ -36,6 +36,10 @@ rules:
   - jobs
   - cronjobs
   verbs: ["get", "list", "watch"]
+- apiGroups: ["storage.k8s.io"]
+  resources:
+    - storageclasses
+  verbs: ["get", "list", "watch"]
 - apiGroups:
   - ""
   resources:

diff --git a/go.mod b/go.mod
@@ -160,8 +160,14 @@ require (
 
 require (
 	github.com/elastic/bayeux v1.0.5
+<<<<<<< HEAD
 	github.com/elastic/elastic-agent-autodiscover v0.2.1
 	github.com/elastic/elastic-agent-libs v0.2.9
+=======
+	github.com/elastic/elastic-agent-autodiscover v0.3.0
+	github.com/elastic/elastic-agent-libs v0.2.11
+	github.com/elastic/elastic-agent-shipper-client v0.4.0
+>>>>>>> d20fff40fa (Add enricher to storageclass metricset (#33032))
 	github.com/elastic/elastic-agent-system-metrics v0.4.4
 	github.com/shirou/gopsutil/v3 v3.21.12
 	go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0

diff --git a/go.sum b/go.sum
@@ -528,14 +528,21 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0=
 github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
-github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUdOu0EVlpj53CqCsbU0E4=
-github.com/elastic/elastic-agent-autodiscover v0.2.1/go.mod h1:gPnzzfdYNdgznAb+iG9eyyXaQXBbAMHa+Y6Z8hXfcGY=
+github.com/elastic/elastic-agent-autodiscover v0.3.0 h1:kdpNnIDnVk7gvQxxR6PzZY7aM8LyMTRkwI/p+FNS17s=
+github.com/elastic/elastic-agent-autodiscover v0.3.0/go.mod h1:p3MSf9813JEnolCTD0GyVAr3+Eptg2zQ9aZVFjl4tJ4=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6 h1:nFvXHBjYK3e9+xF0WKDeAKK4aOO51uC28s+L9rBmilo=
 github.com/elastic/elastic-agent-client/v7 v7.0.0-20210727140539-f0905d9377f6/go.mod h1:uh/Gj9a0XEbYoM4NYz4LvaBVARz3QXLmlNjsrKY9fTc=
+<<<<<<< HEAD
 github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQHpitSp8q2zzv1Dqg+s=
 github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
 github.com/elastic/elastic-agent-libs v0.2.9 h1:7jOCqNqEWG0kJb3fa8/SC6beSiys1TmAylH9+hWTnrM=
 github.com/elastic/elastic-agent-libs v0.2.9/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
+=======
+github.com/elastic/elastic-agent-libs v0.2.11 h1:ZeYn35Kxt+IdtMPmE01TaDeaahCg/z7MkGPVWUo6Lp4=
+github.com/elastic/elastic-agent-libs v0.2.11/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
+github.com/elastic/elastic-agent-shipper-client v0.4.0 h1:nsTJF9oo4RHLl+zxFUZqNHaE86C6Ba5aImfegcEf6Sk=
+github.com/elastic/elastic-agent-shipper-client v0.4.0/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q=
+>>>>>>> d20fff40fa (Add enricher to storageclass metricset (#33032))
 github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0=
 github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI=
 github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=

diff --git a/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml b/metricbeat/module/kubernetes/_meta/test/docs/01_playground/metricbeat.yaml
@@ -37,6 +37,9 @@ data:
                     - state_resourcequota
                     - state_statefulset
                     - state_service
+                    - state_persistentvolume
+                    - state_persistentvolumeclaim
+                    - state_storageclass
                 - module: kubernetes
                   metricsets:
                     - apiserver
@@ -293,6 +296,10 @@ rules:
       - jobs
       - cronjobs
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources:
+      - storageclasses
+    verbs: ["get", "list", "watch"]
   - apiGroups:
       - ""
     resources:

diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0 b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0
@@ -7,5 +7,3 @@ kube_storageclass_created{storageclass="test_storageclass"} 1.501569018e+09
 # HELP kube_storageclass_labels Kubernetes labels converted to Prometheus labels.
 # TYPE kube_storageclass_labels gauge
 kube_storageclass_labels{storageclass="test_storageclass",label_foo="bar"} 1
-
-
diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.unit.v1.8.0.expected
@@ -1,6 +1,6 @@
 [
 	{
-		"RootFields": {},
+		"RootFields": null,
 		"ModuleFields": {
 			"labels": {
 				"foo": "bar"

diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v1.8.0.expected
@@ -1,6 +1,6 @@
 [
 	{
-		"RootFields": {},
+		"RootFields": null,
 		"ModuleFields": {
 			"labels": {
 				"addonmanager_kubernetes_io_mode": "EnsureExists"

diff --git a/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected b/metricbeat/module/kubernetes/state_storageclass/_meta/test/ksm.v2.0.0.expected
@@ -1,11 +1,13 @@
 [
 	{
-		"RootFields": {},
+		"RootFields": null,
 		"ModuleFields": null,
 		"MetricSetFields": {
 			"created": "2021-08-12T14:24:52.000Z",
 			"name": "standard",
-			"provisioner": "rancher.io/local-path"
+			"provisioner": "rancher.io/local-path",
+			"reclaim_policy": "Delete",
+			"volume_binding_mode": "WaitForFirstConsumer"
 		},
 		"Index": "",
 		"ID": "",

diff --git a/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go b/metricbeat/module/kubernetes/state_storageclass/state_storageclass.go
@@ -23,6 +23,8 @@ import (
 	p "github.com/elastic/beats/v7/metricbeat/helper/prometheus"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	k8smod "github.com/elastic/beats/v7/metricbeat/module/kubernetes"
+	"github.com/elastic/beats/v7/metricbeat/module/kubernetes/util"
+	"github.com/elastic/elastic-agent-autodiscover/kubernetes"
 )
 
 func init() {
@@ -38,6 +40,7 @@ type StorageClassMetricSet struct {
 	prometheus p.Prometheus
 	mapping    *p.MetricsMapping
 	mod        k8smod.Module
+	enricher   util.Enricher
 }
 
 // NewStorageClassMetricSet returns a prometheus based metricset for Storage classes
@@ -54,6 +57,7 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) {
 		BaseMetricSet: base,
 		prometheus:    prometheus,
 		mod:           mod,
+		enricher:      util.NewResourceMetadataEnricher(base, &kubernetes.StorageClass{}, mod.GetMetricsRepo(), false),
 		mapping: &p.MetricsMapping{
 			Metrics: map[string]p.MetricMap{
 				"kube_storageclass_info": p.InfoMetric(),
@@ -66,10 +70,12 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) {
 				"kube_storageclass_created": p.Metric("created", p.OpUnixTimestampValue()),
 			},
 			Labels: map[string]p.LabelMap{
-				"storageclass":      p.KeyLabel("name"),
-				"provisioner":       p.Label("provisioner"),
-				"reclaimPolicy":     p.Label("reclaim_policy"),
-				"volumeBindingMode": p.Label("volume_binding_mode"),
+				"storageclass":        p.KeyLabel("name"),
+				"provisioner":         p.Label("provisioner"),
+				"reclaimPolicy":       p.Label("reclaim_policy"),
+				"reclaim_policy":      p.Label("reclaim_policy"),
+				"volumeBindingMode":   p.Label("volume_binding_mode"),
+				"volume_binding_mode": p.Label("volume_binding_mode"),
 			},
 		},
 	}, nil
@@ -78,6 +84,8 @@ func NewStorageClassMetricSet(base mb.BaseMetricSet) (mb.MetricSet, error) {
 // Fetch prometheus metrics and treats those prefixed by mb.ModuleDataKey as
 // module rooted fields at the event that gets reported
 func (m *StorageClassMetricSet) Fetch(reporter mb.ReporterV2) {
+	m.enricher.Start()
+
 	families, err := m.mod.GetStateMetricsFamilies(m.prometheus)
 	if err != nil {
 		m.Logger().Error(err)
@@ -91,12 +99,24 @@ func (m *StorageClassMetricSet) Fetch(reporter mb.ReporterV2) {
 		return
 	}
 
+	m.enricher.Enrich(events)
+
 	for _, event := range events {
-		event[mb.NamespaceKey] = "storageclass"
-		reported := reporter.Event(mb.TransformMapStrToEvent("kubernetes", event, nil))
-		if !reported {
+
+		e, err := util.CreateEvent(event, "kubernetes.storageclass")
+		if err != nil {
+			m.Logger().Error(err)
+		}
+
+		if reported := reporter.Event(e); !reported {
 			m.Logger().Debug("error trying to emit event")
 			return
 		}
 	}
 }
+
+// Close stops this metricset
+func (m *StorageClassMetricSet) Close() error {
+	m.enricher.Stop()
+	return nil
+}
diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go
@@ -155,6 +155,8 @@ func NewResourceMetadataEnricher(
 				m[id] = metaGen.Generate("persistentvolume", r)
 			case *kubernetes.PersistentVolumeClaim:
 				m[id] = metaGen.Generate("persistentvolumeclaim", r)
+			case *kubernetes.StorageClass:
+				m[id] = metaGen.Generate("storageclass", r)
 			default:
 				m[id] = metaGen.Generate(r.GetObjectKind().GroupVersionKind().Kind, r)
 			}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,5 +7,3 @@ kube_storageclass_created{storageclass="test_storageclass"} 1.501569018e+09
		# HELP kube_storageclass_labels Kubernetes labels converted to Prometheus labels.
		# TYPE kube_storageclass_labels gauge
		kube_storageclass_labels{storageclass="test_storageclass",label_foo="bar"} 1