From bb4ce60c360a1721e15c54d370fe79298a8e2e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Tue, 15 Jan 2019 22:10:23 +0100 Subject: [PATCH] Fix Pod UID automatic enriching (#10081) * Fix Pod UID automatic enriching Kubernetes Pod events where not being enriched correctly. Before this change, metadata was being added at the module level, which is correct for all metricsets, except `pod` and `state_pod`. For these metricsets metadata must be added at the root level because we are already under `kubernetes.pod`. Existing metadata was being ignored. This change takes this special case into account and updates events in the right place, ensuring the events get the metadata correctly. (cherry picked from commit c60282aab58a5d9fa73199197bc0fba38c963983) --- CHANGELOG.next.asciidoc | 2 ++ .../module/kubernetes/pod/_meta/data.json | 1 + .../kubernetes/state_pod/_meta/data.json | 1 + .../module/kubernetes/util/kubernetes.go | 20 ++++++++++++- .../module/kubernetes/util/kubernetes_test.go | 30 +++++++++++++++++-- 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index dc44a085748..0980b23ef18 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -44,7 +44,9 @@ https://github.com/elastic/beats/compare/v6.5.4...6.5[Check the HEAD diff] *Journalbeat* *Metricbeat* + - Fix panics in vsphere module when certain values where not returned by the API. {pull}9784[9784] +- Fix pod UID metadata enrichment in Kubernetes module. {pull}10081[10081] *Packetbeat* diff --git a/metricbeat/module/kubernetes/pod/_meta/data.json b/metricbeat/module/kubernetes/pod/_meta/data.json index c610844749e..24d9ea6a5bd 100644 --- a/metricbeat/module/kubernetes/pod/_meta/data.json +++ b/metricbeat/module/kubernetes/pod/_meta/data.json @@ -12,6 +12,7 @@ }, "pod": { "name": "nginx-3137573019-pcfzh", + "uid": "b89a812e-18cd-11e9-b333-080027190d51", "network": { "rx": { "bytes": 18999261, diff --git a/metricbeat/module/kubernetes/state_pod/_meta/data.json b/metricbeat/module/kubernetes/state_pod/_meta/data.json index 7b07380ecc8..53cc1da64c3 100644 --- a/metricbeat/module/kubernetes/state_pod/_meta/data.json +++ b/metricbeat/module/kubernetes/state_pod/_meta/data.json @@ -14,6 +14,7 @@ "host_ip": "192.168.99.100", "ip": "172.17.0.3", "name": "tiller-deploy-3067024529-1gp80", + "uid": "659419d5-e27a-11e8-98fa-080027190d51", "status": { "phase": "running", "ready": "true", diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 8047e08bbb0..91ae3979f28 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -59,6 +59,7 @@ type enricher struct { watcher kubernetes.Watcher watcherStarted bool watcherStartedLock sync.Mutex + isPod bool } // GetWatcher initializes a kubernetes watcher with the given @@ -157,6 +158,12 @@ func NewResourceMetadataEnricher( }, ) + // Configure the enricher for Pods, so pod specific metadata ends up in the right place when + // calling Enrich + if _, ok := res.(*kubernetes.Pod); ok { + enricher.isPod = true + } + return enricher } @@ -243,7 +250,7 @@ func buildMetadataEnricher( watcher kubernetes.Watcher, update func(map[string]common.MapStr, kubernetes.Resource), delete func(map[string]common.MapStr, kubernetes.Resource), - index func(e common.MapStr) string) Enricher { + index func(e common.MapStr) string) *enricher { enricher := enricher{ metadata: map[string]common.MapStr{}, @@ -298,6 +305,17 @@ func (m *enricher) Enrich(events []common.MapStr) { defer m.RUnlock() for _, event := range events { if meta := m.metadata[m.index(event)]; meta != nil { + if m.isPod { + // apply pod meta at metricset level + if podMeta, ok := meta["pod"].(common.MapStr); ok { + event.DeepUpdate(podMeta) + } + + // don't apply pod metadata to module level + meta = meta.Clone() + delete(meta, "pod") + } + event.DeepUpdate(common.MapStr{ mb.ModuleDataKey: meta, }) diff --git a/metricbeat/module/kubernetes/util/kubernetes_test.go b/metricbeat/module/kubernetes/util/kubernetes_test.go index c04f0ab4f59..d2ef22efa72 100644 --- a/metricbeat/module/kubernetes/util/kubernetes_test.go +++ b/metricbeat/module/kubernetes/util/kubernetes_test.go @@ -31,6 +31,7 @@ func TestBuildMetadataEnricher(t *testing.T) { watcher := mockWatcher{} funcs := mockFuncs{} resource := &mockResource{ + uid: "mockuid", name: "enrich", namespace: "default", labels: map[string]string{ @@ -59,6 +60,23 @@ func TestBuildMetadataEnricher(t *testing.T) { {"name": "unknown"}, { "name": "enrich", + "_module": common.MapStr{"label": "value", "pod": common.MapStr{"name": "enrich", "uid": "mockuid"}}, + }, + }, events) + + // Enrich a pod (metadata goes in root level) + events = []common.MapStr{ + {"name": "unknown"}, + {"name": "enrich"}, + } + enricher.isPod = true + enricher.Enrich(events) + + assert.Equal(t, []common.MapStr{ + {"name": "unknown"}, + { + "name": "enrich", + "uid": "mockuid", "_module": common.MapStr{"label": "value"}, }, }, events) @@ -87,7 +105,12 @@ type mockFuncs struct { func (f *mockFuncs) update(m map[string]common.MapStr, obj kubernetes.Resource) { f.updated = obj - meta := common.MapStr{} + meta := common.MapStr{ + "pod": common.MapStr{ + "name": obj.GetMetadata().GetName(), + "uid": obj.GetMetadata().GetUid(), + }, + } for k, v := range obj.GetMetadata().Labels { meta[k] = v } @@ -105,12 +128,13 @@ func (f *mockFuncs) index(m common.MapStr) string { } type mockResource struct { - name, namespace string - labels map[string]string + name, namespace, uid string + labels map[string]string } func (r *mockResource) GetMetadata() *v1.ObjectMeta { return &v1.ObjectMeta{ + Uid: &r.uid, Name: &r.name, Namespace: &r.namespace, Labels: r.labels,