Fix annotation enrichment (#29605)

* add annotation enrichment Signed-off-by: Tetiana Kravchenko <[email protected]> * add pr number Signed-off-by: Tetiana Kravchenko <[email protected]> * fix tests Signed-off-by: Tetiana Kravchenko <[email protected]> * update changelog message Signed-off-by: Tetiana Kravchenko <[email protected]> * clean up changelog after rebase Signed-off-by: Tetiana Kravchenko <[email protected]> (cherry picked from commit 2917392) # Conflicts: # libbeat/common/kubernetes/metadata/pod_test.go
elastic · Jan 13, 2022 · b4ca76b · b4ca76b
1 parent d2b677f
commit b4ca76b
Show file tree

Hide file tree

Showing 6 changed files with 206 additions and 12 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -11,6 +11,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - Add job.name in pods controlled by Jobs {pull}28954[28954]
+- Change Docker base image from CentOS 7 to Ubuntu 20.04 {pull}29681[29681]
+- Enrich kubernetes metadata with node annotations. {pull}29605[29605]
 
 *Auditbeat*
 

diff --git a/libbeat/common/kubernetes/metadata/metadata.go b/libbeat/common/kubernetes/metadata/metadata.go
@@ -72,11 +72,16 @@ func WithFields(key string, value interface{}) FieldOptions {
 	}
 }
 
-// WithLabels FieldOption allows adding labels under sub-resource(kind)
+// WithMetadata FieldOption allows adding labels and annotations under sub-resource(kind)
 // example if kind=namespace namespace.labels key will be added
-func WithLabels(kind string) FieldOptions {
+func WithMetadata(kind string) FieldOptions {
 	return func(meta common.MapStr) {
-		safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"])
+		if meta["labels"] != nil {
+			safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"])
+		}
+		if meta["annotations"] != nil {
+			safemapstr.Put(meta, strings.ToLower(kind)+".annotations", meta["annotations"])
+		}
 	}
 }
 

diff --git a/libbeat/common/kubernetes/metadata/node_test.go b/libbeat/common/kubernetes/metadata/node_test.go
@@ -52,7 +52,10 @@ func TestNode_Generate(t *testing.T) {
 					Labels: map[string]string{
 						"foo": "bar",
 					},
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"key1": "value1",
+						"key2": "value2",
+					},
 				},
 				TypeMeta: metav1.TypeMeta{
 					Kind:       "Node",
@@ -71,11 +74,16 @@ func TestNode_Generate(t *testing.T) {
 				"labels": common.MapStr{
 					"foo": "bar",
 				},
+				"annotations": common.MapStr{
+					"key2": "value2",
+				},
 			}},
 		},
 	}
 
-	cfg := common.NewConfig()
+	cfg, _ := common.NewConfigFrom(Config{
+		IncludeAnnotations: []string{"key2"},
+	})
 	metagen := NewNodeMetadataGenerator(cfg, nil, client)
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
@@ -102,7 +110,9 @@ func TestNode_GenerateFromName(t *testing.T) {
 					Labels: map[string]string{
 						"foo": "bar",
 					},
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"key": "value",
+					},
 				},
 				TypeMeta: metav1.TypeMeta{
 					Kind:       "Node",
@@ -121,12 +131,17 @@ func TestNode_GenerateFromName(t *testing.T) {
 				"labels": common.MapStr{
 					"foo": "bar",
 				},
+				"annotations": common.MapStr{
+					"key": "value",
+				},
 			},
 		},
 	}
 
 	for _, test := range tests {
-		cfg := common.NewConfig()
+		cfg, _ := common.NewConfigFrom(Config{
+			IncludeAnnotations: []string{"key"},
+		})
 		nodes := cache.NewStore(cache.MetaNamespaceKeyFunc)
 		nodes.Add(test.input)
 		metagen := NewNodeMetadataGenerator(cfg, nodes, client)

diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go
@@ -93,7 +93,7 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) common.
 	}
 
 	if p.node != nil {
-		meta := p.node.GenerateFromName(po.Spec.NodeName, WithLabels("node"))
+		meta := p.node.GenerateFromName(po.Spec.NodeName, WithMetadata("node"))
 		if meta != nil {
 			out.Put("node", meta["node"])
 		} else {

diff --git a/libbeat/common/kubernetes/metadata/pod_test.go b/libbeat/common/kubernetes/metadata/pod_test.go
@@ -673,7 +673,172 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) {
 		namespaces.Add(test.namespace)
 		nsMeta := NewNamespaceMetadataGenerator(config, namespaces, client)
 
+<<<<<<< HEAD
 		metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta)
+=======
+		metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, addResourceMetadata)
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.output, metagen.Generate(test.input))
+		})
+	}
+}
+
+func TestPod_GenerateWithNodeNamespaceWithAddResourceConfig(t *testing.T) {
+	client := k8sfake.NewSimpleClientset()
+	uid := "005f3b90-4b9d-12f8-acf0-31020a840133"
+	namespace := "default"
+	name := "obj"
+	boolean := true
+
+	tests := []struct {
+		input     kubernetes.Resource
+		node      kubernetes.Resource
+		namespace kubernetes.Resource
+		output    common.MapStr
+		name      string
+	}{
+		{
+			name: "test simple object",
+			input: &v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      name,
+					UID:       types.UID(uid),
+					Namespace: namespace,
+					Labels: map[string]string{
+						"app.kubernetes.io/component": "exporter",
+					},
+					Annotations: map[string]string{
+						"app": "production",
+					},
+					OwnerReferences: []metav1.OwnerReference{
+						{
+							APIVersion: "apps",
+							Kind:       "ReplicaSet",
+							Name:       "nginx-rs",
+							UID:        "005f3b90-4b9d-12f8-acf0-31020a8409087",
+							Controller: &boolean,
+						},
+					},
+				},
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Pod",
+					APIVersion: "v1",
+				},
+
+				Spec: v1.PodSpec{
+					NodeName: "testnode",
+				},
+				Status: v1.PodStatus{PodIP: "127.0.0.5"},
+			},
+			node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "testnode",
+					UID:  types.UID(uid),
+					Labels: map[string]string{
+						"nodekey":  "nodevalue",
+						"nodekey2": "nodevalue2",
+					},
+					Annotations: map[string]string{
+						"node.annotation": "node.value",
+					},
+				},
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Node",
+					APIVersion: "v1",
+				},
+				Status: v1.NodeStatus{
+					Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}},
+				},
+			},
+			namespace: &v1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: namespace,
+					UID:  types.UID(uid),
+					Labels: map[string]string{
+						"app.kubernetes.io/name": "kube-state-metrics",
+						"nskey2":                 "nsvalue2",
+					},
+					Annotations: map[string]string{
+						"ns.annotation": "ns.value",
+					},
+				},
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "Namespace",
+					APIVersion: "v1",
+				},
+			},
+			output: common.MapStr{"kubernetes": common.MapStr{
+				"pod": common.MapStr{
+					"name": "obj",
+					"uid":  uid,
+					"ip":   "127.0.0.5",
+				},
+				"namespace":     "default",
+				"namespace_uid": uid,
+				"namespace_labels": common.MapStr{
+					"app_kubernetes_io/name": "kube-state-metrics",
+				},
+				"namespace_annotations": common.MapStr{
+					"ns_annotation": "ns.value",
+				},
+				"node": common.MapStr{
+					"name": "testnode",
+					"uid":  uid,
+					"labels": common.MapStr{
+						"nodekey2": "nodevalue2",
+					},
+					"hostname": "node1",
+					"annotations": common.MapStr{
+						"node_annotation": "node.value",
+					},
+				},
+				"labels": common.MapStr{
+					"app_kubernetes_io/component": "exporter",
+				},
+				"annotations": common.MapStr{
+					"app": "production",
+				},
+				"replicaset": common.MapStr{
+					"name": "nginx-rs",
+				},
+			}},
+		},
+	}
+
+	for _, test := range tests {
+		config, err := common.NewConfigFrom(map[string]interface{}{
+			"include_annotations": []string{"app"},
+		})
+
+		assert.NoError(t, err)
+
+		namespaceConfig, _ := common.NewConfigFrom(map[string]interface{}{
+			"include_labels":      []string{"app.kubernetes.io/name"},
+			"include_annotations": []string{"ns.annotation"},
+		})
+		nodeConfig, _ := common.NewConfigFrom(map[string]interface{}{
+			"include_labels":      []string{"nodekey2"},
+			"include_annotations": []string{"node.annotation"},
+		})
+		metaConfig := AddResourceMetadataConfig{
+			Namespace:  namespaceConfig,
+			Node:       nodeConfig,
+			Deployment: false,
+		}
+
+		pods := cache.NewStore(cache.MetaNamespaceKeyFunc)
+		pods.Add(test.input)
+
+		nodes := cache.NewStore(cache.MetaNamespaceKeyFunc)
+		nodes.Add(test.node)
+		nodeMeta := NewNodeMetadataGenerator(nodeConfig, nodes, client)
+
+		namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc)
+		namespaces.Add(test.namespace)
+		nsMeta := NewNamespaceMetadataGenerator(namespaceConfig, namespaces, client)
+
+		metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, &metaConfig)
+>>>>>>> 2917392a01 (Fix annotation enrichment (#29605))
 		t.Run(test.name, func(t *testing.T) {
 			assert.Equal(t, test.output, metagen.Generate(test.input))
 		})

diff --git a/libbeat/common/kubernetes/metadata/service_test.go b/libbeat/common/kubernetes/metadata/service_test.go
@@ -279,7 +279,9 @@ func TestService_GenerateWithNamespace(t *testing.T) {
 					Labels: map[string]string{
 						"nskey": "nsvalue",
 					},
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"ns.annotation": "value",
+					},
 				},
 				TypeMeta: metav1.TypeMeta{
 					Kind:       "Namespace",
@@ -309,21 +311,26 @@ func TestService_GenerateWithNamespace(t *testing.T) {
 					"namespace_labels": common.MapStr{
 						"nskey": "nsvalue",
 					},
+					"namespace_annotations": common.MapStr{
+						"ns_annotation": "value",
+					},
 				},
 			},
 		},
 	}
 
 	for _, test := range tests {
-		cfg := common.NewConfig()
+		nsConfig, _ := common.NewConfigFrom(map[string]interface{}{
+			"include_annotations": []string{"ns.annotation"},
+		})
 		services := cache.NewStore(cache.MetaNamespaceKeyFunc)
 		services.Add(test.input)
 
 		namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc)
 		namespaces.Add(test.namespace)
-		nsMeta := NewNamespaceMetadataGenerator(cfg, namespaces, client)
+		nsMeta := NewNamespaceMetadataGenerator(nsConfig, namespaces, client)
 
-		metagen := NewServiceMetadataGenerator(cfg, services, nsMeta, client)
+		metagen := NewServiceMetadataGenerator(nsConfig, services, nsMeta, client)
 		t.Run(test.name, func(t *testing.T) {
 			assert.Equal(t, test.output, metagen.Generate(test.input))
 		})