Fix k8s pod annotations tier in metadata (elastic#16554)

(cherry picked from commit 83e7957)

CHANGELOG.next.asciidoc

@@ -78,6 +78,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937}
 - Fix loading processors from annotation hints. {pull}16348[16348]
 - Fix k8s pods labels broken schema. {pull}16480[16480]
+- Fix k8s pods annotations broken schema. {pull}16554[16554]
 - Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450}
 
 *Auditbeat*

libbeat/common/kubernetes/metadata/pod.go

@@ -50,7 +50,7 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map
 
 	out := p.resource.Generate("pod", obj, opts...)
 	// TODO: remove this call when moving to 8.0
-	out = p.exportPodLabels(out)
+	out = p.exportPodLabelsAndAnnotations(out)
 
 	if p.node != nil {
 		meta := p.node.GenerateFromName(po.Spec.NodeName)
@@ -92,13 +92,20 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr
 	return nil
 }
 
-func (p *pod) exportPodLabels(in common.MapStr) common.MapStr {
+func (p *pod) exportPodLabelsAndAnnotations(in common.MapStr) common.MapStr {
 	labels, err := in.GetValue("pod.labels")
 	if err != nil {
 		return in
 	}
 	in.Put("labels", labels)
 	in.Delete("pod.labels")
 
+	annotations, err := in.GetValue("pod.annotations")
+	if err != nil {
+		return in
+	}
+	in.Put("annotations", annotations)
+	in.Delete("pod.annotations")
+
 	return in
 }

libbeat/common/kubernetes/metadata/pod_test.go

@@ -53,7 +53,9 @@ func TestPod_Generate(t *testing.T) {
 					Labels: map[string]string{
 						"foo": "bar",
 					},
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"app": "production",
+					},
 				},
 				TypeMeta: metav1.TypeMeta{
 					Kind:       "Pod",
@@ -71,6 +73,9 @@ func TestPod_Generate(t *testing.T) {
 				"labels": common.MapStr{
 					"foo": "bar",
 				},
+				"annotations": common.MapStr{
+					"app": "production",
+				},
 				"namespace": "default",
 				"node": common.MapStr{
 					"name": "testnode",
@@ -87,7 +92,9 @@ func TestPod_Generate(t *testing.T) {
 					Labels: map[string]string{
 						"foo": "bar",
 					},
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"app": "production",
+					},
 					OwnerReferences: []metav1.OwnerReference{
 						{
 							APIVersion: "apps",
@@ -121,12 +128,19 @@ func TestPod_Generate(t *testing.T) {
 				"labels": common.MapStr{
 					"foo": "bar",
 				},
+				"annotations": common.MapStr{
+					"app": "production",
+				},
 			},
 		},
 	}
 
-	cfg := common.NewConfig()
-	metagen := NewPodMetadataGenerator(cfg, nil, nil, nil)
+	config, err := common.NewConfigFrom(map[string]interface{}{
+		"include_annotations": []string{"app"},
+	})
+	assert.Nil(t, err)
+
+	metagen := NewPodMetadataGenerator(config, nil, nil, nil)
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			assert.Equal(t, test.output, metagen.Generate(test.input))
@@ -154,7 +168,9 @@ func TestPod_GenerateFromName(t *testing.T) {
 					Labels: map[string]string{
 						"foo": "bar",
 					},
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"app": "production",
+					},
 				},
 				TypeMeta: metav1.TypeMeta{
 					Kind:       "Pod",
@@ -176,6 +192,9 @@ func TestPod_GenerateFromName(t *testing.T) {
 				"labels": common.MapStr{
 					"foo": "bar",
 				},
+				"annotations": common.MapStr{
+					"app": "production",
+				},
 			},
 		},
 		{
@@ -188,7 +207,9 @@ func TestPod_GenerateFromName(t *testing.T) {
 					Labels: map[string]string{
 						"foo": "bar",
 					},
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"app": "production",
+					},
 					OwnerReferences: []metav1.OwnerReference{
 						{
 							APIVersion: "apps",
@@ -222,15 +243,21 @@ func TestPod_GenerateFromName(t *testing.T) {
 				"labels": common.MapStr{
 					"foo": "bar",
 				},
+				"annotations": common.MapStr{
+					"app": "production",
+				},
 			},
 		},
 	}
 
 	for _, test := range tests {
-		cfg := common.NewConfig()
+		config, err := common.NewConfigFrom(map[string]interface{}{
+			"include_annotations": []string{"app"},
+		})
+		assert.Nil(t, err)
 		pods := cache.NewStore(cache.MetaNamespaceKeyFunc)
 		pods.Add(test.input)
-		metagen := NewPodMetadataGenerator(cfg, pods, nil, nil)
+		metagen := NewPodMetadataGenerator(config, pods, nil, nil)
 
 		accessor, err := meta.Accessor(test.input)
 		require.Nil(t, err)
@@ -262,7 +289,9 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) {
 					Labels: map[string]string{
 						"foo": "bar",
 					},
-					Annotations: map[string]string{},
+					Annotations: map[string]string{
+						"app": "production",
+					},
 				},
 				TypeMeta: metav1.TypeMeta{
 					Kind:       "Pod",
@@ -320,24 +349,30 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) {
 				"labels": common.MapStr{
 					"foo": "bar",
 				},
+				"annotations": common.MapStr{
+					"app": "production",
+				},
 			},
 		},
 	}
 
 	for _, test := range tests {
-		cfg := common.NewConfig()
+		config, err := common.NewConfigFrom(map[string]interface{}{
+			"include_annotations": []string{"app"},
+		})
+		assert.Nil(t, err)
 		pods := cache.NewStore(cache.MetaNamespaceKeyFunc)
 		pods.Add(test.input)
 
 		nodes := cache.NewStore(cache.MetaNamespaceKeyFunc)
 		nodes.Add(test.node)
-		nodeMeta := NewNodeMetadataGenerator(cfg, nodes)
+		nodeMeta := NewNodeMetadataGenerator(config, nodes)
 
 		namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc)
 		namespaces.Add(test.namespace)
-		nsMeta := NewNamespaceMetadataGenerator(cfg, namespaces)
+		nsMeta := NewNamespaceMetadataGenerator(config, namespaces)
 
-		metagen := NewPodMetadataGenerator(cfg, pods, nodeMeta, nsMeta)
+		metagen := NewPodMetadataGenerator(config, pods, nodeMeta, nsMeta)
 		t.Run(test.name, func(t *testing.T) {
 			assert.Equal(t, test.output, metagen.Generate(test.input))
 		})

libbeat/processors/add_kubernetes_metadata/indexers_test.go

@@ -257,7 +257,7 @@ func TestFilteredGenMeta(t *testing.T) {
 	assert.Equal(t, ok, true)
 	assert.Equal(t, len(labelMap), 2)
 
-	rawAnnotations, _ := indexers[0].Data.GetValue("pod.annotations")
+	rawAnnotations, _ := indexers[0].Data.GetValue("annotations")
 	assert.Nil(t, rawAnnotations)
 
 	config, err := common.NewConfigFrom(map[string]interface{}{
@@ -284,7 +284,7 @@ func TestFilteredGenMeta(t *testing.T) {
 	ok, _ = labelMap.HasKey("foo")
 	assert.Equal(t, ok, true)
 
-	rawAnnotations, _ = indexers[0].Data.GetValue("pod.annotations")
+	rawAnnotations, _ = indexers[0].Data.GetValue("annotations")
 	assert.NotNil(t, rawAnnotations)
 	annotationsMap, ok := rawAnnotations.(common.MapStr)