Migrate add_docker_metadata to ECS (elastic#9412)

Migrate the docker fields to ECS container fields. * docker.container.id -> container.id * docker.container.image -> container.image.name * docker.container.name -> container.name * docker.container.labels -> container.labels
urso · Dec 27, 2018 · f43683f · f43683f
1 parent d0b54f2
commit f43683f
Show file tree

Hide file tree

Showing 22 changed files with 159 additions and 139 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -18,6 +18,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename beat.timezone to event.timezone. {pull}9458[9458]
 - Use _doc as document type. {pull}9056[9056]{pull}9573[9573]
 - Update to Golang 1.11.3. {pull}9560[9560]
+- Migrate add_docker_metadata fields to ECS.
 
 *Auditbeat*
 

diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc
@@ -2670,30 +2670,27 @@ Docker stats collected from Docker.
 *`docker.container.id`*::
 +
 --
-type: keyword
-
-Unique container id.
+type: alias
 
+alias to: container.id
 
 --
 
 *`docker.container.image`*::
 +
 --
-type: keyword
-
-Name of the image the container was built on.
+type: alias
 
+alias to: container.image.name
 
 --
 
 *`docker.container.name`*::
 +
 --
-type: keyword
-
-Container name.
+type: alias
 
+alias to: container.name
 
 --
 

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -54,6 +54,28 @@
   alias6: true
   alias: true
 
+# Docker processor
+- from: docker.container.id
+  to: container.id
+  alias: true
+  alias6: true
+
+- from: docker.container.image
+  to: container.image.name
+  alias6: true
+  alias: true
+
+- from: docker.container.name
+  to: container.name
+  alias6: true
+  alias: true
+
+- from: docker.container.labels  # TODO: How to map these?
+  to: container.labels
+  alias6: false
+  alias: true
+
+
 # Filebeat modules
 
 ## Suricata module

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -696,30 +696,27 @@ Docker stats collected from Docker.
 *`docker.container.id`*::
 +
 --
-type: keyword
-
-Unique container id.
+type: alias
 
+alias to: container.id
 
 --
 
 *`docker.container.image`*::
 +
 --
-type: keyword
-
-Name of the image the container was built on.
+type: alias
 
+alias to: container.image.name
 
 --
 
 *`docker.container.name`*::
 +
 --
-type: keyword
-
-Container name.
+type: alias
 
+alias to: container.name
 
 --
 

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc
@@ -261,30 +261,27 @@ Docker stats collected from Docker.
 *`docker.container.id`*::
 +
 --
-type: keyword
-
-Unique container id.
+type: alias
 
+alias to: container.id
 
 --
 
 *`docker.container.image`*::
 +
 --
-type: keyword
-
-Name of the image the container was built on.
+type: alias
 
+alias to: container.image.name
 
 --
 
 *`docker.container.name`*::
 +
 --
-type: keyword
-
-Container name.
+type: alias
 
+alias to: container.name
 
 --
 

diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go
diff --git a/journalbeat/docs/fields.asciidoc b/journalbeat/docs/fields.asciidoc
@@ -542,30 +542,27 @@ Docker stats collected from Docker.
 *`docker.container.id`*::
 +
 --
-type: keyword
-
-Unique container id.
+type: alias
 
+alias to: container.id
 
 --
 
 *`docker.container.image`*::
 +
 --
-type: keyword
-
-Name of the image the container was built on.
+type: alias
 
+alias to: container.image.name
 
 --
 
 *`docker.container.name`*::
 +
 --
-type: keyword
-
-Container name.
+type: alias
 
+alias to: container.name
 
 --
 

diff --git a/journalbeat/include/fields.go b/journalbeat/include/fields.go
diff --git a/libbeat/processors/add_docker_metadata/_meta/fields.yml b/libbeat/processors/add_docker_metadata/_meta/fields.yml
@@ -9,18 +9,21 @@
       type: group
       fields:
         - name: container.id
-          type: keyword
-          description: >
-            Unique container id.
+          type: alias
+          path: container.id
+          migration: true
+
         - name: container.image
-          type: keyword
-          description: >
-            Name of the image the container was built on.
+          type: alias
+          path: container.image.name
+          migration: true
+
         - name: container.name
-          type: keyword
-          description: >
-            Container name.
-        - name: container.labels
+          type: alias
+          path: container.name
+          migration: true
+
+        - name: container.labels  # TODO: How to map these?
           type: object
           object_type: keyword
           description: >

diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go
@@ -39,7 +39,7 @@ import (
 
 const (
 	processorName         = "add_docker_metadata"
-	dockerContainerIDKey  = "docker.container.id"
+	dockerContainerIDKey  = "container.id"
 	cgroupCacheExpiration = 5 * time.Minute
 )
 
@@ -89,7 +89,7 @@ func buildDockerMetadataProcessor(cfg *common.Config, watcherConstructor docker.
 			"field":     "source",
 			"separator": string(os.PathSeparator),
 			"index":     config.SourceIndex,
-			"target":    "docker.container.id",
+			"target":    dockerContainerIDKey,
 		})
 		sourceProcessor, err = actions.NewExtractField(procConf)
 		if err != nil {
@@ -168,10 +168,6 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) {
 	container := d.watcher.Container(cid)
 	if container != nil {
 		meta := common.MapStr{}
-		metaIface, ok := event.Fields["docker"]
-		if ok {
-			meta = metaIface.(common.MapStr)
-		}
 
 		if len(container.Labels) > 0 {
 			labels := common.MapStr{}
@@ -187,9 +183,9 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) {
 		}
 
 		meta.Put("container.id", container.ID)
-		meta.Put("container.image", container.Image)
+		meta.Put("container.image.name", container.Image)
 		meta.Put("container.name", container.Name)
-		event.Fields["docker"] = meta.Clone()
+		event.Fields.DeepUpdate(meta.Clone())
 	} else {
 		d.log.Debugf("Container not found: cid=%s", cid)
 	}

diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go
@@ -129,21 +129,21 @@ func TestMatchContainer(t *testing.T) {
 	assert.NoError(t, err, "processing an event")
 
 	assert.EqualValues(t, common.MapStr{
-		"docker": common.MapStr{
-			"container": common.MapStr{
-				"id":    "container_id",
-				"image": "image",
-				"labels": common.MapStr{
-					"a": common.MapStr{
-						"x": "1",
-					},
-					"b": common.MapStr{
-						"value": "2",
-						"foo":   "3",
-					},
+		"container": common.MapStr{
+			"id": "container_id",
+			"image": common.MapStr{
+				"name": "image",
+			},
+			"labels": common.MapStr{
+				"a": common.MapStr{
+					"x": "1",
+				},
+				"b": common.MapStr{
+					"value": "2",
+					"foo":   "3",
 				},
-				"name": "name",
 			},
+			"name": "name",
 		},
 		"foo": "container_id",
 	}, result.Fields)
@@ -178,17 +178,17 @@ func TestMatchContainerWithDedot(t *testing.T) {
 	assert.NoError(t, err, "processing an event")
 
 	assert.EqualValues(t, common.MapStr{
-		"docker": common.MapStr{
-			"container": common.MapStr{
-				"id":    "container_id",
-				"image": "image",
-				"labels": common.MapStr{
-					"a_x":   "1",
-					"b":     "2",
-					"b_foo": "3",
-				},
-				"name": "name",
+		"container": common.MapStr{
+			"id": "container_id",
+			"image": common.MapStr{
+				"name": "image",
 			},
+			"labels": common.MapStr{
+				"a_x":   "1",
+				"b":     "2",
+				"b_foo": "3",
+			},
+			"name": "name",
 		},
 		"foo": "container_id",
 	}, result.Fields)
@@ -228,16 +228,16 @@ func TestMatchSource(t *testing.T) {
 	assert.NoError(t, err, "processing an event")
 
 	assert.EqualValues(t, common.MapStr{
-		"docker": common.MapStr{
-			"container": common.MapStr{
-				"id":    "FABADA",
-				"image": "image",
-				"labels": common.MapStr{
-					"a": "1",
-					"b": "2",
-				},
-				"name": "name",
+		"container": common.MapStr{
+			"id": "FABADA",
+			"image": common.MapStr{
+				"name": "image",
+			},
+			"labels": common.MapStr{
+				"a": "1",
+				"b": "2",
 			},
+			"name": "name",
 		},
 		"source": inputSource,
 	}, result.Fields)
@@ -291,16 +291,16 @@ func TestMatchPIDs(t *testing.T) {
 	assert.NoError(t, err, "initializing add_docker_metadata processor")
 
 	dockerMetadata := common.MapStr{
-		"docker": common.MapStr{
-			"container": common.MapStr{
-				"id":    "FABADA",
-				"image": "image",
-				"labels": common.MapStr{
-					"a": "1",
-					"b": "2",
-				},
-				"name": "name",
+		"container": common.MapStr{
+			"id": "FABADA",
+			"image": common.MapStr{
+				"name": "image",
+			},
+			"labels": common.MapStr{
+				"a": "1",
+				"b": "2",
 			},
+			"name": "name",
 		},
 	}