From 6e8ed83cdb6c0eb7f477eee72ca14d0299d765d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 2 Jan 2019 23:42:39 +0100 Subject: [PATCH 01/11] Add docker `event` metricset This metricset will retrieve events coming from the Docker events API [0]. Example output: ``` "docker": { "event": { "id": "8c229155b039c2adcb4fab1f987f35a0d1f913dfaa95f3113ed6e4f91eb5398c", "from": "busybox", "type": "container", "action": "die", "actor": { "id": "8c229155b039c2adcb4fab1f987f35a0d1f913dfaa95f3113ed6e4f91eb5398c", "attributes": { "image": "busybox", "name": "distracted_lichterman", "exitCode": "0" } }, "time": "2019-01-02T22:41:02.000Z", "status": "die" } } ``` Actor attributes will container labels in the case of container events, the module will perform dedotting (if enabled) on them. [0] https://docs.docker.com/engine/api/v1.37/#operation/SystemEvents --- metricbeat/docs/fields.asciidoc | 94 ++++++++++++++ metricbeat/docs/modules/docker.asciidoc | 5 + metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list.go | 1 + metricbeat/metricbeat.reference.yml | 1 + .../module/docker/_meta/config.reference.yml | 1 + metricbeat/module/docker/_meta/config.yml | 1 + .../module/docker/event/_meta/data.json | 60 +++++++++ .../module/docker/event/_meta/docs.asciidoc | 1 + .../module/docker/event/_meta/fields.yml | 42 +++++++ metricbeat/module/docker/event/event.go | 119 ++++++++++++++++++ metricbeat/module/docker/fields.go | 2 +- metricbeat/modules.d/docker.yml.disabled | 1 + 13 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 metricbeat/module/docker/event/_meta/data.json create mode 100644 metricbeat/module/docker/event/_meta/docs.asciidoc create mode 100644 metricbeat/module/docker/event/_meta/fields.yml create mode 100644 metricbeat/module/docker/event/event.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 3f58984ee371..45a2bc6c7227 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2760,6 +2760,100 @@ type: scaled_float Number of reads and writes per second +-- + +[float] +== event fields + +Docker event + + + +*`docker.event.status`*:: ++ +-- +type: keyword + +Event status + + +-- + +*`docker.event.id`*:: ++ +-- +type: keyword + +Event id when available + + +-- + +*`docker.event.from`*:: ++ +-- +type: keyword + +Event source + + +-- + +*`docker.event.type`*:: ++ +-- +type: keyword + +The type of object emitting the event + + +-- + +*`docker.event.action`*:: ++ +-- +type: keyword + +The type of event + + +-- + +*`docker.event.time`*:: ++ +-- +type: date + +Timestamp of event + + +-- + +[float] +== actor fields + +Actor + + + +*`docker.event.actor.id`*:: ++ +-- +type: keyword + +The ID of the object emitting the event + + +-- + +*`docker.event.actor.attributes`*:: ++ +-- +type: keyword + +Various key/value attributes of the object, depending on its type + + -- [float] diff --git a/metricbeat/docs/modules/docker.asciidoc b/metricbeat/docs/modules/docker.asciidoc index 689b1d31de11..b6c042364751 100644 --- a/metricbeat/docs/modules/docker.asciidoc +++ b/metricbeat/docs/modules/docker.asciidoc @@ -38,6 +38,7 @@ metricbeat.modules: - "container" - "cpu" - "diskio" + - "event" - "healthcheck" - "info" #- "image" @@ -71,6 +72,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -87,6 +90,8 @@ include::docker/cpu.asciidoc[] include::docker/diskio.asciidoc[] +include::docker/event.asciidoc[] + include::docker/healthcheck.asciidoc[] include::docker/image.asciidoc[] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index cf35597b884f..7bbc284c2ff3 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -22,9 +22,10 @@ This file is generated! See scripts/docs_collector.py |<> beta[] |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.8+| .8+| |<> +.9+| .9+| |<> |<> |<> +|<> experimental[] |<> |<> |<> diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index e2afb2060f6a..f08861b04c21 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -46,6 +46,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/docker/container" _ "github.com/elastic/beats/metricbeat/module/docker/cpu" _ "github.com/elastic/beats/metricbeat/module/docker/diskio" + _ "github.com/elastic/beats/metricbeat/module/docker/event" _ "github.com/elastic/beats/metricbeat/module/docker/healthcheck" _ "github.com/elastic/beats/metricbeat/module/docker/image" _ "github.com/elastic/beats/metricbeat/module/docker/info" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f1a7c6563c2d..26ad57ed02de 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -167,6 +167,7 @@ metricbeat.modules: - "container" - "cpu" - "diskio" + - "event" - "healthcheck" - "info" #- "image" diff --git a/metricbeat/module/docker/_meta/config.reference.yml b/metricbeat/module/docker/_meta/config.reference.yml index 2ac7c913ab32..f651b1385afa 100644 --- a/metricbeat/module/docker/_meta/config.reference.yml +++ b/metricbeat/module/docker/_meta/config.reference.yml @@ -3,6 +3,7 @@ - "container" - "cpu" - "diskio" + - "event" - "healthcheck" - "info" #- "image" diff --git a/metricbeat/module/docker/_meta/config.yml b/metricbeat/module/docker/_meta/config.yml index 44d6b61b58b7..340f2106bfc5 100644 --- a/metricbeat/module/docker/_meta/config.yml +++ b/metricbeat/module/docker/_meta/config.yml @@ -3,6 +3,7 @@ # - container # - cpu # - diskio + # - event # - healthcheck # - info # - memory diff --git a/metricbeat/module/docker/event/_meta/data.json b/metricbeat/module/docker/event/_meta/data.json new file mode 100644 index 000000000000..a589298fa64b --- /dev/null +++ b/metricbeat/module/docker/event/_meta/data.json @@ -0,0 +1,60 @@ +{ + "@timestamp": "2019-01-02T22:41:02.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "7.0.0" + }, + "host": { + "os": { + "platform": "ubuntu", + "version": "17.10 (Artful Aardvark)", + "family": "debian", + "name": "Ubuntu", + "kernel": "4.13.0-46-generic", + "codename": "artful" + }, + "id": "28dac5af0b8f4f638a4b4b7c31cf9321", + "containerized": false, + "name": "X1", + "architecture": "x86_64" + }, + "ecs": { + "version": "1.0.0-beta2" + }, + "agent": { + "hostname": "X1", + "version": "7.0.0", + "type": "metricbeat" + }, + "event": { + "dataset": "docker.event", + "module": "docker" + }, + "metricset": { + "name": "event" + }, + "service": { + "type": "docker", + "address": "/var/run/docker.sock" + }, + "docker": { + "event": { + "status": "", + "id": "", + "from": "", + "type": "network", + "action": "disconnect", + "actor": { + "id": "71bd6e24455add714d79af625628c2f8fd9a2de3ec7f4f0a618bd799da820bf6", + "attributes": { + "type": "bridge", + "container": "8c229155b039c2adcb4fab1f987f35a0d1f913dfaa95f3113ed6e4f91eb5398c", + "name": "bridge" + } + }, + "time": "2019-01-02T22:41:02.000Z" + } + } + } + \ No newline at end of file diff --git a/metricbeat/module/docker/event/_meta/docs.asciidoc b/metricbeat/module/docker/event/_meta/docs.asciidoc new file mode 100644 index 000000000000..0d4cc7a3db3f --- /dev/null +++ b/metricbeat/module/docker/event/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the event metricset of the module docker. diff --git a/metricbeat/module/docker/event/_meta/fields.yml b/metricbeat/module/docker/event/_meta/fields.yml new file mode 100644 index 000000000000..16e3677847a6 --- /dev/null +++ b/metricbeat/module/docker/event/_meta/fields.yml @@ -0,0 +1,42 @@ +- name: event + type: group + description: > + Docker event + fields: + - name: status + type: keyword + description: > + Event status + - name: id + type: keyword + description: > + Event id when available + - name: from + type: keyword + description: > + Event source + - name: type + type: keyword + description: > + The type of object emitting the event + - name: action + type: keyword + description: > + The type of event + - name: time + type: date + description: > + Timestamp of event + - name: actor + type: group + description: > + Actor + fields: + - name: id + type: keyword + description: > + The ID of the object emitting the event + - name: attributes + type: keyword + description: > + Various key/value attributes of the object, depending on its type diff --git a/metricbeat/module/docker/event/event.go b/metricbeat/module/docker/event/event.go new file mode 100644 index 000000000000..8301885a3c77 --- /dev/null +++ b/metricbeat/module/docker/event/event.go @@ -0,0 +1,119 @@ +package event + +import ( + "context" + "fmt" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/events" + "github.com/docker/docker/client" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/docker" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("docker", "event", New, + mb.WithHostParser(docker.HostParser), + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + dockerClient *client.Client + dedot bool + logger *logp.Logger +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + config := docker.DefaultConfig() + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + client, err := docker.NewDockerClient(base.HostData().URI, config) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + dockerClient: client, + dedot: config.DeDot, + logger: logp.NewLogger("docker"), + }, nil +} + +// Run listens for docker events and reports them +func (m *MetricSet) Run(reporter mb.PushReporterV2) { + ctx, cancel := context.WithCancel(context.Background()) + options := types.EventsOptions{ + Since: fmt.Sprintf("%d", time.Now().Unix()), + } + + for { + events, errors := m.dockerClient.Events(ctx, options) + + WATCH: + for { + select { + case event := <-events: + m.logger.Debug("Got a new docker event: %v", event) + m.reportEvent(reporter, event) + + case err := <-errors: + // Restart watch call + m.logger.Error("Error watching for docker events: %v", err) + time.Sleep(1 * time.Second) + break WATCH + + case <-reporter.Done(): + m.logger.Debug("docker", "event watcher stopped") + cancel() + return + } + } + } +} + +func (m *MetricSet) reportEvent(reporter mb.PushReporterV2, event events.Message) { + time := time.Unix(event.Time, 0) + + attributes := common.MapStr{} + for k, v := range event.Actor.Attributes { + if m.dedot { + k = common.DeDot(k) + } + attributes[k] = v + } + + reporter.Event(mb.Event{ + Timestamp: time, + MetricSetFields: common.MapStr{ + "status": event.Status, + "id": event.ID, + "from": event.From, + "type": event.Type, + "action": event.Action, + "actor": common.MapStr{ + "id": event.Actor.ID, + "attributes": attributes, + }, + "time": time, + }, + }) +} diff --git a/metricbeat/module/docker/fields.go b/metricbeat/module/docker/fields.go index c544b4ee9972..ead8bceebbda 100644 --- a/metricbeat/module/docker/fields.go +++ b/metricbeat/module/docker/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsm09v27gSwO/5FIO+wwMeGhtvUezBhwV2WyzWh7ZB25wNmhpJXEuklqTiuJ9+wT+yZYuSLEV2kqI+WvHMj/OfpHILG9wtIBJ0g/IGQDOd4QLefLBfvLkBiFBRyQrNBF/AbzcAAO4hKE20AiqyDKnGCGIpcv9sdgMgMUOicAEJuQFQqZB6RQWPWbKAmGQKbwBihlmkFlbqLXCSY43FfPSuMBKkKAv/TYDHfJY8FjIn5msgPLJwTGlGFZC1KLUX+18FsuSc8QSo4JowjlLNvJQ6TZ1o/5f7JyGwDria0fayIEctGd0rN59jk1WfU6xjtDwnPDp6VsFtcLcV8vRZB6L5vHcCQadEw5YowEekpXEv46BTbKxjFuaSSDSGuSKicRjUB6IRtik6goMJDZ/XFMYwUVCqKa1TqXaSw1pZsSJRJFEpvIju5R3s5besm30/NXE4YIetmX3HUNhCS5DWiaQQehWfmuMAlgmeBB72sNnPN6FJ5uBEDCTLbJTELENVBW1LtB4Bbi/B9tVTHYhsYqXkAWGNyKvwBSGBpoQnGIFinKJ7wAQPO1iTJBxaREqyG+bgZU4StBJnzdJXlE8pel9KrlmO8P7ufpp6t0HJMZsVVAdXryjJMFrFmSCnf+DawwIKlBS5JsnAGnS3/531p1kV454HVEEohj3liTWjm7DHAtHVl5F392DlnUegdkpj/gJsZvPUwTvrmazwdF3kU9vOiXUmDCsuFcoXYDBvJkPT5WBLe7EA69Nu3focxvq2j6dSkaSFjgqJs/+14on139h45L5cXdnbx4nBlEPvWlS7y/uXNTwoPpX5GuWB1IdHAHU/yTO1YeJJQzNTG1jOP0/TPCSS8EQ6Yir6ndIyLzPbu41cBVEpzbbC1LSMxfuuH9o8tIHWYUVxiWHp4MRR0Ae89U43httewCph2n58xgL+MD+18OPZZXMD0os+yLa0lBK59jYuTP1EKk62afWoDGdxR+mJsJBITfQt4NfZu7GZPAh0K1nDbpPkjxX86hJoHPVLySBDr5G/giTydj4nOJ87jc5DVWWeE3m6SZuwExH+WnPKtHpRoLQ731ebW7Y7VU54HUlWM3pP9Npx/5nyrBHeAdaKM0WS6ZSmSDdPmkAvcmwbE5YxnigtkWyCxmRcY9KIjB5LUcFpaf1pFGAEp8u/3MnoXwdze9lARYRB3fiAPLwdG1H+rDBv7KGHksijVeA0GrqOqs9AOrUH8igsqeYMqa9BYhV1s4hSF+Wpd6AnOEagtOjZu+aR6VUjguok4QwZZ5RGuO6P8vPjXf3Y+mHlTFM72GR72G8perCM7FAqYBFyzWLWPCfvyyTfRy4TNnDP2T9lxXqAhIQ9IIeyEByYVi1H5nXMglyQcnkA8w3eAr8FFgPTJqKVVm/dNek2ZTR1Q4DvwG5xEZNIdbazCpGflrQLXq/ZSz+W1+7ZHFH/HduEd03uKsJe5biQHBqHD0zqsjGfwNRXOdY0HXdJmJQZCZWmiS+7DAvJMqCEphg5LAVEKUGZ3RBo0QyyA3fQnxlZYzb2WHHU7LD0Ncjo7YG71r0X4/GTDi6XPBZVuYc1URiB4JBqXajFfB4JqmbuhYQZFfkcecI4ziXGKJFTnJOCzd3zlcRcaFyRgq0e/j/75d38P/OIqSIju1t3j3G7ZRHessP7D099o6B6LWKqpP78gNIG6dHl+eDULkipGiUPJsgpl1J8v8twigLvhzSZ/LskV4Bqf2ulSaW0KIqrmMprOosqtHG8BJPts12m6hmkRpUwP6H4aS8VSgeHqTCHrdpBluG3NK3WcFqalS7HXBwdRg2udR+thPHDbesOeUZF2bJR7Dxr6DTQn4SZUlRy3fbuUsZyFtYacEfXoVEPibebVRcmkWqyIvzl61fv6nHVd1T2TnCi5uLZk0tUNqtAobYzUMek35hT4JzggXOudc9E/9gCXRPcdp/f1Dna8ffK7TzHuz4nj8/g+I/ksaIOXO/Di3S1BW11L7y0dDqxa8XGUW+FfNKp6ScnYuKewEzNjgkN58aovl2B7kVbVS3zAh+Qk084715yKnLTsr0j/IvWh7PuoQn8XLchp7MIqxZmZbYnRyS759eepB5B5jUeCAtCN9gslbXzSSlFY4sEk42zTrzZM56P5P/gCiN2N1PtJPk6CfO51In4ERNGVAt7sQmzJ3w5CXM+0vUSppvp0GDWomz5Z45rdRn38vnxv1nYW6PT49/Xk0VTtZ3pwuFnu7lMu7li+rT0nB8wfaZqQtOnz8/mM7L5/BsAAP//24R2lA==" + return "eJzsm0uP2koWgPf9K44yi5FGadCMolmwGGkmmdGwSNJKuu8WFeVjqItd5VsPaPLrr+phMHbZgDF0dxQvMZzz1anzqgf3sMLtBBJBVyjvADTTGU7g3Sf3wbs7gAQVlazQTPAJ/OsOAMC/BKWJVkBFliHVmEAqRR7eje4AJGZIFE5gQe4A1FJIPaOCp2wxgZRkCu8AUoZZoiZO6j1wkmOFxT56W1gJUpgifBLhsc+Up0LmxH4MhCcOjinNqAIyF0YHsX9VIA3njC+ACq4J4yjVKEip0lSJdt/cvYmBdcBVjLaTBTlqyehOuX0OTVY+daxDtDwnPDl4V8KtcLsRsv6uA9E+H71A0EuiYUMU4DNSY6eXcdBLbIxjFOeSSDTGuRKi8TyoT0QjbJboCfYmtHxBUxzDeoFRQ1qnVO0lx7WyYkaSRKJSeBXd0wfYyW8ZN/tRN3HcYc8bM/uBMbeFFietEkkh9Cytm2MPlgm+iLw8wuaeR6FJ5uFECiTLnJekLENVOm2Ltx4Abq7B9j1Q7YlcYC3JGmGOyEv3BSGBLglfYAKKcYr+BRM8PsGaLOKuRaQk2/MmeJqTBTqJo2bqK8wlSe+b4ZrlCB8fnobJdyuUHLNRQXV09IqSDJNZmglS/4IvDxMoUFLkmizOzEEPu9+5+bSjYjzwgCoIxfhMBWLN6Co+YxHvOhaRD0/g5J1GoLZKY/4KbObi1MN769moCHRd5EPbzov1JowrNgrlKzBYMJOl6ZpgR3s1Bzum3U3rSxjrcedPRpFFCx0VEkd/a8UT89+x8cp/OLvxbB8GBlMevWtQ7VN+fFjnO8UXk89R7kmDe0RQd508UysmLmqamVrBdPx1mOIhkcQ70h5d0b8pNbnJXO22chUkRtplhc1pGUt3VT+2eGgDrcKK4hrN0n4Se0Hv8eZb3WhujwKWAdP24xMG8B/7Uwffn102FyBH0c+yLTVSItfBxoXNn0hFbZlW9cp4FHekngQLidR63wT+OfrQN5LPAt1I1rDbIPHjBL+5AOpH/VoiyNJr5G8giIKdT3HOlw6j01CVyXMi64u0ASsR4W81pmypFwVKt/J9s7HlqlM5CW8jyCpGP+K9rt1/oThruHeEteTENXJ9Ue/pNzrrcroazKG3G/9rdcek7nYaB9z59cpY4jdayZqwjMwzjOpNpcgHH6YwksbVWdnDqXtcovu59Si/JgLMmdZliNZnfM9BqJV5HZJ2rXZRGNV5/kb6I8tRaZIX3RoJ1aKelC6oT01hx+pMw6/hiKFP4IBg8OmnMv+eNvkHhtFasrnpKi4X8f1GJBNGWSHjNckMVjQeUr+3ORV5YrkFB6bVYZSUxEskmV7SJdLVAMlw4NOrlLCM8YXSEskq6m+Ma1w0CuQRS1LBqXFlzSrABOrDv17G/v/e3EE2UJHEs1rM1XpHmRMWjH3u2QzyZBbJJdCVaE5AqtsDeRKXVJkMqW9B4hR1swijC9NMBINEeRWlRc9uap6ZnjU8qEoSj5B+Rmm46y4f54ebm33zh5MzTO5oaX16hI4tCh4sI1uUCliCXLOUNY8Lj0VSaKev4zbwxNkfpmTdQ8KCrZGDKUIdiJ8cVjELckXK6R4sVCwH/B5YCkxbj1Zavfe3RTZLRpd+LRQWIn5wCZNIdbZ1CpHXU9oVbxm4uw8sr1w38ETHrxoMeOTuT2TdibZ3yXP9cM2kNo1lGgx9ou1M03GkjguTkVhqGvjM37KQLANK6BITj6WAKCUoc/siWjSdbM8dnc+MzDHre7rSq3eYhhxk9R6Bu9XxP+PpRec3U56KMt3DnChMbLO61LpQk/E4EVSN/L2sERX5GPmCcRxLTFEipzgmBRv79zOJudA4IwWbrf8++seH8V/GCVNFRrb3/jj3fsMSvGf7a2CXXqwqb4cNFdRf1yidkx7cITo7tAtiVCPlwQAx5UOK7zZbvKLINbkmU7hSdwOo9st7TSqlRVHcxFRB00lUsf2zazC5OttlqmvsIYUOJXR7S6F0tJmKc7isHWU5/7C61RpeSzPT5ZiLgz35s3PdZyehf3PbukIeUWFaFoqdW66dBvofYTYVGa7brnBmLGdxrZHp6No7P0IS7ObUxUmkGiwJf/v+PUx1v+zbK3oHOFjw/hzIJSoXVaBQux6oo9Nv9ClwivPAKbdbTkT/3AJdEdx2ramps/fEPym/8uw/9Tl5foGJ/0yeS+rILSd4lVPtQFunF15bONXsWrJx1BshL9o1/eJFDFwTmM3ZKaEDHomUoDvRTlVLvxA/AInH5AXHflNORW5LdpiI8H+T/ZHfuQH8UofC9V6ElQNzMtuDI5Hd/euRoO5BFjTuCQtCV9hMlZX9SSlFY4kEg7WzXrw74DgZKXzhBi12N1NlJ/k2AfPV6IX4GQNGlAN7tQGzI3w9AXM60u0CpptpX2DmwrT8p+1WVcb/B+fw32bu1Ki+/ft2omiosjOcO/wqN9cpNzcMn5aa8xOGz1BFaPjw+VV8ehafPwMAAP//xPSx5w==" } diff --git a/metricbeat/modules.d/docker.yml.disabled b/metricbeat/modules.d/docker.yml.disabled index 216a7a30f3c0..f414279857e6 100644 --- a/metricbeat/modules.d/docker.yml.disabled +++ b/metricbeat/modules.d/docker.yml.disabled @@ -6,6 +6,7 @@ # - container # - cpu # - diskio + # - event # - healthcheck # - info # - memory From be75f8b3fc2fef4151005de5ff751e1b8d65245c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 2 Jan 2019 23:59:34 +0100 Subject: [PATCH 02/11] Add tests --- .../module/docker/event/_meta/data.json | 83 +++++++------------ .../docker/event/event_integration_test.go | 71 ++++++++++++++++ metricbeat/tests/system/test_docker.py | 26 ++++++ 3 files changed, 127 insertions(+), 53 deletions(-) create mode 100644 metricbeat/module/docker/event/event_integration_test.go diff --git a/metricbeat/module/docker/event/_meta/data.json b/metricbeat/module/docker/event/_meta/data.json index a589298fa64b..913f4d8b4563 100644 --- a/metricbeat/module/docker/event/_meta/data.json +++ b/metricbeat/module/docker/event/_meta/data.json @@ -1,60 +1,37 @@ { - "@timestamp": "2019-01-02T22:41:02.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "7.0.0" - }, - "host": { - "os": { - "platform": "ubuntu", - "version": "17.10 (Artful Aardvark)", - "family": "debian", - "name": "Ubuntu", - "kernel": "4.13.0-46-generic", - "codename": "artful" - }, - "id": "28dac5af0b8f4f638a4b4b7c31cf9321", - "containerized": false, - "name": "X1", - "architecture": "x86_64" - }, - "ecs": { - "version": "1.0.0-beta2" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "agent": { - "hostname": "X1", - "version": "7.0.0", - "type": "metricbeat" + "hostname": "host.example.com", + "name": "host.example.com" }, - "event": { - "dataset": "docker.event", - "module": "docker" + "docker": { + "event": { + "action": "exec_create: /bin/sh -c curl -f http://localhost", + "actor": { + "attributes": { + "com_docker_compose_config-hash": "304a077f5d6b5673cdc2ef567dcb762ddac985514d3b73c1563c2f1ae2f4bbe9", + "com_docker_compose_container-number": "1", + "com_docker_compose_oneoff": "False", + "com_docker_compose_project": "metricbeat", + "com_docker_compose_service": "apache", + "com_docker_compose_version": "1.21.0", + "image": "metricbeat_apache", + "name": "metricbeat_apache_1" + }, + "id": "a591e701988ebca0a62b892f99b42fd492fc3fd4581cd630f18a53abd710dbd0" + }, + "from": "metricbeat_apache", + "id": "a591e701988ebca0a62b892f99b42fd492fc3fd4581cd630f18a53abd710dbd0", + "status": "exec_create: /bin/sh -c curl -f http://localhost", + "time": "2019-01-02T23:29:38Z", + "type": "container" + } }, - "metricset": { - "name": "event" + "event": { + "dataset": "event", + "module": "docker" }, "service": { - "type": "docker", - "address": "/var/run/docker.sock" - }, - "docker": { - "event": { - "status": "", - "id": "", - "from": "", - "type": "network", - "action": "disconnect", - "actor": { - "id": "71bd6e24455add714d79af625628c2f8fd9a2de3ec7f4f0a618bd799da820bf6", - "attributes": { - "type": "bridge", - "container": "8c229155b039c2adcb4fab1f987f35a0d1f913dfaa95f3113ed6e4f91eb5398c", - "name": "bridge" - } - }, - "time": "2019-01-02T22:41:02.000Z" - } + "type": "docker" } - } - \ No newline at end of file +} \ No newline at end of file diff --git a/metricbeat/module/docker/event/event_integration_test.go b/metricbeat/module/docker/event/event_integration_test.go new file mode 100644 index 000000000000..9a2538f6207e --- /dev/null +++ b/metricbeat/module/docker/event/event_integration_test.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package event + +import ( + "testing" + "time" + + "github.com/elastic/beats/auditbeat/core" + "github.com/elastic/beats/libbeat/tests/compose" + "github.com/elastic/beats/metricbeat/mb" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + ms := mbtest.NewPushMetricSetV2(t, getConfig()) + var events []mb.Event + done := make(chan interface{}) + go func() { + events = mbtest.RunPushMetricSetV2(10*time.Second, 1, ms) + close(done) + }() + + compose.EnsureUp(t, "apache") + <-done + + if len(events) == 0 { + t.Fatal("received no events") + } + assertNoErrors(t, events) + + beatEvent := mbtest.StandardizeEvent(ms, events[0], core.AddDatasetToEvent) + mbtest.WriteEventToDataJSON(t, beatEvent, "") +} + +func assertNoErrors(t *testing.T, events []mb.Event) { + t.Helper() + + for _, e := range events { + t.Log(e) + + if e.Error != nil { + t.Errorf("received error: %+v", e.Error) + } + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "docker", + "metricsets": []string{"event"}, + "hosts": []string{"unix:///var/run/docker.sock"}, + } +} diff --git a/metricbeat/tests/system/test_docker.py b/metricbeat/tests/system/test_docker.py index df0e615e8743..c757eea06f64 100644 --- a/metricbeat/tests/system/test_docker.py +++ b/metricbeat/tests/system/test_docker.py @@ -210,6 +210,32 @@ def test_image_fields(self): self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_event_fields(self): + """ + test event fields + """ + self.render_config_template(modules=[{ + "name": "docker", + "metricsets": ["event"], + "hosts": ["unix:///var/run/docker.sock"], + "period": "10s", + }]) + + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + self.assert_no_logged_warnings(["Container stopped when recovering stats", + "An error occurred while getting docker stats"]) + + output = self.read_output_json() + evt = output[0] + + if 'attributes' in evt["docker"]["event"]["actor"]: + del evt["docker"]["event"]["actor"]["attributes"] + + self.assert_fields_are_documented(evt) + def remove_labels(self, evt): if 'labels' in evt["docker"]["container"]: From 8f647ab2282d0044fdf1cb436e1caec01a5d413d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 3 Jan 2019 00:34:46 +0100 Subject: [PATCH 03/11] Update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 38d02bd0b5dc..2a8d3d4a3f79 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -99,6 +99,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `key` metricset to the Redis module. {issue}9582[9582] {pull}9657[9657] {pull}9746[9746] - Add `socket_summary` metricset to system defaults, removing experimental tag and supporting Windows {pull}9709[9709] +- Add docker `event` metricset. {pull}9856[9856] *Packetbeat* From a9b753ebaed105cb5571594f7438366809ddfc44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 3 Jan 2019 00:38:47 +0100 Subject: [PATCH 04/11] Add missing header --- metricbeat/module/docker/event/event.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/metricbeat/module/docker/event/event.go b/metricbeat/module/docker/event/event.go index 8301885a3c77..7177b5311d6c 100644 --- a/metricbeat/module/docker/event/event.go +++ b/metricbeat/module/docker/event/event.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package event import ( From 820c7b17955c7d1049bf1e307c615ecb6032a146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 3 Jan 2019 00:47:40 +0100 Subject: [PATCH 05/11] Add missing file --- metricbeat/docs/modules/docker/event.asciidoc | 23 +++++++++++++++++++ x-pack/metricbeat/metricbeat.reference.yml | 1 + 2 files changed, 24 insertions(+) create mode 100644 metricbeat/docs/modules/docker/event.asciidoc diff --git a/metricbeat/docs/modules/docker/event.asciidoc b/metricbeat/docs/modules/docker/event.asciidoc new file mode 100644 index 000000000000..a8923803b044 --- /dev/null +++ b/metricbeat/docs/modules/docker/event.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-docker-event]] +=== Docker event metricset + +experimental[] + +include::../../../module/docker/event/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/docker/event/_meta/data.json[] +---- diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1dea6f6a90b0..954e8e42d19b 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -167,6 +167,7 @@ metricbeat.modules: - "container" - "cpu" - "diskio" + - "event" - "healthcheck" - "info" #- "image" From 93667329985aeab472153cb81e09e623dae5d31f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 3 Jan 2019 12:07:13 +0100 Subject: [PATCH 06/11] Adapt to ECS --- metricbeat/docs/fields.asciidoc | 18 ++++++++++++----- metricbeat/docs/modules/docker/event.asciidoc | 2 -- metricbeat/docs/modules_list.asciidoc | 2 +- .../module/docker/event/_meta/data.json | 12 +++++------ .../module/docker/event/_meta/fields.yml | 20 ++++++++++++++----- metricbeat/module/docker/event/event.go | 14 ++++++++----- metricbeat/module/docker/fields.go | 2 +- 7 files changed, 45 insertions(+), 25 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 45a2bc6c7227..4b8d1575044d 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -2782,7 +2782,9 @@ Event status *`docker.event.id`*:: + -- -type: keyword +type: alias + +alias to: event.id Event id when available @@ -2802,7 +2804,9 @@ Event source *`docker.event.type`*:: + -- -type: keyword +type: alias + +alias to: event.kind The type of object emitting the event @@ -2812,7 +2816,9 @@ The type of object emitting the event *`docker.event.action`*:: + -- -type: keyword +type: alias + +alias to: event.action The type of event @@ -2822,7 +2828,9 @@ The type of event *`docker.event.time`*:: + -- -type: date +type: alias + +alias to: event.created Timestamp of event @@ -2849,7 +2857,7 @@ The ID of the object emitting the event *`docker.event.actor.attributes`*:: + -- -type: keyword +type: object Various key/value attributes of the object, depending on its type diff --git a/metricbeat/docs/modules/docker/event.asciidoc b/metricbeat/docs/modules/docker/event.asciidoc index a8923803b044..fd187b0fb3bf 100644 --- a/metricbeat/docs/modules/docker/event.asciidoc +++ b/metricbeat/docs/modules/docker/event.asciidoc @@ -5,8 +5,6 @@ This file is generated! See scripts/docs_collector.py [[metricbeat-metricset-docker-event]] === Docker event metricset -experimental[] - include::../../../module/docker/event/_meta/docs.asciidoc[] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 7bbc284c2ff3..80ff5cbd7be3 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -25,7 +25,7 @@ This file is generated! See scripts/docs_collector.py .9+| .9+| |<> |<> |<> -|<> experimental[] +|<> |<> |<> |<> diff --git a/metricbeat/module/docker/event/_meta/data.json b/metricbeat/module/docker/event/_meta/data.json index 913f4d8b4563..6885e3890dee 100644 --- a/metricbeat/module/docker/event/_meta/data.json +++ b/metricbeat/module/docker/event/_meta/data.json @@ -6,7 +6,6 @@ }, "docker": { "event": { - "action": "exec_create: /bin/sh -c curl -f http://localhost", "actor": { "attributes": { "com_docker_compose_config-hash": "304a077f5d6b5673cdc2ef567dcb762ddac985514d3b73c1563c2f1ae2f4bbe9", @@ -18,17 +17,18 @@ "image": "metricbeat_apache", "name": "metricbeat_apache_1" }, - "id": "a591e701988ebca0a62b892f99b42fd492fc3fd4581cd630f18a53abd710dbd0" + "id": "b7bd093f744ec9e2da9aea475c19c3a7fd9a4bdb392d79c7f1745498a7cc5415" }, "from": "metricbeat_apache", - "id": "a591e701988ebca0a62b892f99b42fd492fc3fd4581cd630f18a53abd710dbd0", - "status": "exec_create: /bin/sh -c curl -f http://localhost", - "time": "2019-01-02T23:29:38Z", - "type": "container" + "status": "exec_create: /bin/sh -c curl -f http://localhost" } }, "event": { + "action": "exec_create: /bin/sh -c curl -f http://localhost", + "created": "2019-01-03T11:12:57Z", "dataset": "event", + "id": "b7bd093f744ec9e2da9aea475c19c3a7fd9a4bdb392d79c7f1745498a7cc5415", + "kind": "container", "module": "docker" }, "service": { diff --git a/metricbeat/module/docker/event/_meta/fields.yml b/metricbeat/module/docker/event/_meta/fields.yml index 16e3677847a6..44b87db4c2da 100644 --- a/metricbeat/module/docker/event/_meta/fields.yml +++ b/metricbeat/module/docker/event/_meta/fields.yml @@ -2,13 +2,16 @@ type: group description: > Docker event + release: ga fields: - name: status type: keyword description: > Event status - name: id - type: keyword + type: alias + path: event.id + migration: true description: > Event id when available - name: from @@ -16,15 +19,21 @@ description: > Event source - name: type - type: keyword + type: alias + path: event.kind + migration: true description: > The type of object emitting the event - name: action - type: keyword + type: alias + path: event.action + migration: true description: > The type of event - name: time - type: date + type: alias + path: event.created + migration: true description: > Timestamp of event - name: actor @@ -37,6 +46,7 @@ description: > The ID of the object emitting the event - name: attributes - type: keyword + type: object + object_type: keyword description: > Various key/value attributes of the object, depending on its type diff --git a/metricbeat/module/docker/event/event.go b/metricbeat/module/docker/event/event.go index 7177b5311d6c..03cf839f06a1 100644 --- a/metricbeat/module/docker/event/event.go +++ b/metricbeat/module/docker/event/event.go @@ -110,7 +110,7 @@ func (m *MetricSet) Run(reporter mb.PushReporterV2) { func (m *MetricSet) reportEvent(reporter mb.PushReporterV2, event events.Message) { time := time.Unix(event.Time, 0) - attributes := common.MapStr{} + attributes := make(map[string]string, len(event.Actor.Attributes)) for k, v := range event.Actor.Attributes { if m.dedot { k = common.DeDot(k) @@ -120,17 +120,21 @@ func (m *MetricSet) reportEvent(reporter mb.PushReporterV2, event events.Message reporter.Event(mb.Event{ Timestamp: time, + RootFields: common.MapStr{ + "event": common.MapStr{ + "kind": event.Type, + "action": event.Action, + "created": time, + "id": event.ID, + }, + }, MetricSetFields: common.MapStr{ "status": event.Status, - "id": event.ID, "from": event.From, - "type": event.Type, - "action": event.Action, "actor": common.MapStr{ "id": event.Actor.ID, "attributes": attributes, }, - "time": time, }, }) } diff --git a/metricbeat/module/docker/fields.go b/metricbeat/module/docker/fields.go index ead8bceebbda..cfb3c3e2bf56 100644 --- a/metricbeat/module/docker/fields.go +++ b/metricbeat/module/docker/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsm0uP2koWgPf9K44yi5FGadCMolmwGGkmmdGwSNJKuu8WFeVjqItd5VsPaPLrr+phMHbZgDF0dxQvMZzz1anzqgf3sMLtBBJBVyjvADTTGU7g3Sf3wbs7gAQVlazQTPAJ/OsOAMC/BKWJVkBFliHVmEAqRR7eje4AJGZIFE5gQe4A1FJIPaOCp2wxgZRkCu8AUoZZoiZO6j1wkmOFxT56W1gJUpgifBLhsc+Up0LmxH4MhCcOjinNqAIyF0YHsX9VIA3njC+ACq4J4yjVKEip0lSJdt/cvYmBdcBVjLaTBTlqyehOuX0OTVY+daxDtDwnPDl4V8KtcLsRsv6uA9E+H71A0EuiYUMU4DNSY6eXcdBLbIxjFOeSSDTGuRKi8TyoT0QjbJboCfYmtHxBUxzDeoFRQ1qnVO0lx7WyYkaSRKJSeBXd0wfYyW8ZN/tRN3HcYc8bM/uBMbeFFietEkkh9Cytm2MPlgm+iLw8wuaeR6FJ5uFECiTLnJekLENVOm2Ltx4Abq7B9j1Q7YlcYC3JGmGOyEv3BSGBLglfYAKKcYr+BRM8PsGaLOKuRaQk2/MmeJqTBTqJo2bqK8wlSe+b4ZrlCB8fnobJdyuUHLNRQXV09IqSDJNZmglS/4IvDxMoUFLkmizOzEEPu9+5+bSjYjzwgCoIxfhMBWLN6Co+YxHvOhaRD0/g5J1GoLZKY/4KbObi1MN769moCHRd5EPbzov1JowrNgrlKzBYMJOl6ZpgR3s1Bzum3U3rSxjrcedPRpFFCx0VEkd/a8UT89+x8cp/OLvxbB8GBlMevWtQ7VN+fFjnO8UXk89R7kmDe0RQd508UysmLmqamVrBdPx1mOIhkcQ70h5d0b8pNbnJXO22chUkRtplhc1pGUt3VT+2eGgDrcKK4hrN0n4Se0Hv8eZb3WhujwKWAdP24xMG8B/7Uwffn102FyBH0c+yLTVSItfBxoXNn0hFbZlW9cp4FHekngQLidR63wT+OfrQN5LPAt1I1rDbIPHjBL+5AOpH/VoiyNJr5G8giIKdT3HOlw6j01CVyXMi64u0ASsR4W81pmypFwVKt/J9s7HlqlM5CW8jyCpGP+K9rt1/oThruHeEteTENXJ9Ue/pNzrrcroazKG3G/9rdcek7nYaB9z59cpY4jdayZqwjMwzjOpNpcgHH6YwksbVWdnDqXtcovu59Si/JgLMmdZliNZnfM9BqJV5HZJ2rXZRGNV5/kb6I8tRaZIX3RoJ1aKelC6oT01hx+pMw6/hiKFP4IBg8OmnMv+eNvkHhtFasrnpKi4X8f1GJBNGWSHjNckMVjQeUr+3ORV5YrkFB6bVYZSUxEskmV7SJdLVAMlw4NOrlLCM8YXSEskq6m+Ma1w0CuQRS1LBqXFlzSrABOrDv17G/v/e3EE2UJHEs1rM1XpHmRMWjH3u2QzyZBbJJdCVaE5AqtsDeRKXVJkMqW9B4hR1swijC9NMBINEeRWlRc9uap6ZnjU8qEoSj5B+Rmm46y4f54ebm33zh5MzTO5oaX16hI4tCh4sI1uUCliCXLOUNY8Lj0VSaKev4zbwxNkfpmTdQ8KCrZGDKUIdiJ8cVjELckXK6R4sVCwH/B5YCkxbj1Zavfe3RTZLRpd+LRQWIn5wCZNIdbZ1CpHXU9oVbxm4uw8sr1w38ETHrxoMeOTuT2TdibZ3yXP9cM2kNo1lGgx9ou1M03GkjguTkVhqGvjM37KQLANK6BITj6WAKCUoc/siWjSdbM8dnc+MzDHre7rSq3eYhhxk9R6Bu9XxP+PpRec3U56KMt3DnChMbLO61LpQk/E4EVSN/L2sERX5GPmCcRxLTFEipzgmBRv79zOJudA4IwWbrf8++seH8V/GCVNFRrb3/jj3fsMSvGf7a2CXXqwqb4cNFdRf1yidkx7cITo7tAtiVCPlwQAx5UOK7zZbvKLINbkmU7hSdwOo9st7TSqlRVHcxFRB00lUsf2zazC5OttlqmvsIYUOJXR7S6F0tJmKc7isHWU5/7C61RpeSzPT5ZiLgz35s3PdZyehf3PbukIeUWFaFoqdW66dBvofYTYVGa7brnBmLGdxrZHp6No7P0IS7ObUxUmkGiwJf/v+PUx1v+zbK3oHOFjw/hzIJSoXVaBQux6oo9Nv9ClwivPAKbdbTkT/3AJdEdx2ramps/fEPym/8uw/9Tl5foGJ/0yeS+rILSd4lVPtQFunF15bONXsWrJx1BshL9o1/eJFDFwTmM3ZKaEDHomUoDvRTlVLvxA/AInH5AXHflNORW5LdpiI8H+T/ZHfuQH8UofC9V6ElQNzMtuDI5Hd/euRoO5BFjTuCQtCV9hMlZX9SSlFY4kEg7WzXrw74DgZKXzhBi12N1NlJ/k2AfPV6IX4GQNGlAN7tQGzI3w9AXM60u0CpptpX2DmwrT8p+1WVcb/B+fw32bu1Ki+/ft2omiosjOcO/wqN9cpNzcMn5aa8xOGz1BFaPjw+VV8ehafPwMAAP//xPSx5w==" + return "eJzsm0uP47gRgO/9KwqbQ4Bg2kaCRQ4+BEh2EsSH2RnszORq0FTJrlgiFZLqbu+vD/iQLUuUZMuyu3uxOlp28WO9+fAj7HC/gETyHaoHAEMmwwX88NF98MMDQIKaKyoMSbGAvz0AAPiXoA0zGrjMMuQGE0iVzMO72QOAwgyZxgVs2AOA3kplVlyKlDYLSFmm8QEgJcwSvXBSH0GwHGss9jH7wkpQsizCJxEe+yxFKlXO7MfAROLgSBviGthaliaI/aMGVQpBYgNcCsNIoNKzIKVOUyc6fPPwJgbWA1dT2kEW5GgU8cPg9jlVWfU0sU7R8pyJ5ORdBbfD/bNUzXc9iPb5yQsEs2UGnpkGfEFeWvOSALPF1jxmcS6FzGCcK2EGL4P6yAzC8xY9wVGFli+MFMewXlDqKbVTDe0lx0elYsWSRKHWeJOxl1/gIL9j3vRrU8Vxh71szvQrxtwWOpy0TqSkNKu0qY4jWCbFJvJygM0936RhmYeTKbAsc16SUoa6ctoObz0BfL4F29dAdSRygbVlTwhrRFG5L0gFfMvEBhPQJDj6FyRF3MCGbeKuxZRi+8sMvMzZBp3EWTv1FeU1Se+XUhjKEX768n2afLdDJTCbFdxEZ685yzBZpZlkzS/48rCAAhVHYdjmwhz05fA7Z087KxKBB3TBOMYtFYgN8V3cYhHvGorIL9/ByTuPQO+1wfwN6MzFqYf32rNREej6yKfWnRfrVRgfuNSo3oDCgposTZ+BHe3NHGxodGfW11DWt4M/lZptOui4VDj7UyeeXP8XW6/8h6s7W/s0MEh79L5JdZt8eFqXO8XPZb5GdSQN7hFBPXTypHckr2qaSe9gOf88TfFQyOId6Yiu6O+cl3mZudpt5WpISmWXFTanZZQeqn5s8dAFWoeVxS2apaMRR0Ef8dZ702puBwGrgOn68RkT+If9qYMfz67aC5BB9It0y0ulUJig48LmT+SysUyre2U8intST4KFQm69bwF/nf04NpIvAn1W1NLbJPHjBL+7ABpH/VYiyNIbFO8giIKez3HO1w6j81B1medMNRdpE1YiJt5rTNlSLwtUbuX7bmPLVafKCO8jyGpKH/Be1+6/Upy13DvCWnHiEwpzVe/pNzqbci5vO6fehPynJYpJPew/xrtclhFrYhTMbIOuZq2f5bTxTrEAo8oLFzeekhK/b8ueGGVsnWEUOFUyn1w/slQ8PpyVPUpDO2pttV+lo29bdCNb3/arM8CcjKmSRdP3jlNg3MocNYnoTyebRjeyXduOAo6fJVxHTDlqw/KiH5lxI5vJ+Yo63RY2VG9bIQkDoXEGBwSLLT9Wdeg81ztRjDGK1mVfkY1uQUBzG+KqWfyHKZKltkLmTywrscZ1OrcPtgKhSOzspAAy+jQJVPPaIsvMlm+R7yYoHROf9aWMMhIbbRSyXdQrSRjctNqJAU1yKXjpmgA7ACbQnP7tKtm/j+oOsoHLJJ60Yw45OhadsKDsS0+yUCSryBEm9J1vnoHU1AeKJC6pZgxl7kHiBupnkaUpyliwTxDldZSOcQ6meSGzanlQnSQeIeOU0nLXQ9bOT7eCx+YPJ2ea3NHREo4IHVs6PFjG9qg0UILCUErtw9WhSAqLj9u4DXwX9L+yYj1CwoaeUEBZhDoQP2etYxbshpTLI1ioWA74A1AKZKxHa6M/+Ls1z1viW79yDMs2P7mEFHKT7d2AKJop7YZ3MtxNEcprlzM80fDFjAkvKPjza3f+713yUj98ImXK1qIWpj7/d6rpuYCAmzJjsdQ08Q0Jy8KyDDjjW0w8lgamteTkdpGMbDvZkTtqz4ytMRt7FjWqd1iGHGTHHYC712UJEulVp11Lkcoq3cOaaUxss7o1ptCL+TyRXM/8LbYZl/kcxYYEzhWmqFBwnLOC5v79SmEuDa5YQaunP8/+8uP8D/OEdJGx/aM//H58pgQf6Xhp7tpraNVduqmC+vMTKuekJzeuLg7tgpW6lfJggpjyISUOW1N+oMilwjZTuIB4B6juq45tKm1kUdxFVWGks6hiu423YHJ1tk9VA43UqBQWOpTQ7W2lNtFmKs7hsnaU5fKj/U5t+FHamS7HXJ6cYFyc6z45CeOb284V8ozLsmOh2LtB3augfzGyqagUpuvCa0Y5xUeNmKPvpGGAJOjNDRcnUXqyJPzL16/B1OOy76joneAYxvtzIFeoXVSBRuN6oJ5OP7phNeg8cM5doDPRP3VA1wR3XQJrjzna8N+1X3mON33OXl7B8J/YS0UduRMGb9LUDrTTvPDWwqmh14pNoHmW6qpd05+9iIlrAtmcnTIej41RdbsCPYh2Q3X0C/FDmnhMXnFIuhRc5rZkB0OEf+ccD0gvDeDXOkJv9iJUTczJ7A6ORPX3rwNBPYIsjHgkLBjfYTtV1vYnlZKtJRJM1s568e6A42yk8IU7tNj9TLWd5PsEzOfSbORvMWBkNbE3GzAHwrcTMOcj3S9g+pmOBWYty45/AN6ryvh/LJ3+N8+dGjW3f99PFE1VdqZzh9/LzW3KzR3Dp6Pm/AbDZ6oiNH34/F58Rhaf/wcAAP//98L9Zw==" } From 6bb547f0a4b427e0c3ce4d2e2400ff4e324c95f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 3 Jan 2019 12:39:48 +0100 Subject: [PATCH 07/11] Use a cheaper event for tests --- .../module/docker/event/_meta/data.json | 2 +- .../docker/event/event_integration_test.go | 29 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/docker/event/_meta/data.json b/metricbeat/module/docker/event/_meta/data.json index 6885e3890dee..6e7785de12e8 100644 --- a/metricbeat/module/docker/event/_meta/data.json +++ b/metricbeat/module/docker/event/_meta/data.json @@ -25,7 +25,7 @@ }, "event": { "action": "exec_create: /bin/sh -c curl -f http://localhost", - "created": "2019-01-03T11:12:57Z", + "created": "2019-01-03T11:39:08Z", "dataset": "event", "id": "b7bd093f744ec9e2da9aea475c19c3a7fd9a4bdb392d79c7f1745498a7cc5415", "kind": "container", diff --git a/metricbeat/module/docker/event/event_integration_test.go b/metricbeat/module/docker/event/event_integration_test.go index 9a2538f6207e..dd98ab968da8 100644 --- a/metricbeat/module/docker/event/event_integration_test.go +++ b/metricbeat/module/docker/event/event_integration_test.go @@ -23,8 +23,12 @@ import ( "testing" "time" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" + "golang.org/x/net/context" + "github.com/elastic/beats/auditbeat/core" - "github.com/elastic/beats/libbeat/tests/compose" "github.com/elastic/beats/metricbeat/mb" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) @@ -38,7 +42,7 @@ func TestData(t *testing.T) { close(done) }() - compose.EnsureUp(t, "apache") + createEvent(t) <-done if len(events) == 0 { @@ -62,6 +66,27 @@ func assertNoErrors(t *testing.T, events []mb.Event) { } } +func createEvent(t *testing.T) { + client, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + defer client.Close() + + _, err = client.ImagePull(context.Background(), "busybox", types.ImagePullOptions{}) + if err != nil { + t.Fatal(err) + } + + _, err = client.ContainerCreate(context.Background(), &container.Config{ + Image: "busybox", + Cmd: []string{"echo", "foo"}, + }, nil, nil, "") + if err != nil { + t.Fatal(err) + } +} + func getConfig() map[string]interface{} { return map[string]interface{}{ "module": "docker", From 1eea2a7bc17cb420ce4a1b4a10446f7da2983cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 3 Jan 2019 12:40:00 +0100 Subject: [PATCH 08/11] Close docker client on module shutdown --- metricbeat/module/docker/event/event.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metricbeat/module/docker/event/event.go b/metricbeat/module/docker/event/event.go index 03cf839f06a1..9fd16a29c363 100644 --- a/metricbeat/module/docker/event/event.go +++ b/metricbeat/module/docker/event/event.go @@ -82,6 +82,8 @@ func (m *MetricSet) Run(reporter mb.PushReporterV2) { Since: fmt.Sprintf("%d", time.Now().Unix()), } + defer m.dockerClient.Close() + for { events, errors := m.dockerClient.Events(ctx, options) From ed19c9ee421657f9244e565317c407496560b15c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 3 Jan 2019 12:45:06 +0100 Subject: [PATCH 09/11] Update data --- .../module/docker/event/_meta/data.json | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/metricbeat/module/docker/event/_meta/data.json b/metricbeat/module/docker/event/_meta/data.json index 6e7785de12e8..305af3a02182 100644 --- a/metricbeat/module/docker/event/_meta/data.json +++ b/metricbeat/module/docker/event/_meta/data.json @@ -8,26 +8,20 @@ "event": { "actor": { "attributes": { - "com_docker_compose_config-hash": "304a077f5d6b5673cdc2ef567dcb762ddac985514d3b73c1563c2f1ae2f4bbe9", - "com_docker_compose_container-number": "1", - "com_docker_compose_oneoff": "False", - "com_docker_compose_project": "metricbeat", - "com_docker_compose_service": "apache", - "com_docker_compose_version": "1.21.0", - "image": "metricbeat_apache", - "name": "metricbeat_apache_1" + "image": "busybox", + "name": "wizardly_knuth" }, - "id": "b7bd093f744ec9e2da9aea475c19c3a7fd9a4bdb392d79c7f1745498a7cc5415" + "id": "9d4c3af1d4a0268940205aad04a3c6bf434b32f45e0784dec5c283201ef299fc" }, - "from": "metricbeat_apache", - "status": "exec_create: /bin/sh -c curl -f http://localhost" + "from": "busybox", + "status": "create" } }, "event": { - "action": "exec_create: /bin/sh -c curl -f http://localhost", - "created": "2019-01-03T11:39:08Z", + "action": "create", + "created": "2019-01-03T12:44:52+01:00", "dataset": "event", - "id": "b7bd093f744ec9e2da9aea475c19c3a7fd9a4bdb392d79c7f1745498a7cc5415", + "id": "9d4c3af1d4a0268940205aad04a3c6bf434b32f45e0784dec5c283201ef299fc", "kind": "container", "module": "docker" }, From a8199324f636c3b74f1cdc4bdba4576b8f143f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 3 Jan 2019 13:03:49 +0100 Subject: [PATCH 10/11] clean created container --- metricbeat/module/docker/event/event_integration_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/docker/event/event_integration_test.go b/metricbeat/module/docker/event/event_integration_test.go index dd98ab968da8..198564278704 100644 --- a/metricbeat/module/docker/event/event_integration_test.go +++ b/metricbeat/module/docker/event/event_integration_test.go @@ -78,13 +78,15 @@ func createEvent(t *testing.T) { t.Fatal(err) } - _, err = client.ContainerCreate(context.Background(), &container.Config{ + resp, err := client.ContainerCreate(context.Background(), &container.Config{ Image: "busybox", Cmd: []string{"echo", "foo"}, }, nil, nil, "") if err != nil { t.Fatal(err) } + + client.ContainerRemove(context.Background(), resp.ID, types.ContainerRemoveOptions{}) } func getConfig() map[string]interface{} { From 8a490c860812198c289f06e9c54aea025f351c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Thu, 3 Jan 2019 14:49:33 +0100 Subject: [PATCH 11/11] fix image pull --- metricbeat/module/docker/event/event_integration_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/docker/event/event_integration_test.go b/metricbeat/module/docker/event/event_integration_test.go index 198564278704..361cb0e5c95a 100644 --- a/metricbeat/module/docker/event/event_integration_test.go +++ b/metricbeat/module/docker/event/event_integration_test.go @@ -20,6 +20,8 @@ package event import ( + "io" + "os" "testing" "time" @@ -73,10 +75,12 @@ func createEvent(t *testing.T) { } defer client.Close() - _, err = client.ImagePull(context.Background(), "busybox", types.ImagePullOptions{}) + reader, err := client.ImagePull(context.Background(), "busybox", types.ImagePullOptions{}) if err != nil { t.Fatal(err) } + io.Copy(os.Stdout, reader) + reader.Close() resp, err := client.ContainerCreate(context.Background(), &container.Config{ Image: "busybox",