Cherry-pick elastic#9408 to 6.x: Add event.dataset field (elastic#9561)

* Add event.dataset field (elastic#9408)

This brings us closer to ECS compliance by adding the event.dataset field.

All heartbeat events are considered part of the 'uptime' dataset, even though their data may differ.

The key thing defining this dataset is the presence of the `monitor.status` field

(cherry picked from commit 5e54acb)

* Fix test failures by cleaning more keys

* Add event.dataset to common fields

filebeat/tests/system/test_modules.py

@@ -190,7 +190,7 @@ def _test_expected_events(self, test_file, objects):
 
 def clean_keys(obj):
     # These keys are host dependent
-    host_keys = ["host.name", "beat.hostname", "beat.name"]
+    host_keys = ["host.name", "beat.hostname", "beat.name", "agent.ephemeral_id", "agent.id"]
     # The create timestamps area always new
     time_keys = ["read_timestamp", "event.created"]
     # source path and beat.version can be different for each run

heartbeat/_meta/fields.common.yml

@@ -2,6 +2,14 @@
   title: "Common heartbeat monitor"
   description:
   fields:
+    - name: event
+      type: group
+      description: Used for context information about the log or metric event itself.
+      fields:
+        - name: dataset
+          type: keyword
+          description: Name of the dataset.
+
     - name: monitor
       type: group
       description: >

heartbeat/docs/fields.asciidoc

@@ -329,6 +329,21 @@ Region in which this host is running.
 None
 
 
+[float]
+== event fields
+
+Used for context information about the log or metric event itself.
+
+
+*`event.dataset`*::
++
+--
+type: keyword
+
+Name of the dataset.
+
+--
+
 [float]
 == monitor fields
 

heartbeat/monitors/task.go

@@ -123,9 +123,11 @@ func (t *task) makeSchedulerTaskFunc() scheduler.TaskFunc {
 // Start schedules this task for execution.
 func (t *task) Start() {
 	var err error
+
 	t.client, err = t.monitor.pipelineConnector.ConnectWith(beat.ClientConfig{
 		EventMetadata: t.config.EventMetadata,
 		Processor:     t.processors,
+		Fields:        common.MapStr{"event": common.MapStr{"dataset": "uptime"}},
 	})
 	if err != nil {
 		logp.Err("could not start monitor: %v", err)

heartbeat/tests/system/test_base.py

@@ -4,6 +4,7 @@
 from heartbeat import BaseTest
 from elasticsearch import Elasticsearch
 from beat.beat import INTEGRATION_TESTS
+import nose.tools
 
 
 class Test(BaseTest):
@@ -130,3 +131,33 @@ def test_template(self):
         assert exit_code == 0
         assert self.log_contains('Loaded index template')
         assert len(es.cat.templates(name='heartbeat-*', h='name')) > 0
+
+    def test_dataset(self):
+        """
+        Test that event.dataset is set to `uptime`
+        """
+        self.render_config_template(
+            monitors=[
+                {
+                    "type": "http",
+                    "urls": ["http://localhost:9200"]
+                },
+                {
+                    "type": "tcp",
+                    "hosts": ["localhost:9200"]
+                }
+            ]
+        )
+
+        try:
+            heartbeat_proc = self.start_beat()
+            self.wait_until(lambda: self.output_lines() >= 2)
+        finally:
+            heartbeat_proc.check_kill_and_wait()
+
+        for output in self.read_output():
+            nose.tools.assert_equal(
+                output["event.dataset"],
+                "uptime",
+                "Check for event.dataset in {} failed".format(output)
+            )