Remove agent.hostname field from libbeat

agent.hostname is not part of ECS so this removes the field. Instead the agent.name field can be used for this purpose and it it part of ECS. This add a migration alias pointing from agent.hostname to agent.name. Closes #16377
elastic · May 6, 2020 · 4af663b · 4af663b
1 parent 9e7321a
commit 4af663b
Show file tree

Hide file tree

Showing 36 changed files with 98 additions and 72 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -15,6 +15,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change aws_elb autodiscover provider field name from elb_listener.* to aws.elb.*. {issue}16219[16219] {pull}16402{16402}
 - Remove `AddDockerMetadata` and `AddKubernetesMetadata` processors from the `script` processor. They can still be used as normal processors in the configuration. {issue}16349[16349] {pull}16514[16514]
 - Introduce APM libbeat instrumentation, active when running the beat with ELASTIC_APM_ACTIVE=true. {pull}17938[17938]
+- Remove the non-ECS `agent.hostname` field. Use the `agent.name` or `agent.id` fields for an identifier. {issue}16377[16377] {pull}18328[18328]
 
 *Auditbeat*
 

diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc
@@ -2458,10 +2458,9 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
-
+type: alias
 
-type: keyword
+alias to: agent.name
 
 --
 

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/auditbeat/module/auditd/_meta/kibana/7/dashboard/auditbeat-kernel-executions.json b/auditbeat/module/auditd/_meta/kibana/7/dashboard/auditbeat-kernel-executions.json
@@ -164,7 +164,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "process.args", 
                     "auditd.summary.actor.primary", 
                     "auditd.summary.actor.secondary", 

diff --git a/auditbeat/module/auditd/_meta/kibana/7/dashboard/auditbeat-kernel-overview.json b/auditbeat/module/auditd/_meta/kibana/7/dashboard/auditbeat-kernel-overview.json
@@ -146,7 +146,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "auditd.summary.actor.primary", 
                     "auditd.summary.actor.secondary", 
                     "event.action", 

diff --git a/auditbeat/module/auditd/_meta/kibana/7/dashboard/auditbeat-kernel-sockets.json b/auditbeat/module/auditd/_meta/kibana/7/dashboard/auditbeat-kernel-sockets.json
@@ -451,7 +451,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "auditd.summary.how", 
                     "auditd.summary.object.primary", 
                     "auditd.summary.object.secondary", 
@@ -566,7 +566,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "auditd.summary.how", 
                     "auditd.summary.object.primary", 
                     "auditd.summary.object.secondary", 
@@ -673,7 +673,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname", 
+                    "agent.name", 
                     "auditd.summary.how", 
                     "auditd.summary.object.primary", 
                     "auditd.summary.object.secondary", 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -3112,10 +3112,9 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
-
+type: alias
 
-type: keyword
+alias to: agent.name
 
 --
 

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/osquery/_meta/kibana/7/dashboard/osquery-rootkit.json b/filebeat/module/osquery/_meta/kibana/7/dashboard/osquery-rootkit.json
@@ -113,7 +113,7 @@
                             "id": "1", 
                             "params": {
                                 "customLabel": "Hosts", 
-                                "field": "agent.hostname"
+                                "field": "agent.name"
                             }, 
                             "schema": "metric", 
                             "type": "cardinality"
@@ -185,7 +185,7 @@
                 "columns": [
                     "osquery.result.name", 
                     "osquery.result.columns.path", 
-                    "agent.hostname"
+                    "agent.name"
                 ], 
                 "description": "", 
                 "hits": 0, 

diff --git a/filebeat/module/santa/_meta/kibana/7/dashboard/filebeat-santa-log-overview.json b/filebeat/module/santa/_meta/kibana/7/dashboard/filebeat-santa-log-overview.json
@@ -269,7 +269,7 @@
                             "id": "1",
                             "params": {
                                 "customLabel": "Hosts Reporting",
-                                "field": "agent.hostname"
+                                "field": "agent.name"
                             },
                             "schema": "metric",
                             "type": "cardinality"
@@ -373,7 +373,7 @@
         {
             "attributes": {
                 "columns": [
-                    "agent.hostname",
+                    "agent.name",
                     "process.executable",
                     "user.name",
                     "certificate.common_name"

diff --git a/filebeat/tests/system/test_fields.py b/filebeat/tests/system/test_fields.py
@@ -60,10 +60,9 @@ def test_custom_fields_under_root(self):
         assert doc["timestamp"] == 2
         assert "fields" not in doc
 
-    def test_beat_fields(self):
+    def test_agent_name_custom(self):
         """
-        Checks that it's possible to set a custom shipper name. Also
-        tests that agent.hostname has values.
+        Checks that it's possible to set a custom agent name.
         """
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/test.log",
@@ -81,5 +80,23 @@ def test_beat_fields(self):
         doc = output[0]
         assert doc["host.name"] == "testShipperName"
         assert doc["agent.name"] == "testShipperName"
-        assert doc["agent.hostname"] == socket.gethostname()
         assert "fields" not in doc
+
+    def test_agent_name_default(self):
+        """
+        Checks that agent.name defaults to the hostname.
+        """
+        self.render_config_template(
+            path=os.path.abspath(self.working_dir) + "/test.log",
+        )
+
+        with open(self.working_dir + "/test.log", "w") as f:
+            f.write("test message\n")
+
+        filebeat = self.start_beat()
+        self.wait_until(lambda: self.output_has(lines=1))
+        filebeat.check_kill_and_wait()
+
+        output = self.read_output()
+        doc = output[0]
+        assert doc["agent.name"] == socket.gethostname()
diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -214,7 +214,7 @@ def _test_expected_events(self, test_file, objects):
 
 def clean_keys(obj):
     # These keys are host dependent
-    host_keys = ["host.name", "agent.name", "agent.hostname", "agent.type", "agent.ephemeral_id", "agent.id"]
+    host_keys = ["host.name", "agent.name", "agent.type", "agent.ephemeral_id", "agent.id"]
     # The create timestamps area always new
     time_keys = ["event.created"]
     # source path and agent.version can be different for each run

diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc
@@ -40,10 +40,9 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
-
+type: alias
 
-type: keyword
+alias to: agent.name
 
 --
 

diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go
diff --git a/journalbeat/docs/fields.asciidoc b/journalbeat/docs/fields.asciidoc
@@ -33,10 +33,9 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
-
+type: alias
 
-type: keyword
+alias to: agent.name
 
 --
 

diff --git a/journalbeat/include/fields.go b/journalbeat/include/fields.go
diff --git a/libbeat/_meta/fields.common.yml b/libbeat/_meta/fields.common.yml
@@ -5,10 +5,9 @@
     Contains common beat fields available in all event types.
   fields:
     - name: agent.hostname
-      type: keyword
-      description: >
-        Deprecated - use agent.name or agent.id to identify an agent.
-        Hostname of the agent.
+      type: alias
+      path: agent.name
+      migration: true
 
     - name: beat.timezone
       type: alias

diff --git a/libbeat/publisher/processing/default.go b/libbeat/publisher/processing/default.go
@@ -149,9 +149,6 @@ func WithAgentMeta() modifier {
 			"name":         info.Hostname,
 			"type":         info.Beat,
 			"version":      info.Version,
-			// hostname is deprecated. To be removed for 8.0. It's not in ECS.
-			// See https://github.com/elastic/beats/issues/16377.
-			"hostname": info.Hostname,
 		}
 		if info.Name != "" {
 			metadata["name"] = info.Name

diff --git a/libbeat/publisher/processing/default_test.go b/libbeat/publisher/processing/default_test.go
@@ -183,7 +183,6 @@ func TestProcessorsConfigs(t *testing.T) {
 				},
 				"agent": common.MapStr{
 					"ephemeral_id": "123e4567-e89b-12d3-a456-426655440000",
-					"hostname":     "test.host.name",
 					"name":         "test.host.name",
 					"id":           "123e4567-e89b-12d3-a456-426655440001",
 					"type":         "test",
@@ -210,7 +209,6 @@ func TestProcessorsConfigs(t *testing.T) {
 				},
 				"agent": common.MapStr{
 					"ephemeral_id": "123e4567-e89b-12d3-a456-426655440000",
-					"hostname":     "test.host.name",
 					"name":         "other.test.host.name",
 					"id":           "123e4567-e89b-12d3-a456-426655440001",
 					"type":         "test",

diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py
@@ -17,7 +17,7 @@
 
 
 BEAT_REQUIRED_FIELDS = ["@timestamp",
-                        "agent.type", "agent.hostname", "agent.version"]
+                        "agent.type", "agent.name", "agent.version"]
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -4631,10 +4631,9 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
-
+type: alias
 
-type: keyword
+alias to: agent.name
 
 --
 

diff --git a/metricbeat/module/docker/_meta/kibana/7/dashboard/Metricbeat-docker-overview.json b/metricbeat/module/docker/_meta/kibana/7/dashboard/Metricbeat-docker-overview.json
@@ -249,7 +249,7 @@
                             "id": "2", 
                             "params": {
                                 "customLabel": "Hosts", 
-                                "field": "agent.hostname", 
+                                "field": "agent.name", 
                                 "order": "desc", 
                                 "orderBy": "1", 
                                 "size": 5

diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc
@@ -452,10 +452,9 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
-
+type: alias
 
-type: keyword
+alias to: agent.name
 
 --
 

diff --git a/packetbeat/include/fields.go b/packetbeat/include/fields.go
diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py
@@ -9,15 +9,15 @@
 from beat.beat import Proc
 
 TRANS_REQUIRED_FIELDS = ["@timestamp", "type", "status",
-                         "agent.type", "agent.hostname", "agent.version",
+                         "agent.type", "agent.name", "agent.version",
                          "event.kind", "event.category", "event.dataset", "event.start",
                          "source.ip", "destination.ip",
                          "client.ip", "server.ip",
                          "network.type", "network.transport", "network.community_id",
                          ]
 
 FLOWS_REQUIRED_FIELDS = ["@timestamp", "type",
-                         "agent.type", "agent.hostname", "agent.version",
+                         "agent.type", "agent.name", "agent.version",
                          "event.kind", "event.category", "event.dataset", "event.action", "event.start", "event.end", "event.duration",
                          "source.ip", "destination.ip",
                          "flow.id",

diff --git a/packetbeat/tests/system/test_0099_golden_files.py b/packetbeat/tests/system/test_0099_golden_files.py
@@ -64,7 +64,6 @@ def clean_keys(obj):
     keys = [
         "@metadata.version",
         "agent.ephemeral_id",
-        "agent.hostname",
         "agent.id",
         "agent.name",
         "agent.type",

diff --git a/script/ecs-migration-8x.yml b/script/ecs-migration-8x.yml
@@ -0,0 +1,17 @@
+# The ECS migration file contains the information about all the Beats fields
+# which were changed between 7.x and 8.0.
+#
+# This file will be used to generate breaking change documentation and migrate
+# Kibana dashboards. See script/renamed_fields.py and script/kibana-migration.py.
+#
+# The format of the file is as following:
+#
+# - from: <original-name-in-7x> # Field name in 7.x.
+#   to: <new-name-in-8x>        # Field name in 8.0.
+#   alias: <true|false>         # True if a migration alias exists in fields.yml.
+#   beat: <beatname>            # Omit for libbeat fields.
+
+## Libbeat
+- from: agent.hostname
+  to: agent.name
+  alias: true
diff --git a/script/kibana-migration.py b/script/kibana-migration.py
@@ -1,21 +1,23 @@
 import yaml
 import glob
+import argparse
 
 
-def migration():
+def migration(append_ecs):
     print("Start Kibana files migration")
 
     print("Migrate all fields to the ECS fields")
     migration_fields = read_migration_fields()
     rename_entries(migration_fields)
 
-    print("Postfix all ids with -ecs")
-    ids = get_replaceable_ids()
-    rename_entries(ids)
+    if append_ecs:
+        print("Postfix all ids with -ecs")
+        ids = get_replaceable_ids()
+        rename_entries(ids)
 
-    print("Postfix all titles with ` ECS`")
-    titles = get_replacable_titles()
-    rename_entries(titles)
+        print("Postfix all titles with ` ECS`")
+        titles = get_replacable_titles()
+        rename_entries(titles)
 
 
 def get_replaceable_ids():
@@ -39,7 +41,7 @@ def get_replaceable_ids():
 
 def read_migration_fields():
     migration_fields = {}
-    migration_yml = "../dev-tools/ecs-migration.yml"
+    migration_yml = "ecs-migration-8x.yml"
     with open(migration_yml, 'r') as f:
         migration = yaml.safe_load(f)
         for k in migration:
@@ -105,7 +107,11 @@ def get_files():
 
 
 if __name__ == "__main__":
-    migration()
+    parser = argparse.ArgumentParser(description='Migrate field names in dashboards')
+    parser.add_argument('--append-ecs', action='store_true', help='append "-ecs" to the end of all viz identifiers')
+    args = parser.parse_args()
+
+    migration(args.append_ecs)
 
 
 # There are more id's, do they matter?

diff --git a/script/renamed_fields.py b/script/renamed_fields.py
@@ -27,7 +27,7 @@ def get_table(migration_fields):
 
 def read_migration_fields(beat):
     migration_fields = {}
-    migration_yml = "../dev-tools/ecs-migration.yml"
+    migration_yml = "ecs-migration-8x.yml"
     with open(migration_yml, 'r') as f:
         migration = yaml.safe_load(f)
         for k in migration:

diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc
@@ -36,10 +36,9 @@ Contains common beat fields available in all event types.
 *`agent.hostname`*::
 +
 --
-Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
-
+type: alias
 
-type: keyword
+alias to: agent.name
 
 --
 

diff --git a/winlogbeat/include/fields.go b/winlogbeat/include/fields.go
diff --git a/x-pack/filebeat/module/netflow/_meta/kibana/7/dashboard/filebeat-netflow-flow-exporters.json b/x-pack/filebeat/module/netflow/_meta/kibana/7/dashboard/filebeat-netflow-flow-exporters.json
@@ -234,7 +234,7 @@
               "id": "2",
               "params": {
                 "customLabel": "Flow Exporter",
-                "field": "agent.hostname",
+                "field": "agent.name",
                 "order": "desc",
                 "orderBy": "1",
                 "size": 50