[vSphere] Add unique document id for all the data-streams

packages/vsphere/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.16.3-next"
+  changes:
+    - description: Added the id field to all data streams for improved identification and tracking.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11534
 - version: "1.16.2"
   changes:
     - description: Apply doc guidelines and general editing.

packages/vsphere/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json

@@ -45,6 +45,7 @@
                         },
                         "enabled": false
                     },
+                    "id": "domain-c1",
                     "name": "Cluster_1",
                     "host": {
                         "names": [

packages/vsphere/data_stream/cluster/_dev/test/pipeline/test-cluster-metrics.json-expected.json

@@ -34,6 +34,7 @@
                             "VM_Host_1"
                         ]
                     },
+                    "id": "domain-c1",
                     "name": "Cluster_1",
                     "network": {
                         "count": 1,
@@ -83,4 +84,4 @@
             }
         }
     ]
-}
+}

packages/vsphere/data_stream/cluster/fields/fields.yml

@@ -24,23 +24,29 @@
           type: long
           metric_type: gauge
           description: >
-            Number of Datastores associated with the cluster.
+            Number of datastores associated with the cluster.
         - name: names
           type: keyword
           description: >
-            List of all the Datastore names associated with the cluster.
+            List of all the datastore names associated with the cluster.
     - name: host
       type: group
       fields:
         - name: count
           type: long
           metric_type: gauge
           description: >
-            Number of Hosts associated with the cluster.
+            Number of hosts associated with the cluster.
         - name: names
           type: keyword
           description: >
             List of all the host names associated with the cluster.
+    - name: id
+      type: keyword
+      description: >
+        Unique cluster ID.
+      # Reason to add as a dimension field: to uniquely identify the cluster ID.
+      dimension: true
     - name: name
       type: keyword
       description: >
@@ -54,11 +60,11 @@
           type: long
           metric_type: gauge
           description: >
-            Number of Networks associated with the cluster.
+            Number of networks associated with the cluster.
         - name: names
           type: keyword
           description: >
-            List of all the Network names associated with the cluster.
+            List of all the network names associated with the cluster.
     - name: triggered_alarms
       type: group
       description: >

packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json

@@ -66,6 +66,7 @@
                         "names": "DC0_H0"
                     },
                     "name": "LocalDS_0",
+                    "id": "datastore-0",
                     "read": {
                         "bytes": 1024
                     },

packages/vsphere/data_stream/datastore/_dev/test/pipeline/test-datastore.json-expected.json

@@ -41,6 +41,7 @@
                         "count": 1,
                         "names": "DC0_H0"
                     },
+                    "id": "datastore-0",
                     "name": "LocalDS_0",
                     "read": {
                         "bytes": 1024
@@ -100,4 +101,4 @@
             }
         }
     ]
-}
+}

packages/vsphere/data_stream/datastore/fields/fields.yml

@@ -67,6 +67,12 @@
       type: keyword
       description: >
         List of all the host names associated with the datastore.
+    - name: id
+      type: keyword
+      description: >
+        Unique datastore ID.
+      # Reason to add as a dimension field: to uniquely identify the datastore ID.
+      dimension: true
     - name: read.bytes
       type: long
       metric_type: gauge

packages/vsphere/data_stream/datastorecluster/_dev/test/pipeline/test-datastorecluster-metrics.json

@@ -21,6 +21,7 @@
                             "status": "yellow"
                         }
                     ],
+                    "id": "group-p1",
                     "name": "datastore_cluster1",
                     "capacity": {
                         "bytes": 8795019280384

packages/vsphere/data_stream/datastorecluster/_dev/test/pipeline/test-datastorecluster-metrics.json-expected.json

@@ -23,6 +23,7 @@
                     "free_space": {
                         "bytes": 8788836876288
                     },
+                    "id": "group-p1",
                     "name": "datastore_cluster1",
                     "triggered_alarms": [
                         {
@@ -49,4 +50,4 @@
             }
         }
     ]
-}
+}

packages/vsphere/data_stream/datastorecluster/fields/fields.yml

@@ -19,23 +19,29 @@
       fields:
         - name: names
           type: keyword
-          description: List of all the Datastore names associated with the Datastore Cluster.
+          description: List of all the datastore names associated with the datastore cluster.
         - name: count
           type: long
           metric_type: gauge
-          description: Number of datastores in the Datastore Cluster.
+          description: Number of datastores in the datastore cluster.
     - name: free_space.bytes
       type: long
       metric_type: gauge
       unit: byte
       description: >
         Total free space on this storage pod, in bytes.
       format: bytes
+    - name: id
+      type: keyword
+      description: >
+        Unique datastore cluster ID.
+      # Reason to add as a dimension field: to uniquely identify the datastore cluster ID.
+      dimension: true
     - name: name
       type: keyword
       # Reason to add as a dimension field: unique identifier of the host
       dimension: true
-      description: The Datastore Cluster name.
+      description: The datastore cluster name.
     - name: triggered_alarms
       type: group
       description: >

packages/vsphere/data_stream/host/_dev/test/pipeline/test-host.json

@@ -155,6 +155,7 @@
             "network_names": [
                 "VM Network"
             ],
+            "id": "host-0",
             "name": "DC0_H0",
             "status": "green",
             "uptime": 1728865

packages/vsphere/data_stream/host/_dev/test/pipeline/test-host.json-expected.json

@@ -57,6 +57,7 @@
                             "bytes": 248000
                         }
                     },
+                    "id": "host-0",
                     "memory": {
                         "free": {
                             "bytes": 2822230016
@@ -178,4 +179,4 @@
             }
         }
     ]
-}
+}

packages/vsphere/data_stream/host/fields/fields.yml

@@ -64,6 +64,12 @@
       metric_type: gauge
       unit: byte
       description: Sum of disk read and write rates each second in bytes.
+    - name: id
+      type: keyword
+      description: >
+        Unique host ID.
+      # Reason to add as a dimension field: to uniquely identify the host ID.
+      dimension: true
     - name: memory.used.bytes
       type: long
       metric_type: gauge

packages/vsphere/data_stream/host/sample_event.json

@@ -1,48 +1,48 @@
 {
-    "@timestamp": "2024-10-03T04:09:10.462Z",
+    "@timestamp": "2024-09-02T06:04:26.077Z",
     "agent": {
-        "ephemeral_id": "c13fd262-b655-4a62-a5f7-dda2f497332f",
-        "id": "036bbeff-9b4f-497d-846d-d5562d328109",
+        "ephemeral_id": "1acd79f0-0361-47dd-8d47-32064268713f",
+        "id": "958125fc-776b-4be3-b0ce-2762148f5f6d",
         "name": "docker-fleet-agent",
         "type": "metricbeat",
-        "version": "8.15.2"
+        "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "vsphere.host",
-        "namespace": "default",
+        "namespace": "50835",
         "type": "metrics"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "036bbeff-9b4f-497d-846d-d5562d328109",
-        "snapshot": true,
-        "version": "8.15.2"
+        "id": "958125fc-776b-4be3-b0ce-2762148f5f6d",
+        "snapshot": false,
+        "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "vsphere.host",
-        "duration": 83416967,
-        "ingested": "2024-10-03T04:09:20Z",
+        "duration": 32951837,
+        "ingested": "2024-09-02T06:04:38Z",
         "module": "vsphere"
     },
     "host": {
         "architecture": "x86_64",
         "containerized": true,
         "hostname": "docker-fleet-agent",
-        "id": "57723763cd1b4ff48e54a505de4ebe6c",
+        "id": "8259e024976a406e8a54cdbffeb84fec",
         "ip": [
-            "172.18.0.7"
+            "192.168.253.6"
         ],
         "mac": [
-            "02-42-AC-12-00-07"
+            "02-42-C0-A8-FD-06"
         ],
         "name": "docker-fleet-agent",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.15.153.1-microsoft-standard",
+            "kernel": "3.10.0-1160.118.1.el7.x86_64",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
@@ -51,15 +51,12 @@
     },
     "metricset": {
         "name": "host",
-        "period": 20000
+        "period": 10000
     },
     "service": {
-        "address": "https://172.18.0.4:8989/sdk",
+        "address": "https://svc-vsphere-metrics:8989/sdk",
         "type": "vsphere"
     },
-    "tags": [
-        "vsphere-host"
-    ],
     "vsphere": {
         "host": {
             "cpu": {
@@ -74,26 +71,6 @@
                     "pct": 0.015
                 }
             },
-            "datastore": {
-                "count": 4,
-                "names": [
-                    "LocalDS_0",
-                    "LocalDS_1",
-                    "LocalDS_2",
-                    "LocalDS_3"
-                ]
-            },
-            "disk": {
-                "read": {
-                    "bytes": 142336
-                },
-                "total": {
-                    "bytes": 1492992
-                },
-                "write": {
-                    "bytes": 1895424
-                }
-            },
             "memory": {
                 "free": {
                     "bytes": 2822230016
@@ -107,52 +84,7 @@
                 }
             },
             "name": "DC0_H0",
-            "network": {
-                "bandwidth": {
-                    "received": {
-                        "bytes": 702464
-                    },
-                    "total": {
-                        "bytes": 1538048
-                    },
-                    "transmitted": {
-                        "bytes": 564224
-                    }
-                },
-                "count": 3,
-                "names": [
-                    "DC0_DVPG0",
-                    "DVS0-DVUplinks-10",
-                    "VM Network"
-                ],
-                "packets": {
-                    "multicast": {
-                        "received": {
-                            "count": 97
-                        }
-                    },
-                    "received": {
-                        "count": 11933
-                    },
-                    "transmitted": {
-                        "count": 8801
-                    }
-                }
-            },
-            "network_names": [
-                "DC0_DVPG0",
-                "DVS0-DVUplinks-10",
-                "VM Network"
-            ],
-            "status": "gray",
-            "uptime": 77229,
-            "vm": {
-                "count": 2,
-                "names": [
-                    "DC0_H0_VM0",
-                    "DC0_H0_VM1"
-                ]
-            }
+            "network_names": "VM Network"
         }
     }
 }

packages/vsphere/data_stream/network/_dev/test/pipeline/test-network.json

@@ -25,6 +25,7 @@
                     "config": {
                         "status": "green"
                     },
+                    "id": "network-1",
                     "name": "VM Network",
                     "status": "green",
                     "type": "Network"

packages/vsphere/data_stream/network/_dev/test/pipeline/test-network.json-expected.json

@@ -15,6 +15,7 @@
                     "config": {
                         "status": "green"
                     },
+                    "id": "network-1",
                     "name": "VM Network",
                     "status": "green",
                     "triggered_alarms": [
@@ -43,4 +44,4 @@
             }
         }
     ]
-}
+}