Update fields to use mapstr in vSphere virtualmachine metricset (#40707)

* Convert fields to mapstr * lint changes * Address comments, update changelog * lint change * Address CI failure (cherry picked from commit e7637c0)
elastic · Sep 14, 2024 · 26aa189 · 26aa189
1 parent b5f61eb
commit 26aa189
Show file tree

Hide file tree

Showing 9 changed files with 165 additions and 127 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -36,6 +36,21 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add metrics for the vSphere Virtualmachine metricset. {pull}40485[40485]
 - Add new metrics for the vSphere Datastore metricset. {pull}40441[40441]
 - Update metrics for the vSphere Host metricset. {pull}40429[40429]
+- Add support of Graphite series 1.1.0+ tagging extension for statsd module. {pull}39619[39619]
+- Allow metricsets to report their status via control v2 protocol. {pull}40025[40025]
+- Remove fallback to the node limit for the `kubernetes.pod.cpu.usage.limit.pct` and `kubernetes.pod.memory.usage.limit.pct` metrics calculation
+- Add support for Kibana status metricset in v8 format {pull}40275[40275]
+- Add new metrics for the vSphere Datastore metricset. {pull}40441[40441]
+- Update metrics for the vSphere Host metricset. {pull}40429[40429]
+- Mark system process metricsets as running if metrics are partially available {pull}40565[40565]
+- Added back `elasticsearch.node.stats.jvm.mem.pools.*` to the `node_stats` metricset {pull}40571[40571]
+
+
+*Osquerybeat*
+
+- Add action responses data stream, allowing osquerybeat to post action results directly to elasticsearch. {pull}39143[39143]
+- Disable allow_unsafe osquery configuration. {pull}40130[40130]
+- Upgrade to osquery 5.12.1. {pull}40368[40368]
 
 *Osquerybeat*
 
@@ -211,6 +226,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add new metricset resourcepool for the vSphere module. {pull}40456[40456]
 - Add new metricset cluster for the vSphere module. {pull}40536[40536]
 - Add support for snapshot in vSphere virtualmachine metricset {pull}40683[40683]
+- Log the total time taken for GCP `ListTimeSeries` and `AggregatedList` requests {pull}40661[40661]
+- Add metrics for the vSphere Virtualmachine metricset. {pull}40485[40485]
+- Add support for snapshot in vSphere virtualmachine metricset {pull}40683[40683]
+- Update fields to use mapstr in vSphere virtualmachine metricset  {pull}40707[40707]
 
 *Metricbeat*
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -67914,6 +67914,7 @@ virtualmachine
 
 
 
+
 *`vsphere.virtualmachine.host.id`*::
 +
 --

diff --git a/metricbeat/module/vsphere/fields.go b/metricbeat/module/vsphere/fields.go
diff --git a/metricbeat/module/vsphere/virtualmachine/_meta/data.json b/metricbeat/module/vsphere/virtualmachine/_meta/data.json
@@ -14,73 +14,80 @@
         "type": "vsphere"
     },
     "vsphere": {
-        "virtualmachine": {
-            "network.names": [
-              "PROD_VCF"
-            ],
-            "os": "Microsoft Windows 10 (64-bit)",
-            "datastore": {
-              "count": 1
-            },
-            "memory": {
-              "used": {
-                "guest": {
-                  "bytes": 0
-                },
-                "host": {
-                  "bytes": 0
-                }
-              },
-              "total": {
-                "guest": {
-                  "bytes": 4294967296
-                }
-              },
-              "free": {
-                "guest": {
-                  "bytes": 4294967296
-                }
-              }
-            },
-            "network": {
-              "count": 1
-            },
-            "host.hostname": "phx-w1c1",
-            "name": "phx-14",
-            "cpu": {
-              "free": {
-                "mhz": 0
-              },
-              "used": {
-                "mhz": 0
-              },
-              "total": {
-                "mhz": 0
-              }
+      "virtualmachine": {
+        "name": "xt0nmfpv9",
+        "uptime": 5348978,
+        "status": "green",
+        "host": {
+          "id":       "host-32",
+          "hostname": "phx-w1c1-esxi04.com"
+        },
+        "cpu": {
+          "free": {
+            "mhz": 0
+          },
+          "used": {
+            "mhz": 161
+          },
+          "total": {
+            "mhz": 0
+          }
+        },
+        "network": {
+          "names": [
+            "PROD_VCF_VMS"
+          ],
+          "count": 1
+        },
+        "memory": {
+          "used": {
+            "guest": {
+              "bytes": 686817280
             },
-            "uptime": 0,
-            "status": "green",
-            "network_names": [
-              "PROD_VCF"
-            ],
-            "datastore.names": [
-              "VxRailtec-Virtual-SAN-Datastore-247df-bc1d-5aad2"
-            ],
-            "host.id": "host-20",
-            "snapshot.info": [
-              {
-                "Name":        "Snapshot_1",
-                "Description": "Test snapshot 1",
-                "CreateTime": "2024-09-01T12:34:56Z"
-
-              },
-              {
-                "Name":        "Snapshot_2",
-                "Description": "Test snapshot 2",
-                "CreateTime": "2024-09-03T2:34:56Z"
-              }
-            ],
-            "snapshot.count": 2
+            "host": {
+              "bytes": 29027729408
+            }
+          },
+          "total": {
+            "guest": {
+              "bytes": 68719476736
+            }
+          },
+          "free": {
+            "guest": {
+              "bytes": 68032659456
+            }
           }
-        }
+        },
+        "network_names": [
+          "PROD_VCF_VMS"
+        ],
+        "datastore": {
+          "count": 1,
+          "names": [
+            "VxRailtoup-Virtual-Datastore-bc1d-5aa310fb"
+          ]
+        },
+        "os": "CentOS 4/5/6/7 (64-bit)",
+        "snapshot": {
+          "info": [
+            {
+            "id":         123,
+            "name":        "Snapshot_1",
+            "description": "Test snapshot 1",
+            "createtime":  "2024-09-01T12:34:56Z",
+            "state":       "PoweredOff"
+            },
+            {
+            "id":          745,
+            "name":       "Snapshot_2",
+            "description": "Test snapshot 2",
+            "createtime":  "2024-09-03T2:34:56Z",
+            "state":       "PoweredOn"
+            }
+          ],
+          "count": 2
+      }
+      }
     }
+  }
diff --git a/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml b/metricbeat/module/vsphere/virtualmachine/_meta/fields.yml
@@ -4,14 +4,17 @@
     virtualmachine
   release: ga
   fields:
-    - name: host.id
-      type: keyword
-      description: >
-        Host id.
-    - name: host.hostname
-      type: keyword
-      description: >
-        Hostname of the host.
+    - name: host
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          description: >
+            Host id.
+        - name: hostname
+          type: keyword
+          description: >
+            Hostname of the host.
     - name: name
       type: keyword
       description: >

diff --git a/metricbeat/module/vsphere/virtualmachine/data.go b/metricbeat/module/vsphere/virtualmachine/data.go
@@ -32,12 +32,14 @@ func (m *MetricSet) mapEvent(data VMData) mapstr.M {
 	freeMemory := max(0, totalMemory-usedMemory)
 
 	event := mapstr.M{
-		"name":          data.VM.Summary.Config.Name,
-		"os":            data.VM.Summary.Config.GuestFullName,
-		"uptime":        data.VM.Summary.QuickStats.UptimeSeconds,
-		"status":        data.VM.Summary.OverallStatus,
-		"host.id":       data.HostID,
-		"host.hostname": data.HostName,
+		"name":   data.VM.Summary.Config.Name,
+		"os":     data.VM.Summary.Config.GuestFullName,
+		"uptime": data.VM.Summary.QuickStats.UptimeSeconds,
+		"status": data.VM.Summary.OverallStatus,
+		"host": mapstr.M{
+			"id":       data.HostID,
+			"hostname": data.HostName,
+		},
 		"cpu": mapstr.M{
 			"used":  mapstr.M{"mhz": usedCPU},
 			"total": mapstr.M{"mhz": totalCPU},
@@ -55,26 +57,22 @@ func (m *MetricSet) mapEvent(data VMData) mapstr.M {
 				"guest": mapstr.M{"bytes": freeMemory},
 			},
 		},
-		"network": mapstr.M{
-			"count": len(data.NetworkNames),
-		},
-		"datastore": mapstr.M{
-			"count": len(data.DatastoreNames),
-		},
 	}
 	if len(data.CustomFields) > 0 {
 		event["custom_fields"] = data.CustomFields
 	}
 	if len(data.NetworkNames) > 0 {
+		event.Put("network.count", len(data.NetworkNames))
+		event.Put("network.names", data.NetworkNames)
 		event["network_names"] = data.NetworkNames
-		event["network.names"] = data.NetworkNames
 	}
 	if len(data.DatastoreNames) > 0 {
-		event["datastore.names"] = data.DatastoreNames
+		event.Put("datastore.count", len(data.DatastoreNames))
+		event.Put("datastore.names", data.DatastoreNames)
 	}
 	if len(data.Snapshots) > 0 {
-		event["snapshot.info"] = data.Snapshots
-		event["snapshot.count"] = len(data.Snapshots)
+		event.Put("snapshot.count", len(data.Snapshots))
+		event.Put("snapshot.info", data.Snapshots)
 	}
 
 	return event

diff --git a/metricbeat/module/vsphere/virtualmachine/data_test.go b/metricbeat/module/vsphere/virtualmachine/data_test.go
@@ -81,12 +81,10 @@ func TestEventMapping(t *testing.T) {
 
 	// Expected event structure
 	expectedEvent := mapstr.M{
-		"name":          "localhost.localdomain",
-		"os":            "otherGuest",
-		"uptime":        int32(10),
-		"status":        types.ManagedEntityStatus("green"),
-		"host.id":       "host-1234",
-		"host.hostname": "test-host",
+		"name":   "localhost.localdomain",
+		"os":     "otherGuest",
+		"uptime": int32(10),
+		"status": types.ManagedEntityStatus("green"),
 		"cpu": mapstr.M{
 			"used":  mapstr.M{"mhz": int32(30)},
 			"total": mapstr.M{"mhz": int32(2294)},
@@ -112,36 +110,42 @@ func TestEventMapping(t *testing.T) {
 				},
 			},
 		},
+		"host": mapstr.M{
+			"id":       "host-1234",
+			"hostname": "test-host",
+		},
 		"network": mapstr.M{
 			"count": 2,
+			"names": []string{"network-1", "network-2"},
 		},
 		"datastore": mapstr.M{
 			"count": 2,
+			"names": []string{"ds1", "ds2"},
 		},
 		"custom_fields": mapstr.M{
 			"customField1": "value1",
 			"customField2": "value2",
 		},
-		"network.names":   []string{"network-1", "network-2"},
-		"network_names":   []string{"network-1", "network-2"},
-		"datastore.names": []string{"ds1", "ds2"},
-		"snapshot.info": []VMSnapshotData{
-			{
-				ID:          123,
-				Name:        "Snapshot_1",
-				Description: "Test snapshot 1",
-				CreateTime:  time.Time{},
-				State:       types.VirtualMachinePowerStatePoweredOff,
-			},
-			{
-				ID:          745,
-				Name:        "Snapshot_2",
-				Description: "Test snapshot 2",
-				CreateTime:  time.Time{},
-				State:       types.VirtualMachinePowerStatePoweredOn,
+		"network_names": []string{"network-1", "network-2"},
+		"snapshot": mapstr.M{
+			"info": []VMSnapshotData{
+				{
+					ID:          123,
+					Name:        "Snapshot_1",
+					Description: "Test snapshot 1",
+					CreateTime:  time.Time{},
+					State:       types.VirtualMachinePowerStatePoweredOff,
+				},
+				{
+					ID:          745,
+					Name:        "Snapshot_2",
+					Description: "Test snapshot 2",
+					CreateTime:  time.Time{},
+					State:       types.VirtualMachinePowerStatePoweredOn,
+				},
 			},
+			"count": 2,
 		},
-		"snapshot.count": 2,
 	}
 
 	// Assert that the output event matches the expected event

diff --git a/metricbeat/module/vsphere/virtualmachine/virtualmachine.go b/metricbeat/module/vsphere/virtualmachine/virtualmachine.go
@@ -61,11 +61,11 @@ type VMData struct {
 }
 
 type VMSnapshotData struct {
-	ID          int32
-	Name        string
-	Description string
-	CreateTime  time.Time
-	State       types.VirtualMachinePowerState
+	ID          int32                          `json:"id"`
+	Name        string                         `json:"name"`
+	Description string                         `json:"description"`
+	CreateTime  time.Time                      `json:"createtime"`
+	State       types.VirtualMachinePowerState `json:"state"`
 }
 
 // New creates a new instance of the MetricSet.

diff --git a/metricbeat/module/vsphere/virtualmachine/virtualmachine_test.go b/metricbeat/module/vsphere/virtualmachine/virtualmachine_test.go
@@ -47,15 +47,21 @@ func TestFetchEventContents(t *testing.T) {
 	event := events[0].MetricSetFields
 
 	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event.StringToPrint())
-
-	assert.EqualValues(t, "ha-host", event["host.id"])
-	assert.EqualValues(t, "localhost.localdomain", event["host.hostname"])
 	assert.EqualValues(t, "green", event["status"])
 	assert.EqualValues(t, 0, event["uptime"])
 	assert.True(t, strings.Contains(event["name"].(string), "ha-host_VM"))
-	expectedDatastoreNames := []string{"LocalDS_0"}
-	actualDatastoreNames := event["datastore.names"].([]string)
-	assert.EqualValues(t, expectedDatastoreNames, actualDatastoreNames)
+
+	host := event["host"].(mapstr.M)
+	assert.EqualValues(t, "ha-host", host["id"])
+	assert.EqualValues(t, "localhost.localdomain", host["hostname"])
+
+	datastore := event["datastore"].(mapstr.M)
+	assert.EqualValues(t, []string{"LocalDS_0"}, datastore["names"])
+	assert.EqualValues(t, 1, datastore["count"])
+
+	network := event["network"].(mapstr.M)
+	assert.EqualValues(t, []string{"VM Network"}, network["names"])
+	assert.EqualValues(t, 1, network["count"])
 
 	cpu := event["cpu"].(mapstr.M)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -67914,6 +67914,7 @@ virtualmachine




		`vsphere.virtualmachine.host.id`::
		+
		--
Expand Down