Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add host inventory metrics to googlecloud compute metricset #20391

Merged
merged 6 commits into from
Aug 20, 2020
Merged

Add host inventory metrics to googlecloud compute metricset #20391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a2e66a7
Add host inventory metrics to googlecloud compute metricset
kaiyan-sheng Jul 31, 2020
179fead
Merge remote-tracking branch 'upstream/master' into add_host_data_gcp
kaiyan-sheng Aug 4, 2020
6cb36d2
Merge remote-tracking branch 'upstream/master' into add_host_data_gcp
kaiyan-sheng Aug 10, 2020
383cc9a
Merge remote-tracking branch 'upstream/master' into add_host_data_gcp
kaiyan-sheng Aug 19, 2020
cf7f963
Change host.cpu.pct to be in between 0 to 1 and add cloud.region
kaiyan-sheng Aug 19, 2020
1286bd4
add changelog
kaiyan-sheng Aug 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -646,6 +646,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077]
- Add host inventory metrics into aws ec2 metricset. {pull}20171[20171]
- Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547]
- Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391]

*Packetbeat*

Expand Down
50 changes: 19 additions & 31 deletions x-pack/metricbeat/module/googlecloud/compute/_meta/data.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,16 @@
"id": "elastic-observability"
},
"instance": {
"id": "7692260917184259934",
"name": "instance-test-ks"
"id": "2438671519308240892",
"name": "heartbeat-mumbai"
},
"machine": {
"type": "e2-standard-2"
"type": "f1-micro"
},
"provider": "googlecloud"
},
"cloud.availability_zone": "us-central1-a",
"cloud.availability_zone": "asia-south1-c",
"cloud.region": "asia-south1",
"event": {
"dataset": "googlecloud.compute",
"duration": 115000,
Expand All @@ -23,50 +24,37 @@
"compute": {
"firewall": {
"dropped_bytes_count": {
"value": 171
"value": 435
},
"dropped_packets_count": {
"value": 4
"value": 7
}
},
"instance": {
"cpu": {
"reserved_cores": {
"value": 2
"value": 0.2
},
"usage_time": {
"value": 0.1803964574089818
"value": 0.8653667340986431
},
"utilization": {
"value": 0.001503303811741515
}
},
"memory": {
"balloon": {
"ram_size": {
"value": 7896264704
},
"ram_used": {
"value": 416518144
},
"swap_in_bytes_count": {
"value": 0
},
"swap_out_bytes_count": {
"value": 0
}
"value": 0.07211389450822024
}
},
"uptime": {
"value": 60.000000000000455
"value": 60
}
}
},
"labels": {
"user": {
"created-by": "ks"
}
}
"labels": {}
},
"host": {
"cpu": {
"pct": 0.07211389450822024
},
"id": "2438671519308240892",
"name": "heartbeat-mumbai"
},
"metricset": {
"name": "compute",
Expand Down
50 changes: 19 additions & 31 deletions x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,16 @@
"id": "elastic-observability"
},
"instance": {
"id": "7692260917184259934",
"name": "instance-test-ks"
"id": "7925831082803123728",
"name": "heartbeat-tokyo"
},
"machine": {
"type": "e2-standard-2"
"type": "f1-micro"
},
"provider": "googlecloud"
},
"cloud.availability_zone": "us-central1-a",
"cloud.availability_zone": "asia-northeast1-b",
"cloud.region": "asia-northeast1",
"event": {
"dataset": "googlecloud.compute",
"duration": 115000,
Expand All @@ -23,50 +24,37 @@
"compute": {
"firewall": {
"dropped_bytes_count": {
"value": 171
"value": 422
},
"dropped_packets_count": {
"value": 4
"value": 7
}
},
"instance": {
"cpu": {
"reserved_cores": {
"value": 2
"value": 0.2
},
"usage_time": {
"value": 0.1803964574089818
"value": 0.8921228461549617
},
"utilization": {
"value": 0.001503303811741515
}
},
"memory": {
"balloon": {
"ram_size": {
"value": 7896264704
},
"ram_used": {
"value": 416518144
},
"swap_in_bytes_count": {
"value": 0
},
"swap_out_bytes_count": {
"value": 0
}
"value": 0.07434357051291347
}
},
"uptime": {
"value": 60.000000000000455
"value": 60
}
}
},
"labels": {
"user": {
"created-by": "ks"
}
}
"labels": {}
},
"host": {
"cpu": {
"pct": 0.07434357051291347
},
"id": "7925831082803123728",
"name": "heartbeat-tokyo"
},
"metricset": {
"name": "compute",
Expand Down
30 changes: 20 additions & 10 deletions x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,16 @@
"id": "elastic-observability"
},
"instance": {
"id": "7692260917184259934",
"name": "instance-test-ks"
"id": "7925831082803123728",
"name": "heartbeat-tokyo"
},
"machine": {
"type": "e2-standard-2"
"type": "f1-micro"
},
"provider": "googlecloud"
},
"cloud.availability_zone": "us-central1-a",
"cloud.availability_zone": "asia-northeast1-b",
"cloud.region": "asia-northeast1",
Copy link
Member

@jsoriano jsoriano Aug 20, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nothing to change here, just thinking loud. It might be interesting to have cloud-agnostic common values for regions. So a user can more easily group metrics coming from asia-northeast1 in GCP, and ap-northeast-1 in AWS.

Imagine for example being able to ingest with something like this metadata:

"cloud.region": {
    "name": "asia-northeast1",
    "area": "asia-pacific",
    "country": "japan",
    "city": "tokyo",
}

And if it were ingested from AWS:

"cloud.region": {
    "name": "ap-northeast-1",
    "area": "asia-pacific",
    "country": "japan",
    "city": "tokyo",
}

Not sure if cloud providers have APIs for this information.

Though maybe for that we can add geodata through an ingest pipeline in the future.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jsoriano Yeah good point! Different cloud has different names for regions. I don't see a specific API for GCP though but I think the geoip processor should be sufficient if we see good use cases in the future!

"event": {
"dataset": "googlecloud.compute",
"duration": 115000,
Expand All @@ -30,25 +31,34 @@
"value": 0
},
"write_bytes_count": {
"value": 158506
"value": 0
},
"write_ops_count": {
"value": 25
"value": 0
}
}
}
},
"labels": {
"metrics": {
"device_name": "instance-test-ks",
"device_name": "disk-1",
"device_type": "permanent",
"storage_type": "pd-standard"
},
"user": {
"created-by": "ks"
}
}
},
"host": {
"disk": {
"read": {
"bytes": 0
},
"write": {
"bytes": 0
}
},
"id": "7925831082803123728",
"name": "heartbeat-tokyo"
},
"metricset": {
"name": "compute",
"period": 10000
Expand Down
48 changes: 18 additions & 30 deletions x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,16 @@
"id": "elastic-observability"
},
"instance": {
"id": "7692260917184259934",
"name": "instance-test-ks"
"id": "7925831082803123728",
"name": "heartbeat-tokyo"
},
"machine": {
"type": "e2-standard-2"
"type": "f1-micro"
},
"provider": "googlecloud"
},
"cloud.availability_zone": "us-central1-a",
"cloud.availability_zone": "asia-northeast1-b",
"cloud.region": "asia-northeast1",
"event": {
"dataset": "googlecloud.compute",
"duration": 115000,
Expand All @@ -23,50 +24,37 @@
"compute": {
"firewall": {
"dropped_bytes_count": {
"value": 277
"value": 422
},
"dropped_packets_count": {
"value": 5
"value": 7
}
},
"instance": {
"cpu": {
"reserved_cores": {
"value": 2
"value": 0.2
},
"usage_time": {
"value": 0.18404532833979204
"value": 0.8921228461549617
},
"utilization": {
"value": 0.001533711069498267
}
},
"memory": {
"balloon": {
"ram_size": {
"value": 7896264704
},
"ram_used": {
"value": 416260096
},
"swap_in_bytes_count": {
"value": 0
},
"swap_out_bytes_count": {
"value": 0
}
"value": 0.07434357051291347
}
},
"uptime": {
"value": 60
}
}
},
"labels": {
"user": {
"created-by": "ks"
}
}
"labels": {}
},
"host": {
"cpu": {
"pct": 0.07434357051291347
},
"id": "7925831082803123728",
"name": "heartbeat-tokyo"
},
"metricset": {
"name": "compute",
Expand Down
Loading