From 3ba2d229af00c6b203a823ffa6cb6906189e9a80 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 27 Apr 2020 09:45:41 -0600 Subject: [PATCH] [Metricbeat GCP] Improve integration test to generate more specific data-*.json (#17906) (#18011) * Improve TestData integration test to generate more specific data-*.json * rerun TestData to generate data.json files (cherry picked from commit 3bba66d9915affc6a5596db48ea016c36bbfbf8b) --- .../googlecloud/compute/_meta/data.json | 6 +- .../googlecloud/compute/_meta/data_cpu.json | 83 ++++++++--------- .../googlecloud/compute/_meta/data_disk.json | 59 ++++++++++++ .../compute/_meta/data_disk_01.json | 74 --------------- .../compute/_meta/data_disk_02.json | 77 --------------- .../compute/_meta/data_firewall.json | 93 ++++++++----------- .../compute/_meta/data_instance.json | 73 --------------- .../compute/_meta/data_network.json | 54 +++++++++++ .../compute/compute_integration_test.go | 29 +++++- .../googlecloud/loadbalancing/_meta/data.json | 2 +- .../loadbalancing/_meta/data_l3.json | 50 ++++++++++ .../loadbalancing_integration_test.go | 32 ++++++- .../module/googlecloud/pubsub/_meta/data.json | 4 +- .../pubsub/_meta/data_subscription.json | 35 +++++++ .../googlecloud/pubsub/_meta/data_topic.json | 43 +++++++++ .../pubsub/pubsub_integration_test.go | 32 ++++++- .../googlecloud/storage/_meta/data.json | 15 ++- .../storage/_meta/data_network.json | 38 ++++++++ .../storage/_meta/data_storage.json | 39 ++++++++ .../storage/storage_integration_test.go | 28 +++++- 20 files changed, 518 insertions(+), 348 deletions(-) create mode 100644 x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json delete mode 100644 x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk_01.json delete mode 100644 x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk_02.json delete mode 100644 x-pack/metricbeat/module/googlecloud/compute/_meta/data_instance.json create mode 100644 x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json create mode 100644 x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data_l3.json create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json create mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json create mode 100644 x-pack/metricbeat/module/googlecloud/storage/_meta/data_network.json create mode 100644 x-pack/metricbeat/module/googlecloud/storage/_meta/data_storage.json diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json index f1f9f8f386b..4a3ab08217c 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json @@ -5,8 +5,8 @@ "id": "elastic-observability" }, "instance": { - "id": "6889336735612324102", - "name": "gke-dev-oblt-dev-oblt-pool-83a8831b-kd53" + "id": "1174463293187628268", + "name": "gke-observability-8--observability-8--bc1afd95-ngmh" }, "machine": { "type": "n1-standard-4" @@ -40,7 +40,7 @@ }, "labels": { "metrics": { - "device_name": "disk-4", + "device_name": "disk-2", "device_type": "permanent", "storage_type": "pd-standard" }, diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json index 8496bfd79b1..1d3a120a218 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json @@ -1,69 +1,62 @@ { - "@timestamp": "2020-01-08T16:06:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "host": { - "name": "mcastro", - "id": "54f70115bae545cbac2b150f254472a0", - "containerized": false, - "hostname": "mcastro", - "architecture": "x86_64", - "os": { - "version": "", - "family": "", - "name": "Antergos Linux", - "kernel": "5.4.3-arch1-1", - "platform": "antergos" - } - }, - "agent": { - "ephemeral_id": "8b802033-b611-414b-bcaf-1aa19e5f5901", - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", - "version": "8.0.0", - "type": "metricbeat" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "cloud": { "account": { - "id": "elastic-metricbeat" + "id": "elastic-observability" }, - "provider": "googlecloud", "instance": { - "name": "instance-1", - "id": "4503798379141677974" + "id": "1174463293187628268", + "name": "gke-observability-8--observability-8--bc1afd95-ngmh" }, "machine": { - "type": "f1-micro" + "type": "n1-standard-4" }, - "availability_zone": "us-central1-a" + "provider": "googlecloud" }, + "cloud.availability_zone": "europe-west1-c", "event": { - "duration": 1398412653, "dataset": "googlecloud.compute", + "duration": 115000, "module": "googlecloud" }, - "metricset": { - "name": "compute", - "period": 300000 - }, "googlecloud": { "compute": { + "firewall": { + "dropped_bytes_count": { + "value": 181 + }, + "dropped_packets_count": { + "value": 3 + } + }, "instance": { "cpu": { - "reserved_cores": 0.2, - "utilization": 0.005524845140497596 + "reserved_cores": { + "value": 4 + }, + "usage_time": { + "value": 63.478293027728796 + }, + "utilization": { + "value": 0.26449288761553663 + } + }, + "uptime": { + "value": 60 } } }, - "labels": {} + "labels": { + "user": { + "goog-gke-node": "" + } + } + }, + "metricset": { + "name": "compute", + "period": 10000 }, "service": { "type": "googlecloud" - }, - "ecs": { - "version": "1.2.0" } -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json new file mode 100644 index 00000000000..8da39b6ab7d --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json @@ -0,0 +1,59 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-observability" + }, + "instance": { + "id": "8390997210852978465", + "name": "gke-observability-7--observability-7--3dd3e39b-0jm5" + }, + "machine": { + "type": "n1-standard-4" + }, + "provider": "googlecloud" + }, + "cloud.availability_zone": "europe-west1-c", + "event": { + "dataset": "googlecloud.compute", + "duration": 115000, + "module": "googlecloud" + }, + "googlecloud": { + "compute": { + "instance": { + "disk": { + "read_bytes_count": { + "value": 0 + }, + "read_ops_count": { + "value": 0 + }, + "write_bytes_count": { + "value": 0 + }, + "write_ops_count": { + "value": 0 + } + } + } + }, + "labels": { + "metrics": { + "device_name": "gke-observability-7-1--pvc-65581044-7d5d-11ea-8cd9-42010af0011c", + "device_type": "permanent", + "storage_type": "pd-standard" + }, + "user": { + "goog-gke-node": "" + } + } + }, + "metricset": { + "name": "compute", + "period": 10000 + }, + "service": { + "type": "googlecloud" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk_01.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk_01.json deleted file mode 100644 index 038c451d934..00000000000 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk_01.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "@timestamp": "2020-01-08T16:05:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "agent": { - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", - "version": "8.0.0", - "type": "metricbeat", - "ephemeral_id": "8b802033-b611-414b-bcaf-1aa19e5f5901" - }, - "ecs": { - "version": "1.2.0" - }, - "googlecloud": { - "labels": { - "metrics": { - "device_type": "permanent", - "storage_type": "pd-standard", - "device_name": "instance-1" - } - }, - "compute": { - "instance": { - "disk": { - "write_bytes_count": 945853 - } - } - } - }, - "service": { - "type": "googlecloud" - }, - "cloud": { - "account": { - "id": "elastic-metricbeat" - }, - "provider": "googlecloud", - "instance": { - "name": "instance-1", - "id": "4503798379141677974" - }, - "machine": { - "type": "f1-micro" - }, - "availability_zone": "us-central1-a" - }, - "metricset": { - "name": "compute", - "period": 300000 - }, - "event": { - "module": "googlecloud", - "duration": 1398637364, - "dataset": "googlecloud.compute" - }, - "host": { - "containerized": false, - "hostname": "mcastro", - "architecture": "x86_64", - "os": { - "platform": "antergos", - "version": "", - "family": "", - "name": "Antergos Linux", - "kernel": "5.4.3-arch1-1" - }, - "name": "mcastro", - "id": "54f70115bae545cbac2b150f254472a0" - } -} diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk_02.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk_02.json deleted file mode 100644 index d3d0bc9c5ba..00000000000 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk_02.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "@timestamp": "2020-01-08T16:04:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "service": { - "type": "googlecloud" - }, - "ecs": { - "version": "1.2.0" - }, - "host": { - "os": { - "name": "Antergos Linux", - "kernel": "5.4.3-arch1-1", - "platform": "antergos", - "version": "", - "family": "" - }, - "id": "54f70115bae545cbac2b150f254472a0", - "containerized": false, - "name": "mcastro", - "hostname": "mcastro", - "architecture": "x86_64" - }, - "agent": { - "ephemeral_id": "8b802033-b611-414b-bcaf-1aa19e5f5901", - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", - "version": "8.0.0", - "type": "metricbeat" - }, - "cloud": { - "availability_zone": "us-central1-a", - "account": { - "id": "elastic-metricbeat" - }, - "provider": "googlecloud", - "instance": { - "id": "4503798379141677974", - "name": "instance-1" - }, - "machine": { - "type": "f1-micro" - } - }, - "metricset": { - "name": "compute", - "period": 300000 - }, - "event": { - "dataset": "googlecloud.compute", - "module": "googlecloud", - "duration": 1398743696 - }, - "googlecloud": { - "labels": { - "metrics": { - "device_name": "instance-1", - "device_type": "permanent", - "storage_type": "pd-standard" - } - }, - "compute": { - "instance": { - "disk": { - "write_ops_count": 140, - "read_ops_count": 2897, - "read_bytes_count": 71574649, - "write_bytes_count": 2557677 - } - } - } - } -} diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json index ee219ec74e0..04b750b8b7d 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json @@ -1,75 +1,62 @@ { - "@timestamp": "2020-01-08T16:05:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "ecs": { - "version": "1.2.0" - }, - "host": { - "containerized": false, - "name": "mcastro", - "hostname": "mcastro", - "architecture": "x86_64", - "os": { - "version": "", - "family": "", - "name": "Antergos Linux", - "kernel": "5.4.3-arch1-1", - "platform": "antergos" - }, - "id": "54f70115bae545cbac2b150f254472a0" - }, - "agent": { - "type": "metricbeat", - "ephemeral_id": "8b802033-b611-414b-bcaf-1aa19e5f5901", - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", - "version": "8.0.0" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "cloud": { - "availability_zone": "us-central1-a", "account": { - "id": "elastic-metricbeat" + "id": "elastic-observability" }, - "provider": "googlecloud", "instance": { - "id": "4503798379141677974", - "name": "instance-1" + "id": "2528596280375797115", + "name": "gke-dev-next-oblt-dev-next-oblt-pool-404d7f0c-cpj6" }, "machine": { - "type": "f1-micro" - } + "type": "n1-standard-4" + }, + "provider": "googlecloud" }, + "cloud.availability_zone": "europe-west1-c", "event": { "dataset": "googlecloud.compute", - "module": "googlecloud", - "duration": 1397755844 - }, - "metricset": { - "name": "compute", - "period": 300000 + "duration": 115000, + "module": "googlecloud" }, "googlecloud": { - "labels": {}, "compute": { + "firewall": { + "dropped_bytes_count": { + "value": 386 + }, + "dropped_packets_count": { + "value": 7 + } + }, "instance": { - "uptime": 60.00000000000001, "cpu": { - "reserved_cores": 0.2, - "utilization": 0.38202685489490784, - "usage_time": 0.06629814168597115 + "reserved_cores": { + "value": 4 + }, + "usage_time": { + "value": 106.88293868489563 + }, + "utilization": { + "value": 0.4453455778537318 + } + }, + "uptime": { + "value": 60 } - }, - "firewall": { - "dropped_packets_count": 3 + } + }, + "labels": { + "user": { + "goog-gke-node": "" } } }, + "metricset": { + "name": "compute", + "period": 10000 + }, "service": { "type": "googlecloud" } -} - +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_instance.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_instance.json deleted file mode 100644 index 4306273f73c..00000000000 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_instance.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "@timestamp": "2020-01-08T16:04:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "ecs": { - "version": "1.2.0" - }, - "host": { - "os": { - "platform": "antergos", - "version": "", - "family": "", - "name": "Antergos Linux", - "kernel": "5.4.3-arch1-1" - }, - "id": "54f70115bae545cbac2b150f254472a0", - "containerized": false, - "hostname": "mcastro", - "name": "mcastro", - "architecture": "x86_64" - }, - "agent": { - "ephemeral_id": "8b802033-b611-414b-bcaf-1aa19e5f5901", - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", - "version": "8.0.0", - "type": "metricbeat" - }, - "cloud": { - "provider": "googlecloud", - "instance": { - "id": "4503798379141677974", - "name": "instance-1" - }, - "machine": { - "type": "f1-micro" - }, - "availability_zone": "us-central1-a", - "account": { - "id": "elastic-metricbeat" - } - }, - "event": { - "module": "googlecloud", - "duration": 1397750508, - "dataset": "googlecloud.compute" - }, - "metricset": { - "period": 300000, - "name": "compute" - }, - "googlecloud": { - "labels": {}, - "compute": { - "firewall": { - "dropped_bytes_count": 0, - "dropped_packets_count": 0 - }, - "instance": { - "uptime": 46.181442, - "cpu": { - "usage_time": 4.5843222587388945 - } - } - } - }, - "service": { - "type": "googlecloud" - } -} diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json new file mode 100644 index 00000000000..d543fc2382f --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json @@ -0,0 +1,54 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-observability" + }, + "instance": { + "id": "7208038667777737825", + "name": "gke-dev-next-oblt-dev-next-oblt-pool-404d7f0c-fgxk" + }, + "machine": { + "type": "n1-standard-4" + }, + "provider": "googlecloud" + }, + "cloud.availability_zone": "europe-west1-c", + "event": { + "dataset": "googlecloud.compute", + "duration": 115000, + "module": "googlecloud" + }, + "googlecloud": { + "compute": { + "instance": { + "network": { + "received_bytes_count": { + "value": 17913 + }, + "received_packets_count": { + "value": 128 + }, + "sent_bytes_count": { + "value": 841 + } + } + } + }, + "labels": { + "metrics": { + "loadbalanced": "true" + }, + "user": { + "goog-gke-node": "" + } + } + }, + "metricset": { + "name": "compute", + "period": 10000 + }, + "service": { + "type": "googlecloud" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go index a4d47d95dd6..be2dd08cdec 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go @@ -8,14 +8,39 @@ package compute import ( + "fmt" "testing" + "github.com/elastic/beats/v7/libbeat/common" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/stackdriver" ) func TestData(t *testing.T) { + metricPrefixIs := func(metricPrefix string) func(e common.MapStr) bool { + return func(e common.MapStr) bool { + v, err := e.GetValue(metricPrefix) + return err == nil && v != nil + } + } + + dataFiles := []struct { + metricPrefix string + path string + }{ + {"googlecloud.compute.instance", "./_meta/data.json"}, + {"googlecloud.compute.instance.disk", "./_meta/data_disk.json"}, + {"googlecloud.compute.instance.network", "./_meta/data_network.json"}, + {"googlecloud.compute.instance.cpu", "./_meta/data_cpu.json"}, + {"googlecloud.compute.firewall", "./_meta/data_firewall.json"}, + } + config := stackdriver.GetConfigForTest(t, "compute") - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + + for _, df := range dataFiles { + metricSet := mbtest.NewFetcher(t, config) + t.Run(fmt.Sprintf("metric prefix: %s", df.metricPrefix), func(t *testing.T) { + metricSet.WriteEventsCond(t, df.path, metricPrefixIs(df.metricPrefix)) + }) + } } diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json index f9335292924..97a9192732a 100644 --- a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json @@ -49,4 +49,4 @@ "service": { "type": "googlecloud" } -} \ No newline at end of file +} diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data_l3.json b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data_l3.json new file mode 100644 index 00000000000..9a58a5ebc5b --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data_l3.json @@ -0,0 +1,50 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-observability" + }, + "provider": "googlecloud" + }, + "event": { + "dataset": "googlecloud.loadbalancing", + "duration": 115000, + "module": "googlecloud" + }, + "googlecloud": { + "labels": { + "metrics": { + "client_network": "UNKNOWN", + "client_subnetwork": "REMOTE_IS_EXTERNAL", + "client_zone": "UNKNOWN" + }, + "resource": { + "backend_name": "ocp-be-c5kjr-master-us-central1-c", + "backend_scope": "us-central1-c", + "backend_scope_type": "ZONE", + "backend_subnetwork_name": "ocp-be-c5kjr-master-subnet", + "backend_target_name": "ocp-be-c5kjr-api-internal", + "backend_target_type": "BACKEND_SERVICE", + "backend_type": "INSTANCE_GROUP", + "forwarding_rule_name": "ocp-be-c5kjr-api-internal", + "load_balancer_name": "ocp-be-c5kjr-api-internal", + "network_name": "ocp-be-c5kjr-network", + "region": "us-central1" + } + }, + "loadbalancing": { + "l3": { + "internal": { + "egress_packets_count": 394 + } + } + } + }, + "metricset": { + "name": "loadbalancing", + "period": 10000 + }, + "service": { + "type": "googlecloud" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/loadbalancing_integration_test.go b/x-pack/metricbeat/module/googlecloud/loadbalancing/loadbalancing_integration_test.go index 80a5e99c57e..c070d96a736 100644 --- a/x-pack/metricbeat/module/googlecloud/loadbalancing/loadbalancing_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/loadbalancing_integration_test.go @@ -8,14 +8,38 @@ package loadbalancing import ( + "fmt" "testing" + "github.com/elastic/beats/v7/libbeat/common" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/stackdriver" ) func TestData(t *testing.T) { - config := googlecloud.GetConfigForTest(t, "loadbalancing") - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricPrefixIs := func(metricPrefix string) func(e common.MapStr) bool { + return func(e common.MapStr) bool { + v, err := e.GetValue(metricPrefix) + return err == nil && v != nil + } + } + + dataFiles := []struct { + metricPrefix string + path string + }{ + {"googlecloud.loadbalancing", "./_meta/data.json"}, + {"googlecloud.loadbalancing.https", "./_meta/data_https.json"}, + {"googlecloud.loadbalancing.l3", "./_meta/data_l3.json"}, + {"googlecloud.loadbalancing.tcp_ssl_proxy", "./_meta/data_tcp_ssl_proxy.json"}, + } + + config := stackdriver.GetConfigForTest(t, "loadbalancing") + + for _, df := range dataFiles { + metricSet := mbtest.NewFetcher(t, config) + t.Run(fmt.Sprintf("metric prefix: %s", df.metricPrefix), func(t *testing.T) { + metricSet.WriteEventsCond(t, df.path, metricPrefixIs(df.metricPrefix)) + }) + } } diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json index fd0dd71838a..c1a0365401d 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json @@ -14,12 +14,12 @@ "googlecloud": { "labels": { "resource": { - "subscription_id": "test-ks" + "subscription_id": "test-subscription-1" } }, "pubsub": { "subscription": { - "oldest_unacked_message_age": { + "backlog_bytes": { "value": 0 } } diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json new file mode 100644 index 00000000000..13c2724143f --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json @@ -0,0 +1,35 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-observability" + }, + "provider": "googlecloud" + }, + "event": { + "dataset": "googlecloud.pubsub", + "duration": 115000, + "module": "googlecloud" + }, + "googlecloud": { + "labels": { + "resource": { + "subscription_id": "test-ks" + } + }, + "pubsub": { + "subscription": { + "backlog_bytes": { + "value": 0 + } + } + } + }, + "metricset": { + "name": "pubsub", + "period": 10000 + }, + "service": { + "type": "googlecloud" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json new file mode 100644 index 00000000000..7f296406136 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json @@ -0,0 +1,43 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-observability" + }, + "provider": "googlecloud" + }, + "event": { + "dataset": "googlecloud.pubsub", + "duration": 115000, + "module": "googlecloud" + }, + "googlecloud": { + "labels": { + "resource": { + "topic_id": "test-ks" + } + }, + "pubsub": { + "topic": { + "message_sizes": { + "bucket_options": { + "Options": { + "ExponentialBuckets": { + "num_finite_buckets": 16, + "growth_factor": 4, + "scale": 1 + } + } + } + } + } + } + }, + "metricset": { + "name": "pubsub", + "period": 10000 + }, + "service": { + "type": "googlecloud" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go b/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go index 90032d50310..6d739326dea 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/pubsub/pubsub_integration_test.go @@ -8,14 +8,38 @@ package pubsub import ( + "fmt" "testing" + "github.com/elastic/beats/v7/libbeat/common" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/stackdriver" ) func TestData(t *testing.T) { - config := googlecloud.GetConfigForTest(t, "pubsub") - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + metricPrefixIs := func(metricPrefix string) func(e common.MapStr) bool { + return func(e common.MapStr) bool { + v, err := e.GetValue(metricPrefix) + return err == nil && v != nil + } + } + + dataFiles := []struct { + metricPrefix string + path string + }{ + {"googlecloud.pubsub", "./_meta/data.json"}, + {"googlecloud.pubsub.snapshot", "./_meta/data_snapshot.json"}, + {"googlecloud.pubsub.subscription", "./_meta/data_subscription.json"}, + {"googlecloud.pubsub.topic", "./_meta/data_topic.json"}, + } + + config := stackdriver.GetConfigForTest(t, "pubsub") + + for _, df := range dataFiles { + metricSet := mbtest.NewFetcher(t, config) + t.Run(fmt.Sprintf("metric prefix: %s", df.metricPrefix), func(t *testing.T) { + metricSet.WriteEventsCond(t, df.path, metricPrefixIs(df.metricPrefix)) + }) + } } diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json b/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json index 02541f8a1e7..679102209b2 100644 --- a/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/data.json @@ -2,7 +2,7 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "cloud": { "account": { - "id": "elastic-observability" + "id": "elastic-apm" }, "provider": "googlecloud" }, @@ -14,18 +14,17 @@ "googlecloud": { "labels": { "metrics": { - "method": "GetBucketMetadata", - "response_code": "OK" + "storage_class": "MULTI_REGIONAL" }, "resource": { - "bucket_name": "ocp-be-c5kjr-image-registry-us-central1-dsoafnbgctvfimpavswkgn", - "location": "us-central1" + "bucket_name": "artifacts.elastic-apm.appspot.com", + "location": "us" } }, "storage": { - "network": { - "sent_bytes_count": { - "value": 2637 + "storage": { + "object_count": { + "value": 15 } } } diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/data_network.json b/x-pack/metricbeat/module/googlecloud/storage/_meta/data_network.json new file mode 100644 index 00000000000..7bb1c6a4c86 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/data_network.json @@ -0,0 +1,38 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-observability" + }, + "provider": "googlecloud" + }, + "event": { + "dataset": "googlecloud.storage", + "duration": 115000, + "module": "googlecloud" + }, + "googlecloud": { + "labels": { + "metrics": { + "method": "GetBucketMetadata", + "response_code": "OK" + }, + "resource": { + "bucket_name": "ocp-be-c5kjr-image-registry-us-central1-dsoafnbgctvfimpavswkgn", + "location": "us-central1" + } + }, + "storage": { + "network": { + "received_bytes_count": 0 + } + } + }, + "metricset": { + "name": "storage", + "period": 10000 + }, + "service": { + "type": "googlecloud" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/storage/_meta/data_storage.json b/x-pack/metricbeat/module/googlecloud/storage/_meta/data_storage.json new file mode 100644 index 00000000000..f98a6a6f744 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/storage/_meta/data_storage.json @@ -0,0 +1,39 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-observability" + }, + "provider": "googlecloud" + }, + "event": { + "dataset": "googlecloud.storage", + "duration": 115000, + "module": "googlecloud" + }, + "googlecloud": { + "labels": { + "metrics": { + "storage_class": "MULTI_REGIONAL" + }, + "resource": { + "bucket_name": "fstuermer-log-data-categorization-7-6-0", + "location": "us" + } + }, + "storage": { + "storage": { + "total_bytes": { + "value": 4472520191 + } + } + } + }, + "metricset": { + "name": "storage", + "period": 10000 + }, + "service": { + "type": "googlecloud" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go b/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go index 0f2b010f5f2..7d40e7b2bf9 100644 --- a/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/storage/storage_integration_test.go @@ -8,14 +8,38 @@ package storage import ( + "fmt" "testing" + "github.com/elastic/beats/v7/libbeat/common" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/x-pack/metricbeat/module/googlecloud/stackdriver" ) func TestData(t *testing.T) { + metricPrefixIs := func(metricPrefix string) func(e common.MapStr) bool { + return func(e common.MapStr) bool { + v, err := e.GetValue(metricPrefix) + return err == nil && v != nil + } + } + + dataFiles := []struct { + metricPrefix string + path string + }{ + {"googlecloud.storage", "./_meta/data.json"}, + {"googlecloud.storage.authz", "./_meta/data_authz.json"}, + {"googlecloud.storage.network", "./_meta/data_network.json"}, + {"googlecloud.storage.storage", "./_meta/data_storage.json"}, + } + config := stackdriver.GetConfigForTest(t, "storage") - metricSet := mbtest.NewFetcher(t, config) - metricSet.WriteEvents(t, "/") + + for _, df := range dataFiles { + metricSet := mbtest.NewFetcher(t, config) + t.Run(fmt.Sprintf("metric prefix: %s", df.metricPrefix), func(t *testing.T) { + metricSet.WriteEventsCond(t, df.path, metricPrefixIs(df.metricPrefix)) + }) + } }