From b442a44c660e1355b2081d7879dfeb5a5d892bc9 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 19 Jan 2022 14:33:51 -0800 Subject: [PATCH 1/8] Add firestore metricset --- .../module/gcp/firestore/_meta/data.json | 39 ++++++++++++ .../gcp/firestore/_meta/data_document.json | 39 ++++++++++++ .../module/gcp/firestore/_meta/docs.asciidoc | 13 ++++ .../module/gcp/firestore/_meta/fields.yml | 14 +++++ .../firestore/firestore_integration_test.go | 59 +++++++++++++++++++ .../module/gcp/firestore/firestore_test.go | 21 +++++++ .../module/gcp/firestore/manifest.yml | 11 ++++ 7 files changed, 196 insertions(+) create mode 100644 x-pack/metricbeat/module/gcp/firestore/_meta/data.json create mode 100644 x-pack/metricbeat/module/gcp/firestore/_meta/data_document.json create mode 100644 x-pack/metricbeat/module/gcp/firestore/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/gcp/firestore/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/gcp/firestore/firestore_integration_test.go create mode 100644 x-pack/metricbeat/module/gcp/firestore/firestore_test.go create mode 100644 x-pack/metricbeat/module/gcp/firestore/manifest.yml diff --git a/x-pack/metricbeat/module/gcp/firestore/_meta/data.json b/x-pack/metricbeat/module/gcp/firestore/_meta/data.json new file mode 100644 index 00000000000..3eeb092ce76 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/firestore/_meta/data.json @@ -0,0 +1,39 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-apm" + }, + "provider": "gcp" + }, + "event": { + "dataset": "gcp.firestore", + "duration": 115000, + "module": "gcp" + }, + "gcp": { + "labels": { + "metrics": { + "storage_class": "MULTI_REGIONAL" + }, + "resource": { + "bucket_name": "artifacts.elastic-apm.appspot.com", + "location": "us" + } + }, + "firestore": { + "document": { + "delete_count": { + "value": 15 + } + } + } + }, + "metricset": { + "name": "firestore", + "period": 10000 + }, + "service": { + "type": "gcp" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/firestore/_meta/data_document.json b/x-pack/metricbeat/module/gcp/firestore/_meta/data_document.json new file mode 100644 index 00000000000..3eeb092ce76 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/firestore/_meta/data_document.json @@ -0,0 +1,39 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "cloud": { + "account": { + "id": "elastic-apm" + }, + "provider": "gcp" + }, + "event": { + "dataset": "gcp.firestore", + "duration": 115000, + "module": "gcp" + }, + "gcp": { + "labels": { + "metrics": { + "storage_class": "MULTI_REGIONAL" + }, + "resource": { + "bucket_name": "artifacts.elastic-apm.appspot.com", + "location": "us" + } + }, + "firestore": { + "document": { + "delete_count": { + "value": 15 + } + } + } + }, + "metricset": { + "name": "firestore", + "period": 10000 + }, + "service": { + "type": "gcp" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/firestore/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/firestore/_meta/docs.asciidoc new file mode 100644 index 00000000000..8496de82389 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/firestore/_meta/docs.asciidoc @@ -0,0 +1,13 @@ +Firestore metricset fetches metrics from https://cloud.google.com/firestore/[Firestore] in Google Cloud Platform. + +The `firestore` metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-firestore[GCP Firestore Monitoring API]. The field names have been left untouched for people already familiar with them. + +You can specify a single region to fetch metrics like `us-central1`. Be aware that GCP Storage does not use zones so `us-central1-a` will return nothing. If no region is specified, it will return metrics from all buckets. + +[float] +=== Metrics +Here is a list of metrics collected by `firestore` metricset: + +- `firestore.document.delete_count`: The number of successful document deletes. +- `firestore.document.read_count`: The number of successful document reads from queries or lookups. +- `firestore.document.write_count`: The number of successful document writes. diff --git a/x-pack/metricbeat/module/gcp/firestore/_meta/fields.yml b/x-pack/metricbeat/module/gcp/firestore/_meta/fields.yml new file mode 100644 index 00000000000..99ed9bf5e17 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/firestore/_meta/fields.yml @@ -0,0 +1,14 @@ +- name: firestore + description: Google Cloud Firestore metrics + release: beta + type: group + fields: + - name: document.delete.count + type: long + description: The number of successful document deletes. + - name: document.read.count + type: long + description: The number of successful document reads from queries or lookups. + - name: document.write.count + type: long + description: The number of successful document writes. diff --git a/x-pack/metricbeat/module/gcp/firestore/firestore_integration_test.go b/x-pack/metricbeat/module/gcp/firestore/firestore_integration_test.go new file mode 100644 index 00000000000..8783f3144fd --- /dev/null +++ b/x-pack/metricbeat/module/gcp/firestore/firestore_integration_test.go @@ -0,0 +1,59 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build integration && gcp +// +build integration,gcp + +package firestore + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics" +) + +func TestFetch(t *testing.T) { + config := metrics.GetConfigForTest(t, "firestore") + fmt.Printf("%+v\n", config) + + metricSet := mbtest.NewReportingMetricSetV2WithContext(t, config) + events, errs := mbtest.ReportingFetchV2WithContext(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} + +func TestData(t *testing.T) { + metricPrefixIs := func(metricPrefix string) func(e common.MapStr) bool { + return func(e common.MapStr) bool { + v, err := e.GetValue(metricPrefix) + return err == nil && v != nil + } + } + + dataFiles := []struct { + metricPrefix string + path string + }{ + {"gcp.firestore", "./_meta/data.json"}, + {"gcp.firestore.document", "./_meta/data_document.json"}, + } + + config := metrics.GetConfigForTest(t, "firestore") + + for _, df := range dataFiles { + metricSet := mbtest.NewFetcher(t, config) + t.Run(fmt.Sprintf("metric prefix: %s", df.metricPrefix), func(t *testing.T) { + metricSet.WriteEventsCond(t, df.path, metricPrefixIs(df.metricPrefix)) + }) + } +} diff --git a/x-pack/metricbeat/module/gcp/firestore/firestore_test.go b/x-pack/metricbeat/module/gcp/firestore/firestore_test.go new file mode 100644 index 00000000000..ea7d802296d --- /dev/null +++ b/x-pack/metricbeat/module/gcp/firestore/firestore_test.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package firestore + +import ( + "os" + + "github.com/elastic/beats/v7/metricbeat/mb" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/gcp/firestore/manifest.yml b/x-pack/metricbeat/module/gcp/firestore/manifest.yml new file mode 100644 index 00000000000..95760a8e894 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/firestore/manifest.yml @@ -0,0 +1,11 @@ +default: false +input: + module: gcp + metricset: metrics + defaults: + metrics: + - service: firestore + metric_types: + - "document/delete_count" + - "document/read_count" + - "document/write_count" From c1ae15334a229fd947dbbc7d5a0d9a7887d1fcf5 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 19 Jan 2022 14:35:00 -0800 Subject: [PATCH 2/8] Add firestore metricset reference --- x-pack/metricbeat/metricbeat.reference.yml | 1 + x-pack/metricbeat/module/gcp/_meta/config.yml | 1 + x-pack/metricbeat/module/gcp/constants.go | 1 + x-pack/metricbeat/modules.d/gcp.yml.disabled | 1 + 4 files changed, 4 insertions(+) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 4971d923273..cb59390723f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -552,6 +552,7 @@ metricbeat.modules: metricsets: - pubsub - loadbalancing + - firestore zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" diff --git a/x-pack/metricbeat/module/gcp/_meta/config.yml b/x-pack/metricbeat/module/gcp/_meta/config.yml index 00295e795d4..632e9e23538 100644 --- a/x-pack/metricbeat/module/gcp/_meta/config.yml +++ b/x-pack/metricbeat/module/gcp/_meta/config.yml @@ -11,6 +11,7 @@ metricsets: - pubsub - loadbalancing + - firestore zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" diff --git a/x-pack/metricbeat/module/gcp/constants.go b/x-pack/metricbeat/module/gcp/constants.go index 3776263b7b6..c8bd1812ff8 100644 --- a/x-pack/metricbeat/module/gcp/constants.go +++ b/x-pack/metricbeat/module/gcp/constants.go @@ -22,6 +22,7 @@ const ( ServiceLoadBalancing = "loadbalancing" ServicePubsub = "pubsub" ServiceStorage = "storage" + ServiceFirestore = "firestore" ) //Paths within the GCP monitoring.TimeSeries response, if converted to JSON, where you can find each ECS field required for the output event diff --git a/x-pack/metricbeat/modules.d/gcp.yml.disabled b/x-pack/metricbeat/modules.d/gcp.yml.disabled index d7d873be6d1..638846d0b96 100644 --- a/x-pack/metricbeat/modules.d/gcp.yml.disabled +++ b/x-pack/metricbeat/modules.d/gcp.yml.disabled @@ -14,6 +14,7 @@ metricsets: - pubsub - loadbalancing + - firestore zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" From 6eb5ee16972ef8d6d2fd8d51598ad2ceafb44291 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 19 Jan 2022 14:35:34 -0800 Subject: [PATCH 3/8] Add firestore metricset metrics mapping --- x-pack/metricbeat/module/gcp/metrics/response_parser.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/metricbeat/module/gcp/metrics/response_parser.go b/x-pack/metricbeat/module/gcp/metrics/response_parser.go index 02fa9893cad..e0f968f77a5 100644 --- a/x-pack/metricbeat/module/gcp/metrics/response_parser.go +++ b/x-pack/metricbeat/module/gcp/metrics/response_parser.go @@ -213,6 +213,11 @@ var reMapping = map[string]string{ "storage.object_count.value": "storage.object.count", "storage.total_byte_seconds.value": "storage.total_byte_seconds.bytes", "storage.total_bytes.value": "storage.total.bytes", + + // gcp.firestore metricset + "document.delete_count.value": "document.delete.count", + "document.read_count.value": "document.read.count", + "document.write_count.value": "document.write.count", } func remap(l *logp.Logger, s string) string { From bf5e2fb82d00103437de84545df6c0445c246310 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 19 Jan 2022 14:35:49 -0800 Subject: [PATCH 4/8] Add firestore filter logic --- x-pack/metricbeat/module/gcp/metrics/metrics_requester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go index d518acc0014..d25701236e4 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go +++ b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go @@ -124,7 +124,7 @@ func (r *metricsRequester) getFilterForMetric(serviceName, m string) (f string) } f = fmt.Sprintf("%s AND resource.label.location=starts_with(\"%s\")", f, zone) } - case gcp.ServicePubsub, gcp.ServiceLoadBalancing, gcp.ServiceCloudFunctions: + case gcp.ServicePubsub, gcp.ServiceLoadBalancing, gcp.ServiceCloudFunctions, gcp.ServiceFirestore: return case gcp.ServiceStorage: if r.config.Region == "" { From 75de4c7050afae85509e61a01f7e818e67dfa42b Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 19 Jan 2022 14:36:18 -0800 Subject: [PATCH 5/8] Update gcp fields & module --- x-pack/metricbeat/module/gcp/fields.go | 2 +- x-pack/metricbeat/module/gcp/module.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/gcp/fields.go b/x-pack/metricbeat/module/gcp/fields.go index a5b2d2a836a..d9d6047873f 100644 --- a/x-pack/metricbeat/module/gcp/fields.go +++ b/x-pack/metricbeat/module/gcp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGcp returns asset data. // This is the base64 encoded zlib format compressed contents of module/gcp. func AssetGcp() string { - return "eJzcXVtz27byf/en2OlLkv84yr89Z85D5kxnUqftyZy49YyTvLIguJJQgQALgHaUT38GN14kSqJ4kZMmT7bE3d/esLvAgn4JG9y+hhUtrgAMMxxfw7NfpVxxhBsuywzuODFLqfJnVwAKORKNryFFQ64AMtRUscIwKV7Dj1cAAL/e3EEus5LjFcCSIc/0a/fBSxAkx8jK/jPbwv6sZBl/0/x+8xlOUuS6+nV8VKZ/IjWNX3fgif88LsGMVEysIEejGNX7lHchNGGUGtXi/1ofHYRi//lfJv4bG9w+SpV1Es7RkIwYMhdxK+ostPVWG8xnIa1Qy1JRnIx4JPxdpRD//7vTftWim8kydd7d8WmSk6JgYhW++l2L+BHvvA3uaNbEgEJTKoEZLJXMoRWMb+7ewV8lqu1iT6yUcc7E6hC/Fpmf/HejazSe2Y3wtmKasQqdwRLRUKm9Rq72Tddl9RbWG6mN+64GJigvMwSFq5ITdQ2GfL4Gkv1ZapOjMNdARAZKliKzakelpFp04GHiQTKKSS6FWQ/BFFWmsJDKgKPTxahQ0nkDy4ZwufNPw7u3IJdg1hjNGvmmyKVYaTCyi7mRhvAOvksuiTnM9YN9rOJEclkKs+9gVOZFabCXg934787oYEum8JFwvsiULArMFunWoO6Q3errsOjvBJW5ldo9DoEYpFun/MikB/+kIHSDRifUae+B8LLL93uiCcTOwMOENkRQXNCiXCjUqB4wS6hUqA+C2VvJduD8VuYpKuuJjg5EsiCFg7O2cRr8NPI/Ba3UZIWJYTkuNNIBoD5aArCUCgjnARgToJFKkele7BfFTubox/mDtYAi1P4UxSacS0oMZnBz99Ev30wDLZVCYfjWIis1RoX1UVLG9GahkAz16Bvrfxae92hLyScSS7gX40QW49y4gmBJegTvfgdZoCL2C0eN5FA8KmZwGvktKYMCjOynAMd6Yg04mv1VkGMu1XaRWt+SYqFInmj2BQdCsV7r8kJY2C0qz8E6p/XKT7cL+LBmOqzW1oGl4FsgD4RxknIfbZ9uQ3niU5BVqH0Yf4AlyRnfdmfdwyKVGrOBIt16+HWUWVpPJ41+JEXCxEB/tfbZs4yLGSYCqlWJ2vggZkaDfBRgeYIuCMUnkVaWZkpxY5A6EWuJjXwKeQWaR6k2CyZWCrWeahmmyB5iTW/BBDbnIAlVwcKtTMMRxeJiFCacUDka7U8PqAaCmFgvZ8MpiyP1zHHmdYkVCpiwhsGaaEgRBahSCCZWR13WA0jcMj8Ixs+cFHYNtWRAM0Ex4ngkGrQhymB23aizFnBP8oJjBviAagv/+v/6kzdLgwq0/ZyJ1TVkxBAbqEIaeGCaxSgtCxuY3/9QP7rXc6w2B/uNZ3+sNvgHUCkMYcLrLW+20Y8ImipSVI30zR3cG0I3mWLWuLaXDk+7Xnh/c8iCtE/9+t+fn83SIzvmqFxlaivZZHR1fFPmJSeGPaArSB09W33WtbLLlaGfqBD0s+xJGTjLmRnZdFjYHqmjFgvtmtFofKVhnH1xddiEnYDF7RH37QJCJnOKAkqEjQ/8TBEz+B6IDsZrf2Cfd1ymC8Af/nmGBhX+ZVPzZI1lbe1AucM5Z15rTok6j7vU4p7TOO66DKQIK4XEuGRFxI7jNL0mMHwCv8FijTkqwhNtpLL9t/fgYbXDe0kJh4omBJoh9pjwNcXQVWIfa9TbtGgD1Rnwur5qWrDen8ZCDX3EGNuHxq87MUwKcKaoz5sCfGN5ovfiGfRY2FJmSUpuBhbmdZYo3JafJaWvIVVygwIy2xbaVLEt8Bpy8qdU7iwgZ6L7CGAP4LjIvo2dug/kId44swEulcCCU/+tc1hQ6YjFNfjLNGupQteTjQ4s22noHce1nWegj9kTeO/Rbvqkp9oGtm5mvFPuy9fqrM+ygZCZP0kIO/8k5ThZNdyg2aiMQ/w4zuOxzrMa7CLvvxhcKJQrXVy0z26YbnCLXSF32zsjfc0f9YqO/mucl+3XoQ2PmLgc9ZuGTY+bGjuzv9fJUuHQU5dfFGJDzZ6gxcm7hZoStt8GHHhatOMflwHuEQ88UHCID3nJ1J4xQ3flgTa35QaiDTXK+MBrLG6htpsR4wUS0sgCtU+/1cHustV/U7tjwqkuRkIAVYd00y2444vpGT00ni3F07HE23Hg2VKNU+wotjp9a8DdO3UagV+jMHNhN4oInTNjpodfsCzxGxfDxx1y8hnu/Ajd7/cjfdXiGTGqYOHU2gsdBxRKUtQ6jiyMBZlkBHMpnuwYydme4wPyMBAMHtCokjcKNXHnHSDv4RzQjhcy21stxq9np9aJQmYj46yJ264S02PuWB+mgf0geZlPUS/WiN2EVmgpqiGSMDBiWV4qgTfEG+Hrv3XJ1YzV4yKdwDVLpeaJd1RnKdq1sgn+WIV2fBdxZ2s8DMRV9g4YwoDRRUYbuCRZSjgRtO/U/ntJMvgpPjLjbPXamEIvUkI3KLJk3DZ5OwE25n1IdeSr/XzGfz58uHt17/QCXjHWlBICjk7n7EY6rN7ZSdURWxh5TrcVEPtxF9g+AHUhhR7alx3XpScdlFlhfS4VUELX+MLqEj8bVIJwh//5/Yu+AlzKByhnKIy2UM/T8MymPxfMpcx8KGaCHrsg8n8soheMG+I7CNMhqzztw83dq49v72LO38Ea/LTGHLPCksvHBfwilSXgftLAzDMNzsL1jQ1blBYFZ9SlJdBGIcndTGg/4ZNxw4NtJbQmCKdTwwlBxg2qHjdjQNNcOnpJMq/pgsQz226w7N3omZg/6t69/6nDl54vR9jiRT9xZrZFt2Cng6RCeZkgacKcV+uXDoGGZJ03D2mRaM2TQsnP2wXlUrvbZ0Kgq/UPH1j1bWcatOJ4r0IwqHIm3P0r11za+Ly/fw8exkmcoyJxd6e01tqn24aPljrs+PQBNM5LDyOq7fjp9jxEAh8vYEfqOrdhRpQFigkg3vi+sxEOsjS23cysttqwlSxXa7f0HMD68mAv5l63sN+6dbdoQy7YHnqHQlGmukx7Eb8r0/synbHB1IIUei2Na4y4XI3ayrHPgmZfsB6y07btt8WyG/lwV8NIzbQHIL9xnqTbROGKycF3uwYB3Btx81YJSI6hp1Is2Sopi4xMsu9P49UYT7j0lxeBrolYob72FvcdUnW10fHwt/JRl/y4ukWZJ1Ebo5eWMXZvAmmYfWZIwy0teYbaRMgJWQ27fvRmhfC83qV/ET3Uk4/whyh0H2BDrReHer6iy7TivSB0EwWZMKYq3yB0I+Qjx2zlQ+lN/XO1F9GKtQw5c1uTlvVJ9LOssaVooa5keU4WmwVZQGBaffAi2KMJ7CTwrcGEysHbTi2l+xHd+gL2NeRIdKm8p4TTH7erzMKGuZHxK/BXKQ2B5vb3KeyXXYndnjMSum6BGLE+N0XJkGQJR2NQzRkFRZlyptde8ZYneJ5gZMFoS5aewHOZJTZ0LTHOBM6J/nEtNULk5G5Nets7wLcyY8vtG7p5G78wQVwfEi+pTD2hoPsSxAW3GVIT2GjcRm4/6J3K7wnYFgqNxmCG6iV4gQ+E9pL5zIaJXgOKrJBM2LRWGnf4tEXTyiO95ChFxWs6OS6RGUJxEXO99aBahLmKoS6xdsuNSYUYXYdOK9C4GipI67xjflt1O+EhHxzhgh3yzG22A7JNbK+i5DxpFL6zZJWGID3ziTtWIJDhkgkWpwi6HtXY2M14VW9ntKR8dTqHhtcA9VbX9PmLnGoHemYuB3FOa1oGfwczTm9Cp5lxttPrWXDpNRBjMC+6ccFHwdkGnQD62g8Q2WfcybgClhcccxTGdxaZRD8KkxJD1+51qVWaX8C99C1KdWNS8G39+g4psPXAwp1NNJkpa3h/aOJe82i9wxZIK/aAovWsuzZIigKJgrzkhhUc/VW3sxSdcGJQUDa4KHrLtFEsLaODO2mi+BVxt9bnjCoZF/whHtIuI6Zt9rsLuhGlz07NM+tm67ngx6VNjThHDrBkjwbqaVzCzNn/uqOmSllxrTYyorDNQRhKGdHy+gNS23ZdpDy5j+zu7Pr99ylWOtQ4g8tGJu3sB4/MrEFI8dL68ralVZYN8+22OHN6xI5Ql3KDf1OZ4Y+DnOFc5V1yU6kdX/v7NOO2lk5J9mQuvy+oPymdQMhq8nBGocIA4hC0sVueNeEf2oAan+ndTvT4Y4lxZxGNUwi37e/OJjoLWw/3WzkT9mjjcmMNOnQMZr/sdicL1ZLraLuq29F/cRDMt7BDF05HBnny17mHNUair62xGCOLRpHNvF/Tjotvppht6GeOPRGvlTO3awKmrz0de5hPn4fPCYyqlPAvTLg6AKM1wnYfXq4w3wwbKdjIWydvkRtSW9a9j5a4fbaGKa0m7Sc5mrXMHO+YSJ3hwbYGXZYmpVl/WRDKk5RozJLwV2oIpah1spgCcVXghj9bY53OmVeEv4kTrvCtFBEGM/C8QUuOfAtZ6e5Zhm++uXnfWcTUYtRrzEB9+78cYTV9876xYnW9UO4wkqBGXSBlS0YTiywvzZi1eUercaYjJ1lTQZHjQU1NdBV5B83OJeTdK7zT+OoEt5E7YXe+yXsayPGFMN46Q8fvd+4ie2IaClSQlnSDpgU2vjGGcqJ16w6seyF+aCGkoOgoZGTr/zySu2Ubv6ew8LPWxISRoXCbFdzdggfC44CzLP2r0zLSOYXdenmPSyVJvOz67XjedG8giqmPcF7ZMbzf4Ssz5f8CAAD//6e5oMw=" + return "eJzcXVtvG7mSfs+vKMxLkoWj7Jld7EOwOEDGOTMbbDxjwEleeyh2SeKITXZ4saP8+gPe+iK1rFZf5GSSJ1vqqq+urCqS7Vewxd0bWNPyGYBhhuMbeP6blGuOcM2lzeGWE7OSqnj+DEAhR6LxDSzRkGcAOWqqWGmYFG/gn88AAH67voVC5pbjM4AVQ57rN/6DVyBIgYmV+2d2pftZSZt+0/x+8xlOlsh19ev0qFz+hdQ0ft2BJ/0LuAQzUjGxhgKNYlQfUt6H0IRhNarFf7Q+OgrF/Qu/zMI3trh7kCrvJFygITkxZC7iTtRZaOudNljMQlqhllZRnIx4IvxTpZDw/6fTftWim0u79N7d8WlWkLJkYh2/+lOL+CPeeRPd0WyIAYXGKoE5rJQsoBWMb2/fwxeLarc4EGvJOGdifYxfi8wv4bvJNRrP7Ed4WzHNWIXOYEloqNRBI88OTddl9RbWa6mN/64GJii3OYLCteVEXYEhX6+A5H9ZbQoU5gqIyEFJK3KndlRKqkUHHibuJaOYFVKYzRBMSWUKS6kMeDpdjEolvTewfAiX2/A0vH8HcgVmg8msie8SuRRrDUZ2MTfSEN7Bd8UlMce5fnSPVZxIIa0whw5GZVFag70c7Dp8d0YHWzGFD4TzRa5kWWK+WO4M6g7Znb6Oi/5eUFk4qf3jEInBcueVn5j04J+VhG7R6Ix67d0Tbrt8vyeaSOwMPExoQwTFBS3tQqFGdY95RqVCfRTMQSbbg/O7LZaonCd6OpDIghQezsbFafTTxP8UNKvJGjPDClxopANAfXIEYCUVEM4jMCZAI5Ui173YL8q9laMf54/OAopQ91MSm3AuKTGYw/Xtp5C+mQZqlUJh+M4hsxqTwvooKWd6u1BIhnr0tfM/By94tKMUFhJHuBfjTJbj3LiC4EgGBO//AFmiIu4LjxrJo3hQzOA08jtSBgUY2U8BnvXEGvA0+6ugwEKq3WLpfEuKhSJFptk3HAjFea1fF2Jid6gCB+eczis/3yzg44bpmK2dA0vBd0DuCeNkyUO0fb6J5UlYgpxC3cP4M6xIwfiue9U9LpLVmA8U6SbAr6PM0Xo6afQDKTMmBvqrs8+BZXzMMBFRrS1qE4KYGQ3yQYDjCbokFJ9EWmnNlOKmIPUi1hIb+RTyCjQPUm0XTKwVaj1VGqbI7lNN78BENucgiVXBwmem4YhScTEKE06oHI3up3tUA0FMrJez4djykXrmceZ1iRULmJjDYEM0LBEFKCsEE+tHXTYAyHyaHwTjX5yULoc6MqCZoJhwPBAN2hBlML9q1FkLuCNFyTEHvEe1g//5z/qTtyuDCrT7nIn1FeTEEBeoQhq4Z5qlKLWlC8x//Fw/etBzuIJXG6n6dR2/pm/P0HfUmHJJrWs9FzlyNNjhdUd13pELRe0AllLUemV5xQICC714HIivFueE4RjokCa+WFQMNUgFXMqtLU+BC7XcnOg8hw7nWW+Pus3zP9db/BOoFIYwEYKuaM5gHhA0VaSspjDXt3BnCN3mirnM8Pb2fXraD1IOJ4vOw91Tv/3/v57PMmDxzFH5tsa1Qdno1uraFpYTw+7RdzOenmtd6kbLF1qxGa0Q9EsLJ2XgrGBmZMfqYAeknlrq0mpGo/FZwzj75ov4CdtIhzsg7ttCxjLIKwooES654leKmMM/gOhovPYH7nnPZbrs/fN/n6FBhV9cXTfZVKK2dqTc4ZwzL1SnRJ3HXWpxz5k67LsMLBHWConxlQ4Re47T9JrI8An8BssNFqgIz9zKTtYY4nBg4flBUsKhogmRZow9JkJBOjRLHGJNepsWbaQ6A17flE8LNvjTWKixCR1j+zg16F4YJgU4U9QXTQF+sHWid/KMeixdKbMilpuBXV29SpR+XuxI6StYKrlFAbl8EH6p2JV4BQX5Syq/kVQw0b1/dABwXGTfpDFPCOQh3jizAS61gEWn/luvYVGlI5Jr9JdpcqlrkokaH1iu09B7jrshfpsoDAyewHsfHcWc9FRWYKOZCU55KF9rLHOWDYTMwzZU3DYiS46TVcMNmo3KOMaP5zwe6zzZYB95/2RwoVCudHHRPrthusEtdoXczwZH+lo4JyA6+q9xXnZYhzY8YuJyNEycmx43NXbmfq+zlcKhW3a/KmzOvAJBh5N3CzUl7DBDHrjVuOcflwEeEA/cjfKIj3nJ1J4xQ3cVgDbHcgPRxhplfOA1klus7WbEeIEFaWSB2qff6mB32eq/qd0x4VQXIzGAqh3e6RLu+GJ6Rg9NG5NpazULdhy4MVnjFHuKrbZuG3APtixH4NcozFzYjSJCF8yY6eGXLM/C4GL4WZmCfIXbcP7yj7uRvurwjDjn0t76ih0HlEpS1DqddxkLMssJFlI82TaStz3He+TxNDkEQKNK3iTUxJ13hHyAc0A7Xsr8IFuMz2en8kQp85Fx1sTtssT0mDvywzSw7yW3xRT1Yo3YH++LLUV1AimeNnIsL7WAN8Qb4eu/d8nVjNXHRTqBa5ZKLRDvqM6W6HJlE/xjFdrjU8S90Xg8TVnZO2KIp9Muci6GS5IvCSeC9r3y8UGSHH5Jj8x4MH9jTKkXS0K3KPJs3Ji8vQA2DouRass3Hkv5v48fb1/feb1AUIwzpYSIo9M5u5EOq3f2luqELZ6XX+4qIO7jLrB9AOpSCj20L3tcl4F0VGaF9YVUQAnd4EunS/xqUAnCPf4Xdy/7CnApH6CcoTDaQT1PwzOb/lwwlzLzsZiJeuyCyP9rkbxg3AnQozA9ssrTPl7fvv707jat+XtYo5/WmNOqsOLyYQG/SuUI+J80MPNcg7dwfd3HFaVlyRn1yxJoo5AU/kBxP+GzcSdP20poHT+dTg0nBBl3yvlxM0Y0zdTRS5J5TRclntl2g2XvRs/E/FH3/sMvHb70YjXCFi/7iTOzLboFOx0kFcrLBEkT5rxav3QINCTrvLZKy0xrnpVKft0tKJfaX10UAn2tf3zDqm8706CVjvcqBIOqYMJf3vPNpYvPu7sPEGCcxDkqEvcnpbXWPt80fNTqOPHpA2iclx5HVNvx8815iAQ+XMCO1Hduw4woSxQTQLwOfWcjHKQ1rt3MnbbasJW0641PPUewvjrai/l3dRy2bt0t2pDb2cdewFHapbbLXsRv7fLOLmdsMLUgpd5I4xsjLtejRjnuWdDsG9aH7LRr+12x7I98+HuFpGbaA1AYnGfLXaZwzeTgi4GDAB4ccQtWiUgeQ0+lWLF1ZsucTDL3p+leVSBsw81XoBsi1qivgsVDh1Tdi/U8wisdUFv+uLqFLbKkjdGpZYzdm0AaZp8Z0nBLS56jNglyRtbD7q69XSO8qKf0L5OHBvIJ/hCFHgJsqPXiUM9XtF1WvBeEbpMgE8ZU5RuEboV84JivQyi9rX+uZhGtWMuRMz+adKxPop8lx1rRQl3J8oIstguygMi0+uBltEcT2EngO4MZlYPHTi2lhyO69e39KyiQaKuCp8TdHz9VZnFgbmT6Cnyx0hBojr9PYb9sJvYzZyR00wIxIj83RcmR5BlHY1DNGQWlXXKmN0HxjicEnmBkyWhLlp7AC5lnLnQdMc4Ezon+YSM1QuLkr9wG23vANzJnq91bun2XvjBBXB8TL6tMPaGghxKkhNsMqQlsNG6Q2w96p/J7AnaFQqMxmKF6iV4QAqGdMp+7MNEbQJGXkgm3rFnjN592aFrrSC85rKh4TSfHJVaGWFyktd55UC3CXMVQl1j75cakQoyuQ6cVaFwNFaX13jG/rbqd8JgPjnDBDnnmNtsR2Sa2V2k5zxqF7yyrSkOQnuuJ31YgkOOKCZZOEXQ9qrExzXhdjzNaUr4+vYbGd0j1Vtf06xc51Q70XLk8xDmt6Rj8Hcw4vQm9ZsbZTm9mwaU3QIzBouzGBZ8EZ1v0AuircIDIPeN3xhWwouRYoDChs8glhqMwS2Loxr9rt1rmF3AnQ4tS3ZgUfFe/vkMKbD2w8HsTTWbKGT5smvh3hDrvcAXSmt2jaD3rrw2SskSioLDcsJJjuOp2lqIzTgwKygYXRe+YNootbXJwL00SvyLuc33BqJIp4Q/xkHYZMW2z313QjSh99mqeWYet54Ift2xqxDnWAEf20UA9jUuYOftfv9VUKSvlaiMTCtccxEMpI1resEHq2q6LlCd3id2ty99/n2KlQ40zuGxi0l794IGZDQgpXjlf3rW0yvJhvt0WZ06P2BPqUm7wv1Tm+M9BznCu8i45VGrH1+GcZtxo6ZRkT+byh4KGndIJhKxOHs4oVDyAOARt6pZnXfCPDaDGr/R+Ej1+W2LcXkRjF8KP/f3eRGdhG+D+KHvCAW1KN86gQ4/BHJbdfmehSrmetq+6Pf2XR8H8CBO6uDsyyJO/zxnWGIm+t8ZijCwaRT7zvKYdFz9MMdvQzxwzkaCVM8c1EdP3vhwHmE+/Dp8TGFUpEV6Y8OwIjNYRtrv4coX5zrCRko28dfIOuSG1Zf37aImfszVM6TTpPinQbGTueaeF1BseXGvQZWlizebbglCeLYnGPIt/4oj4l+9miykQVwVu/JtHzum8eUX8g0rxCt9aEWEwh8AbtOTId5Bbf88yfvPt9YfOIqYWo84xA/Ud/uyI0/T1h0bG6nqh3HEkUY26RMpWjGYOWWHNmNy8p9V0pqMgeVNBieNRTU10FXkPzd4l5P0rvNP46gS3kTthd74GfhrI6YUwwTpDj9/v3UUOxDSUqGBp6RZNC2x6YwzlROvWHVj/1xRiCyEFRU8hJ7vwt7X8Ldv0PYVlOGtNTDwyFG+zgr9bcE94OuAsbXh1Wk46T2G3Xt7jl5IsXXb9cTxvujcQpaWPcF7ZMb7f4Tsz5b8DAAD//5UOSUI=" } diff --git a/x-pack/metricbeat/module/gcp/module.yml b/x-pack/metricbeat/module/gcp/module.yml index 6a3ecf77e58..e07d9656454 100644 --- a/x-pack/metricbeat/module/gcp/module.yml +++ b/x-pack/metricbeat/module/gcp/module.yml @@ -5,6 +5,7 @@ metricsets: - loadbalancing - storage - gke + - firestore dashboards: - id: Metricbeat-gcp-gke-overview file: 1ae960c0-f9f8-11eb-bc38-79936db7c106.json From dcf850a0e63d51a9afc3ce1aaa311be7c0132097 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 19 Jan 2022 15:14:38 -0800 Subject: [PATCH 6/8] Update gcp module & metricbeat docs --- metricbeat/docs/fields.asciidoc | 33 +++++++++++++++++++ metricbeat/docs/modules/gcp.asciidoc | 5 +++ .../docs/modules/gcp/firestore.asciidoc | 24 ++++++++++++++ metricbeat/docs/modules_list.asciidoc | 3 +- 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 metricbeat/docs/modules/gcp/firestore.asciidoc diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d1b6ece6ffc..f525089ad93 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -34847,6 +34847,39 @@ type: long -- +[float] +=== firestore + +Google Cloud Firestore metrics + + +*`gcp.firestore.document.delete.count`*:: ++ +-- +The number of successful document deletes. + +type: long + +-- + +*`gcp.firestore.document.read.count`*:: ++ +-- +The number of successful document reads from queries or lookups. + +type: long + +-- + +*`gcp.firestore.document.write.count`*:: ++ +-- +The number of successful document writes. + +type: long + +-- + [float] === gke diff --git a/metricbeat/docs/modules/gcp.asciidoc b/metricbeat/docs/modules/gcp.asciidoc index 164a7829c87..86b6239ae2a 100644 --- a/metricbeat/docs/modules/gcp.asciidoc +++ b/metricbeat/docs/modules/gcp.asciidoc @@ -275,6 +275,7 @@ metricbeat.modules: metricsets: - pubsub - loadbalancing + - firestore zone: "us-central1-a" project_id: "your project id" credentials_file_path: "your JSON credentials file path" @@ -339,6 +340,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -353,6 +356,8 @@ include::gcp/billing.asciidoc[] include::gcp/compute.asciidoc[] +include::gcp/firestore.asciidoc[] + include::gcp/gke.asciidoc[] include::gcp/loadbalancing.asciidoc[] diff --git a/metricbeat/docs/modules/gcp/firestore.asciidoc b/metricbeat/docs/modules/gcp/firestore.asciidoc new file mode 100644 index 00000000000..2e36b795b15 --- /dev/null +++ b/metricbeat/docs/modules/gcp/firestore.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-gcp-firestore]] +[role="xpack"] +=== Google Cloud Platform firestore metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/gcp/firestore/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/gcp/firestore/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 30177e13624..e4f6d15aea9 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -120,8 +120,9 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.7+| .7+| |<> beta[] +.8+| .8+| |<> beta[] |<> beta[] +|<> beta[] |<> beta[] |<> beta[] |<> beta[] From 1c68fef76eb3c6d2a02f5d1da384e074ccac37ac Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 26 Jan 2022 03:56:55 -0800 Subject: [PATCH 7/8] Add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 81271742fd1..60ceff9a1ff 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -194,6 +194,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `elasticsearch.cluster.id` field to Beat and Kibana modules. {pull}29577[29577] - Add `elasticsearch.cluster.id` field to Logstash module. {pull}29625[29625] - Add `xpack.enabled` support for Enterprise Search module. {pull}29871[29871] +- Add gcp firestore metricset. {pull}29918[29918] *Packetbeat* From 52851b642f2fb9644eede02207cbfb0cd2cc9076 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 26 Jan 2022 09:19:37 -0800 Subject: [PATCH 8/8] Add changelog entry in developer.next --- CHANGELOG-developer.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 92566fdf5f9..2757def7829 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -130,6 +130,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Update Go version to 1.17.1. {pull}27543[27543] - Whitelist `GCP_*` environment variables in dev tools {pull}28364[28364] - Add support for `credentials_json` in `gcp` module, all metricsets {pull}29584[29584] +- Add gcp firestore metricset. {pull}29918[29918] ==== Deprecated