From 94e8dc3ed87add11fcfd82c8e9c71feb3a06dde0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 24 Feb 2020 17:31:48 -0800 Subject: [PATCH 01/12] [Debugging only!] Add charts to see memory usage over time. THIS COMMIT MUST BE REMOVED AT THE END!!! --- go.mod | 3 ++- go.sum | 9 +++++++++ libbeat/service/service.go | 6 ++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f44be30d7fc..c96d3cfecf6 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/Azure/go-autorest/autorest/date v0.2.0 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 github.com/Shopify/sarama v0.0.0-00010101000000-000000000000 - github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc github.com/akavel/rsrc v0.8.0 // indirect github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 @@ -114,6 +114,7 @@ require ( github.com/mitchellh/gox v1.0.1 github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51 github.com/mitchellh/mapstructure v1.1.2 + github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/oklog/ulid v1.3.1 github.com/opencontainers/go-digest v1.0.0-rc1.0.20190228220655-ac19fd6e7483 // indirect diff --git a/go.sum b/go.sum index fbce282b594..dd9f80c3120 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWso github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 h1:2Gl9Tray0NEjP9KC0FjdGWlszbmTIsBP3JYzgyFdL4E= github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d h1:g0M6kedfjDpyAAuxqBvJzMNjFzlrQ7Av6LCDFqWierk= github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d/go.mod h1:VykaKG/ofkKje+MSvqjrDsz1wfyHIvEVFljhq2EOZ4g= github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc h1:9iW/Fbn/R/nyUOiqo6AgwBe8uirqUIoTGF3vKG8qjoc= @@ -281,6 +283,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab h1:xveKWz2iaueeTaUgdetzel+U7exyigDYBryyVfV/rZk= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 h1:whypLownH338a3Ork2w9t0KUKtVxbXYySuz7V1YGsJo= github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -376,6 +379,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorhill/cronexpr v0.0.0-20161205141322-d520615e531a h1:yNuTIQkXLNAevCwQJ7ur3ZPoZPhbvAi6QXhJ/ylX6+8= github.com/gorhill/cronexpr v0.0.0-20161205141322-d520615e531a/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= @@ -496,6 +500,8 @@ github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWe github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615 h1:/mD+ABZyXD39BzJI2XyRJlqdZG11gXFo0SSynL+OFeU= +github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -596,6 +602,8 @@ github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522/go.mod h1:tQTYKOQg github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.19.11+incompatible h1:lJHR0foqAjI4exXqWsU3DbH7bX1xvdhGdnXTIARA9W4= github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -780,6 +788,7 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c h1:OYFUffxXPezb7BVTx9AaD4Vl0qtxmklBIkwCKH1YwDY= golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/libbeat/service/service.go b/libbeat/service/service.go index 2d88b25f782..625b2ba3bfd 100644 --- a/libbeat/service/service.go +++ b/libbeat/service/service.go @@ -34,6 +34,7 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" + _ "github.com/mkevac/debugcharts" ) // HandleSignals manages OS signals that ask the service/daemon to stop. @@ -105,6 +106,11 @@ func BeforeRun() { http.DefaultServeMux.ServeHTTP(w, r) }) + // register debugcharts handler + mux.HandleFunc("/debug/charts/", func(w http.ResponseWriter, r *http.Request) { + http.DefaultServeMux.ServeHTTP(w, r) + }) + // register metrics handler mux.HandleFunc("/debug/vars", metricsHandler) From 7e8044ae2d99a07552a55579c75b16012eded543 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 24 Feb 2020 19:30:16 -0800 Subject: [PATCH 02/12] [Debugging only] Disable validation so we can test only index metricset --- metricbeat/module/elasticsearch/elasticsearch.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 9698f6ce003..4718abc006d 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -43,9 +43,9 @@ func init() { // NewModule creates a new module after performing validation. func NewModule(base mb.BaseModule) (mb.Module, error) { - if err := validateXPackMetricsets(base); err != nil { - return nil, err - } + //if err := validateXPackMetricsets(base); err != nil { + // return nil, err + //} return &base, nil } From 4f63066716216fcd59161fcb2a0985b6da3a8f0b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 24 Feb 2020 19:30:42 -0800 Subject: [PATCH 03/12] Streaming parser --- .../module/elasticsearch/index/data_xpack.go | 56 ++++++++++++++++++- .../module/elasticsearch/index/index.go | 2 +- 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data_xpack.go b/metricbeat/module/elasticsearch/index/data_xpack.go index 8f81c930f84..6a791f47302 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack.go +++ b/metricbeat/module/elasticsearch/index/data_xpack.go @@ -18,6 +18,7 @@ package index import ( + "bytes" "encoding/json" "fmt" "strconv" @@ -34,6 +35,10 @@ import ( "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) +var ( + errParse = errors.New("failure parsing Indices Stats Elasticsearch API response") +) + var ( // Based on https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDoc.java#L127-L203 xpackSchema = s.Schema{ @@ -106,8 +111,41 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, return errors.Wrap(err, "failure retrieving cluster state from Elasticsearch") } + dec := json.NewDecoder(bytes.NewReader(content)) + + // read opening `{` + if _, err = dec.Token(); err != nil { + return errors.Wrap(err, errParse.Error()) + } + + // read "indices" + if _, err = dec.Token(); err != nil { + return errors.Wrap(err, errParse.Error()) + } + + // read nested opening `{` + if _, err = dec.Token(); err != nil { + return errors.Wrap(err, errParse.Error()) + } + var errs multierror.Errors - for name, index := range indicesStruct.Indices { + for dec.More() { + // Read index name + tok, err := dec.Token() + if err != nil { + return errors.Wrap(err, errParse.Error()) + } + + name, ok := tok.(string) + if !ok { + return errParse + } + + var index map[string]interface{} + if err = dec.Decode(&index); err != nil { + return errors.Wrap(err, errParse.Error()) + } + event := mb.Event{} indexStats, err := xpackSchema.Apply(index) if err != nil { @@ -134,7 +172,21 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, r.Event(event) } - return errs.Err() + if errs != nil { + return errs.Err() + } + + // read nested closing `}` + if _, err = dec.Token(); err != nil { + return errors.Wrap(err, errParse.Error()) + } + + // read closing `}` + if _, err = dec.Token(); err != nil { + return errors.Wrap(err, errParse.Error()) + } + + return nil } func parseAPIResponse(content []byte, indicesStruct *IndicesStruct) error { diff --git a/metricbeat/module/elasticsearch/index/index.go b/metricbeat/module/elasticsearch/index/index.go index 6936b836b16..cd2dc3ffca0 100644 --- a/metricbeat/module/elasticsearch/index/index.go +++ b/metricbeat/module/elasticsearch/index/index.go @@ -35,7 +35,7 @@ func init() { const ( statsMetrics = "docs,fielddata,indexing,merge,search,segments,store,refresh,query_cache,request_cache" - statsPath = "/_stats/" + statsMetrics + statsPath = "/_stats/" + statsMetrics + "?filter_path=indices" ) // MetricSet type defines all fields of the MetricSet From 3bf526ec6531eb8b2b03ae635029eead897c74bc Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 9 Mar 2020 17:30:56 -0700 Subject: [PATCH 04/12] Removing debugcharts --- libbeat/service/service.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libbeat/service/service.go b/libbeat/service/service.go index 625b2ba3bfd..2d88b25f782 100644 --- a/libbeat/service/service.go +++ b/libbeat/service/service.go @@ -34,7 +34,6 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" - _ "github.com/mkevac/debugcharts" ) // HandleSignals manages OS signals that ask the service/daemon to stop. @@ -106,11 +105,6 @@ func BeforeRun() { http.DefaultServeMux.ServeHTTP(w, r) }) - // register debugcharts handler - mux.HandleFunc("/debug/charts/", func(w http.ResponseWriter, r *http.Request) { - http.DefaultServeMux.ServeHTTP(w, r) - }) - // register metrics handler mux.HandleFunc("/debug/vars", metricsHandler) From ab68e1032345e0b8a884143b6dd80241b2e54df4 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Mar 2020 11:54:55 -0700 Subject: [PATCH 05/12] Replace maps with structs --- .../module/elasticsearch/index/data_xpack.go | 239 ++++++++---------- 1 file changed, 107 insertions(+), 132 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data_xpack.go b/metricbeat/module/elasticsearch/index/data_xpack.go index 6a791f47302..bbb30fbc91e 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack.go +++ b/metricbeat/module/elasticsearch/index/data_xpack.go @@ -18,7 +18,6 @@ package index import ( - "bytes" "encoding/json" "fmt" "strconv" @@ -28,8 +27,6 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" - s "github.com/elastic/beats/v7/libbeat/common/schema" - c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" "github.com/elastic/beats/v7/metricbeat/helper/elastic" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" @@ -39,65 +36,89 @@ var ( errParse = errors.New("failure parsing Indices Stats Elasticsearch API response") ) -var ( - // Based on https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDoc.java#L127-L203 - xpackSchema = s.Schema{ - "uuid": c.Str("uuid"), - "primaries": c.Dict("primaries", indexStatsSchema), - "total": c.Dict("total", indexStatsSchema), - } +// Based on https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDoc.java#L127-L203 +type stats struct { + Indices map[string]index `json:"indices"` +} - indexStatsSchema = s.Schema{ - "docs": c.Dict("docs", s.Schema{ - "count": c.Int("count"), - }), - "fielddata": c.Dict("fielddata", s.Schema{ - "memory_size_in_bytes": c.Int("memory_size_in_bytes"), - "evictions": c.Int("evictions"), - }), - "indexing": c.Dict("indexing", s.Schema{ - "index_total": c.Int("index_total"), - "index_time_in_millis": c.Int("index_time_in_millis"), - "throttle_time_in_millis": c.Int("throttle_time_in_millis"), - }), - "merges": c.Dict("merges", s.Schema{ - "total_size_in_bytes": c.Int("total_size_in_bytes"), - }), - "query_cache": c.Dict("query_cache", cacheStatsSchema), - "request_cache": c.Dict("request_cache", cacheStatsSchema), - "search": c.Dict("search", s.Schema{ - "query_total": c.Int("query_total"), - "query_time_in_millis": c.Int("query_time_in_millis"), - }), - "segments": c.Dict("segments", s.Schema{ - "count": c.Int("count"), - "memory_in_bytes": c.Int("memory_in_bytes"), - "terms_memory_in_bytes": c.Int("terms_memory_in_bytes"), - "stored_fields_memory_in_bytes": c.Int("stored_fields_memory_in_bytes"), - "term_vectors_memory_in_bytes": c.Int("term_vectors_memory_in_bytes"), - "norms_memory_in_bytes": c.Int("norms_memory_in_bytes"), - "points_memory_in_bytes": c.Int("points_memory_in_bytes"), - "doc_values_memory_in_bytes": c.Int("doc_values_memory_in_bytes"), - "index_writer_memory_in_bytes": c.Int("index_writer_memory_in_bytes"), - "version_map_memory_in_bytes": c.Int("version_map_memory_in_bytes"), - "fixed_bit_set_memory_in_bytes": c.Int("fixed_bit_set_memory_in_bytes"), - }), - "store": c.Dict("store", s.Schema{ - "size_in_bytes": c.Int("size_in_bytes"), - }), - "refresh": c.Dict("refresh", s.Schema{ - "external_total_time_in_millis": c.Int("external_total_time_in_millis", s.Optional), - "total_time_in_millis": c.Int("total_time_in_millis"), - }), - } +type index struct { + UUID string `json:"uuid"` + Primaries indexStats `json:"primaries"` + Total indexStats `json:"total"` - cacheStatsSchema = s.Schema{ - "memory_size_in_bytes": c.Int("memory_size_in_bytes"), - "evictions": c.Int("evictions"), - "hit_count": c.Int("hit_count"), - "miss_count": c.Int("miss_count"), - } -) + Index string `json:"index"` + Created int64 `json:"created"` + Status string `json:"status"` + Shards shardStats `json:"shards"` +} + +type indexStats struct { + Docs struct { + Count int `json:"count"` + } `json:"docs"` + FieldData struct { + MemorySizeInBytes int `json:"memory_size_in_bytes"` + Evictions int `json:"evictions"` + } `json:"fielddata"` + Indexing struct { + IndexTotal int `json:"index_total"` + IndexTimeInMillis int `json:"index_time_in_millis"` + ThrottleTimeInMillis int `json:"throttle_time_in_millis"` + } `json:"indexing"` + Merges struct { + TotalSizeInBytes int `json:"total_size_in_bytes"` + } `json:"merges"` + QueryCache cacheStats `json:"query_stats"` + RequestCache cacheStats `json:"request_cache"` + Search struct { + QueryTotal int `json:"query_total"` + QueryTimeInMillis int `json:"query_time_in_millis"` + } `json:"search"` + Segments struct { + Count int `json:"count"` + MemoryInBytes int `json:"memory_in_bytes"` + TermsMemoryInBytes int `json:"terms_memory_in_bytes"` + StoredFieldsMemoryInBytes int `json:"stored_fields_memory_in_bytes"` + TermVectorsMemoryInBytes int `json:"term_vectors_memory_in_bytes"` + NormsMemoryInBytes int `json:"norms_memory_in_bytes"` + PointsMemoryInBytes int `json:"points_memory_in_bytes"` + DocValuesMemoryInBytes int `json:"doc_values_memory_in_bytes"` + IndexWriterMemoryInBytes int `json:"index_writer_memory_in_bytes"` + VersionMapMemoryInBytes int `json:"version_map_memory_in_bytes"` + FixedBitSetMemoryInBytes int `json:"fixed_bit_set_memory_in_bytes"` + } `json:"segments"` + Store struct { + SizeInBytes int `json:"size_in_bytes"` + } `json:"store"` + Refresh struct { + ExternalTotalTimeInMillis int `json:"external_total_time_in_millis"` + TotalTimeInMillis int `json:"total_time_in_millis"` + } `json:"refresh"` +} + +type cacheStats struct { + MemorySizeInBytes int `json:"memory_size_in_bytes"` + Evictions int `json:"evictions"` + HitCount int `json:"hit_count"` + MissCount int `json:"miss_count"` +} + +type shardStats struct { + Total int `json:"total"` + Primaries int `json:"primaries"` + Replicas int `json:"replicas"` + + ActiveTotal int `json:"active_total"` + ActivePrimaries int `json:"active_primaries"` + ActiveReplicas int `json:"active_replicas"` + + UnassignedTotal int `json:"unassigned_total"` + UnassignedPrimaries int `json:"unassigned_primaries"` + UnassignedReplicas int `json:"unassigned_replicas"` + + Initializing int `json:"initializing"` + Relocating int `json:"relocationg"` +} func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { var indicesStruct IndicesStruct @@ -111,50 +132,17 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, return errors.Wrap(err, "failure retrieving cluster state from Elasticsearch") } - dec := json.NewDecoder(bytes.NewReader(content)) - - // read opening `{` - if _, err = dec.Token(); err != nil { - return errors.Wrap(err, errParse.Error()) - } - - // read "indices" - if _, err = dec.Token(); err != nil { - return errors.Wrap(err, errParse.Error()) - } - - // read nested opening `{` - if _, err = dec.Token(); err != nil { - return errors.Wrap(err, errParse.Error()) + var indicesStats stats + if err := json.Unmarshal(content, &indicesStats); err != nil { + return errors.Wrap(err, "failure parsing Indices Stats Elasticsearch API response") } var errs multierror.Errors - for dec.More() { - // Read index name - tok, err := dec.Token() - if err != nil { - return errors.Wrap(err, errParse.Error()) - } - - name, ok := tok.(string) - if !ok { - return errParse - } - - var index map[string]interface{} - if err = dec.Decode(&index); err != nil { - return errors.Wrap(err, errParse.Error()) - } - + for name, idx := range indicesStats.Indices { event := mb.Event{} - indexStats, err := xpackSchema.Apply(index) - if err != nil { - errs = append(errs, errors.Wrap(err, "failure applying index stats schema")) - continue - } - indexStats["index"] = name + idx.Index = name - err = addClusterStateFields(name, indexStats, clusterState) + err = addClusterStateFields(idx, clusterState) if err != nil { errs = append(errs, errors.Wrap(err, "failure adding cluster state fields")) continue @@ -165,7 +153,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, "timestamp": common.Time(time.Now()), "interval_ms": m.Module().Config().Period / time.Millisecond, "type": "index_stats", - "index_stats": indexStats, + "index_stats": idx, } event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) @@ -176,16 +164,6 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, return errs.Err() } - // read nested closing `}` - if _, err = dec.Token(); err != nil { - return errors.Wrap(err, errParse.Error()) - } - - // read closing `}` - if _, err = dec.Token(); err != nil { - return errors.Wrap(err, errParse.Error()) - } - return nil } @@ -195,13 +173,13 @@ func parseAPIResponse(content []byte, indicesStruct *IndicesStruct) error { // Fields added here are based on same fields being added by internal collection in // https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDoc.java#L62-L124 -func addClusterStateFields(indexName string, indexStats, clusterState common.MapStr) error { - indexMetadata, err := getClusterStateMetricForIndex(clusterState, indexName, "metadata") +func addClusterStateFields(idx index, clusterState common.MapStr) error { + indexMetadata, err := getClusterStateMetricForIndex(clusterState, idx.Index, "metadata") if err != nil { return errors.Wrap(err, "failed to get index metadata from cluster state") } - indexRoutingTable, err := getClusterStateMetricForIndex(clusterState, indexName, "routing_table") + indexRoutingTable, err := getClusterStateMetricForIndex(clusterState, idx.Index, "routing_table") if err != nil { return errors.Wrap(err, "failed to get index routing table from cluster state") } @@ -215,7 +193,7 @@ func addClusterStateFields(indexName string, indexStats, clusterState common.Map if err != nil { return errors.Wrap(err, "failed to get index creation time") } - indexStats.Put("created", created) + idx.Created = created // "index_stats.version.created", <--- don't think this is being used in the UI, so can we skip it? // "index_stats.version.upgraded", <--- don't think this is being used in the UI, so can we skip it? @@ -224,13 +202,13 @@ func addClusterStateFields(indexName string, indexStats, clusterState common.Map if err != nil { return errors.Wrap(err, "failed to get index status") } - indexStats.Put("status", status) + idx.Status = status shardStats, err := getIndexShardStats(shards) if err != nil { return errors.Wrap(err, "failed to get index shard stats") } - indexStats.Put("shards", shardStats) + idx.Shards = *shardStats return nil } @@ -293,7 +271,7 @@ func getIndexStatus(shards map[string]interface{}) (string, error) { return "red", nil } -func getIndexShardStats(shards common.MapStr) (common.MapStr, error) { +func getIndexShardStats(shards common.MapStr) (*shardStats, error) { primaries := 0 replicas := 0 @@ -350,21 +328,18 @@ func getIndexShardStats(shards common.MapStr) (common.MapStr, error) { } } - return common.MapStr{ - "total": primaries + replicas, - "primaries": primaries, - "replicas": replicas, - - "active_total": activePrimaries + activeReplicas, - "active_primaries": activePrimaries, - "active_replicas": activeReplicas, - - "unassigned_total": unassignedPrimaries + unassignedReplicas, - "unassigned_primaries": unassignedPrimaries, - "unassigned_replicas": unassignedReplicas, - - "initializing": initializing, - "relocating": relocating, + return &shardStats{ + Total: primaries + replicas, + Primaries: primaries, + Replicas: replicas, + ActiveTotal: activePrimaries + activeReplicas, + ActivePrimaries: activePrimaries, + ActiveReplicas: activeReplicas, + UnassignedTotal: unassignedPrimaries + unassignedReplicas, + UnassignedPrimaries: unassignedPrimaries, + UnassignedReplicas: unassignedReplicas, + Initializing: initializing, + Relocating: relocating, }, nil } From 884614dcb6118c48ba9961d10d65c954135b8282 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Mar 2020 11:56:37 -0700 Subject: [PATCH 06/12] Running go mod tidy --- go.mod | 1 - go.sum | 9 --------- 2 files changed, 10 deletions(-) diff --git a/go.mod b/go.mod index c96d3cfecf6..8726d732295 100644 --- a/go.mod +++ b/go.mod @@ -114,7 +114,6 @@ require ( github.com/mitchellh/gox v1.0.1 github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51 github.com/mitchellh/mapstructure v1.1.2 - github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/oklog/ulid v1.3.1 github.com/opencontainers/go-digest v1.0.0-rc1.0.20190228220655-ac19fd6e7483 // indirect diff --git a/go.sum b/go.sum index dd9f80c3120..84ddeb6da3e 100644 --- a/go.sum +++ b/go.sum @@ -90,8 +90,6 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 h1:2Gl9Tray0NEjP9KC0FjdGWlszbmTIsBP3JYzgyFdL4E= -github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d h1:g0M6kedfjDpyAAuxqBvJzMNjFzlrQ7Av6LCDFqWierk= @@ -283,7 +281,6 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab h1:xveKWz2iaueeTaUgdetzel+U7exyigDYBryyVfV/rZk= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647 h1:whypLownH338a3Ork2w9t0KUKtVxbXYySuz7V1YGsJo= github.com/go-ole/go-ole v1.2.5-0.20190920104607-14974a1cf647/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -379,7 +376,6 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorhill/cronexpr v0.0.0-20161205141322-d520615e531a h1:yNuTIQkXLNAevCwQJ7ur3ZPoZPhbvAi6QXhJ/ylX6+8= github.com/gorhill/cronexpr v0.0.0-20161205141322-d520615e531a/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= @@ -500,8 +496,6 @@ github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWe github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615 h1:/mD+ABZyXD39BzJI2XyRJlqdZG11gXFo0SSynL+OFeU= -github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -602,8 +596,6 @@ github.com/sanathkr/yaml v1.0.1-0.20170819201035-0056894fa522/go.mod h1:tQTYKOQg github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.19.11+incompatible h1:lJHR0foqAjI4exXqWsU3DbH7bX1xvdhGdnXTIARA9W4= github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -788,7 +780,6 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c h1:OYFUffxXPezb7BVTx9AaD4Vl0qtxmklBIkwCKH1YwDY= golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From a767e4c7581b80daa971071b14c0e93e638cc318 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Mar 2020 12:09:14 -0700 Subject: [PATCH 07/12] Pass pointer --- metricbeat/module/elasticsearch/index/data_xpack.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data_xpack.go b/metricbeat/module/elasticsearch/index/data_xpack.go index bbb30fbc91e..ee2caac1f65 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack.go +++ b/metricbeat/module/elasticsearch/index/data_xpack.go @@ -142,7 +142,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, event := mb.Event{} idx.Index = name - err = addClusterStateFields(idx, clusterState) + err = addClusterStateFields(&idx, clusterState) if err != nil { errs = append(errs, errors.Wrap(err, "failure adding cluster state fields")) continue @@ -173,7 +173,7 @@ func parseAPIResponse(content []byte, indicesStruct *IndicesStruct) error { // Fields added here are based on same fields being added by internal collection in // https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDoc.java#L62-L124 -func addClusterStateFields(idx index, clusterState common.MapStr) error { +func addClusterStateFields(idx *index, clusterState common.MapStr) error { indexMetadata, err := getClusterStateMetricForIndex(clusterState, idx.Index, "metadata") if err != nil { return errors.Wrap(err, "failed to get index metadata from cluster state") From e5422ed08d424a20ca88d338f9b91590b64fb4af Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 10 Mar 2020 12:11:34 -0700 Subject: [PATCH 08/12] Uncomment code --- metricbeat/module/elasticsearch/elasticsearch.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 4718abc006d..9698f6ce003 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -43,9 +43,9 @@ func init() { // NewModule creates a new module after performing validation. func NewModule(base mb.BaseModule) (mb.Module, error) { - //if err := validateXPackMetricsets(base); err != nil { - // return nil, err - //} + if err := validateXPackMetricsets(base); err != nil { + return nil, err + } return &base, nil } From adb913f3bb65a95bd9bdbe065d134d7b78dc9aff Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 11 Mar 2020 11:41:43 -0700 Subject: [PATCH 09/12] Reverting unnecessary changes --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8726d732295..f44be30d7fc 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/Azure/go-autorest/autorest/date v0.2.0 github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 github.com/Shopify/sarama v0.0.0-00010101000000-000000000000 - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d + github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc github.com/akavel/rsrc v0.8.0 // indirect github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 diff --git a/go.sum b/go.sum index 84ddeb6da3e..fbce282b594 100644 --- a/go.sum +++ b/go.sum @@ -90,8 +90,8 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6 h1:2Gl9Tray0NEjP9KC0FjdGWlszbmTIsBP3JYzgyFdL4E= +github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d h1:g0M6kedfjDpyAAuxqBvJzMNjFzlrQ7Av6LCDFqWierk= github.com/adriansr/fsnotify v0.0.0-20180417234312-c9bbe1f46f1d/go.mod h1:VykaKG/ofkKje+MSvqjrDsz1wfyHIvEVFljhq2EOZ4g= github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc h1:9iW/Fbn/R/nyUOiqo6AgwBe8uirqUIoTGF3vKG8qjoc= From 62734b7e65505cfc93a21ac06681d6cd6ad451ff Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 11 Mar 2020 11:46:20 -0700 Subject: [PATCH 10/12] Adding CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 854eb4e1672..61d1b0a6553 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -116,6 +116,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix imports after PR was merged before rebase. {pull}16756[16756] - Add dashboard for `redisenterprise` module. {pull}16752[16752] - Dynamically choose a method for the system/service metricset to support older linux distros. {pull}16902[16902] +- Reduce memory usage in `elasticsearch/index` metricset. {issue}16503[16503] {pull}16538[16538] *Packetbeat* From 7bdccfd491c5d3150b638f407ba0df148baa34c2 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 16 Mar 2020 20:00:36 -0700 Subject: [PATCH 11/12] Incorporating benchmark test --- metricbeat/module/elasticsearch/index/data_xpack.go | 11 +++-------- .../module/elasticsearch/index/data_xpack_test.go | 4 ++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data_xpack.go b/metricbeat/module/elasticsearch/index/data_xpack.go index ee2caac1f65..e2a0565a9dd 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack.go +++ b/metricbeat/module/elasticsearch/index/data_xpack.go @@ -121,11 +121,6 @@ type shardStats struct { } func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { - var indicesStruct IndicesStruct - if err := parseAPIResponse(content, &indicesStruct); err != nil { - return errors.Wrap(err, "failure parsing Indices Stats Elasticsearch API response") - } - clusterStateMetrics := []string{"metadata", "routing_table"} clusterState, err := elasticsearch.GetClusterState(m.HTTP, m.HTTP.GetURI(), clusterStateMetrics) if err != nil { @@ -133,7 +128,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, } var indicesStats stats - if err := json.Unmarshal(content, &indicesStats); err != nil { + if err := parseAPIResponse(content, &indicesStats); err != nil { return errors.Wrap(err, "failure parsing Indices Stats Elasticsearch API response") } @@ -167,8 +162,8 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, return nil } -func parseAPIResponse(content []byte, indicesStruct *IndicesStruct) error { - return json.Unmarshal(content, indicesStruct) +func parseAPIResponse(content []byte, indicesStats *stats) error { + return json.Unmarshal(content, indicesStats) } // Fields added here are based on same fields being added by internal collection in diff --git a/metricbeat/module/elasticsearch/index/data_xpack_test.go b/metricbeat/module/elasticsearch/index/data_xpack_test.go index 749467ef72d..1bdc790d9ae 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack_test.go +++ b/metricbeat/module/elasticsearch/index/data_xpack_test.go @@ -31,10 +31,10 @@ func BenchmarkParseAPIResponse(b *testing.B) { content, err := ioutil.ReadFile("_meta/test/stats.800.bench.json") require.NoError(b, err) - var indicesStruct IndicesStruct + var indicesStats stats for i := 0; i < b.N; i++ { - err = parseAPIResponse(content, &indicesStruct) + err = parseAPIResponse(content, &indicesStats) require.NoError(b, err) } From 4620a0f2d304b2f8c18c1650e3b9333e4b4f3599 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 17 Mar 2020 11:22:50 -0700 Subject: [PATCH 12/12] Removing unnecessary nil check --- metricbeat/module/elasticsearch/index/data_xpack.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metricbeat/module/elasticsearch/index/data_xpack.go b/metricbeat/module/elasticsearch/index/data_xpack.go index e2a0565a9dd..160211ea61c 100644 --- a/metricbeat/module/elasticsearch/index/data_xpack.go +++ b/metricbeat/module/elasticsearch/index/data_xpack.go @@ -155,11 +155,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, r.Event(event) } - if errs != nil { - return errs.Err() - } - - return nil + return errs.Err() } func parseAPIResponse(content []byte, indicesStats *stats) error {