Add Gauge for total_fields setting on a index (prometheus-community#411)

* prometheus-community#257: Add gauge for total fields mapped in an index

Signed-off-by: Mike Roest <[email protected]>
Co-authored-by: Marco Sohns <[email protected]>
Co-authored-by: Jeff Nadler <[email protected]>
Signed-off-by: Toomas Ormisson <[email protected]>

README.md

@@ -129,6 +129,7 @@ Further Information
 | elasticsearch_indices_indexing_delete_total                           | counter   | 1           | Total indexing deletes
 | elasticsearch_indices_indexing_index_time_seconds_total               | counter   | 1           | Cumulative index time in seconds
 | elasticsearch_indices_indexing_index_total                            | counter   | 1           | Total index calls
+| elasticsearch_indices_mappings_stats_fields                           | gauge     |             | Count of fields currently mapped by index
 | elasticsearch_indices_merges_docs_total                               | counter   | 1           | Cumulative docs merged
 | elasticsearch_indices_merges_total                                    | counter   | 1           | Total merges
 | elasticsearch_indices_merges_total_size_bytes_total                   | counter   | 1           | Total merge size in bytes
@@ -151,6 +152,7 @@ Further Information
 | elasticsearch_indices_segments_count                                  | gauge     | 1           | Count of index segments on this node
 | elasticsearch_indices_segments_memory_bytes                           | gauge     | 1           | Current memory size of segments in bytes
 | elasticsearch_indices_settings_stats_read_only_indices                | gauge     | 1           | Count of indices that have read_only_allow_delete=true
+| elasticsearch_indices_settings_total_fields                           | gauge     |             | Index setting value for index.mapping.total_fields.limit (total allowable mapped fields in a index)
 | elasticsearch_indices_shards_docs                                     | gauge     | 3           | Count of documents on this shard
 | elasticsearch_indices_shards_docs_deleted                             | gauge     | 3           | Count of deleted documents on each shard
 | elasticsearch_indices_store_size_bytes                                | gauge     | 1           | Current size of stored index data in bytes

collector/indices_mappings.go

@@ -0,0 +1,193 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	defaultIndicesMappingsLabels = []string{"index"}
+)
+
+type indicesMappingsMetric struct {
+	Type  prometheus.ValueType
+	Desc  *prometheus.Desc
+	Value func(indexMapping IndexMapping) float64
+}
+
+// IndicesMappings information struct
+type IndicesMappings struct {
+	logger log.Logger
+	client *http.Client
+	url    *url.URL
+
+	up                              prometheus.Gauge
+	totalScrapes, jsonParseFailures prometheus.Counter
+
+	metrics []*indicesMappingsMetric
+}
+
+// NewIndicesMappings defines Indices IndexMappings Prometheus metrics
+func NewIndicesMappings(logger log.Logger, client *http.Client, url *url.URL) *IndicesMappings {
+	subsystem := "indices_mappings_stats"
+
+	return &IndicesMappings{
+		logger: logger,
+		client: client,
+		url:    url,
+
+		up: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: prometheus.BuildFQName(namespace, subsystem, "up"),
+			Help: "Was the last scrape of the ElasticSearch Indices Mappings endpoint successful.",
+		}),
+		totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: prometheus.BuildFQName(namespace, subsystem, "scrapes_total"),
+			Help: "Current total ElasticSearch Indices Mappings scrapes.",
+		}),
+		jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: prometheus.BuildFQName(namespace, subsystem, "json_parse_failures_total"),
+			Help: "Number of errors while parsing JSON.",
+		}),
+		metrics: []*indicesMappingsMetric{
+			{
+				Type: prometheus.GaugeValue,
+				Desc: prometheus.NewDesc(
+					prometheus.BuildFQName(namespace, subsystem, "fields"),
+					"Current number fields within cluster.",
+					defaultIndicesMappingsLabels, nil,
+				),
+				Value: func(indexMapping IndexMapping) float64 {
+					return countFieldsRecursive(indexMapping.Mappings.Properties, 0)
+				},
+			},
+		},
+	}
+}
+
+func countFieldsRecursive(properties IndexMappingProperties, fieldCounter float64) float64 {
+	// iterate over all properties
+	for _, property := range properties {
+		if property.Type != nil {
+			// property has a type set - counts as a field
+			fieldCounter++
+
+			// iterate over all fields of that property
+			for _, field := range property.Fields {
+				// field has a type set - counts as a field
+				if field.Type != nil {
+					fieldCounter++
+				}
+			}
+		}
+
+		// count recursively in case the property has more properties
+		if property.Properties != nil {
+			fieldCounter = +countFieldsRecursive(property.Properties, fieldCounter)
+		}
+	}
+
+	return fieldCounter
+}
+
+// Describe add Snapshots metrics descriptions
+func (im *IndicesMappings) Describe(ch chan<- *prometheus.Desc) {
+	for _, metric := range im.metrics {
+		ch <- metric.Desc
+	}
+
+	ch <- im.up.Desc()
+	ch <- im.totalScrapes.Desc()
+	ch <- im.jsonParseFailures.Desc()
+}
+
+func (im *IndicesMappings) getAndParseURL(u *url.URL) (*IndicesMappingsResponse, error) {
+	res, err := im.client.Get(u.String())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get from %s://%s:%s%s: %s",
+			u.Scheme, u.Hostname(), u.Port(), u.Path, err)
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode)
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		_ = level.Warn(im.logger).Log("msg", "failed to read response body", "err", err)
+		return nil, err
+	}
+
+	err = res.Body.Close()
+	if err != nil {
+		_ = level.Warn(im.logger).Log("msg", "failed to close response body", "err", err)
+		return nil, err
+	}
+
+	var imr IndicesMappingsResponse
+	if err := json.Unmarshal(body, &imr); err != nil {
+		im.jsonParseFailures.Inc()
+		return nil, err
+	}
+
+	return &imr, nil
+}
+
+func (im *IndicesMappings) fetchAndDecodeIndicesMappings() (*IndicesMappingsResponse, error) {
+	u := *im.url
+	u.Path = path.Join(u.Path, "/_all/_mappings")
+	return im.getAndParseURL(&u)
+}
+
+// Collect gets all indices mappings metric values
+func (im *IndicesMappings) Collect(ch chan<- prometheus.Metric) {
+
+	im.totalScrapes.Inc()
+	defer func() {
+		ch <- im.up
+		ch <- im.totalScrapes
+		ch <- im.jsonParseFailures
+	}()
+
+	indicesMappingsResponse, err := im.fetchAndDecodeIndicesMappings()
+	if err != nil {
+		im.up.Set(0)
+		_ = level.Warn(im.logger).Log(
+			"msg", "failed to fetch and decode cluster mappings stats",
+			"err", err,
+		)
+		return
+	}
+	im.up.Set(1)
+
+	for _, metric := range im.metrics {
+		for indexName, mappings := range *indicesMappingsResponse {
+			ch <- prometheus.MustNewConstMetric(
+				metric.Desc,
+				metric.Type,
+				metric.Value(mappings),
+				indexName,
+			)
+		}
+	}
+}

collector/indices_mappings_response.go

@@ -0,0 +1,47 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+// IndicesMappingsResponse is a representation of elasticsearch mappings for each index
+type IndicesMappingsResponse map[string]IndexMapping
+
+// IndexMapping defines the struct of the tree for the mappings of each index
+type IndexMapping struct {
+	Mappings IndexMappings `json:"mappings"`
+}
+
+// IndexMappings defines all index mappings
+type IndexMappings struct {
+	Properties IndexMappingProperties `json:"properties"`
+}
+
+// IndexMappingProperties defines all the properties of the current mapping
+type IndexMappingProperties map[string]*IndexMappingProperty
+
+// IndexMappingFields defines all the fields of the current mapping
+type IndexMappingFields map[string]*IndexMappingField
+
+// IndexMappingProperty defines a single property of the current index properties
+type IndexMappingProperty struct {
+	Type       *string                `json:"type"`
+	Properties IndexMappingProperties `json:"properties"`
+	Fields     IndexMappingFields     `json:"fields"`
+}
+
+// IndexMappingField defines a single property of the current index field
+type IndexMappingField struct {
+	Type       *string                `json:"type"`
+	Properties IndexMappingProperties `json:"properties"`
+	Fields     IndexMappingFields     `json:"fields"`
+}