From b07852317c1c29d68d805ca6b2500984960f289d Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Tue, 25 Jan 2022 16:49:18 +0100 Subject: [PATCH] Stack Monitoring: Add Enterprise Search monitoring index templates (#82743) --- .../resources/monitoring-ent-search-mb.json | 721 ++++++++++++++++++ .../MonitoringTemplateRegistry.java | 15 +- .../http/HttpExporterResourceTests.java | 2 +- 3 files changed, 736 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json diff --git a/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json new file mode 100644 index 0000000000000..fac2a026c405c --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json @@ -0,0 +1,721 @@ +{ + "index_patterns": [".monitoring-ent-search-${xpack.stack.monitoring.template.version}-*"], + "version": ${xpack.stack.monitoring.template.release.version}, + "_meta": { + "description": "Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring", + "managed": true + }, + "template": { + "mappings": { + "properties": { + "enterprisesearch": { + "properties": { + "cluster_uuid": { + "type": "keyword", + "ignore_above": 1024 + }, + "health": { + "properties": { + "crawler": { + "properties": { + "workers": { + "properties": { + "active": { + "type": "long" + }, + "available": { + "type": "long" + }, + "pool_size": { + "type": "long" + } + } + } + } + }, + "jvm": { + "properties": { + "gc": { + "properties": { + "collection_count": { + "type": "long" + }, + "collection_time": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "memory_usage": { + "properties": { + "heap_committed": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "heap_init": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "heap_max": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "heap_used": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "non_heap_committed": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "non_heap_init": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "object_pending_finalization_count": { + "type": "long" + } + } + }, + "threads": { + "properties": { + "current": { + "type": "long" + }, + "daemon": { + "type": "long" + }, + "max": { + "type": "long" + }, + "total_started": { + "type": "long" + } + } + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "process": { + "properties": { + "filebeat": { + "properties": { + "pid": { + "type": "long" + }, + "restart_count": { + "type": "long" + }, + "time_since_last_restart": { + "properties": { + "sec": { + "type": "long" + } + } + } + } + }, + "pid": { + "type": "long" + }, + "uptime": { + "properties": { + "sec": { + "type": "long" + } + } + } + } + }, + "version": { + "properties": { + "build_hash": { + "type": "keyword", + "ignore_above": 1024 + }, + "number": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "stats": { + "properties": { + "connectors": { + "properties": { + "job_store": { + "properties": { + "job_types": { + "properties": { + "delete": { + "type": "long" + }, + "full": { + "type": "long" + }, + "incremental": { + "type": "long" + }, + "permissions": { + "type": "long" + } + } + }, + "waiting": { + "type": "long" + }, + "working": { + "type": "long" + } + } + }, + "pool": { + "properties": { + "extract_worker_pool": { + "properties": { + "busy": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "queue_depth": { + "type": "long" + }, + "size": { + "type": "long" + }, + "total_completed": { + "type": "long" + }, + "total_scheduled": { + "type": "long" + } + } + }, + "publish_worker_pool": { + "properties": { + "busy": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "queue_depth": { + "type": "long" + }, + "size": { + "type": "long" + }, + "total_completed": { + "type": "long" + }, + "total_scheduled": { + "type": "long" + } + } + }, + "subextract_worker_pool": { + "properties": { + "busy": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "queue_depth": { + "type": "long" + }, + "size": { + "type": "long" + }, + "total_completed": { + "type": "long" + }, + "total_scheduled": { + "type": "long" + } + } + } + } + } + } + }, + "http": { + "properties": { + "connections": { + "properties": { + "current": { + "type": "long" + }, + "max": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "network": { + "properties": { + "received": { + "properties": { + "bytes": { + "type": "long" + }, + "bytes_per_sec": { + "type": "long" + } + } + }, + "sent": { + "properties": { + "bytes": { + "type": "long" + }, + "bytes_per_sec": { + "type": "long" + } + } + } + } + }, + "request_duration": { + "properties": { + "max": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "mean": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "std_dev": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "responses": { + "properties": { + "1xx": { + "type": "long" + }, + "2xx": { + "type": "long" + }, + "3xx": { + "type": "long" + }, + "4xx": { + "type": "long" + }, + "5xx": { + "type": "long" + } + } + } + } + }, + "product_usage": { + "properties": { + "app_search": { + "properties": { + "total_engines": { + "type": "long" + } + } + }, + "workplace_search": { + "properties": { + "total_org_sources": { + "type": "long" + }, + "total_private_sources": { + "type": "long" + } + } + } + } + }, + "queues": { + "properties": { + "engine_destroyer": { + "properties": { + "count": { + "type": "long" + } + } + }, + "failed": { + "properties": { + "count": { + "type": "long" + } + } + }, + "mailer": { + "properties": { + "count": { + "type": "long" + } + } + }, + "process_crawl": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "agent": { + "properties": { + "id": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "ephemeral_id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "process": { + "properties": { + "pid": { + "type": "long" + } + } + }, + "@timestamp": { + "type": "date" + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "type": "keyword", + "ignore_above": 1024 + }, + "agent_id_status": { + "type": "keyword", + "ignore_above": 1024 + }, + "category": { + "type": "keyword", + "ignore_above": 1024 + }, + "code": { + "type": "keyword", + "ignore_above": 1024 + }, + "created": { + "type": "date" + }, + "dataset": { + "type": "keyword", + "ignore_above": 1024 + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "ingested": { + "type": "date" + }, + "kind": { + "type": "keyword", + "ignore_above": 1024 + }, + "module": { + "type": "keyword", + "ignore_above": 1024 + }, + "original": { + "type": "keyword", + "index": false, + "doc_values": false, + "ignore_above": 1024 + }, + "outcome": { + "type": "keyword", + "ignore_above": 1024 + }, + "provider": { + "type": "keyword", + "ignore_above": 1024 + }, + "reason": { + "type": "keyword", + "ignore_above": 1024 + }, + "reference": { + "type": "keyword", + "ignore_above": 1024 + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "url": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "metricset": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "period": { + "type": "long" + } + } + }, + "service": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "environment": { + "type": "keyword", + "ignore_above": 1024 + }, + "ephemeral_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "node": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "origin": { + "properties": { + "address": { + "type": "keyword", + "ignore_above": 1024 + }, + "environment": { + "type": "keyword", + "ignore_above": 1024 + }, + "ephemeral_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "node": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "state": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "state": { + "type": "keyword", + "ignore_above": 1024 + }, + "target": { + "properties": { + "address": { + "type": "keyword", + "ignore_above": 1024 + }, + "environment": { + "type": "keyword", + "ignore_above": 1024 + }, + "ephemeral_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "node": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "state": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "architecture": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "settings": { + "index.mapping.total_fields.limit": 2000 + } + }, + "data_stream": {} +} diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java index fe54eac3bebe2..bea869222c38e 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @@ -207,6 +207,18 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry { STACK_TEMPLATE_VARIABLES ); + ////////////////////////////////////////////////////////// + // Enterprise Search metricbeat template (for matching ".monitoring-ent-search-8-*" indices) + ////////////////////////////////////////////////////////// + public static final String ENTERPRISE_SEARCH_STACK_INDEX_TEMPLATE_NAME = ".monitoring-ent-search-mb"; + public static final IndexTemplateConfig ENTERPRISE_SEARCH_STACK_INDEX_TEMPLATE = new IndexTemplateConfig( + ENTERPRISE_SEARCH_STACK_INDEX_TEMPLATE_NAME, + "/monitoring-ent-search-mb.json", + STACK_MONITORING_REGISTRY_VERSION, + STACK_MONITORING_REGISTRY_VERSION_VARIABLE, + STACK_TEMPLATE_VARIABLES + ); + public static final String[] TEMPLATE_NAMES = new String[] { ALERTS_INDEX_TEMPLATE_NAME, BEATS_INDEX_TEMPLATE_NAME, @@ -296,7 +308,8 @@ protected List getLegacyTemplateConfigs() { BEATS_STACK_INDEX_TEMPLATE, ES_STACK_INDEX_TEMPLATE, KIBANA_STACK_INDEX_TEMPLATE, - LOGSTASH_STACK_INDEX_TEMPLATE + LOGSTASH_STACK_INDEX_TEMPLATE, + ENTERPRISE_SEARCH_STACK_INDEX_TEMPLATE ); @Override diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java index b09d8fb984fb4..b255bf9a7758c 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java @@ -66,7 +66,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe private final boolean validLicense = randomBoolean(); /** - * kibana, logstash, and beats + * kibana, logstash, beats and enterprise search */ private final int EXPECTED_TEMPLATES = TEMPLATE_NAMES.length; private final int EXPECTED_WATCHES = ClusterAlertsUtil.WATCH_IDS.length;