From 7944d8645fa367b3c551b0b8830e7c77e0a252f1 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Tue, 18 Jan 2022 16:47:56 +0100 Subject: [PATCH 1/5] Stack Monitoring: Add Enterprise Search monitoring index templates --- .../core/monitoring/MonitoredSystem.java | 2 + .../resources/monitoring-ent-search-mb.json | 717 ++++++++++++++++++ .../monitoring/integration/MonitoringIT.java | 2 +- .../MonitoringTemplateRegistry.java | 15 +- .../http/HttpExporterResourceTests.java | 2 +- 5 files changed, 735 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoredSystem.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoredSystem.java index 63a8b6f825020..2610019dda940 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoredSystem.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoredSystem.java @@ -16,6 +16,7 @@ public enum MonitoredSystem { KIBANA("kibana"), LOGSTASH("logstash"), BEATS("beats"), + ENTERPRISE_SEARCH("enterprise_search"), UNKNOWN("unknown"); private final String system; @@ -34,6 +35,7 @@ public static MonitoredSystem fromSystem(String system) { case "kibana" -> KIBANA; case "logstash" -> LOGSTASH; case "beats" -> BEATS; + case "enterprise_search" -> ENTERPRISE_SEARCH; default -> // Return an "unknown" monitored system // that can easily be filtered out if diff --git a/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json new file mode 100644 index 0000000000000..d656ea368955c --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json @@ -0,0 +1,717 @@ +{ + "index_patterns": [".monitoring-ent-search-${xpack.stack.monitoring.template.version}-*"], + "version": ${xpack.stack.monitoring.template.release.version}, + "template": { + "mappings": { + "properties": { + "enterprisesearch": { + "properties": { + "cluster_uuid": { + "type": "keyword", + "ignore_above": 1024 + }, + "health": { + "properties": { + "crawler": { + "properties": { + "workers": { + "properties": { + "active": { + "type": "long" + }, + "available": { + "type": "long" + }, + "pool_size": { + "type": "long" + } + } + } + } + }, + "jvm": { + "properties": { + "gc": { + "properties": { + "collection_count": { + "type": "long" + }, + "collection_time": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "memory_usage": { + "properties": { + "heap_committed": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "heap_init": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "heap_max": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "heap_used": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "non_heap_committed": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "non_heap_init": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "object_pending_finalization_count": { + "type": "long" + } + } + }, + "threads": { + "properties": { + "current": { + "type": "long" + }, + "daemon": { + "type": "long" + }, + "max": { + "type": "long" + }, + "total_started": { + "type": "long" + } + } + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "process": { + "properties": { + "filebeat": { + "properties": { + "pid": { + "type": "long" + }, + "restart_count": { + "type": "long" + }, + "time_since_last_restart": { + "properties": { + "sec": { + "type": "long" + } + } + } + } + }, + "pid": { + "type": "long" + }, + "uptime": { + "properties": { + "sec": { + "type": "long" + } + } + } + } + }, + "version": { + "properties": { + "build_hash": { + "type": "keyword", + "ignore_above": 1024 + }, + "number": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "stats": { + "properties": { + "connectors": { + "properties": { + "job_store": { + "properties": { + "job_types": { + "properties": { + "delete": { + "type": "long" + }, + "full": { + "type": "long" + }, + "incremental": { + "type": "long" + }, + "permissions": { + "type": "long" + } + } + }, + "waiting": { + "type": "long" + }, + "working": { + "type": "long" + } + } + }, + "pool": { + "properties": { + "extract_worker_pool": { + "properties": { + "busy": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "queue_depth": { + "type": "long" + }, + "size": { + "type": "long" + }, + "total_completed": { + "type": "long" + }, + "total_scheduled": { + "type": "long" + } + } + }, + "publish_worker_pool": { + "properties": { + "busy": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "queue_depth": { + "type": "long" + }, + "size": { + "type": "long" + }, + "total_completed": { + "type": "long" + }, + "total_scheduled": { + "type": "long" + } + } + }, + "subextract_worker_pool": { + "properties": { + "busy": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "queue_depth": { + "type": "long" + }, + "size": { + "type": "long" + }, + "total_completed": { + "type": "long" + }, + "total_scheduled": { + "type": "long" + } + } + } + } + } + } + }, + "http": { + "properties": { + "connections": { + "properties": { + "current": { + "type": "long" + }, + "max": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "network": { + "properties": { + "received": { + "properties": { + "bytes": { + "type": "long" + }, + "bytes_per_sec": { + "type": "long" + } + } + }, + "sent": { + "properties": { + "bytes": { + "type": "long" + }, + "bytes_per_sec": { + "type": "long" + } + } + } + } + }, + "request_duration": { + "properties": { + "max": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "mean": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "std_dev": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "responses": { + "properties": { + "1xx": { + "type": "long" + }, + "2xx": { + "type": "long" + }, + "3xx": { + "type": "long" + }, + "4xx": { + "type": "long" + }, + "5xx": { + "type": "long" + } + } + } + } + }, + "product_usage": { + "properties": { + "app_search": { + "properties": { + "total_engines": { + "type": "long" + } + } + }, + "workplace_search": { + "properties": { + "total_org_sources": { + "type": "long" + }, + "total_private_sources": { + "type": "long" + } + } + } + } + }, + "queues": { + "properties": { + "engine_destroyer": { + "properties": { + "count": { + "type": "long" + } + } + }, + "failed": { + "properties": { + "count": { + "type": "long" + } + } + }, + "mailer": { + "properties": { + "count": { + "type": "long" + } + } + }, + "process_crawl": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "agent": { + "properties": { + "id": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "ephemeral_id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "process": { + "properties": { + "pid": { + "type": "long" + } + } + }, + "@timestamp": { + "type": "date" + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "type": "keyword", + "ignore_above": 1024 + }, + "agent_id_status": { + "type": "keyword", + "ignore_above": 1024 + }, + "category": { + "type": "keyword", + "ignore_above": 1024 + }, + "code": { + "type": "keyword", + "ignore_above": 1024 + }, + "created": { + "type": "date" + }, + "dataset": { + "type": "keyword", + "ignore_above": 1024 + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "ingested": { + "type": "date" + }, + "kind": { + "type": "keyword", + "ignore_above": 1024 + }, + "module": { + "type": "keyword", + "ignore_above": 1024 + }, + "original": { + "type": "keyword", + "index": false, + "doc_values": false, + "ignore_above": 1024 + }, + "outcome": { + "type": "keyword", + "ignore_above": 1024 + }, + "provider": { + "type": "keyword", + "ignore_above": 1024 + }, + "reason": { + "type": "keyword", + "ignore_above": 1024 + }, + "reference": { + "type": "keyword", + "ignore_above": 1024 + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "url": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "metricset": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "period": { + "type": "long" + } + } + }, + "service": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "environment": { + "type": "keyword", + "ignore_above": 1024 + }, + "ephemeral_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "node": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "origin": { + "properties": { + "address": { + "type": "keyword", + "ignore_above": 1024 + }, + "environment": { + "type": "keyword", + "ignore_above": 1024 + }, + "ephemeral_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "node": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "state": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "state": { + "type": "keyword", + "ignore_above": 1024 + }, + "target": { + "properties": { + "address": { + "type": "keyword", + "ignore_above": 1024 + }, + "environment": { + "type": "keyword", + "ignore_above": 1024 + }, + "ephemeral_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "node": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "state": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "architecture": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "settings": { + "index.mapping.total_fields.limit": 2000 + } + }, + "data_stream": {} +} diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index 47a9e54644901..3df27c8283771 100644 --- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -456,6 +456,6 @@ private static Map toMap(final ToXContentObject xContentObject) * Returns a {@link MonitoredSystem} supported by the Monitoring Bulk API */ private static MonitoredSystem randomSystem() { - return randomFrom(MonitoredSystem.LOGSTASH, MonitoredSystem.KIBANA, MonitoredSystem.BEATS); + return randomFrom(MonitoredSystem.LOGSTASH, MonitoredSystem.KIBANA, MonitoredSystem.BEATS, MonitoredSystem.ENTERPRISE_SEARCH); } } diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java index 4f4cae3a28f62..2c7c5baf8d405 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @@ -188,6 +188,18 @@ public class MonitoringTemplateRegistry extends IndexTemplateRegistry { STACK_TEMPLATE_VARIABLES ); + ////////////////////////////////////////////////////////// + // Enterprise Search metricbeat template (for matching ".monitoring-ent-search-8-*" indices) + ////////////////////////////////////////////////////////// + public static final String ENTERPRISE_SEARCH_STACK_INDEX_TEMPLATE_NAME = ".monitoring-ent-search-mb"; + public static final IndexTemplateConfig ENTERPRISE_SEARCH_STACK_INDEX_TEMPLATE = new IndexTemplateConfig( + ENTERPRISE_SEARCH_STACK_INDEX_TEMPLATE_NAME, + "/monitoring-ent-search-mb.json", + STACK_MONITORING_REGISTRY_VERSION, + STACK_MONITORING_REGISTRY_VERSION_VARIABLE, + STACK_TEMPLATE_VARIABLES + ); + public static final String[] TEMPLATE_NAMES = new String[] { ALERTS_INDEX_TEMPLATE_NAME, BEATS_INDEX_TEMPLATE_NAME, @@ -257,7 +269,8 @@ protected List getLegacyTemplateConfigs() { BEATS_STACK_INDEX_TEMPLATE, ES_STACK_INDEX_TEMPLATE, KIBANA_STACK_INDEX_TEMPLATE, - LOGSTASH_STACK_INDEX_TEMPLATE + LOGSTASH_STACK_INDEX_TEMPLATE, + ENTERPRISE_SEARCH_STACK_INDEX_TEMPLATE ); @Override diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java index b09d8fb984fb4..b255bf9a7758c 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java @@ -66,7 +66,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe private final boolean validLicense = randomBoolean(); /** - * kibana, logstash, and beats + * kibana, logstash, beats and enterprise search */ private final int EXPECTED_TEMPLATES = TEMPLATE_NAMES.length; private final int EXPECTED_WATCHES = ClusterAlertsUtil.WATCH_IDS.length; From fe3a40ba39e19b9f299b7902999836aab4597f5d Mon Sep 17 00:00:00 2001 From: cdelgado Date: Thu, 20 Jan 2022 12:17:46 +0100 Subject: [PATCH 2/5] Add _meta template description --- .../plugin/core/src/main/resources/monitoring-beats-mb.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json index 001c6fdfaf3a5..fd33b418d57c1 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json @@ -1,6 +1,10 @@ { "index_patterns": [".monitoring-beats-${xpack.stack.monitoring.template.version}-*"], "version": ${xpack.stack.monitoring.template.release.version}, + "_meta": { + "description": "Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring", + "managed": true + }, "template": { "mappings": { "properties": { From 3edd1cba72f9810c2acdf95cb032316c643ff37a Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 24 Jan 2022 09:56:38 +0100 Subject: [PATCH 3/5] Remove Enterprise Search from MonitoredSystem and MonitoringIT --- .../elasticsearch/xpack/core/monitoring/MonitoredSystem.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoredSystem.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoredSystem.java index 2610019dda940..63a8b6f825020 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoredSystem.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/MonitoredSystem.java @@ -16,7 +16,6 @@ public enum MonitoredSystem { KIBANA("kibana"), LOGSTASH("logstash"), BEATS("beats"), - ENTERPRISE_SEARCH("enterprise_search"), UNKNOWN("unknown"); private final String system; @@ -35,7 +34,6 @@ public static MonitoredSystem fromSystem(String system) { case "kibana" -> KIBANA; case "logstash" -> LOGSTASH; case "beats" -> BEATS; - case "enterprise_search" -> ENTERPRISE_SEARCH; default -> // Return an "unknown" monitored system // that can easily be filtered out if From 462aec6bad21be01ee686d66efd667301e3fd6ed Mon Sep 17 00:00:00 2001 From: cdelgado Date: Mon, 24 Jan 2022 10:43:34 +0100 Subject: [PATCH 4/5] Remove Enterprise Search from MonitoredSystem and MonitoringIT --- .../xpack/monitoring/integration/MonitoringIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index 3df27c8283771..47a9e54644901 100644 --- a/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -456,6 +456,6 @@ private static Map toMap(final ToXContentObject xContentObject) * Returns a {@link MonitoredSystem} supported by the Monitoring Bulk API */ private static MonitoredSystem randomSystem() { - return randomFrom(MonitoredSystem.LOGSTASH, MonitoredSystem.KIBANA, MonitoredSystem.BEATS, MonitoredSystem.ENTERPRISE_SEARCH); + return randomFrom(MonitoredSystem.LOGSTASH, MonitoredSystem.KIBANA, MonitoredSystem.BEATS); } } From 3fa8134249207dba76001099c963a100aa1c1294 Mon Sep 17 00:00:00 2001 From: cdelgado Date: Tue, 25 Jan 2022 10:23:54 +0100 Subject: [PATCH 5/5] Add _meta description for the correct file (monitoring-ent-search-mb.json) --- .../plugin/core/src/main/resources/monitoring-beats-mb.json | 4 ---- .../core/src/main/resources/monitoring-ent-search-mb.json | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json index fd33b418d57c1..001c6fdfaf3a5 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json @@ -1,10 +1,6 @@ { "index_patterns": [".monitoring-beats-${xpack.stack.monitoring.template.version}-*"], "version": ${xpack.stack.monitoring.template.release.version}, - "_meta": { - "description": "Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring", - "managed": true - }, "template": { "mappings": { "properties": { diff --git a/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json b/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json index d656ea368955c..fac2a026c405c 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json @@ -1,6 +1,10 @@ { "index_patterns": [".monitoring-ent-search-${xpack.stack.monitoring.template.version}-*"], "version": ${xpack.stack.monitoring.template.release.version}, + "_meta": { + "description": "Template used by Enterprise Search Metricbeat module monitoring information for Stack Monitoring", + "managed": true + }, "template": { "mappings": { "properties": {