From f8cba32ae7d6fa8c235f380376302f4a5bcf0416 Mon Sep 17 00:00:00 2001 From: Igor Zaytsev Date: Wed, 30 Sep 2020 17:14:11 -0400 Subject: [PATCH 1/7] Memory usage first draft --- x-pack/plugins/monitoring/common/constants.ts | 2 + .../alerts/memory_usage_alert/index.tsx | 29 ++ .../cluster/overview/elasticsearch_panel.js | 2 + x-pack/plugins/monitoring/public/plugin.ts | 10 +- .../elasticsearch/node/advanced/index.js | 3 +- .../public/views/elasticsearch/node/index.js | 3 +- .../public/views/elasticsearch/nodes/index.js | 3 +- .../server/alerts/alerts_factory.ts | 3 + .../server/alerts/cpu_usage_alert.ts | 4 +- .../server/alerts/disk_usage_alert.ts | 14 +- .../plugins/monitoring/server/alerts/index.ts | 1 + .../server/alerts/memory_usage_alert.ts | 343 ++++++++++++++++++ .../monitoring/server/alerts/types.d.ts | 36 +- .../alerts/fetch_memory_usage_node_stats.ts | 114 ++++++ 14 files changed, 538 insertions(+), 29 deletions(-) create mode 100644 x-pack/plugins/monitoring/public/alerts/memory_usage_alert/index.tsx create mode 100644 x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts create mode 100644 x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts diff --git a/x-pack/plugins/monitoring/common/constants.ts b/x-pack/plugins/monitoring/common/constants.ts index 6eb0d6e93d58a..aa60a4027220d 100644 --- a/x-pack/plugins/monitoring/common/constants.ts +++ b/x-pack/plugins/monitoring/common/constants.ts @@ -236,6 +236,7 @@ export const ALERT_NODES_CHANGED = `${ALERT_PREFIX}alert_nodes_changed`; export const ALERT_ELASTICSEARCH_VERSION_MISMATCH = `${ALERT_PREFIX}alert_elasticsearch_version_mismatch`; export const ALERT_KIBANA_VERSION_MISMATCH = `${ALERT_PREFIX}alert_kibana_version_mismatch`; export const ALERT_LOGSTASH_VERSION_MISMATCH = `${ALERT_PREFIX}alert_logstash_version_mismatch`; +export const ALERT_MEMORY_USAGE = `${ALERT_PREFIX}alert_jvm_memory_usage`; /** * A listing of all alert types @@ -249,6 +250,7 @@ export const ALERTS = [ ALERT_ELASTICSEARCH_VERSION_MISMATCH, ALERT_KIBANA_VERSION_MISMATCH, ALERT_LOGSTASH_VERSION_MISMATCH, + ALERT_MEMORY_USAGE, ]; /** diff --git a/x-pack/plugins/monitoring/public/alerts/memory_usage_alert/index.tsx b/x-pack/plugins/monitoring/public/alerts/memory_usage_alert/index.tsx new file mode 100644 index 0000000000000..dd60967a3458b --- /dev/null +++ b/x-pack/plugins/monitoring/public/alerts/memory_usage_alert/index.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { validate } from '../components/duration/validation'; +import { Expression, Props } from '../components/duration/expression'; + +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { AlertTypeModel } from '../../../../triggers_actions_ui/public/types'; + +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { MemoryUsageAlert } from '../../../server/alerts'; + +export function createMemoryUsageAlertType(): AlertTypeModel { + return { + id: MemoryUsageAlert.TYPE, + name: MemoryUsageAlert.LABEL, + iconClass: 'bell', + alertParamsExpression: (props: Props) => ( + + ), + validate, + defaultActionMessage: '{{context.internalFullMessage}}', + requiresAppContext: true, + }; +} diff --git a/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js b/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js index 61a24f31ca89a..49028fc7e46f3 100644 --- a/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js +++ b/x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js @@ -41,6 +41,7 @@ import { ALERT_CLUSTER_HEALTH, ALERT_CPU_USAGE, ALERT_DISK_USAGE, + ALERT_MEMORY_USAGE, ALERT_NODES_CHANGED, ALERT_ELASTICSEARCH_VERSION_MISMATCH, } from '../../../../common/constants'; @@ -159,6 +160,7 @@ const OVERVIEW_PANEL_ALERTS = [ALERT_CLUSTER_HEALTH, ALERT_LICENSE_EXPIRATION]; const NODES_PANEL_ALERTS = [ ALERT_CPU_USAGE, ALERT_DISK_USAGE, + ALERT_MEMORY_USAGE, ALERT_NODES_CHANGED, ALERT_ELASTICSEARCH_VERSION_MISMATCH, ]; diff --git a/x-pack/plugins/monitoring/public/plugin.ts b/x-pack/plugins/monitoring/public/plugin.ts index a9c26ca7103a2..7cf98e79e9ca8 100644 --- a/x-pack/plugins/monitoring/public/plugin.ts +++ b/x-pack/plugins/monitoring/public/plugin.ts @@ -25,6 +25,7 @@ import { TriggersAndActionsUIPublicPluginSetup } from '../../triggers_actions_ui import { createCpuUsageAlertType } from './alerts/cpu_usage_alert'; import { createLegacyAlertTypes } from './alerts/legacy_alert'; import { createDiskUsageAlertType } from './alerts/disk_usage_alert'; +import { createMemoryUsageAlertType } from './alerts/memory_usage_alert'; interface MonitoringSetupPluginDependencies { home?: HomePublicPluginSetup; @@ -71,11 +72,14 @@ export class MonitoringPlugin }); } - plugins.triggers_actions_ui.alertTypeRegistry.register(createCpuUsageAlertType()); - plugins.triggers_actions_ui.alertTypeRegistry.register(createDiskUsageAlertType()); + const { alertTypeRegistry } = plugins.triggers_actions_ui; + alertTypeRegistry.register(createCpuUsageAlertType()); + alertTypeRegistry.register(createDiskUsageAlertType()); + alertTypeRegistry.register(createMemoryUsageAlertType()); + const legacyAlertTypes = createLegacyAlertTypes(); for (const legacyAlertType of legacyAlertTypes) { - plugins.triggers_actions_ui.alertTypeRegistry.register(legacyAlertType); + alertTypeRegistry.register(legacyAlertType); } const app: App = { diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js index 8c30e4a2c1b07..7c13997ee1152 100644 --- a/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js @@ -21,6 +21,7 @@ import { CODE_PATH_ELASTICSEARCH, ALERT_CPU_USAGE, ALERT_DISK_USAGE, + ALERT_MEMORY_USAGE, } from '../../../../../common/constants'; function getPageData($injector) { @@ -71,7 +72,7 @@ uiRoutes.when('/elasticsearch/nodes/:node/advanced', { alerts: { shouldFetch: true, options: { - alertTypeIds: [ALERT_CPU_USAGE, ALERT_DISK_USAGE], + alertTypeIds: [ALERT_CPU_USAGE, ALERT_DISK_USAGE, ALERT_MEMORY_USAGE], filters: [ { nodeUuid: nodeName, diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/node/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/node/index.js index ed2603e6dfff3..74738bdddab5d 100644 --- a/x-pack/plugins/monitoring/public/views/elasticsearch/node/index.js +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/node/index.js @@ -22,6 +22,7 @@ import { CODE_PATH_ELASTICSEARCH, ALERT_CPU_USAGE, ALERT_DISK_USAGE, + ALERT_MEMORY_USAGE, } from '../../../../common/constants'; uiRoutes.when('/elasticsearch/nodes/:node', { @@ -55,7 +56,7 @@ uiRoutes.when('/elasticsearch/nodes/:node', { alerts: { shouldFetch: true, options: { - alertTypeIds: [ALERT_CPU_USAGE, ALERT_DISK_USAGE], + alertTypeIds: [ALERT_CPU_USAGE, ALERT_DISK_USAGE, ALERT_MEMORY_USAGE], filters: [ { nodeUuid: nodeName, diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/nodes/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/nodes/index.js index 66fcac43e4fc5..d54f663ac8050 100644 --- a/x-pack/plugins/monitoring/public/views/elasticsearch/nodes/index.js +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/nodes/index.js @@ -20,6 +20,7 @@ import { CODE_PATH_ELASTICSEARCH, ALERT_CPU_USAGE, ALERT_DISK_USAGE, + ALERT_MEMORY_USAGE, } from '../../../../common/constants'; uiRoutes.when('/elasticsearch/nodes', { @@ -87,7 +88,7 @@ uiRoutes.when('/elasticsearch/nodes', { alerts: { shouldFetch: true, options: { - alertTypeIds: [ALERT_CPU_USAGE, ALERT_DISK_USAGE], + alertTypeIds: [ALERT_CPU_USAGE, ALERT_DISK_USAGE, ALERT_MEMORY_USAGE], }, }, }); diff --git a/x-pack/plugins/monitoring/server/alerts/alerts_factory.ts b/x-pack/plugins/monitoring/server/alerts/alerts_factory.ts index 6b1c0d5fffe18..8f7fb0b7d9736 100644 --- a/x-pack/plugins/monitoring/server/alerts/alerts_factory.ts +++ b/x-pack/plugins/monitoring/server/alerts/alerts_factory.ts @@ -7,6 +7,7 @@ import { CpuUsageAlert, DiskUsageAlert, + MemoryUsageAlert, NodesChangedAlert, ClusterHealthAlert, LicenseExpirationAlert, @@ -20,6 +21,7 @@ import { ALERT_LICENSE_EXPIRATION, ALERT_CPU_USAGE, ALERT_DISK_USAGE, + ALERT_MEMORY_USAGE, ALERT_NODES_CHANGED, ALERT_LOGSTASH_VERSION_MISMATCH, ALERT_KIBANA_VERSION_MISMATCH, @@ -32,6 +34,7 @@ export const BY_TYPE = { [ALERT_LICENSE_EXPIRATION]: LicenseExpirationAlert, [ALERT_CPU_USAGE]: CpuUsageAlert, [ALERT_DISK_USAGE]: DiskUsageAlert, + [ALERT_MEMORY_USAGE]: MemoryUsageAlert, [ALERT_NODES_CHANGED]: NodesChangedAlert, [ALERT_LOGSTASH_VERSION_MISMATCH]: LogstashVersionMismatchAlert, [ALERT_KIBANA_VERSION_MISMATCH]: KibanaVersionMismatchAlert, diff --git a/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.ts b/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.ts index 4228354f52748..2e72537b2a3ee 100644 --- a/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.ts @@ -193,13 +193,13 @@ export class CpuUsageAlert extends BaseAlert { i18n.translate('xpack.monitoring.alerts.cpuUsage.ui.nextSteps.hotThreads', { defaultMessage: '#start_linkCheck hot threads#end_link', }), - `{elasticWebsiteUrl}/guide/en/elasticsearch/reference/{docLinkVersion}/cluster-nodes-hot-threads.html` + `{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/cluster-nodes-hot-threads.html` ), createLink( i18n.translate('xpack.monitoring.alerts.cpuUsage.ui.nextSteps.runningTasks', { defaultMessage: '#start_linkCheck long running tasks#end_link', }), - `{elasticWebsiteUrl}/guide/en/elasticsearch/reference/{docLinkVersion}/tasks.html` + `{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/tasks.html` ), ], tokens: [ diff --git a/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.ts b/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.ts index e43dca3ce87b1..2322d6ba99dfe 100644 --- a/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.ts @@ -109,13 +109,13 @@ export class DiskUsageAlert extends BaseAlert { protected filterAlertInstance(alertInstance: RawAlertInstance, filters: CommonAlertFilter[]) { const alertInstanceStates = alertInstance.state?.alertStates as AlertDiskUsageState[]; - const nodeUuid = filters?.find((filter) => filter.nodeUuid); + const nodeFilter = filters?.find((filter) => filter.nodeUuid); - if (!filters || !filters.length || !alertInstanceStates?.length || !nodeUuid) { + if (!filters || !filters.length || !alertInstanceStates?.length || !nodeFilter?.nodeUuid) { return true; } - const nodeAlerts = alertInstanceStates.filter(({ nodeId }) => nodeId === nodeUuid); + const nodeAlerts = alertInstanceStates.filter(({ nodeId }) => nodeId === nodeFilter.nodeUuid); return Boolean(nodeAlerts.length); } @@ -160,7 +160,7 @@ export class DiskUsageAlert extends BaseAlert { i18n.translate('xpack.monitoring.alerts.diskUsage.ui.nextSteps.tuneDisk', { defaultMessage: '#start_linkTune for disk usage#end_link', }), - `{elasticWebsiteUrl}/guide/en/elasticsearch/reference/{docLinkVersion}/tune-for-disk-usage.html` + `{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/tune-for-disk-usage.html` ), createLink( i18n.translate('xpack.monitoring.alerts.diskUsage.ui.nextSteps.identifyIndices', { @@ -173,19 +173,19 @@ export class DiskUsageAlert extends BaseAlert { i18n.translate('xpack.monitoring.alerts.diskUsage.ui.nextSteps.ilmPolicies', { defaultMessage: '#start_linkImplement ILM policies#end_link', }), - `{elasticWebsiteUrl}/guide/en/elasticsearch/reference/{docLinkVersion}/index-lifecycle-management.html` + `{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/index-lifecycle-management.html` ), createLink( i18n.translate('xpack.monitoring.alerts.diskUsage.ui.nextSteps.addMoreNodes', { defaultMessage: '#start_linkAdd more data nodes#end_link', }), - `{elasticWebsiteUrl}/guide/en/elasticsearch/reference/{docLinkVersion}/add-elasticsearch-nodes.html` + `{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/add-elasticsearch-nodes.html` ), createLink( i18n.translate('xpack.monitoring.alerts.diskUsage.ui.nextSteps.resizeYourDeployment', { defaultMessage: '#start_linkResize your deployment (ECE)#end_link', }), - `{elasticWebsiteUrl}/guide/en/cloud-enterprise/current/ece-resize-deployment.html` + `{elasticWebsiteUrl}guide/en/cloud-enterprise/current/ece-resize-deployment.html` ), ], tokens: [ diff --git a/x-pack/plugins/monitoring/server/alerts/index.ts b/x-pack/plugins/monitoring/server/alerts/index.ts index 8fdac65514477..53561af454d48 100644 --- a/x-pack/plugins/monitoring/server/alerts/index.ts +++ b/x-pack/plugins/monitoring/server/alerts/index.ts @@ -7,6 +7,7 @@ export { BaseAlert } from './base_alert'; export { CpuUsageAlert } from './cpu_usage_alert'; export { DiskUsageAlert } from './disk_usage_alert'; +export { MemoryUsageAlert } from './memory_usage_alert'; export { ClusterHealthAlert } from './cluster_health_alert'; export { LicenseExpirationAlert } from './license_expiration_alert'; export { NodesChangedAlert } from './nodes_changed_alert'; diff --git a/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts b/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts new file mode 100644 index 0000000000000..d0517f02915c8 --- /dev/null +++ b/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts @@ -0,0 +1,343 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { IUiSettingsClient, Logger } from 'kibana/server'; +import { i18n } from '@kbn/i18n'; +import { BaseAlert } from './base_alert'; +import { + AlertData, + AlertCluster, + AlertState, + AlertMessage, + AlertMemoryUsageState, + AlertMessageTimeToken, + AlertMessageLinkToken, + AlertInstanceState, +} from './types'; +import { AlertInstance, AlertServices } from '../../../alerts/server'; +import { INDEX_PATTERN_ELASTICSEARCH, ALERT_MEMORY_USAGE } from '../../common/constants'; +import { fetchMemoryUsageNodeStats } from '../lib/alerts/fetch_memory_usage_node_stats'; +import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; +import { AlertMessageTokenType, AlertSeverity, AlertParamType } from '../../common/enums'; +import { RawAlertInstance } from '../../../alerts/common'; +import { CommonAlertFilter, CommonAlertParams, CommonAlertParamDetail } from '../../common/types'; +import { AlertingDefaults, createLink } from './alerts_common'; +import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; + +interface ParamDetails { + [key: string]: CommonAlertParamDetail; +} + +export class MemoryUsageAlert extends BaseAlert { + public static readonly PARAM_DETAILS: ParamDetails = { + threshold: { + label: i18n.translate('xpack.monitoring.alerts.memoryUsage.paramDetails.threshold.label', { + defaultMessage: `Notify when memory usage is over`, + }), + type: AlertParamType.Percentage, + }, + duration: { + label: i18n.translate('xpack.monitoring.alerts.memoryUsage.paramDetails.duration.label', { + defaultMessage: `Look at the average over`, + }), + type: AlertParamType.Duration, + }, + }; + public static paramDetails = MemoryUsageAlert.PARAM_DETAILS; + public static readonly TYPE = ALERT_MEMORY_USAGE; + public static readonly LABEL = i18n.translate('xpack.monitoring.alerts.memoryUsage.label', { + defaultMessage: 'Memory Usage (JVM)', + }); + public type = MemoryUsageAlert.TYPE; + public label = MemoryUsageAlert.LABEL; + + protected defaultParams = { + threshold: 90, + duration: '5m', + }; + + protected actionVariables = [ + { + name: 'nodes', + description: i18n.translate('xpack.monitoring.alerts.memoryUsage.actionVariables.nodes', { + defaultMessage: 'The list of nodes reporting high memory usage.', + }), + }, + { + name: 'count', + description: i18n.translate('xpack.monitoring.alerts.memoryUsage.actionVariables.count', { + defaultMessage: 'The number of nodes reporting high memory usage.', + }), + }, + ...Object.values(AlertingDefaults.ALERT_TYPE.context), + ]; + + protected async fetchData( + params: CommonAlertParams, + callCluster: any, + clusters: AlertCluster[], + uiSettings: IUiSettingsClient, + availableCcs: string[] + ): Promise { + let esIndexPattern = appendMetricbeatIndex(this.config, INDEX_PATTERN_ELASTICSEARCH); + if (availableCcs) { + esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); + } + const { duration, threshold } = params; + const stats = await fetchMemoryUsageNodeStats( + callCluster, + clusters, + esIndexPattern, + duration as string, + this.config.ui.max_bucket_size + ); + + return stats.map((stat) => { + const { clusterUuid, nodeId, memoryUsage, ccs } = stat; + return { + instanceKey: `${clusterUuid}:${nodeId}`, + shouldFire: memoryUsage > threshold, + severity: AlertSeverity.Danger, + meta: stat, + clusterUuid, + ccs, + }; + }); + } + + protected filterAlertInstance(alertInstance: RawAlertInstance, filters: CommonAlertFilter[]) { + const alertInstanceStates = alertInstance.state?.alertStates as AlertMemoryUsageState[]; + const nodeFilter = filters?.find((filter) => filter.nodeUuid); + + if (!filters || !filters.length || !alertInstanceStates?.length || !nodeFilter?.nodeUuid) { + return true; + } + + const nodeAlerts = alertInstanceStates.filter(({ nodeId }) => nodeId === nodeFilter.nodeUuid); + return Boolean(nodeAlerts.length); + } + + protected getDefaultAlertState(cluster: AlertCluster, item: AlertData): AlertState { + const currentState = super.getDefaultAlertState(cluster, item); + currentState.ui.severity = AlertSeverity.Warning; + return currentState; + } + + protected getUiMessage(alertState: AlertState, item: AlertData): AlertMessage { + const stat = item.meta as AlertMemoryUsageState; + if (!alertState.ui.isFiring) { + return { + text: i18n.translate('xpack.monitoring.alerts.memoryUsage.ui.resolvedMessage', { + defaultMessage: `The JVM memory usage on node {nodeName} is now under the threshold, currently reporting at {memoryUsage}% as of #resolved`, + values: { + nodeName: stat.nodeName, + memoryUsage: stat.memoryUsage.toFixed(2), + }, + }), + tokens: [ + { + startToken: '#resolved', + type: AlertMessageTokenType.Time, + isAbsolute: true, + isRelative: false, + timestamp: alertState.ui.resolvedMS, + } as AlertMessageTimeToken, + ], + }; + } + return { + text: i18n.translate('xpack.monitoring.alerts.memoryUsage.ui.firingMessage', { + defaultMessage: `Node #start_link{nodeName}#end_link is reporting JVM memory usage of {memoryUsage}% at #absolute`, + values: { + nodeName: stat.nodeName, + memoryUsage: stat.memoryUsage, + }, + }), + nextSteps: [ + createLink( + i18n.translate('xpack.monitoring.alerts.memoryUsage.ui.nextSteps.tuneThreadPools', { + defaultMessage: '#start_linkTune thread pools#end_link', + }), + `{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/modules-threadpool.html` + ), + createLink( + i18n.translate('xpack.monitoring.alerts.memoryUsage.ui.nextSteps.managingHeap', { + defaultMessage: '#start_linkManaging ES Heap#end_link', + }), + `{elasticWebsiteUrl}blog/a-heap-of-trouble` + ), + createLink( + i18n.translate('xpack.monitoring.alerts.memoryUsage.ui.nextSteps.identifyIndicesShards', { + defaultMessage: '#start_linkIdentify large indices/shards#end_link', + }), + 'elasticsearch/indices', + AlertMessageTokenType.Link + ), + createLink( + i18n.translate('xpack.monitoring.alerts.memoryUsage.ui.nextSteps.addMoreNodes', { + defaultMessage: '#start_linkAdd more data nodes#end_link', + }), + `{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/add-elasticsearch-nodes.html` + ), + createLink( + i18n.translate('xpack.monitoring.alerts.memoryUsage.ui.nextSteps.resizeYourDeployment', { + defaultMessage: '#start_linkResize your deployment (ECE)#end_link', + }), + `{elasticWebsiteUrl}guide/en/cloud-enterprise/current/ece-resize-deployment.html` + ), + ], + tokens: [ + { + startToken: '#absolute', + type: AlertMessageTokenType.Time, + isAbsolute: true, + isRelative: false, + timestamp: alertState.ui.triggeredMS, + } as AlertMessageTimeToken, + { + startToken: '#start_link', + endToken: '#end_link', + type: AlertMessageTokenType.Link, + url: `elasticsearch/nodes/${stat.nodeId}`, + } as AlertMessageLinkToken, + ], + }; + } + + protected executeActions( + instance: AlertInstance, + { alertStates }: AlertInstanceState, + item: AlertData | null, + cluster: AlertCluster + ) { + const firingNodes = alertStates.filter( + (alertState) => alertState.ui.isFiring + ) as AlertMemoryUsageState[]; + const firingCount = firingNodes.length; + + if (firingCount > 0) { + const shortActionText = i18n.translate('xpack.monitoring.alerts.memoryUsage.shortAction', { + defaultMessage: 'Verify memory usage levels across affected nodes.', + }); + const fullActionText = i18n.translate('xpack.monitoring.alerts.memoryUsage.fullAction', { + defaultMessage: 'View nodes', + }); + + const action = `[${fullActionText}](elasticsearch/nodes)`; + const internalShortMessage = i18n.translate( + 'xpack.monitoring.alerts.memoryUsage.firing.internalShortMessage', + { + defaultMessage: `Memory usage alert is firing for {count} node(s) in cluster: {clusterName}. {shortActionText}`, + values: { + count: firingCount, + clusterName: cluster.clusterName, + shortActionText, + }, + } + ); + const internalFullMessage = i18n.translate( + 'xpack.monitoring.alerts.memoryUsage.firing.internalFullMessage', + { + defaultMessage: `Memory usage alert is firing for {count} node(s) in cluster: {clusterName}. {action}`, + values: { + count: firingCount, + clusterName: cluster.clusterName, + action, + }, + } + ); + + instance.scheduleActions('default', { + internalShortMessage, + internalFullMessage: this.isCloud ? internalShortMessage : internalFullMessage, + state: AlertingDefaults.ALERT_STATE.firing, + nodes: firingNodes + .map((state) => `${state.nodeName}:${state.memoryUsage.toFixed(2)}`) + .join(','), + count: firingCount, + clusterName: cluster.clusterName, + action, + actionPlain: shortActionText, + }); + } else { + const resolvedNodes = (alertStates as AlertMemoryUsageState[]) + .filter((state) => !state.ui.isFiring) + .map((state) => `${state.nodeName}:${state.memoryUsage.toFixed(2)}`); + const resolvedCount = resolvedNodes.length; + + if (resolvedCount > 0) { + const internalMessage = i18n.translate( + 'xpack.monitoring.alerts.memoryUsage.resolved.internalMessage', + { + defaultMessage: `Memory usage alert is resolved for {count} node(s) in cluster: {clusterName}.`, + values: { + count: resolvedCount, + clusterName: cluster.clusterName, + }, + } + ); + + instance.scheduleActions('default', { + internalShortMessage: internalMessage, + internalFullMessage: internalMessage, + state: AlertingDefaults.ALERT_STATE.resolved, + nodes: resolvedNodes.join(','), + count: resolvedCount, + clusterName: cluster.clusterName, + }); + } + } + } + + protected async processData( + data: AlertData[], + clusters: AlertCluster[], + services: AlertServices, + logger: Logger, + state: any + ) { + const currentUTC = +new Date(); + for (const cluster of clusters) { + const nodes = data.filter((node) => node.clusterUuid === cluster.clusterUuid); + if (!nodes.length) { + continue; + } + + const firingNodeUuids = nodes + .filter((node) => node.shouldFire) + .map((node) => node.meta.nodeId) + .join(','); + const instanceId = `${this.type}:${cluster.clusterUuid}:${firingNodeUuids}`; + const instance = services.alertInstanceFactory(instanceId); + const newAlertStates: AlertMemoryUsageState[] = []; + + for (const node of nodes) { + const stat = node.meta as AlertMemoryUsageState; + const nodeState = this.getDefaultAlertState(cluster, node) as AlertMemoryUsageState; + nodeState.memoryUsage = stat.memoryUsage; + nodeState.nodeId = stat.nodeId; + nodeState.nodeName = stat.nodeName; + + if (node.shouldFire) { + nodeState.ui.triggeredMS = currentUTC; + nodeState.ui.isFiring = true; + nodeState.ui.severity = node.severity; + newAlertStates.push(nodeState); + } + nodeState.ui.message = this.getUiMessage(nodeState, node); + } + + const alertInstanceState = { alertStates: newAlertStates }; + instance.replaceState(alertInstanceState); + if (newAlertStates.length && !instance.hasScheduledActions()) { + this.executeActions(instance, alertInstanceState, null, cluster); + state.lastExecutedAction = currentUTC; + } + } + + state.lastChecked = currentUTC; + return state; + } +} diff --git a/x-pack/plugins/monitoring/server/alerts/types.d.ts b/x-pack/plugins/monitoring/server/alerts/types.d.ts index b685dcaed790f..c68542073f46a 100644 --- a/x-pack/plugins/monitoring/server/alerts/types.d.ts +++ b/x-pack/plugins/monitoring/server/alerts/types.d.ts @@ -22,16 +22,21 @@ export interface AlertState { ui: AlertUiState; } -export interface AlertCpuUsageState extends AlertState { - cpuUsage: number; +export interface AlertNodeState extends AlertState { nodeId: string; - nodeName: string; + nodeName?: string; +} + +export interface AlertCpuUsageState extends AlertNodeState { + cpuUsage: number; } -export interface AlertDiskUsageState extends AlertState { +export interface AlertDiskUsageState extends AlertNodeState { diskUsage: number; - nodeId: string; - nodeName?: string; +} + +export interface AlertMemoryUsageState extends AlertNodeState { + memoryUsage: number; } export interface AlertUiState { @@ -74,23 +79,26 @@ export interface AlertCluster { clusterName: string; } -export interface AlertCpuUsageNodeStats { +export interface AlertNodeStats { clusterUuid: string; nodeId: string; - nodeName: string; + nodeName?: string; + ccs?: string; +} + +export interface AlertCpuUsageNodeStats extends AlertNodeStats { cpuUsage: number; containerUsage: number; containerPeriods: number; containerQuota: number; - ccs?: string; } -export interface AlertDiskUsageNodeStats { - clusterUuid: string; - nodeId: string; - nodeName: string; +export interface AlertDiskUsageNodeStats extends AlertNodeStats { diskUsage: number; - ccs?: string; +} + +export interface AlertMemoryUsageNodeStats extends AlertNodeStats { + memoryUsage: number; } export interface AlertData { diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts new file mode 100644 index 0000000000000..88edc0eed0d65 --- /dev/null +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; +import { AlertCluster, AlertMemoryUsageNodeStats } from '../../alerts/types'; + +export async function fetchMemoryUsageNodeStats( + callCluster: any, + clusters: AlertCluster[], + index: string, + duration: string, + size: number +): Promise { + const clustersIds = clusters.map((cluster) => cluster.clusterUuid); + const params = { + index, + filterPath: ['aggregations'], + body: { + size: 0, + query: { + bool: { + filter: [ + { + terms: { + cluster_uuid: clustersIds, + }, + }, + { + term: { + type: 'node_stats', + }, + }, + { + range: { + timestamp: { + gte: `now-${duration}`, + }, + }, + }, + ], + }, + }, + aggs: { + clusters: { + terms: { + field: 'cluster_uuid', + size, + }, + aggs: { + nodes: { + terms: { + field: 'source_node.uuid', + size, + }, + aggs: { + index: { + terms: { + field: '_index', + size: 1, + }, + }, + avg_heap: { + avg: { + field: 'node_stats.jvm.mem.heap_used_percent', + }, + }, + cluster_uuid: { + terms: { + field: 'cluster_uuid', + }, + }, + name: { + terms: { + field: 'source_node.name', + size: 1, + }, + }, + }, + }, + }, + }, + }, + }, + }; + + const response = await callCluster('search', params); + const stats: AlertMemoryUsageNodeStats[] = []; + const { buckets: clusterBuckets = [] } = response.aggregations.clusters; + + if (!clusterBuckets.length) { + return stats; + } + + for (const clusterBucket of clusterBuckets) { + for (const node of clusterBucket.nodes.buckets) { + const indexName = get(node, 'index.buckets[0].key', ''); + const memoryUsage = Math.floor(Number(get(node, 'avg_heap.value'))); + if (isNaN(memoryUsage) || memoryUsage === undefined || memoryUsage === null) { + continue; + } + stats.push({ + memoryUsage, + clusterUuid: clusterBucket.key, + nodeId: node.key, + nodeName: get(node, 'name.buckets[0].key'), + ccs: indexName.includes(':') ? indexName.split(':')[0] : null, + }); + } + } + return stats; +} From 5320edc9aff2adc80a8cdb6a2302a85125065a16 Mon Sep 17 00:00:00 2001 From: Igor Zaytsev Date: Wed, 30 Sep 2020 17:20:03 -0400 Subject: [PATCH 2/7] Fixed tests --- .../monitoring/server/alerts/alerts_factory.test.ts | 2 +- .../monitoring/server/alerts/cpu_usage_alert.test.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/monitoring/server/alerts/alerts_factory.test.ts b/x-pack/plugins/monitoring/server/alerts/alerts_factory.test.ts index 60693eb42a30e..ddc8dcafebd21 100644 --- a/x-pack/plugins/monitoring/server/alerts/alerts_factory.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/alerts_factory.test.ts @@ -63,6 +63,6 @@ describe('AlertsFactory', () => { it('should get all', () => { const alerts = AlertsFactory.getAll(); - expect(alerts.length).toBe(8); + expect(alerts.length).toBe(9); }); }); diff --git a/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts index 495fe993cca1b..0911c529a6061 100644 --- a/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts @@ -154,7 +154,7 @@ describe('CpuUsageAlert', () => { endToken: '#end_link', type: 'docLink', partialUrl: - '{elasticWebsiteUrl}/guide/en/elasticsearch/reference/{docLinkVersion}/cluster-nodes-hot-threads.html', + '{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/cluster-nodes-hot-threads.html', }, ], }, @@ -166,7 +166,7 @@ describe('CpuUsageAlert', () => { endToken: '#end_link', type: 'docLink', partialUrl: - '{elasticWebsiteUrl}/guide/en/elasticsearch/reference/{docLinkVersion}/tasks.html', + '{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/tasks.html', }, ], }, @@ -506,7 +506,7 @@ describe('CpuUsageAlert', () => { endToken: '#end_link', type: 'docLink', partialUrl: - '{elasticWebsiteUrl}/guide/en/elasticsearch/reference/{docLinkVersion}/cluster-nodes-hot-threads.html', + '{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/cluster-nodes-hot-threads.html', }, ], }, @@ -518,7 +518,7 @@ describe('CpuUsageAlert', () => { endToken: '#end_link', type: 'docLink', partialUrl: - '{elasticWebsiteUrl}/guide/en/elasticsearch/reference/{docLinkVersion}/tasks.html', + '{elasticWebsiteUrl}guide/en/elasticsearch/reference/{docLinkVersion}/tasks.html', }, ], }, From 509be7eeae0529dd938d14fb5e8a181db2058d4c Mon Sep 17 00:00:00 2001 From: Igor Zaytsev Date: Thu, 1 Oct 2020 21:44:12 -0400 Subject: [PATCH 3/7] CR feedback --- .../monitoring/server/alerts/memory_usage_alert.ts | 10 ++++++++-- .../server/lib/alerts/fetch_memory_usage_node_stats.ts | 7 +++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts b/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts index d0517f02915c8..5b49c27cca799 100644 --- a/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts @@ -25,6 +25,7 @@ import { RawAlertInstance } from '../../../alerts/common'; import { CommonAlertFilter, CommonAlertParams, CommonAlertParamDetail } from '../../common/types'; import { AlertingDefaults, createLink } from './alerts_common'; import { appendMetricbeatIndex } from '../lib/alerts/append_mb_index'; +import { parseDuration } from '../../../alerts/common/parse_duration'; interface ParamDetails { [key: string]: CommonAlertParamDetail; @@ -54,7 +55,7 @@ export class MemoryUsageAlert extends BaseAlert { public label = MemoryUsageAlert.LABEL; protected defaultParams = { - threshold: 90, + threshold: 85, duration: '5m', }; @@ -86,11 +87,16 @@ export class MemoryUsageAlert extends BaseAlert { esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs); } const { duration, threshold } = params; + const parsedDuration = parseDuration(duration as string); + const endMs = +new Date(); + const startMs = endMs - parsedDuration; + const stats = await fetchMemoryUsageNodeStats( callCluster, clusters, esIndexPattern, - duration as string, + startMs, + endMs, this.config.ui.max_bucket_size ); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts index 88edc0eed0d65..dd586b24f8a4f 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts @@ -11,7 +11,8 @@ export async function fetchMemoryUsageNodeStats( callCluster: any, clusters: AlertCluster[], index: string, - duration: string, + startMs: number, + endMs: number, size: number ): Promise { const clustersIds = clusters.map((cluster) => cluster.clusterUuid); @@ -36,7 +37,9 @@ export async function fetchMemoryUsageNodeStats( { range: { timestamp: { - gte: `now-${duration}`, + format: 'epoch_millis', + gte: startMs, + lte: endMs, }, }, }, From 7ddae02f92cc95a88a372039dafc709877401a50 Mon Sep 17 00:00:00 2001 From: Igor Zaytsev Date: Thu, 1 Oct 2020 22:41:27 -0400 Subject: [PATCH 4/7] Feedback and tests --- .../server/alerts/alerts_factory.test.ts | 2 +- .../monitoring/server/alerts/base_alert.ts | 9 +++++++++ .../server/alerts/memory_usage_alert.ts | 8 +++++++- .../missing_monitoring_data_alert.test.ts | 10 +++++----- .../alerts/missing_monitoring_data_alert.ts | 17 ++++------------- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/monitoring/server/alerts/alerts_factory.test.ts b/x-pack/plugins/monitoring/server/alerts/alerts_factory.test.ts index ddc8dcafebd21..f486061109b39 100644 --- a/x-pack/plugins/monitoring/server/alerts/alerts_factory.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/alerts_factory.test.ts @@ -63,6 +63,6 @@ describe('AlertsFactory', () => { it('should get all', () => { const alerts = AlertsFactory.getAll(); - expect(alerts.length).toBe(9); + expect(alerts.length).toBe(10); }); }); diff --git a/x-pack/plugins/monitoring/server/alerts/base_alert.ts b/x-pack/plugins/monitoring/server/alerts/base_alert.ts index 61486626040f7..b1327719f3169 100644 --- a/x-pack/plugins/monitoring/server/alerts/base_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/base_alert.ts @@ -377,4 +377,13 @@ export class BaseAlert { ) { throw new Error('Child classes must implement `executeActions`'); } + + protected createGlobalStateLink(link: string, clusterUuid: string, ccs?: string) { + const globalState = [`cluster_uuid:${clusterUuid}`]; + if (ccs) { + globalState.push(`ccs:${ccs}`); + } + globalState.push('refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)'); + return `${this.kibanaUrl}/app/monitoring#/${link}?_g=(${globalState.toString()})`; + } } diff --git a/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts b/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts index 5b49c27cca799..5e49b2fc65917 100644 --- a/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts @@ -231,7 +231,13 @@ export class MemoryUsageAlert extends BaseAlert { defaultMessage: 'View nodes', }); - const action = `[${fullActionText}](elasticsearch/nodes)`; + const ccs = alertStates.find((state) => state.ccs)?.ccs; + const globalStateLink = this.createGlobalStateLink( + 'elasticsearch/nodes', + cluster.clusterUuid, + ccs + ); + const action = `[${fullActionText}](${globalStateLink})`; const internalShortMessage = i18n.translate( 'xpack.monitoring.alerts.memoryUsage.firing.internalShortMessage', { diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts index 4c06d9718c455..34dd31a9451ef 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts @@ -234,9 +234,9 @@ describe('MissingMonitoringDataAlert', () => { ], }); expect(scheduleActions).toHaveBeenCalledWith('default', { - internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#overview?_g=(cluster_uuid:abc123))`, + internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, - action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#overview?_g=(cluster_uuid:abc123))`, + action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName, @@ -414,9 +414,9 @@ describe('MissingMonitoringDataAlert', () => { } as any); const count = 1; expect(scheduleActions).toHaveBeenCalledWith('default', { - internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#overview?_g=(cluster_uuid:abc123,ccs:testCluster))`, + internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,ccs:testCluster,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, internalShortMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, - action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#overview?_g=(cluster_uuid:abc123,ccs:testCluster))`, + action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,ccs:testCluster,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName, @@ -446,7 +446,7 @@ describe('MissingMonitoringDataAlert', () => { expect(scheduleActions).toHaveBeenCalledWith('default', { internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, - action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#overview?_g=(cluster_uuid:abc123))`, + action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName, diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts index 6017314f332e6..75dee475e7525 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts @@ -309,13 +309,6 @@ export class MissingMonitoringDataAlert extends BaseAlert { return; } - const ccs = instanceState.alertStates.reduce((accum: string, state): string => { - if (state.ccs) { - return state.ccs; - } - return accum; - }, ''); - const firingCount = instanceState.alertStates.filter((alertState) => alertState.ui.isFiring) .length; const firingStackProducts = instanceState.alertStates @@ -336,12 +329,10 @@ export class MissingMonitoringDataAlert extends BaseAlert { const fullActionText = i18n.translate('xpack.monitoring.alerts.missingData.fullAction', { defaultMessage: 'View what monitoring data we do have for these stack products.', }); - const globalState = [`cluster_uuid:${cluster.clusterUuid}`]; - if (ccs) { - globalState.push(`ccs:${ccs}`); - } - const url = `${this.kibanaUrl}/app/monitoring#overview?_g=(${globalState.join(',')})`; - const action = `[${fullActionText}](${url})`; + + const ccs = instanceState.alertStates.find((state) => state.ccs)?.ccs; + const globalStateLink = this.createGlobalStateLink('overview', cluster.clusterUuid, ccs); + const action = `[${fullActionText}](${globalStateLink})`; const internalShortMessage = i18n.translate( 'xpack.monitoring.alerts.missingData.firing.internalShortMessage', { From 635dfd048b5bf9a7d8910d57192e8ee10608ed1f Mon Sep 17 00:00:00 2001 From: Igor Zaytsev Date: Fri, 2 Oct 2020 14:06:43 -0400 Subject: [PATCH 5/7] Added size to optimize query --- .../server/lib/alerts/fetch_memory_usage_node_stats.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts index dd586b24f8a4f..c6843c3ed5f12 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_memory_usage_node_stats.ts @@ -73,6 +73,7 @@ export async function fetchMemoryUsageNodeStats( cluster_uuid: { terms: { field: 'cluster_uuid', + size: 1, }, }, name: { From 5d28f0096352709144d59b04845eb82b5507dc5e Mon Sep 17 00:00:00 2001 From: Igor Zaytsev Date: Fri, 2 Oct 2020 15:04:16 -0400 Subject: [PATCH 6/7] Removed scheduled check --- x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts | 2 -- .../plugins/monitoring/server/alerts/disk_usage_alert.test.ts | 2 -- x-pack/plugins/monitoring/server/alerts/disk_usage_alert.ts | 2 +- x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts | 2 +- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts index 0911c529a6061..a53ae1f9d0dd5 100644 --- a/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.test.ts @@ -78,7 +78,6 @@ describe('CpuUsageAlert', () => { }; const kibanaUrl = 'http://localhost:5601'; - const hasScheduledActions = jest.fn(); const replaceState = jest.fn(); const scheduleActions = jest.fn(); const getState = jest.fn(); @@ -87,7 +86,6 @@ describe('CpuUsageAlert', () => { callCluster: jest.fn(), alertInstanceFactory: jest.fn().mockImplementation(() => { return { - hasScheduledActions, replaceState, scheduleActions, getState, diff --git a/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.test.ts index 546399f666b6c..e3d69820ebb05 100644 --- a/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.test.ts @@ -89,7 +89,6 @@ describe('DiskUsageAlert', () => { }; const kibanaUrl = 'http://localhost:5601'; - const hasScheduledActions = jest.fn(); const replaceState = jest.fn(); const scheduleActions = jest.fn(); const getState = jest.fn(); @@ -98,7 +97,6 @@ describe('DiskUsageAlert', () => { callCluster: jest.fn(), alertInstanceFactory: jest.fn().mockImplementation(() => { return { - hasScheduledActions, replaceState, scheduleActions, getState, diff --git a/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.ts b/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.ts index 2322d6ba99dfe..c577550de8617 100644 --- a/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/disk_usage_alert.ts @@ -331,7 +331,7 @@ export class DiskUsageAlert extends BaseAlert { const alertInstanceState = { alertStates: newAlertStates }; instance.replaceState(alertInstanceState); - if (newAlertStates.length && !instance.hasScheduledActions()) { + if (newAlertStates.length) { this.executeActions(instance, alertInstanceState, null, cluster); state.lastExecutedAction = currentUTC; } diff --git a/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts b/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts index 5e49b2fc65917..8dc707afab1e1 100644 --- a/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/memory_usage_alert.ts @@ -343,7 +343,7 @@ export class MemoryUsageAlert extends BaseAlert { const alertInstanceState = { alertStates: newAlertStates }; instance.replaceState(alertInstanceState); - if (newAlertStates.length && !instance.hasScheduledActions()) { + if (newAlertStates.length) { this.executeActions(instance, alertInstanceState, null, cluster); state.lastExecutedAction = currentUTC; } From 72823a07a69de796b5d723f79e7cbc10f5a3c19a Mon Sep 17 00:00:00 2001 From: Igor Zaytsev Date: Fri, 2 Oct 2020 17:02:36 -0400 Subject: [PATCH 7/7] Removed globalstate date --- x-pack/plugins/monitoring/server/alerts/base_alert.ts | 1 - .../alerts/missing_monitoring_data_alert.test.ts | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/monitoring/server/alerts/base_alert.ts b/x-pack/plugins/monitoring/server/alerts/base_alert.ts index b1327719f3169..c92291cf72093 100644 --- a/x-pack/plugins/monitoring/server/alerts/base_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/base_alert.ts @@ -383,7 +383,6 @@ export class BaseAlert { if (ccs) { globalState.push(`ccs:${ccs}`); } - globalState.push('refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)'); return `${this.kibanaUrl}/app/monitoring#/${link}?_g=(${globalState.toString()})`; } } diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts index 34dd31a9451ef..6ed237a055b5c 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts @@ -234,9 +234,9 @@ describe('MissingMonitoringDataAlert', () => { ], }); expect(scheduleActions).toHaveBeenCalledWith('default', { - internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, + internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, - action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, + action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName, @@ -414,9 +414,9 @@ describe('MissingMonitoringDataAlert', () => { } as any); const count = 1; expect(scheduleActions).toHaveBeenCalledWith('default', { - internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,ccs:testCluster,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, + internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,ccs:testCluster))`, internalShortMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, - action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,ccs:testCluster,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, + action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,ccs:testCluster))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName, @@ -446,7 +446,7 @@ describe('MissingMonitoringDataAlert', () => { expect(scheduleActions).toHaveBeenCalledWith('default', { internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, - action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123,refreshInterval:(pause:!f,value:10000),time:(from:now-1h,to:now)))`, + action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName,