[Monitoring][Alerting] CCR read exceptions alert

x-pack/plugins/monitoring/common/constants.ts

@@ -251,6 +251,7 @@ export const ALERT_MEMORY_USAGE = `${ALERT_PREFIX}alert_jvm_memory_usage`;
 export const ALERT_MISSING_MONITORING_DATA = `${ALERT_PREFIX}alert_missing_monitoring_data`;
 export const ALERT_THREAD_POOL_SEARCH_REJECTIONS = `${ALERT_PREFIX}alert_thread_pool_search_rejections`;
 export const ALERT_THREAD_POOL_WRITE_REJECTIONS = `${ALERT_PREFIX}alert_thread_pool_write_rejections`;
+export const ALERT_CCR_READ_EXCEPTIONS = `${ALERT_PREFIX}ccr_read_exceptions`;
 
 /**
  * Legacy alerts details/label for server and public use
@@ -451,6 +452,25 @@ export const ALERT_DETAILS = {
         'Alert when the number of rejections in the write thread pool exceeds the threshold.',
     }),
   },
+  [ALERT_CCR_READ_EXCEPTIONS]: {
+    paramDetails: {
+      duration: {
+        label: i18n.translate(
+          'xpack.monitoring.alerts.ccrReadExceptions.paramDetails.duration.label',
+          {
+            defaultMessage: `In the last`,
+          }
+        ),
+        type: AlertParamType.Duration,
+      },
+    },
+    label: i18n.translate('xpack.monitoring.alerts.ccrReadExceptions.label', {
+      defaultMessage: 'CCR read exceptions',
+    }),
+    description: i18n.translate('xpack.monitoring.alerts.ccrReadExceptions.description', {
+      defaultMessage: 'Alert if any CCR read exceptions have been detected.',
+    }),
+  },
 };
 
 export const ALERT_PANEL_MENU = [
@@ -485,6 +505,7 @@ export const ALERT_PANEL_MENU = [
       { alertName: ALERT_LICENSE_EXPIRATION },
       { alertName: ALERT_THREAD_POOL_SEARCH_REJECTIONS },
       { alertName: ALERT_THREAD_POOL_WRITE_REJECTIONS },
+      { alertName: ALERT_CCR_READ_EXCEPTIONS },
     ],
   },
 ];
@@ -505,6 +526,7 @@ export const ALERTS = [
   ALERT_MISSING_MONITORING_DATA,
   ALERT_THREAD_POOL_SEARCH_REJECTIONS,
   ALERT_THREAD_POOL_WRITE_REJECTIONS,
+  ALERT_CCR_READ_EXCEPTIONS,
 ];
 
 /**

x-pack/plugins/monitoring/common/types/alerts.ts

@@ -22,6 +22,7 @@ export interface CommonAlertState {
 
 export interface CommonAlertFilter {
   nodeUuid?: string;
+  shardId?: string;
 }
 
 export interface CommonAlertParamDetail {
@@ -103,6 +104,7 @@ export interface AlertUiState {
 
 export interface AlertMessage {
   text: string; // Do this. #link this is a link #link
+  code?: string;
   nextSteps?: AlertMessage[];
   tokens?: AlertMessageToken[];
 }
@@ -165,6 +167,20 @@ export interface AlertMemoryUsageNodeStats extends AlertNodeStats {
 export interface AlertMissingData extends AlertNodeStats {
   gapDuration: number;
 }
+export interface CCRReadExceptionsStats {
+  remoteCluster: string;
+  followerIndex: string;
+  shardId: number;
+  leaderIndex: string;
+  lastReadException: { type: string; reason: string };
+  clusterUuid: string;
+  ccs: string;
+}
+
+export interface CCRReadExceptionsUIMeta extends CCRReadExceptionsStats {
+  instanceId: string;
+  itemLabel: string;
+}
 
 export interface AlertData {
   nodeName?: string;

x-pack/plugins/monitoring/public/alerts/callout.tsx

@@ -15,6 +15,7 @@ import {
   EuiFlexGroup,
   EuiFlexItem,
   EuiIcon,
+  EuiCodeBlock,
 } from '@elastic/eui';
 import { replaceTokens } from './lib/replace_tokens';
 import { AlertMessage } from '../../common/types/alerts';
@@ -66,12 +67,24 @@ export const AlertsCallout: React.FC<Props> = (props: Props) => {
       </div>
     );
 
+    const { code } = status.state.state.ui.message;
     const accordion = (
       <EuiAccordion
         id={`monitoringAlertCallout_${index}`}
         buttonContent={buttonContent}
         paddingSize="s"
       >
+        {code?.length ? (
+          <EuiCodeBlock
+            fontSize="s"
+            paddingSize="s"
+            language="json"
+            isCopyable={true}
+            overflowHeight={300}
+          >
+            {code}
+          </EuiCodeBlock>
+        ) : null}
         <EuiListGroup
           flush={true}
           bordered={true}
@@ -84,11 +97,19 @@ export const AlertsCallout: React.FC<Props> = (props: Props) => {
             paddingLeft: `0.5rem`,
           }}
         >
-          {(status.state.state.ui.message.nextSteps || []).map((step: AlertMessage) => {
-            return <EuiListGroupItem onClick={() => {}} label={replaceTokens(step)} />;
-          })}
+          {(status.state.state.ui.message.nextSteps || []).map(
+            (step: AlertMessage, stepIndex: number) => {
+              return (
+                <EuiListGroupItem
+                  onClick={() => {}}
+                  label={replaceTokens(step)}
+                  key={index + stepIndex}
+                />
+              );
+            }
+          )}
           <EuiListGroupItem
-            label={<AlertConfiguration alert={status.alert.rawAlert} compressed />}
+            label={<AlertConfiguration alert={status.alert.rawAlert} key={index} compressed />}
           />
         </EuiListGroup>
       </EuiAccordion>

x-pack/plugins/monitoring/public/alerts/ccr_read_exceptions_alert/index.tsx

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { Expression, Props } from '../components/duration/expression';
+import { AlertTypeModel, ValidationResult } from '../../../../triggers_actions_ui/public';
+import { ALERT_CCR_READ_EXCEPTIONS, ALERT_DETAILS } from '../../../common/constants';
+
+interface ValidateOptions {
+  duration: string;
+}
+
+const validate = (inputValues: ValidateOptions): ValidationResult => {
+  const validationResult = { errors: {} };
+  const errors: { [key: string]: string[] } = {
+    duration: [],
+  };
+  if (!inputValues.duration) {
+    errors.duration.push(
+      i18n.translate('xpack.monitoring.alerts.validation.duration', {
+        defaultMessage: 'A valid duration is required.',
+      })
+    );
+  }
+  validationResult.errors = errors;
+  return validationResult;
+};
+
+export function createCCRReadExceptionsAlertType(): AlertTypeModel {
+  return {
+    id: ALERT_CCR_READ_EXCEPTIONS,
+    name: ALERT_DETAILS[ALERT_CCR_READ_EXCEPTIONS].label,
+    description: ALERT_DETAILS[ALERT_CCR_READ_EXCEPTIONS].description,
+    iconClass: 'bell',
+    documentationUrl(docLinks) {
+      return `${docLinks.ELASTIC_WEBSITE_URL}guide/en/kibana/${docLinks.DOC_LINK_VERSION}/kibana-alerts.html`;
+    },
+    alertParamsExpression: (props: Props) => (
+      <Expression {...props} paramDetails={ALERT_DETAILS[ALERT_CCR_READ_EXCEPTIONS].paramDetails} />
+    ),
+    validate,
+    defaultActionMessage: '{{context.internalFullMessage}}',
+    requiresAppContext: true,
+  };
+}

x-pack/plugins/monitoring/public/alerts/lib/get_alert_panels_by_category.tsx

@@ -171,6 +171,7 @@ export function getAlertPanelsByCategory(
       for (const { alert, states } of category.alerts) {
         const items = [];
         for (const alertState of states.filter(({ state }) => stateFilter(state))) {
+          const { nodeName, itemLabel } = alertState.state;
           items.push({
             name: (
               <Fragment>
@@ -188,7 +189,7 @@ export function getAlertPanelsByCategory(
                     )}
                   </EuiText>
                 </EuiToolTip>
-                <EuiText size="s">{alertState.state.nodeName}</EuiText>
+                <EuiText size="s">{nodeName || itemLabel}</EuiText>
               </Fragment>
             ),
             panel: ++tertiaryPanelIndex,

x-pack/plugins/monitoring/public/alerts/lib/get_alert_panels_by_node.tsx

@@ -69,10 +69,11 @@ export function getAlertPanelsByNode(
           const states = (statesByNodes[nodeUuid] as CommonAlertState[]).filter(({ state }) =>
             stateFilter(state)
           );
+          const { nodeName, itemLabel } = states[0].state;
           return {
             name: (
               <EuiText>
-                {states[0].state.nodeName} ({states.length})
+                {nodeName || itemLabel} ({states.length})
               </EuiText>
             ),
             panel: index + 1,
@@ -86,7 +87,8 @@ export function getAlertPanelsByNode(
       let title = '';
       for (const { alert, states } of alertsForNode) {
         for (const alertState of states) {
-          title = alertState.state.nodeName;
+          const { nodeName, itemLabel } = alertState.state;
+          title = nodeName || itemLabel;
           panelItems.push({
             name: (
               <Fragment>

x-pack/plugins/monitoring/public/alerts/lib/replace_tokens.tsx

@@ -77,6 +77,7 @@ export function replaceTokens(alertMessage: AlertMessage): JSX.Element | string
     }
 
     const url = linkToken.partialUrl
+      .replace('{basePath}', Legacy.shims.getBasePath())
       .replace('{elasticWebsiteUrl}', Legacy.shims.docLinks.ELASTIC_WEBSITE_URL)
       .replace('{docLinkVersion}', Legacy.shims.docLinks.DOC_LINK_VERSION);
     const index = text.indexOf(linkPart[0]);

x-pack/plugins/monitoring/public/alerts/panel.tsx

@@ -10,6 +10,7 @@ import {
   EuiHorizontalRule,
   EuiListGroup,
   EuiListGroupItem,
+  EuiCodeBlock,
 } from '@elastic/eui';
 
 import { CommonAlert, CommonAlertState, AlertMessage } from '../../common/types/alerts';
@@ -47,12 +48,24 @@ export const AlertPanel: React.FC<Props> = (props: Props) => {
       </EuiListGroup>
     ) : null;
 
+  const { code } = alertState.state.ui.message;
   return (
     <Fragment>
       <div style={{ padding: '1rem' }}>
         <EuiTitle size="xs">
           <h5>{replaceTokens(alertState.state.ui.message)}</h5>
         </EuiTitle>
+        {code?.length ? (
+          <EuiCodeBlock
+            fontSize="s"
+            paddingSize="s"
+            language="json"
+            isCopyable={true}
+            overflowHeight={150}
+          >
+            {code}
+          </EuiCodeBlock>
+        ) : null}
         {nextStepsUi ? <EuiSpacer size="s" /> : null}
         {nextStepsUi}
       </div>

x-pack/plugins/monitoring/public/components/cluster/overview/elasticsearch_panel.js

@@ -47,6 +47,7 @@ import {
   ALERT_NODES_CHANGED,
   ALERT_ELASTICSEARCH_VERSION_MISMATCH,
   ALERT_MISSING_MONITORING_DATA,
+  ALERT_CCR_READ_EXCEPTIONS,
 } from '../../../../common/constants';
 import { AlertsBadge } from '../../../alerts/badge';
 import { shouldShowAlertBadge } from '../../../alerts/lib/should_show_alert_badge';
@@ -159,7 +160,11 @@ function renderLog(log) {
   );
 }
 
-const OVERVIEW_PANEL_ALERTS = [ALERT_CLUSTER_HEALTH, ALERT_LICENSE_EXPIRATION];
+const OVERVIEW_PANEL_ALERTS = [
+  ALERT_CLUSTER_HEALTH,
+  ALERT_LICENSE_EXPIRATION,
+  ALERT_CCR_READ_EXCEPTIONS,
+];
 
 const NODES_PANEL_ALERTS = [
   ALERT_CPU_USAGE,