[Enterprise Search] Add stats overview to indices page (#145282)

x-pack/plugins/enterprise_search/common/stats.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface SyncJobsStats {
+  connected: number;
+  errors: number;
+  in_progress: number;
+  incomplete: number;
+  long_running: number;
+  orphaned_jobs: number;
+}

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/stats/fetch_sync_jobs_stats_api_logic.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { SyncJobsStats } from '../../../../../common/stats';
+
+import { createApiLogic } from '../../../shared/api_logic/create_api_logic';
+import { HttpLogic } from '../../../shared/http';
+
+export type FetchSyncJobsStatsResponse = SyncJobsStats;
+
+export const fetchSyncJobsStats = async () => {
+  const route = '/internal/enterprise_search/stats/sync_jobs';
+  return await HttpLogic.values.http.get<FetchSyncJobsStatsResponse>(route);
+};
+
+export const FetchSyncJobsStatsApiLogic = createApiLogic(
+  ['enterprise_search_content', 'fetch_sync_jobs_stats_api_logic'],
+  fetchSyncJobsStats
+);

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_stats.tsx

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { useEffect } from 'react';
+
+import { useActions, useValues } from 'kea';
+
+import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiStat } from '@elastic/eui';
+
+import { i18n } from '@kbn/i18n';
+
+import { Status } from '../../../../../common/types/api';
+
+import { FetchSyncJobsStatsApiLogic } from '../../api/stats/fetch_sync_jobs_stats_api_logic';
+
+export const IndicesStats: React.FC = () => {
+  const { makeRequest } = useActions(FetchSyncJobsStatsApiLogic);
+  const { data, status } = useValues(FetchSyncJobsStatsApiLogic);
+  const isLoading = status === Status.LOADING;
+
+  useEffect(() => {
+    makeRequest({});
+  }, []);
+
+  return (
+    <EuiFlexGroup direction="column">
+      <EuiFlexItem>
+        <EuiFlexGroup>
+          <EuiFlexItem>
+            <EuiPanel
+              color={data?.connected ? 'success' : 'subdued'}
+              hasShadow={false}
+              paddingSize="l"
+            >
+              <EuiStat
+                description={i18n.translate(
+                  'xpack.enterpriseSearch.content.searchIndices.jobStats.connectedMethods',
+                  {
+                    defaultMessage: 'Connected ingest methods',
+                  }
+                )}
+                isLoading={isLoading}
+                title={data?.connected}
+              />
+            </EuiPanel>
+          </EuiFlexItem>
+          <EuiFlexItem>
+            <EuiPanel
+              color={data?.incomplete ? 'warning' : 'subdued'}
+              hasShadow={false}
+              paddingSize="l"
+            >
+              <EuiStat
+                description={i18n.translate(
+                  'xpack.enterpriseSearch.content.searchIndices.jobStats.incompleteMethods',
+                  {
+                    defaultMessage: 'Incomplete ingest methods',
+                  }
+                )}
+                isLoading={isLoading}
+                title={data?.incomplete}
+              />
+            </EuiPanel>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </EuiFlexItem>
+      <EuiFlexItem>
+        <EuiFlexGroup>
+          <EuiFlexItem>
+            <EuiPanel color="subdued" hasShadow={false} paddingSize="l">
+              <EuiStat
+                description={i18n.translate(
+                  'xpack.enterpriseSearch.content.searchIndices.jobStats.runningSyncs',
+                  {
+                    defaultMessage: 'Running syncs',
+                  }
+                )}
+                isLoading={isLoading}
+                title={data?.in_progress}
+              />
+            </EuiPanel>
+          </EuiFlexItem>
+          <EuiFlexItem>
+            <EuiPanel
+              color={data?.long_running ? 'warning' : 'subdued'}
+              hasShadow={false}
+              paddingSize="l"
+            >
+              <EuiStat
+                description={i18n.translate(
+                  'xpack.enterpriseSearch.content.searchIndices.jobStats.longRunningSyncs',
+                  {
+                    defaultMessage: 'Long running syncs',
+                  }
+                )}
+                isLoading={isLoading}
+                title={data?.long_running}
+              />
+            </EuiPanel>
+          </EuiFlexItem>
+          <EuiFlexItem>
+            <EuiPanel
+              color={data?.orphaned_jobs ? 'warning' : 'subdued'}
+              hasShadow={false}
+              paddingSize="l"
+            >
+              <EuiStat
+                description={i18n.translate(
+                  'xpack.enterpriseSearch.content.searchIndices.jobStats.orphanedSyncs',
+                  {
+                    defaultMessage: 'Orphaned syncs',
+                  }
+                )}
+                isLoading={isLoading}
+                title={data?.orphaned_jobs}
+              />
+            </EuiPanel>
+          </EuiFlexItem>
+          <EuiFlexItem>
+            <EuiPanel color={data?.errors ? 'danger' : 'subdued'} hasShadow={false} paddingSize="l">
+              <EuiStat
+                description={i18n.translate(
+                  'xpack.enterpriseSearch.content.searchIndices.jobStats.errorSyncs',
+                  {
+                    defaultMessage: 'Syncs errors',
+                  }
+                )}
+                isLoading={isLoading}
+                title={data?.errors}
+              />
+            </EuiPanel>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </EuiFlexItem>
+    </EuiFlexGroup>
+  );
+};

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/search_indices.tsx

@@ -36,6 +36,7 @@ import { EnterpriseSearchContentPageTemplate } from '../layout/page_template';
 
 import { DeleteIndexModal } from './delete_index_modal';
 import { IndicesLogic } from './indices_logic';
+import { IndicesStats } from './indices_stats';
 import { IndicesTable } from './indices_table';
 
 import './search_indices.scss';
@@ -148,6 +149,9 @@ export const SearchIndices: React.FC = () => {
                 </EuiCallOut>
               </EuiFlexItem>
             )}
+            <EuiFlexItem>
+              <IndicesStats />
+            </EuiFlexItem>
             <EuiFlexItem>
               <EuiFlexGroup justifyContent="spaceBetween" alignItems="center">
                 <EuiFlexItem grow={false}>

x-pack/plugins/enterprise_search/server/lib/stats/get_sync_jobs.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+
+import { IScopedClusterClient } from '@kbn/core/server';
+
+import { CONNECTORS_INDEX, CONNECTORS_JOBS_INDEX } from '../..';
+import { SyncJobsStats } from '../../../common/stats';
+
+import { ConnectorStatus, SyncStatus } from '../../../common/types/connectors';
+
+export const fetchSyncJobsStats = async (client: IScopedClusterClient): Promise<SyncJobsStats> => {
+  const connectorIdsResult = await client.asCurrentUser.search({
+    index: CONNECTORS_INDEX,
+    scroll: '10s',
+    stored_fields: [],
+  });
+  const ids = connectorIdsResult.hits.hits.map((hit) => hit._id);
+  const orphanedJobsCountResponse = await client.asCurrentUser.count({
+    index: CONNECTORS_JOBS_INDEX,
+    query: {
+      terms: {
+        'connector.id': ids,
+      },
+    },
+  });
+
+  const inProgressJobsCountResponse = await client.asCurrentUser.count({
+    index: CONNECTORS_JOBS_INDEX,
+    query: {
+      term: {
+        status: SyncStatus.IN_PROGRESS,
+      },
+    },
+  });
+
+  const longRunningProgressJobsCountResponse = await client.asCurrentUser.count({
+    index: CONNECTORS_JOBS_INDEX,
+    query: {
+      bool: {
+        filter: [
+          {
+            term: {
+              status: SyncStatus.IN_PROGRESS,
+            },
+          },
+          {
+            range: {
+              last_seen: {
+                lt: moment().subtract(1, 'day').toISOString(),
+              },
+            },
+          },
+        ],
+      },
+    },
+  });
+
+  const errorResponse = await client.asCurrentUser.count({
+    index: CONNECTORS_JOBS_INDEX,
+    query: {
+      term: {
+        status: SyncStatus.ERROR,
+      },
+    },
+  });
+
+  const connectedResponse = await client.asCurrentUser.count({
+    index: CONNECTORS_INDEX,
+    query: {
+      bool: {
+        filter: [
+          {
+            term: {
+              status: ConnectorStatus.CONNECTED,
+            },
+          },
+          {
+            range: {
+              last_seen: {
+                gte: moment().subtract(30, 'minutes').toISOString(),
+              },
+            },
+          },
+        ],
+      },
+    },
+  });
+
+  const incompleteResponse = await client.asCurrentUser.count({
+    index: CONNECTORS_INDEX,
+    query: {
+      bool: {
+        should: [
+          {
+            bool: {
+              must_not: {
+                terms: {
+                  status: [ConnectorStatus.CONNECTED, ConnectorStatus.ERROR],
+                },
+              },
+            },
+          },
+          {
+            range: {
+              last_seen: {
+                gt: moment().subtract(30, 'minutes').toISOString(),
+              },
+            },
+          },
+        ],
+      },
+    },
+  });
+
+  const response = {
+    connected: connectedResponse.count,
+    errors: errorResponse.count,
+    in_progress: inProgressJobsCountResponse.count,
+    incomplete: incompleteResponse.count,
+    long_running: longRunningProgressJobsCountResponse.count,
+    orphaned_jobs: orphanedJobsCountResponse.count,
+  };
+
+  return response;
+};

x-pack/plugins/enterprise_search/server/plugin.ts

@@ -55,6 +55,7 @@ import { registerConfigDataRoute } from './routes/enterprise_search/config_data'
 import { registerConnectorRoutes } from './routes/enterprise_search/connectors';
 import { registerCrawlerRoutes } from './routes/enterprise_search/crawler/crawler';
 import { registerCreateAPIKeyRoute } from './routes/enterprise_search/create_api_key';
+import { registerStatsRoutes } from './routes/enterprise_search/stats';
 import { registerTelemetryRoute } from './routes/enterprise_search/telemetry';
 import { registerWorkplaceSearchRoutes } from './routes/workplace_search';
 
@@ -189,6 +190,7 @@ export class EnterpriseSearchPlugin implements Plugin {
     registerConnectorRoutes(dependencies);
     registerCrawlerRoutes(dependencies);
     registerAnalyticsRoutes(dependencies);
+    registerStatsRoutes(dependencies);
 
     getStartServices().then(([, { security: securityStart }]) => {
       registerCreateAPIKeyRoute(dependencies, securityStart);

x-pack/plugins/enterprise_search/server/routes/enterprise_search/stats.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { fetchSyncJobsStats } from '../../lib/stats/get_sync_jobs';
+import { RouteDependencies } from '../../plugin';
+import { elasticsearchErrorHandler } from '../../utils/elasticsearch_error_handler';
+
+export function registerStatsRoutes({ router, log }: RouteDependencies) {
+  router.get(
+    {
+      path: '/internal/enterprise_search/stats/sync_jobs',
+      validate: {},
+    },
+    elasticsearchErrorHandler(log, async (context, request, response) => {
+      const { client } = (await context.core).elasticsearch;
+      const body = await fetchSyncJobsStats(client);
+      return response.ok({ body });
+    })
+  );
+}