From bb0365e43a4335880973e85da18f7506491e59dd Mon Sep 17 00:00:00 2001 From: Byron Hulcher Date: Tue, 26 Jul 2022 07:50:32 -0400 Subject: [PATCH] [Enterprise Search] Display most recent crawl request status in Indices and Crawl Request tables (#137128) --- .../enterprise_search/common/types/crawler.ts | 20 ++++ .../api/crawler/types.ts | 17 +-- .../crawler/crawl_requests_panel/constants.ts | 13 +++ .../crawl_requests_table.tsx | 6 +- .../search_indices/indices_table.tsx | 21 ++-- .../utils/crawler_status_helpers.ts | 28 +++++ .../server/lib/connectors/fetch_connectors.ts | 31 ++---- .../server/lib/crawler/fetch_crawlers.ts | 102 ++++++++++++++++++ .../enterprise_search/server/lib/fetch_all.ts | 37 +++++++ .../server/lib/indices/fetch_index.test.ts | 27 +++-- .../server/lib/indices/fetch_index.ts | 10 +- .../routes/enterprise_search/indices.ts | 3 + 12 files changed, 249 insertions(+), 66 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/crawler_status_helpers.ts create mode 100644 x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawlers.ts create mode 100644 x-pack/plugins/enterprise_search/server/lib/fetch_all.ts diff --git a/x-pack/plugins/enterprise_search/common/types/crawler.ts b/x-pack/plugins/enterprise_search/common/types/crawler.ts index afe4078f1421f..b4d4841062d71 100644 --- a/x-pack/plugins/enterprise_search/common/types/crawler.ts +++ b/x-pack/plugins/enterprise_search/common/types/crawler.ts @@ -5,7 +5,27 @@ * 2.0. */ +// See SharedTogo::Crawler::Status for details on how these are generated +export enum CrawlerStatus { + Pending = 'pending', + Suspended = 'suspended', + Starting = 'starting', + Running = 'running', + Suspending = 'suspending', + Canceling = 'canceling', + Success = 'success', + Failed = 'failed', + Canceled = 'canceled', + Skipped = 'skipped', +} + +export interface CrawlRequest { + id: string; + configuration_oid: string; + status: CrawlerStatus; +} export interface Crawler { id: string; index_name: string; + most_recent_crawl_request_status?: CrawlerStatus; } diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/types.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/types.ts index a9119d6fb9e02..7439c9ede1d8f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/types.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/crawler/types.ts @@ -5,6 +5,10 @@ * 2.0. */ import { Meta } from '../../../../../common/types'; +import { CrawlerStatus } from '../../../../../common/types/crawler'; + +// TODO remove this proxy export, which will affect a lot of files +export { CrawlerStatus }; export enum CrawlerPolicies { allow = 'allow', @@ -51,19 +55,6 @@ export type CrawlerDomainValidationStepName = | 'networkConnectivity' | 'indexingRestrictions' | 'contentVerification'; -// See SharedTogo::Crawler::Status for details on how these are generated -export enum CrawlerStatus { - Pending = 'pending', - Suspended = 'suspended', - Starting = 'starting', - Running = 'running', - Suspending = 'suspending', - Canceling = 'canceling', - Success = 'success', - Failed = 'failed', - Canceled = 'canceled', - Skipped = 'skipped', -} export type CrawlEventStage = 'crawl' | 'process'; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_requests_panel/constants.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_requests_panel/constants.ts index d736e04981ba1..ac76850eadf2d 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_requests_panel/constants.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_requests_panel/constants.ts @@ -60,3 +60,16 @@ export const readableCrawlerStatuses: { [key in CrawlerStatus]: string } = { { defaultMessage: 'Skipped' } ), }; + +export const crawlStatusColors: { [key in CrawlerStatus]: 'default' | 'danger' | 'success' } = { + [CrawlerStatus.Pending]: 'default', + [CrawlerStatus.Suspended]: 'default', + [CrawlerStatus.Starting]: 'default', + [CrawlerStatus.Running]: 'default', + [CrawlerStatus.Suspending]: 'default', + [CrawlerStatus.Canceling]: 'default', + [CrawlerStatus.Success]: 'success', + [CrawlerStatus.Failed]: 'danger', + [CrawlerStatus.Canceled]: 'default', + [CrawlerStatus.Skipped]: 'default', +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_requests_panel/crawl_requests_table.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_requests_panel/crawl_requests_table.tsx index b72d93d0f2cd4..66c11b32d6d16 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_requests_panel/crawl_requests_table.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/crawler/crawl_requests_panel/crawl_requests_table.tsx @@ -25,7 +25,7 @@ import { CrawlEvent } from '../../../../api/crawler/types'; import { CrawlDetailLogic } from '../crawl_details_flyout/crawl_detail_logic'; import { CrawlerLogic } from '../crawler_logic'; -import { readableCrawlerStatuses } from './constants'; +import { crawlStatusColors, readableCrawlerStatuses } from './constants'; import { CrawlEventTypeBadge } from './crawl_event_type_badge'; export const CrawlRequestsTable: React.FC = () => { @@ -84,7 +84,9 @@ export const CrawlRequestsTable: React.FC = () => { name: i18n.translate('xpack.enterpriseSearch.crawler.crawlRequestsTable.column.status', { defaultMessage: 'Status', }), - render: (status: CrawlEvent['status']) => readableCrawlerStatuses[status], + render: (status: CrawlEvent['status']) => ( + {readableCrawlerStatuses[status]} + ), }, ]; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_table.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_table.tsx index 1e2cbd003c689..2533b07fd06a7 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_table.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_indices/indices_table.tsx @@ -24,8 +24,9 @@ import { Meta } from '../../../../../common/types'; import { EuiLinkTo, EuiButtonIconTo } from '../../../shared/react_router_helpers'; import { convertMetaToPagination } from '../../../shared/table_pagination'; import { SEARCH_INDEX_PATH } from '../../routes'; -import { ElasticsearchViewIndex, IngestionMethod, IngestionStatus } from '../../types'; -import { ingestionMethodToText } from '../../utils/indices'; +import { ElasticsearchViewIndex, IngestionMethod } from '../../types'; +import { crawlerStatusToColor, crawlerStatusToText } from '../../utils/crawler_status_helpers'; +import { ingestionMethodToText, isCrawlerIndex } from '../../utils/indices'; import { ingestionStatusToColor, ingestionStatusToText, @@ -119,18 +120,22 @@ const columns: Array> = [ truncateText: true, }, { - field: 'ingestionStatus', name: i18n.translate( 'xpack.enterpriseSearch.content.searchIndices.ingestionStatus.columnTitle', { defaultMessage: 'Ingestion status', } ), - render: (ingestionStatus: IngestionStatus) => ( - - {ingestionStatusToText(ingestionStatus)} - - ), + render: (index: ElasticsearchViewIndex) => + isCrawlerIndex(index) ? ( + + {crawlerStatusToText(index.crawler?.most_recent_crawl_request_status)} + + ) : ( + + {ingestionStatusToText(index.ingestionStatus)} + + ), truncateText: true, }, diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/crawler_status_helpers.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/crawler_status_helpers.ts new file mode 100644 index 0000000000000..666f45a355919 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/crawler_status_helpers.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +import { CrawlerStatus } from '../api/crawler/types'; +import { + crawlStatusColors, + readableCrawlerStatuses, +} from '../components/search_index/crawler/crawl_requests_panel/constants'; + +export function crawlerStatusToText(crawlerStatus?: CrawlerStatus): string { + return crawlerStatus + ? readableCrawlerStatuses[crawlerStatus] + : i18n.translate('xpack.enterpriseSearch.content.searchIndices.ingestionStatus.idle.label', { + defaultMessage: 'Idle', + }); +} + +export function crawlerStatusToColor( + crawlerStatus?: CrawlerStatus +): 'default' | 'danger' | 'success' { + return crawlerStatus ? crawlStatusColors[crawlerStatus] : 'default'; +} diff --git a/x-pack/plugins/enterprise_search/server/lib/connectors/fetch_connectors.ts b/x-pack/plugins/enterprise_search/server/lib/connectors/fetch_connectors.ts index 5ab0828bce7f2..4ef696dc3cd46 100644 --- a/x-pack/plugins/enterprise_search/server/lib/connectors/fetch_connectors.ts +++ b/x-pack/plugins/enterprise_search/server/lib/connectors/fetch_connectors.ts @@ -10,10 +10,10 @@ import { IScopedClusterClient } from '@kbn/core/server'; import { CONNECTORS_INDEX } from '../..'; import { Connector, ConnectorDocument } from '../../../common/types/connectors'; -import { isNotNullish } from '../../../common/utils/is_not_nullish'; import { setupConnectorsIndices } from '../../index_management/setup_indices'; import { isIndexNotFoundException } from '../../utils/identify_exceptions'; +import { fetchAll } from '../fetch_all'; export const fetchConnectorById = async ( client: IScopedClusterClient, @@ -63,31 +63,12 @@ export const fetchConnectors = async ( client: IScopedClusterClient, indexNames?: string[] ): Promise => { + const query: QueryDslQueryContainer = indexNames + ? { terms: { index_name: indexNames } } + : { match_all: {} }; + try { - const connectorResult = await client.asCurrentUser.search({ - from: 0, - index: CONNECTORS_INDEX, - query: { match_all: {} }, - size: 1000, - }); - let connectors = connectorResult.hits.hits; - let length = connectors.length; - const query: QueryDslQueryContainer = indexNames - ? { terms: { index_name: indexNames } } - : { match_all: {} }; - while (length >= 1000) { - const newConnectorResult = await client.asCurrentUser.search({ - from: 0, - index: CONNECTORS_INDEX, - query, - size: 1000, - }); - connectors = connectors.concat(newConnectorResult.hits.hits); - length = newConnectorResult.hits.hits.length; - } - return connectors - .map(({ _source, _id }) => (_source ? { ..._source, id: _id } : undefined)) - .filter(isNotNullish); + return await fetchAll(client, CONNECTORS_INDEX, query); } catch (error) { if (isIndexNotFoundException(error)) { await setupConnectorsIndices(client.asCurrentUser); diff --git a/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawlers.ts b/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawlers.ts new file mode 100644 index 0000000000000..c0d3c7344d1a8 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawlers.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { IScopedClusterClient } from '@kbn/core/server'; + +import { Crawler, CrawlRequest } from '../../../common/types/crawler'; +import { fetchAll } from '../fetch_all'; + +const CRAWLER_CONFIGURATIONS_INDEX = '.ent-search-actastic-crawler2_configurations'; +const CRAWLER_CRAWL_REQUESTS_INDEX = '.ent-search-actastic-crawler2_crawl_requests'; + +export const fetchMostRecentCrawlerRequestByConfigurationId = async ( + client: IScopedClusterClient, + configurationId: string +): Promise => { + try { + const crawlRequestResult = await client.asCurrentUser.search({ + index: CRAWLER_CRAWL_REQUESTS_INDEX, + query: { term: { configuration_oid: configurationId } }, + sort: 'created_at:desc', + }); + const result = crawlRequestResult.hits.hits[0]?._source; + + return result; + } catch (error) { + return undefined; + } +}; + +export const fetchCrawlerByIndexName = async ( + client: IScopedClusterClient, + indexName: string +): Promise => { + let crawler: Crawler | undefined; + try { + const crawlerResult = await client.asCurrentUser.search({ + index: CRAWLER_CONFIGURATIONS_INDEX, + query: { term: { index_name: indexName } }, + }); + crawler = crawlerResult.hits.hits[0]?._source; + } catch (error) { + return undefined; + } + + if (crawler) { + try { + const mostRecentCrawlRequest = await fetchMostRecentCrawlerRequestByConfigurationId( + client, + crawler.id + ); + + return { + ...crawler, + most_recent_crawl_request_status: mostRecentCrawlRequest?.status, + }; + } catch (error) { + return crawler; + } + } + + return undefined; +}; + +export const fetchCrawlers = async ( + client: IScopedClusterClient, + indexNames?: string[] +): Promise => { + const query: QueryDslQueryContainer = indexNames + ? { terms: { index_name: indexNames } } + : { match_all: {} }; + let crawlers: Crawler[]; + try { + crawlers = await fetchAll(client, CRAWLER_CONFIGURATIONS_INDEX, query); + } catch (error) { + return []; + } + + try { + // TODO replace this with an aggregation query + const crawlersWithStatuses = await Promise.all( + crawlers.map(async (crawler): Promise => { + const mostRecentCrawlRequest = await fetchMostRecentCrawlerRequestByConfigurationId( + client, + crawler.id + ); + + return { + ...crawler, + most_recent_crawl_request_status: mostRecentCrawlRequest?.status, + }; + }) + ); + return crawlersWithStatuses; + } catch (error) { + return crawlers; + } +}; diff --git a/x-pack/plugins/enterprise_search/server/lib/fetch_all.ts b/x-pack/plugins/enterprise_search/server/lib/fetch_all.ts new file mode 100644 index 0000000000000..b000ee731ff5c --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/lib/fetch_all.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isNotNullish } from '@opentelemetry/sdk-metrics-base/build/src/utils'; + +import { QueryDslQueryContainer, SearchHit } from '@elastic/elasticsearch/lib/api/types'; +import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; + +// TODO add safety to prevent an OOM error if the query results are too enough + +export const fetchAll = async ( + client: IScopedClusterClient, + index: string, + query: QueryDslQueryContainer +): Promise => { + let hits: Array> = []; + let accumulator: Array> = []; + + do { + const connectorResult = await client.asCurrentUser.search({ + from: accumulator.length, + index, + query, + size: 1000, + }); + hits = connectorResult.hits.hits; + accumulator = accumulator.concat(hits); + } while (hits.length >= 1000); + + return accumulator + .map(({ _source, _id }) => (_source ? { ..._source, id: _id } : undefined)) + .filter(isNotNullish); +}; diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.test.ts index 0cde2901736c7..1de3b0372a47a 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.test.ts @@ -9,6 +9,7 @@ import { ByteSizeValue } from '@kbn/config-schema'; import { IScopedClusterClient } from '@kbn/core/server'; import { fetchConnectorByIndexName } from '../connectors/fetch_connectors'; +import { fetchCrawlerByIndexName } from '../crawler/fetch_crawlers'; import { fetchIndex } from './fetch_index'; @@ -16,6 +17,10 @@ jest.mock('../connectors/fetch_connectors', () => ({ fetchConnectorByIndexName: jest.fn(), })); +jest.mock('../crawler/fetch_crawlers', () => ({ + fetchCrawlerByIndexName: jest.fn(), +})); + describe('fetchIndex lib function', () => { const mockClient = { asCurrentUser: { @@ -76,9 +81,7 @@ describe('fetchIndex lib function', () => { index_name: { aliases: [], data: 'full index' }, }) ); - mockClient.asCurrentUser.search.mockImplementation(() => - Promise.resolve({ hits: { hits: [] } }) - ); + (fetchCrawlerByIndexName as jest.Mock).mockImplementationOnce(() => Promise.resolve(undefined)); (fetchConnectorByIndexName as jest.Mock).mockImplementationOnce(() => Promise.resolve(undefined) ); @@ -95,9 +98,6 @@ describe('fetchIndex lib function', () => { index_name: { aliases: [], data: 'full index' }, }) ); - mockClient.asCurrentUser.search.mockImplementation(() => - Promise.resolve({ hits: { hits: [] } }) - ); (fetchConnectorByIndexName as jest.Mock).mockImplementationOnce(() => Promise.resolve({ doc: 'doc' }) ); @@ -114,17 +114,24 @@ describe('fetchIndex lib function', () => { index_name: { aliases: [], data: 'full index' }, }) ); + (fetchCrawlerByIndexName as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + id: '1234', + }) + ); (fetchConnectorByIndexName as jest.Mock).mockImplementationOnce(() => Promise.resolve(undefined) ); - mockClient.asCurrentUser.search.mockImplementation(() => ({ - hits: { hits: [{ _source: 'source' }] }, - })); mockClient.asCurrentUser.indices.stats.mockImplementation(() => Promise.resolve(statsResponse)); await expect( fetchIndex(mockClient as unknown as IScopedClusterClient, 'index_name') - ).resolves.toEqual({ ...result, crawler: 'source' }); + ).resolves.toEqual({ + ...result, + crawler: { + id: '1234', + }, + }); }); it('should throw a 404 error if the index cannot be fonud', async () => { mockClient.asCurrentUser.indices.get.mockImplementation(() => Promise.resolve({})); diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.ts b/x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.ts index 182780b572956..b4845cb72391a 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/fetch_index.ts @@ -7,10 +7,9 @@ import { IScopedClusterClient } from '@kbn/core/server'; -import { Crawler } from '../../../common/types/crawler'; import { ElasticsearchIndexWithIngestion } from '../../../common/types/indices'; - import { fetchConnectorByIndexName } from '../connectors/fetch_connectors'; +import { fetchCrawlerByIndexName } from '../crawler/fetch_crawlers'; import { mapIndexStats } from './fetch_indices'; @@ -35,12 +34,7 @@ export const fetchIndex = async ( }; } - const crawlerResult = await client.asCurrentUser.search({ - index: '.ent-search-actastic-crawler2_configurations', - query: { term: { index_name: index } }, - }); - const crawler = crawlerResult.hits.hits[0]?._source; - + const crawler = await fetchCrawlerByIndexName(client, index); if (crawler) { return { ...indexResult, crawler }; } diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts index b673e93d7524b..c6cdf47f50f0b 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts @@ -10,6 +10,7 @@ import { schema } from '@kbn/config-schema'; import { ErrorCode } from '../../../common/types/error_codes'; import { fetchConnectors } from '../../lib/connectors/fetch_connectors'; +import { fetchCrawlers } from '../../lib/crawler/fetch_crawlers'; import { createApiIndex } from '../../lib/indices/create_index'; import { fetchIndex } from '../../lib/indices/fetch_index'; @@ -68,9 +69,11 @@ export function registerIndexRoutes({ router }: RouteDependencies) { const selectedIndices = totalIndices.slice(startIndex, endIndex); const indexNames = selectedIndices.map(({ name }) => name); const connectors = await fetchConnectors(client, indexNames); + const crawlers = await fetchCrawlers(client, indexNames); const indices = selectedIndices.map((index) => ({ ...index, connector: connectors.find((connector) => connector.index_name === index.name), + crawler: crawlers.find((crawler) => crawler.index_name === index.name), })); return response.ok({ body: {