-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Enterprise Search] Display most recent crawl request status in Indic…
…es and Crawl Request tables (#137128)
- Loading branch information
Byron Hulcher
authored
Jul 26, 2022
1 parent
d96fbc4
commit bb0365e
Showing
12 changed files
with
249 additions
and
66 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
28 changes: 28 additions & 0 deletions
28
...rise_search/public/applications/enterprise_search_content/utils/crawler_status_helpers.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0; you may not use this file except in compliance with the Elastic License | ||
| * 2.0. | ||
| */ | ||
|
|
||
| import { i18n } from '@kbn/i18n'; | ||
|
|
||
| import { CrawlerStatus } from '../api/crawler/types'; | ||
| import { | ||
| crawlStatusColors, | ||
| readableCrawlerStatuses, | ||
| } from '../components/search_index/crawler/crawl_requests_panel/constants'; | ||
|
|
||
| export function crawlerStatusToText(crawlerStatus?: CrawlerStatus): string { | ||
| return crawlerStatus | ||
| ? readableCrawlerStatuses[crawlerStatus] | ||
| : i18n.translate('xpack.enterpriseSearch.content.searchIndices.ingestionStatus.idle.label', { | ||
| defaultMessage: 'Idle', | ||
| }); | ||
| } | ||
|
|
||
| export function crawlerStatusToColor( | ||
| crawlerStatus?: CrawlerStatus | ||
| ): 'default' | 'danger' | 'success' { | ||
| return crawlerStatus ? crawlStatusColors[crawlerStatus] : 'default'; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
102 changes: 102 additions & 0 deletions
102
x-pack/plugins/enterprise_search/server/lib/crawler/fetch_crawlers.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0; you may not use this file except in compliance with the Elastic License | ||
| * 2.0. | ||
| */ | ||
|
|
||
| import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; | ||
| import { IScopedClusterClient } from '@kbn/core/server'; | ||
|
|
||
| import { Crawler, CrawlRequest } from '../../../common/types/crawler'; | ||
| import { fetchAll } from '../fetch_all'; | ||
|
|
||
| const CRAWLER_CONFIGURATIONS_INDEX = '.ent-search-actastic-crawler2_configurations'; | ||
| const CRAWLER_CRAWL_REQUESTS_INDEX = '.ent-search-actastic-crawler2_crawl_requests'; | ||
|
|
||
| export const fetchMostRecentCrawlerRequestByConfigurationId = async ( | ||
| client: IScopedClusterClient, | ||
| configurationId: string | ||
| ): Promise<CrawlRequest | undefined> => { | ||
| try { | ||
| const crawlRequestResult = await client.asCurrentUser.search<CrawlRequest>({ | ||
| index: CRAWLER_CRAWL_REQUESTS_INDEX, | ||
| query: { term: { configuration_oid: configurationId } }, | ||
| sort: 'created_at:desc', | ||
| }); | ||
| const result = crawlRequestResult.hits.hits[0]?._source; | ||
|
|
||
| return result; | ||
| } catch (error) { | ||
| return undefined; | ||
| } | ||
| }; | ||
|
|
||
| export const fetchCrawlerByIndexName = async ( | ||
| client: IScopedClusterClient, | ||
| indexName: string | ||
| ): Promise<Crawler | undefined> => { | ||
| let crawler: Crawler | undefined; | ||
| try { | ||
| const crawlerResult = await client.asCurrentUser.search<Crawler>({ | ||
| index: CRAWLER_CONFIGURATIONS_INDEX, | ||
| query: { term: { index_name: indexName } }, | ||
| }); | ||
| crawler = crawlerResult.hits.hits[0]?._source; | ||
| } catch (error) { | ||
| return undefined; | ||
| } | ||
|
|
||
| if (crawler) { | ||
| try { | ||
| const mostRecentCrawlRequest = await fetchMostRecentCrawlerRequestByConfigurationId( | ||
| client, | ||
| crawler.id | ||
| ); | ||
|
|
||
| return { | ||
| ...crawler, | ||
| most_recent_crawl_request_status: mostRecentCrawlRequest?.status, | ||
| }; | ||
| } catch (error) { | ||
| return crawler; | ||
| } | ||
| } | ||
|
|
||
| return undefined; | ||
| }; | ||
|
|
||
| export const fetchCrawlers = async ( | ||
| client: IScopedClusterClient, | ||
| indexNames?: string[] | ||
| ): Promise<Crawler[]> => { | ||
| const query: QueryDslQueryContainer = indexNames | ||
| ? { terms: { index_name: indexNames } } | ||
| : { match_all: {} }; | ||
| let crawlers: Crawler[]; | ||
| try { | ||
| crawlers = await fetchAll<Crawler>(client, CRAWLER_CONFIGURATIONS_INDEX, query); | ||
| } catch (error) { | ||
| return []; | ||
| } | ||
|
|
||
| try { | ||
| // TODO replace this with an aggregation query | ||
| const crawlersWithStatuses = await Promise.all( | ||
| crawlers.map(async (crawler): Promise<Crawler> => { | ||
| const mostRecentCrawlRequest = await fetchMostRecentCrawlerRequestByConfigurationId( | ||
| client, | ||
| crawler.id | ||
| ); | ||
|
|
||
| return { | ||
| ...crawler, | ||
| most_recent_crawl_request_status: mostRecentCrawlRequest?.status, | ||
| }; | ||
| }) | ||
| ); | ||
| return crawlersWithStatuses; | ||
| } catch (error) { | ||
| return crawlers; | ||
| } | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0; you may not use this file except in compliance with the Elastic License | ||
| * 2.0. | ||
| */ | ||
|
|
||
| import { isNotNullish } from '@opentelemetry/sdk-metrics-base/build/src/utils'; | ||
|
|
||
| import { QueryDslQueryContainer, SearchHit } from '@elastic/elasticsearch/lib/api/types'; | ||
| import { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; | ||
|
|
||
| // TODO add safety to prevent an OOM error if the query results are too enough | ||
|
|
||
| export const fetchAll = async <T>( | ||
| client: IScopedClusterClient, | ||
| index: string, | ||
| query: QueryDslQueryContainer | ||
| ): Promise<T[]> => { | ||
| let hits: Array<SearchHit<T>> = []; | ||
| let accumulator: Array<SearchHit<T>> = []; | ||
|
|
||
| do { | ||
| const connectorResult = await client.asCurrentUser.search<T>({ | ||
| from: accumulator.length, | ||
| index, | ||
| query, | ||
| size: 1000, | ||
| }); | ||
| hits = connectorResult.hits.hits; | ||
| accumulator = accumulator.concat(hits); | ||
| } while (hits.length >= 1000); | ||
|
|
||
| return accumulator | ||
| .map(({ _source, _id }) => (_source ? { ..._source, id: _id } : undefined)) | ||
| .filter(isNotNullish); | ||
| }; |
Oops, something went wrong.