diff --git a/x-pack/plugins/aiops/common/api/log_categorization/create_categorize_query.ts b/x-pack/plugins/aiops/common/api/log_categorization/create_categorize_query.ts new file mode 100644 index 0000000000000..8b9e3a95c653f --- /dev/null +++ b/x-pack/plugins/aiops/common/api/log_categorization/create_categorize_query.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { cloneDeep } from 'lodash'; + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; + +export function createCategorizeQuery( + queryIn: QueryDslQueryContainer, + timeField: string, + from: number | undefined, + to: number | undefined +) { + const query = cloneDeep(queryIn); + + if (query.bool === undefined) { + query.bool = {}; + } + if (query.bool.must === undefined) { + query.bool.must = []; + if (query.match_all !== undefined) { + query.bool.must.push({ match_all: query.match_all }); + delete query.match_all; + } + } + if (query.multi_match !== undefined) { + query.bool.should = { + multi_match: query.multi_match, + }; + delete query.multi_match; + } + + (query.bool.must as QueryDslQueryContainer[]).push({ + range: { + [timeField]: { + gte: from, + lte: to, + format: 'epoch_millis', + }, + }, + }); + + return query; +} diff --git a/x-pack/plugins/aiops/common/api/log_categorization/create_category_request.ts b/x-pack/plugins/aiops/common/api/log_categorization/create_category_request.ts new file mode 100644 index 0000000000000..38acb5029e830 --- /dev/null +++ b/x-pack/plugins/aiops/common/api/log_categorization/create_category_request.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; + +import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; + +import { createCategorizeQuery } from './create_categorize_query'; + +const CATEGORY_LIMIT = 1000; +const EXAMPLE_LIMIT = 1; + +export function createCategoryRequest( + index: string, + field: string, + timeField: string, + from: number | undefined, + to: number | undefined, + queryIn: QueryDslQueryContainer, + wrap: ReturnType['wrap'], + intervalMs?: number +) { + const query = createCategorizeQuery(queryIn, timeField, from, to); + const aggs = { + categories: { + categorize_text: { + field, + size: CATEGORY_LIMIT, + }, + aggs: { + hit: { + top_hits: { + size: EXAMPLE_LIMIT, + sort: [timeField], + _source: field, + }, + }, + ...(intervalMs + ? { + sparkline: { + date_histogram: { + field: timeField, + fixed_interval: `${intervalMs}ms`, + }, + }, + } + : {}), + }, + }, + }; + + return { + params: { + index, + size: 0, + body: { + query, + aggs: wrap(aggs), + }, + }, + }; +} diff --git a/x-pack/plugins/aiops/common/api/log_categorization/get_category_query.ts b/x-pack/plugins/aiops/common/api/log_categorization/get_category_query.ts new file mode 100644 index 0000000000000..ba6e2886f6a9e --- /dev/null +++ b/x-pack/plugins/aiops/common/api/log_categorization/get_category_query.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Category } from './types'; + +export const QUERY_MODE = { + INCLUDE: 'should', + EXCLUDE: 'must_not', +} as const; +export type QueryMode = typeof QUERY_MODE[keyof typeof QUERY_MODE]; + +export const getCategoryQuery = ( + field: string, + categories: Category[], + mode: QueryMode = QUERY_MODE.INCLUDE +) => ({ + bool: { + [mode]: categories.map(({ key: query }) => ({ + match: { + [field]: { + auto_generate_synonyms_phrase_query: false, + fuzziness: 0, + operator: 'and', + query, + }, + }, + })), + }, +}); diff --git a/x-pack/plugins/aiops/common/api/log_categorization/process_category_results.ts b/x-pack/plugins/aiops/common/api/log_categorization/process_category_results.ts new file mode 100644 index 0000000000000..347ba5711719d --- /dev/null +++ b/x-pack/plugins/aiops/common/api/log_categorization/process_category_results.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get } from 'lodash'; + +import { estypes } from '@elastic/elasticsearch'; + +import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; + +import type { Category, CategoriesAgg, CatResponse, SparkLinesPerCategory } from './types'; + +export function processCategoryResults( + result: CatResponse, + field: string, + unwrap: ReturnType['unwrap'] +) { + const sparkLinesPerCategory: SparkLinesPerCategory = {}; + const { aggregations } = result.rawResponse; + if (aggregations === undefined) { + throw new Error('processCategoryResults failed, did not return aggregations.'); + } + const { + categories: { buckets }, + } = unwrap( + aggregations as unknown as Record + ) as CategoriesAgg; + + const categories: Category[] = buckets.map((b) => { + sparkLinesPerCategory[b.key] = + b.sparkline === undefined + ? {} + : b.sparkline.buckets.reduce>((acc2, cur2) => { + acc2[cur2.key] = cur2.doc_count; + return acc2; + }, {}); + + return { + key: b.key, + count: b.doc_count, + examples: b.hit.hits.hits.map((h) => get(h._source, field)), + }; + }); + return { + categories, + sparkLinesPerCategory, + }; +} diff --git a/x-pack/plugins/aiops/common/api/log_categorization/types.ts b/x-pack/plugins/aiops/common/api/log_categorization/types.ts new file mode 100644 index 0000000000000..83e16d8ada7d0 --- /dev/null +++ b/x-pack/plugins/aiops/common/api/log_categorization/types.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; + +export interface Category { + key: string; + count: number; + examples: string[]; + sparkline?: Array<{ doc_count: number; key: number; key_as_string: string }>; +} + +export interface CategoriesAgg { + categories: { + buckets: Array<{ + key: string; + doc_count: number; + hit: { hits: { hits: Array<{ _source: { message: string } }> } }; + sparkline: { + buckets: Array<{ key_as_string: string; key: number; doc_count: number }>; + }; + }>; + }; +} + +interface CategoriesSampleAgg { + sample: CategoriesAgg; +} + +export interface CatResponse { + rawResponse: estypes.SearchResponseBody; +} + +export type SparkLinesPerCategory = Record>; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx b/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx index 1dc30c04253bc..346fc7f5b3562 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx @@ -6,8 +6,7 @@ */ import React, { FC, useMemo, useState } from 'react'; -import { i18n } from '@kbn/i18n'; -import type { TimefilterContract } from '@kbn/data-plugin/public'; + import { useEuiBackgroundColor, EuiInMemoryTable, @@ -19,14 +18,25 @@ import { EuiSpacer, } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import type { TimefilterContract } from '@kbn/data-plugin/public'; import { DataViewField } from '@kbn/data-views-plugin/common'; import { Filter } from '@kbn/es-query'; import { useTableState } from '@kbn/ml-in-memory-table'; -import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links'; -import { MiniHistogram } from '../../mini_histogram'; + +import type { + Category, + SparkLinesPerCategory, +} from '../../../../common/api/log_categorization/types'; + import { useEuiTheme } from '../../../hooks/use_eui_theme'; import type { LogCategorizationAppState } from '../../../application/utils/url_state'; -import type { EventRate, Category, SparkLinesPerCategory } from '../use_categorize_request'; + +import { MiniHistogram } from '../../mini_histogram'; + +import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links'; +import type { EventRate } from '../use_categorize_request'; + import { getLabels } from './labels'; import { TableHeader } from './table_header'; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx b/x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx index d34861b924ccb..859eaed0fec57 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx @@ -9,11 +9,15 @@ import React, { FC, useMemo } from 'react'; import { i18n } from '@kbn/i18n'; import { DocumentCountChart as DocumentCountChartRoot } from '@kbn/aiops-components'; + +import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types'; + import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; +import { DocumentCountStats } from '../../get_document_stats'; + import { TotalCountHeader } from '../document_count_content/total_count_header'; -import type { Category, SparkLinesPerCategory } from './use_categorize_request'; + import type { EventRate } from './use_categorize_request'; -import { DocumentCountStats } from '../../get_document_stats'; interface Props { totalCount: number; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx index 599a669197ee5..28c1350a88141 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx @@ -5,10 +5,7 @@ * 2.0. */ import React, { FC, useState, useEffect, useCallback, useRef, useMemo } from 'react'; -import type { SavedSearch } from '@kbn/saved-search-plugin/public'; -import type { DataView, DataViewField } from '@kbn/data-views-plugin/public'; -import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n-react'; + import { EuiTitle, EuiFlyoutHeader, @@ -18,26 +15,33 @@ import { useEuiTheme, } from '@elastic/eui'; +import type { SavedSearch } from '@kbn/saved-search-plugin/public'; +import type { DataView, DataViewField } from '@kbn/data-views-plugin/public'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; import { buildEmptyFilter, Filter } from '@kbn/es-query'; - import { usePageUrlState } from '@kbn/ml-url-state'; import type { FieldValidationResults } from '@kbn/ml-category-validator'; + +import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types'; + +import { + type LogCategorizationPageUrlState, + getDefaultLogCategorizationAppState, +} from '../../application/utils/url_state'; +import { createMergedEsQuery } from '../../application/utils/search_utils'; import { useData } from '../../hooks/use_data'; import { useSearch } from '../../hooks/use_search'; +import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; + import { useCategorizeRequest } from './use_categorize_request'; -import type { EventRate, Category, SparkLinesPerCategory } from './use_categorize_request'; +import type { EventRate } from './use_categorize_request'; import { CategoryTable } from './category_table'; -import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; import { InformationText } from './information_text'; -import { createMergedEsQuery } from '../../application/utils/search_utils'; import { SamplingMenu } from './sampling_menu'; import { TechnicalPreviewBadge } from './technical_preview_badge'; import { LoadingCategorization } from './loading_categorization'; import { useValidateFieldRequest } from './use_validate_category_field'; -import { - type LogCategorizationPageUrlState, - getDefaultLogCategorizationAppState, -} from '../../application/utils/url_state'; import { FieldValidationCallout } from './category_validation_callout'; export interface LogCategorizationPageProps { diff --git a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx index 9539ff607e05e..5ccdca64d1036 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx +++ b/x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx @@ -24,9 +24,11 @@ import { Filter, Query } from '@kbn/es-query'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { usePageUrlState, useUrlState } from '@kbn/ml-url-state'; - import type { FieldValidationResults } from '@kbn/ml-category-validator'; import type { SearchQueryLanguage } from '@kbn/ml-query-utils'; + +import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types'; + import { useDataSource } from '../../hooks/use_data_source'; import { useData } from '../../hooks/use_data'; import { useSearch } from '../../hooks/use_search'; @@ -39,7 +41,7 @@ import { import { SearchPanel } from '../search_panel'; import { PageHeader } from '../page_header'; -import type { EventRate, Category, SparkLinesPerCategory } from './use_categorize_request'; +import type { EventRate } from './use_categorize_request'; import { useCategorizeRequest } from './use_categorize_request'; import { CategoryTable } from './category_table'; import { DocumentCountChart } from './document_count_chart'; diff --git a/x-pack/plugins/aiops/public/components/log_categorization/use_categorize_request.ts b/x-pack/plugins/aiops/public/components/log_categorization/use_categorize_request.ts index 3108ec9391cef..8179751266e6e 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/use_categorize_request.ts +++ b/x-pack/plugins/aiops/public/components/log_categorization/use_categorize_request.ts @@ -5,62 +5,37 @@ * 2.0. */ -import { cloneDeep, get } from 'lodash'; import { useRef, useCallback, useMemo } from 'react'; -import { isCompleteResponse } from '@kbn/data-plugin/public'; + import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; -import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils'; -import { estypes } from '@elastic/elasticsearch'; +import { isCompleteResponse } from '@kbn/data-plugin/public'; import { useStorage } from '@kbn/ml-local-storage'; + +import { createCategoryRequest } from '../../../common/api/log_categorization/create_category_request'; +import { processCategoryResults } from '../../../common/api/log_categorization/process_category_results'; +import type { + Category, + CatResponse, + SparkLinesPerCategory, +} from '../../../common/api/log_categorization/types'; + import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; -import { RandomSampler } from './sampling_menu'; import { type AiOpsKey, type AiOpsStorageMapped, AIOPS_RANDOM_SAMPLING_MODE_PREFERENCE, AIOPS_RANDOM_SAMPLING_PROBABILITY_PREFERENCE, } from '../../types/storage'; -import { RANDOM_SAMPLER_OPTION, DEFAULT_PROBABILITY } from './sampling_menu/random_sampler'; -const CATEGORY_LIMIT = 1000; -const EXAMPLE_LIMIT = 1; - -interface CategoriesAgg { - categories: { - buckets: Array<{ - key: string; - doc_count: number; - hit: { hits: { hits: Array<{ _source: { message: string } }> } }; - sparkline: { - buckets: Array<{ key_as_string: string; key: number; doc_count: number }>; - }; - }>; - }; -} - -interface CategoriesSampleAgg { - sample: CategoriesAgg; -} - -interface CatResponse { - rawResponse: estypes.SearchResponseBody; -} - -export interface Category { - key: string; - count: number; - examples: string[]; - sparkline?: Array<{ doc_count: number; key: number; key_as_string: string }>; -} +import { RandomSampler } from './sampling_menu'; +import { RANDOM_SAMPLER_OPTION, DEFAULT_PROBABILITY } from './sampling_menu/random_sampler'; export type EventRate = Array<{ key: number; docCount: number; }>; -export type SparkLinesPerCategory = Record>; - export function useCategorizeRequest() { const [randomSamplerMode, setRandomSamplerMode] = useStorage< AiOpsKey, @@ -135,129 +110,3 @@ export function useCategorizeRequest() { return { runCategorizeRequest, cancelRequest, randomSampler }; } - -function createCategoryRequest( - index: string, - field: string, - timeField: string, - from: number | undefined, - to: number | undefined, - queryIn: QueryDslQueryContainer, - wrap: ReturnType['wrap'], - intervalMs?: number -) { - const query = createCategorizeQuery(queryIn, timeField, from, to); - const aggs = { - categories: { - categorize_text: { - field, - size: CATEGORY_LIMIT, - }, - aggs: { - hit: { - top_hits: { - size: EXAMPLE_LIMIT, - sort: [timeField], - _source: field, - }, - }, - ...(intervalMs - ? { - sparkline: { - date_histogram: { - field: timeField, - fixed_interval: `${intervalMs}ms`, - }, - }, - } - : {}), - }, - }, - }; - - return { - params: { - index, - size: 0, - body: { - query, - aggs: wrap(aggs), - }, - }, - }; -} - -export function createCategorizeQuery( - queryIn: QueryDslQueryContainer, - timeField: string, - from: number | undefined, - to: number | undefined -) { - const query = cloneDeep(queryIn); - - if (query.bool === undefined) { - query.bool = {}; - } - if (query.bool.must === undefined) { - query.bool.must = []; - if (query.match_all !== undefined) { - query.bool.must.push({ match_all: query.match_all }); - delete query.match_all; - } - } - if (query.multi_match !== undefined) { - query.bool.should = { - multi_match: query.multi_match, - }; - delete query.multi_match; - } - - (query.bool.must as QueryDslQueryContainer[]).push({ - range: { - [timeField]: { - gte: from, - lte: to, - format: 'epoch_millis', - }, - }, - }); - - return query; -} - -function processCategoryResults( - result: CatResponse, - field: string, - unwrap: ReturnType['unwrap'] -) { - const sparkLinesPerCategory: SparkLinesPerCategory = {}; - const { aggregations } = result.rawResponse; - if (aggregations === undefined) { - throw new Error('processCategoryResults failed, did not return aggregations.'); - } - const { - categories: { buckets }, - } = unwrap( - aggregations as unknown as Record - ) as CategoriesAgg; - - const categories: Category[] = buckets.map((b) => { - sparkLinesPerCategory[b.key] = - b.sparkline === undefined - ? {} - : b.sparkline.buckets.reduce>((acc2, cur2) => { - acc2[cur2.key] = cur2.doc_count; - return acc2; - }, {}); - - return { - key: b.key, - count: b.doc_count, - examples: b.hit.hits.hits.map((h) => get(h._source, field)), - }; - }); - return { - categories, - sparkLinesPerCategory, - }; -} diff --git a/x-pack/plugins/aiops/public/components/log_categorization/use_discover_links.ts b/x-pack/plugins/aiops/public/components/log_categorization/use_discover_links.ts index cb3b60268f438..22e8e50ebcf19 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/use_discover_links.ts +++ b/x-pack/plugins/aiops/public/components/log_categorization/use_discover_links.ts @@ -5,15 +5,18 @@ * 2.0. */ -import rison from '@kbn/rison'; import moment from 'moment'; +import rison from '@kbn/rison'; import type { TimeRangeBounds } from '@kbn/data-plugin/common'; import { i18n } from '@kbn/i18n'; import type { Filter } from '@kbn/es-query'; + +import { getCategoryQuery } from '../../../common/api/log_categorization/get_category_query'; +import type { Category } from '../../../common/api/log_categorization/types'; + import type { AiOpsIndexBasedAppState } from '../../application/utils/url_state'; import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; -import type { Category } from './use_categorize_request'; export const QUERY_MODE = { INCLUDE: 'should', @@ -71,20 +74,7 @@ export function createFilter( ): Filter { const selectedRows = category === undefined ? selection : [category]; return { - query: { - bool: { - [mode]: selectedRows.map(({ key: query }) => ({ - match: { - [field]: { - auto_generate_synonyms_phrase_query: false, - fuzziness: 0, - operator: 'and', - query, - }, - }, - })), - }, - }, + query: getCategoryQuery(field, selectedRows, mode), meta: { alias: i18n.translate('xpack.aiops.logCategorization.filterAliasLabel', { defaultMessage: 'Categorization - {field}', diff --git a/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts b/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts index d354ed5f3fbb6..8e0850aa1daa8 100644 --- a/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts +++ b/x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts @@ -8,10 +8,13 @@ import { useRef, useCallback } from 'react'; import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; + import type { FieldValidationResults } from '@kbn/ml-category-validator'; + import { AIOPS_API_ENDPOINT } from '../../../common/api'; +import { createCategorizeQuery } from '../../../common/api/log_categorization/create_categorize_query'; + import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; -import { createCategorizeQuery } from './use_categorize_request'; export function useValidateFieldRequest() { const { http } = useAiopsAppContext();