From 27c0182b49c813518790b5d138f8c453fe82e244 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Mon, 30 Sep 2024 08:59:43 -0400 Subject: [PATCH 01/19] add entities route --- .../src/rest_specs/entity.ts | 58 +++++ .../src/rest_specs/get_entities.ts | 34 +++ .../src/rest_specs/index.ts | 4 + .../public/hooks/query_key_factory.ts | 10 + .../details => }/hooks/use_fetch_alert.tsx | 2 +- .../public/hooks/use_fetch_entities.ts | 77 ++++++ .../alert_details_button.tsx | 2 +- .../clients/create_entities_es_client.ts | 144 +++++++++++ .../server/lib/get_sample_documents.ts | 238 ++++++++++++++++++ .../server/lib/queries/index.ts | 41 +++ ...investigate_app_server_route_repository.ts | 38 +++ .../server/services/get_entities.ts | 219 ++++++++++++++++ .../server/services/get_events.ts | 20 +- 13 files changed, 866 insertions(+), 21 deletions(-) create mode 100644 packages/kbn-investigation-shared/src/rest_specs/entity.ts create mode 100644 packages/kbn-investigation-shared/src/rest_specs/get_entities.ts rename x-pack/plugins/observability_solution/investigate_app/public/{pages/details => }/hooks/use_fetch_alert.tsx (96%) create mode 100644 x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts create mode 100644 x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts create mode 100644 x-pack/plugins/observability_solution/investigate_app/server/lib/get_sample_documents.ts create mode 100644 x-pack/plugins/observability_solution/investigate_app/server/lib/queries/index.ts create mode 100644 x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts diff --git a/packages/kbn-investigation-shared/src/rest_specs/entity.ts b/packages/kbn-investigation-shared/src/rest_specs/entity.ts new file mode 100644 index 0000000000000..f2477b333ec42 --- /dev/null +++ b/packages/kbn-investigation-shared/src/rest_specs/entity.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { z } from '@kbn/zod'; + +const sourceSchema = z.record(z.string(), z.any()); + +const metricsSchema = z.object({ + failedTransactionRate: z.number().optional(), + latency: z.number().optional(), + throughput: z.number().optional(), + logErrorRate: z.number().optional(), + logRate: z.number().optional(), +}); + +const entitySchema = z.object({ + id: z.string(), + definitionId: z.string(), + definitionVersion: z.string(), + displayName: z.string(), + firstSeenTimestamp: z.string(), + lastSeenTimestamp: z.string(), + identityFields: z.array(z.string()), + schemaVersion: z.string(), + type: z.string(), + metrics: metricsSchema, +}); + +const entityDocumentAnalysisSchema = z.object({ + total: z.number(), + sampled: z.number(), + fields: z.array(z.string()), +}); + +const entitySourcesSchema = z.object({ + index: z.string(), + aliases: sourceSchema.optional(), + dataStream: z.string().optional(), + documentAnalysis: entityDocumentAnalysisSchema, +}); + +const entityWithSampleDocumentsSchema = z.intersection( + entitySchema, + z.object({ + sources: z.array(entitySourcesSchema), + }) +); + +type EntityWithSampledDocuments = z.output; + +export { entityWithSampleDocumentsSchema }; +export type { EntityWithSampledDocuments }; diff --git a/packages/kbn-investigation-shared/src/rest_specs/get_entities.ts b/packages/kbn-investigation-shared/src/rest_specs/get_entities.ts new file mode 100644 index 0000000000000..515e0b753d965 --- /dev/null +++ b/packages/kbn-investigation-shared/src/rest_specs/get_entities.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { z } from '@kbn/zod'; +import { entityWithSampleDocumentsSchema } from './entity'; + +const getEntitiesParamsSchema = z + .object({ + query: z + .object({ + 'service.name': z.string(), + 'service.environment': z.string(), + 'host.name': z.string(), + 'container.id': z.string(), + }) + .partial(), + }) + .partial(); + +const getEntitiesResponseSchema = z.object({ + entities: z.array(entityWithSampleDocumentsSchema), +}); + +type GetEntitiesParams = z.infer; +type GetEntitiesResponse = z.output; + +export { getEntitiesParamsSchema, getEntitiesResponseSchema }; +export type { GetEntitiesParams, GetEntitiesResponse }; diff --git a/packages/kbn-investigation-shared/src/rest_specs/index.ts b/packages/kbn-investigation-shared/src/rest_specs/index.ts index 42bec32041af4..d0070c8b8959d 100644 --- a/packages/kbn-investigation-shared/src/rest_specs/index.ts +++ b/packages/kbn-investigation-shared/src/rest_specs/index.ts @@ -27,6 +27,8 @@ export type * from './update_item'; export type * from './update_note'; export type * from './event'; export type * from './get_events'; +export type * from './entity'; +export type * from './get_entities'; export * from './create'; export * from './create_item'; @@ -48,3 +50,5 @@ export * from './update_item'; export * from './update_note'; export * from './event'; export * from './get_events'; +export * from './entity'; +export * from './get_entities'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/hooks/query_key_factory.ts b/x-pack/plugins/observability_solution/investigate_app/public/hooks/query_key_factory.ts index 44352e46997ea..8484bc7d7b156 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/hooks/query_key_factory.ts +++ b/x-pack/plugins/observability_solution/investigate_app/public/hooks/query_key_factory.ts @@ -22,6 +22,16 @@ export const investigationKeys = { [...investigationKeys.detail(investigationId), 'notes'] as const, detailItems: (investigationId: string) => [...investigationKeys.detail(investigationId), 'items'] as const, + entities: ({ + investigationId, + ...params + }: { + investigationId: string; + serviceName?: string; + serviceEnvironment?: string; + hostName?: string; + containerId?: string; + }) => [...investigationKeys.detail(investigationId), 'entities', params] as const, }; export type InvestigationKeys = typeof investigationKeys; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/hooks/use_fetch_alert.tsx b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_alert.tsx similarity index 96% rename from x-pack/plugins/observability_solution/investigate_app/public/pages/details/hooks/use_fetch_alert.tsx rename to x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_alert.tsx index 85246b33bf70d..0c0cda89d3eb8 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/hooks/use_fetch_alert.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_alert.tsx @@ -7,7 +7,7 @@ import { useQuery } from '@tanstack/react-query'; import { BASE_RAC_ALERTS_API_PATH, EcsFieldsResponse } from '@kbn/rule-registry-plugin/common'; -import { useKibana } from '../../../hooks/use_kibana'; +import { useKibana } from './use_kibana'; export interface AlertParams { id?: string; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts new file mode 100644 index 0000000000000..007856109bce0 --- /dev/null +++ b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useQuery } from '@tanstack/react-query'; +import { GetEntitiesResponse } from '@kbn/investigation-shared'; +import { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common'; +import { useKibana } from './use_kibana'; +import { investigationKeys } from './query_key_factory'; + +export interface EntityParams { + investigationId: string; + serviceName?: string; + serviceEnvironment?: string; + hostName?: string; + containerId?: string; +} + +export interface UseFetchAlertResponse { + isInitialLoading: boolean; + isLoading: boolean; + isRefetching: boolean; + isSuccess: boolean; + isError: boolean; + data: EcsFieldsResponse | undefined | null; +} + +export function useFetchEntities({ + investigationId, + serviceName, + serviceEnvironment, + hostName, + containerId, +}: EntityParams): any { + const { + core: { http }, + } = useKibana(); + + const { isInitialLoading, isLoading, isError, isSuccess, isRefetching, data } = useQuery({ + queryKey: investigationKeys.entities({ + investigationId, + serviceName, + serviceEnvironment, + hostName, + containerId, + }), + queryFn: async ({ signal }) => { + return await http.get('/api/observability/investigation/entities', { + query: { + 'service.name': serviceName, + 'service.environment': serviceEnvironment, + 'host.name': hostName, + 'container.id': containerId, + }, + version: '2023-10-31', + signal, + }); + }, + refetchOnWindowFocus: false, + onError: (error: Error) => { + // ignore error + }, + enabled: Boolean(investigationId && (serviceName || hostName || containerId)), + }); + + return { + data, + isInitialLoading, + isLoading, + isRefetching, + isSuccess, + isError, + }; +} diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_header/alert_details_button.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_header/alert_details_button.tsx index ff33ca7949f75..b9c8729e570f2 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_header/alert_details_button.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_header/alert_details_button.tsx @@ -10,8 +10,8 @@ import { alertOriginSchema } from '@kbn/investigation-shared'; import { ALERT_RULE_CATEGORY } from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names'; import React from 'react'; import { useKibana } from '../../../../hooks/use_kibana'; +import { useFetchAlert } from '../../../../hooks/use_fetch_alert'; import { useInvestigation } from '../../contexts/investigation_context'; -import { useFetchAlert } from '../../hooks/use_fetch_alert'; export function AlertDetailsButton() { const { diff --git a/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts b/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts new file mode 100644 index 0000000000000..8d451b9afd96c --- /dev/null +++ b/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; +import type { KibanaRequest } from '@kbn/core/server'; +import { ElasticsearchClient } from '@kbn/core/server'; +import { entitiesAliasPattern, ENTITY_LATEST, ENTITY_HISTORY } from '@kbn/entities-schema'; +import { unwrapEsResponse } from '@kbn/observability-plugin/common/utils/unwrap_es_response'; +import { + MsearchMultisearchBody, + MsearchMultisearchHeader, +} from '@elastic/elasticsearch/lib/api/types'; + +export const SERVICE_ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ + type: 'service', + dataset: ENTITY_LATEST, +}); +export const SERVICE_ENTITIES_HISTORY_ALIAS = entitiesAliasPattern({ + type: 'service', + dataset: ENTITY_HISTORY, +}); +export const HOST_ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ + type: 'host', + dataset: ENTITY_LATEST, +}); +export const HOST_ENTITIES_HISTORY_ALIAS = entitiesAliasPattern({ + type: 'host', + dataset: ENTITY_HISTORY, +}); +export const CONTAINER_ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ + type: 'container', + dataset: ENTITY_LATEST, +}); +export const CONTAINER_ENTITIES_HISTORY_ALIAS = entitiesAliasPattern({ + type: 'container', + dataset: ENTITY_HISTORY, +}); +type LatestAlias = + | typeof SERVICE_ENTITIES_LATEST_ALIAS + | typeof HOST_ENTITIES_LATEST_ALIAS + | typeof CONTAINER_ENTITIES_LATEST_ALIAS; + +type HistoryAlias = + | typeof SERVICE_ENTITIES_HISTORY_ALIAS + | typeof HOST_ENTITIES_HISTORY_ALIAS + | typeof CONTAINER_ENTITIES_HISTORY_ALIAS; + +export function cancelEsRequestOnAbort>( + promise: T, + request: KibanaRequest, + controller: AbortController +): T { + const subscription = request.events.aborted$.subscribe(() => { + controller.abort(); + }); + + return promise.finally(() => subscription.unsubscribe()) as T; +} + +export interface EntitiesESClient { + search( + indexName: string, + searchRequest: TSearchRequest + ): Promise>; + msearch( + allSearches: TSearchRequest[] + ): Promise<{ responses: Array> }>; +} + +export function createEntitiesESClient({ + request, + esClient, +}: { + request: KibanaRequest; + esClient: ElasticsearchClient; +}) { + function search( + indexName: string, + searchRequest: TSearchRequest + ): Promise> { + const controller = new AbortController(); + + const promise = cancelEsRequestOnAbort( + esClient.search( + { ...searchRequest, index: [indexName], ignore_unavailable: true }, + { + signal: controller.signal, + meta: true, + } + ) as unknown as Promise<{ + body: InferSearchResponseOf; + }>, + request, + controller + ); + + return unwrapEsResponse(promise); + } + + return { + async search( + entityIndexAlias: LatestAlias | HistoryAlias, + searchRequest: TSearchRequest + ): Promise> { + return search(entityIndexAlias, searchRequest); + }, + + async msearch( + allSearches: TSearchRequest[] + ): Promise<{ responses: Array> }> { + const searches = allSearches + .map((params) => { + const searchParams: [MsearchMultisearchHeader, MsearchMultisearchBody] = [ + { + index: [SERVICE_ENTITIES_LATEST_ALIAS], + ignore_unavailable: true, + }, + { + ...params.body, + }, + ]; + + return searchParams; + }) + .flat(); + + const promise = esClient.msearch( + { searches }, + { + meta: true, + } + ) as unknown as Promise<{ + body: { responses: Array> }; + }>; + + const { body } = await promise; + return { responses: body.responses }; + }, + }; +} diff --git a/x-pack/plugins/observability_solution/investigate_app/server/lib/get_sample_documents.ts b/x-pack/plugins/observability_solution/investigate_app/server/lib/get_sample_documents.ts new file mode 100644 index 0000000000000..1b56d9f7cbd95 --- /dev/null +++ b/x-pack/plugins/observability_solution/investigate_app/server/lib/get_sample_documents.ts @@ -0,0 +1,238 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { estypes } from '@elastic/elasticsearch'; +import { castArray, sortBy, uniq, partition, shuffle } from 'lodash'; +import { truncateList } from '@kbn/inference-plugin/common/util/truncate_list'; +import { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types'; +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { rangeQuery, excludeFrozenQuery } from './queries'; + +export interface DocumentAnalysis { + total: number; + sampled: number; + fields: Array<{ + name: string; + types: string[]; + cardinality: number | null; + values: Array; + empty: boolean; + }>; +} + +export async function getSampleDocuments({ + esClient, + start, + end, + indexPatterns, + count, + dslFilter, +}: { + esClient: ElasticsearchClient; + start: number; + end: number; + indexPatterns: string[]; + count: number; + dslFilter?: QueryDslQueryContainer[]; +}): Promise<{ + total: number; + samples: Array>; +}> { + return esClient + .search({ + index: indexPatterns, + track_total_hits: true, + size: count, + body: { + query: { + bool: { + should: [ + { + function_score: { + functions: [ + { + random_score: {}, + }, + ], + }, + }, + ], + must: [...rangeQuery(start, end), ...(dslFilter ?? [])], + }, + }, + sort: { + _score: { + order: 'desc', + }, + }, + _source: false, + fields: ['*' as const], + }, + }) + .then((response) => { + const hits = response.hits.total as estypes.SearchTotalHits; + if (hits.value === 0) { + return { + total: 0, + samples: [], + }; + } + return { + total: hits.value, + samples: response.hits.hits.map((hit) => hit.fields ?? {}), + }; + }); +} + +export async function getKeywordAndNumericalFields({ + indexPatterns, + esClient, + start, + end, +}: { + indexPatterns: string[]; + esClient: ElasticsearchClient; + start: number; + end: number; +}): Promise> { + const fieldCaps = await esClient.fieldCaps({ + index: indexPatterns, + fields: '*', + include_empty_fields: false, + types: [ + 'constant_keyword', + 'keyword', + 'integer', + 'long', + 'double', + 'float', + 'byte', + 'boolean', + 'alias', + 'flattened', + 'ip', + 'aggregate_metric_double', + 'histogram', + ], + index_filter: { + bool: { + filter: [...excludeFrozenQuery(), ...rangeQuery(start, end)], + }, + }, + }); + + return Object.entries(fieldCaps.fields).map(([fieldName, fieldSpec]) => { + return { + name: fieldName, + esTypes: Object.keys(fieldSpec), + }; + }); +} + +export function mergeSampleDocumentsWithFieldCaps({ + total, + samples, + fieldCaps, +}: { + total: number; + samples: Array>; + fieldCaps: Array<{ name: string; esTypes?: string[] }>; +}): DocumentAnalysis { + const nonEmptyFields = new Set(); + const fieldValues = new Map>(); + + for (const document of samples) { + Object.keys(document).forEach((field) => { + if (!nonEmptyFields.has(field)) { + nonEmptyFields.add(field); + } + + const values = castArray(document[field]); + + const currentFieldValues = fieldValues.get(field) ?? []; + + values.forEach((value) => { + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + currentFieldValues.push(value); + } + }); + + fieldValues.set(field, currentFieldValues); + }); + } + + const fields = fieldCaps.flatMap((spec) => { + const values = fieldValues.get(spec.name); + + const countByValues = new Map(); + + values?.forEach((value) => { + const currentCount = countByValues.get(value) ?? 0; + countByValues.set(value, currentCount + 1); + }); + + const sortedValues = sortBy( + Array.from(countByValues.entries()).map(([value, count]) => { + return { + value, + count, + }; + }), + 'count', + 'desc' + ); + + return { + name: spec.name, + types: spec.esTypes ?? [], + empty: !nonEmptyFields.has(spec.name), + cardinality: countByValues.size || null, + values: uniq(sortedValues.flatMap(({ value }) => value)), + }; + }); + + return { + total, + sampled: samples.length, + fields, + }; +} + +export function sortAndTruncateAnalyzedFields(analysis: DocumentAnalysis) { + const { fields, ...meta } = analysis; + const [nonEmptyFields, emptyFields] = partition(analysis.fields, (field) => !field.empty); + + const sortedFields = [...shuffle(nonEmptyFields), ...shuffle(emptyFields)]; + + return { + ...meta, + fields: truncateList( + sortedFields.map((field) => { + let name = `${field.name}:${field.types.join(',')}`; + + if (field.empty) { + return `${name} (empty)`; + } + + name += ` - ${field.cardinality} distinct values`; + + if ( + field.values.length && + (field.types.includes('keyword') || field.types.includes('text')) && + field.values.length <= 10 + ) { + return `${name} (${truncateList( + field.values.map((value) => '`' + value + '`'), + field.types.includes('text') ? 2 : 25 + ).join(', ')})`; + } + + return name; + }), + 500 + ).sort(), + }; +} diff --git a/x-pack/plugins/observability_solution/investigate_app/server/lib/queries/index.ts b/x-pack/plugins/observability_solution/investigate_app/server/lib/queries/index.ts new file mode 100644 index 0000000000000..2dff4d40ec850 --- /dev/null +++ b/x-pack/plugins/observability_solution/investigate_app/server/lib/queries/index.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { estypes } from '@elastic/elasticsearch'; + +export function rangeQuery( + start: number, + end: number, + field = '@timestamp' +): estypes.QueryDslQueryContainer[] { + return [ + { + range: { + [field]: { + gte: start, + lte: end, + format: 'epoch_millis', + }, + }, + }, + ]; +} + +export function excludeFrozenQuery(): estypes.QueryDslQueryContainer[] { + return [ + { + bool: { + must_not: [ + { + term: { + _tier: 'data_frozen', + }, + }, + ], + }, + }, + ]; +} diff --git a/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts b/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts index 195fbdb234360..b22b38adc05b0 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts @@ -23,6 +23,8 @@ import { updateInvestigationParamsSchema, getEventsParamsSchema, GetEventsResponse, + getEntitiesParamsSchema, + GetEntitiesResponse, } from '@kbn/investigation-shared'; import { ScopedAnnotationsClient } from '@kbn/observability-plugin/server'; import { createInvestigation } from '../services/create_investigation'; @@ -44,6 +46,8 @@ import { updateInvestigationItem } from '../services/update_investigation_item'; import { updateInvestigationNote } from '../services/update_investigation_note'; import { createInvestigateAppServerRoute } from './create_investigate_app_server_route'; import { getAllInvestigationStats } from '../services/get_all_investigation_stats'; +import { getEntitiesWithSampledDocuments } from '../services/get_entities'; +import { createEntitiesESClient } from '../clients/create_entities_es_client'; const createInvestigationRoute = createInvestigateAppServerRoute({ endpoint: 'POST /api/observability/investigations 2023-10-31', @@ -344,6 +348,39 @@ const getEventsRoute = createInvestigateAppServerRoute({ }, }); +const getEntitiesRoute = createInvestigateAppServerRoute({ + endpoint: 'GET /api/observability/investigation/entities 2023-10-31', + options: { + tags: [], + }, + params: getEntitiesParamsSchema, + handler: async ({ params, context, request }): Promise => { + const core = await context.core; + const esClient = core.elasticsearch.client.asCurrentUser; + const entitiesEsClient = createEntitiesESClient({ request, esClient }); + + const { + 'service.name': serviceName, + 'service.environment': serviceEnvironment, + 'container.id': containerId, + 'host.name': hostName, + } = params?.query ?? {}; + + const { entities } = await getEntitiesWithSampledDocuments({ + context, + serviceName, + serviceEnvironment, + containerId, + hostName, + entitiesEsClient, + }); + + return { + entities, + }; + }, +}); + export function getGlobalInvestigateAppServerRouteRepository() { return { ...createInvestigationRoute, @@ -360,6 +397,7 @@ export function getGlobalInvestigateAppServerRouteRepository() { ...updateInvestigationItemRoute, ...getInvestigationItemsRoute, ...getEventsRoute, + ...getEntitiesRoute, ...getAllInvestigationStatsRoute, ...getAllInvestigationTagsRoute, }; diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts new file mode 100644 index 0000000000000..c3c6395df782d --- /dev/null +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts @@ -0,0 +1,219 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { z } from '@kbn/zod'; +import datemath from '@kbn/datemath'; +import { entityLatestSchema } from '@kbn/entities-schema'; +import { GetEntitiesResponse, EntityWithSampledDocuments } from '@kbn/investigation-shared'; +import { ElasticsearchClient } from '@kbn/core/server'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { IndicesIndexState } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { InvestigateAppRequestHandlerContext } from '../routes/types'; +import { EntitiesESClient } from '../clients/create_entities_es_client'; +import { + SERVICE_ENTITIES_LATEST_ALIAS, + CONTAINER_ENTITIES_LATEST_ALIAS, + HOST_ENTITIES_LATEST_ALIAS, +} from '../clients/create_entities_es_client'; +import { + getSampleDocuments, + getKeywordAndNumericalFields, + mergeSampleDocumentsWithFieldCaps, + sortAndTruncateAnalyzedFields, +} from '../lib/get_sample_documents'; + +type EntitiesLatest = z.infer; + +export async function getEntitiesWithSampledDocuments({ + context, + serviceEnvironment, + serviceName, + containerId, + hostName, + entitiesEsClient, +}: { + context: InvestigateAppRequestHandlerContext; + serviceName?: string; + serviceEnvironment?: string; + containerId?: string; + hostName?: string; + entitiesEsClient: EntitiesESClient; +}): Promise { + const core = await context.core; + const esClient = core.elasticsearch.client.asCurrentUser; + const entityCategoryPromises = getFetchEntitiesPromises({ + entitiesEsClient, + serviceName, + serviceEnvironment, + hostName, + containerId, + }); + + const entityCategory = await Promise.all(entityCategoryPromises); + const discoveredEntities = []; + for (const category of entityCategory) { + for (const entity of category) { + const sourceIndex = entity?.sourceIndex; + + const sources = []; + const indices = await esClient.indices.get({ + index: sourceIndex, + }); + // for all indices related to the entity + for (const index in indices) { + if (index) { + const indexPattern = index; + const source = await sampleEntitySource({ + indexPattern, + index: indices[index], + esClient, + }); + sources.push(source); + } + } + const formattedEntity: EntityWithSampledDocuments = { + identityFields: entity?.entity.identityFields, + id: entity?.entity.id, + definitionId: entity?.entity.definitionId, + firstSeenTimestamp: entity?.entity.firstSeenTimestamp, + lastSeenTimestamp: entity?.entity.lastSeenTimestamp, + displayName: entity?.entity.displayName, + metrics: entity?.entity.metrics, + schemaVersion: entity?.entity.schemaVersion, + definitionVersion: entity?.entity.definitionVersion, + type: entity?.entity.type, + sources, + }; + discoveredEntities.push(formattedEntity); + } + } + return { + entities: discoveredEntities, + }; +} + +const sampleEntitySource = async ({ + indexPattern, + index, + esClient, +}: { + indexPattern: string; + index: IndicesIndexState; + esClient: ElasticsearchClient; +}) => { + const dataStream = index.data_stream; + const { samples, total } = await getSampleDocuments({ + esClient, + indexPatterns: [indexPattern], + count: 500, + start: datemath.parse('now-24h')!.toDate().getTime(), + end: datemath.parse('now')!.toDate().getTime(), + }); + const fieldCaps = await getKeywordAndNumericalFields({ + indexPatterns: [indexPattern], + esClient, + start: datemath.parse('now-24h')!.toDate().getTime(), + end: datemath.parse('now')!.toDate().getTime(), + }); + const documentAnalysis = mergeSampleDocumentsWithFieldCaps({ + total, + samples, + fieldCaps, + }); + const sortedFields = sortAndTruncateAnalyzedFields({ + ...documentAnalysis, + fields: documentAnalysis.fields.filter((field) => !field.empty), + }); + const source = { + index: indexPattern, + aliases: index.aliases, + dataStream, + documentAnalysis: sortedFields, + }; + return source; +}; + +const getFetchEntitiesPromises = ({ + entitiesEsClient, + serviceName, + serviceEnvironment, + hostName, + containerId, +}: { + entitiesEsClient: EntitiesESClient; + serviceName?: string; + hostName?: string; + containerId?: string; + serviceEnvironment?: string; +}): Array>> => { + const shouldFilterForServiceEnvironment = + serviceEnvironment && + serviceName && + serviceEnvironment !== 'ENVIRONMENT_ALL' && + serviceEnvironment !== 'ENVIRONMENT_NOT_DEFINED'; + const containersPromise = getFetchEntityPromise({ + index: CONTAINER_ENTITIES_LATEST_ALIAS, + shouldFetch: Boolean(hostName || containerId), + shouldMatch: [ + ...(hostName ? [{ term: { 'host.name': hostName } }] : []), + ...(containerId ? [{ term: { 'container.id': containerId } }] : []), + ], + entitiesEsClient, + }); + const hostsPromise = getFetchEntityPromise({ + index: HOST_ENTITIES_LATEST_ALIAS, + shouldFetch: Boolean(hostName), + shouldMatch: hostName ? [{ term: { 'host.name': hostName } }] : [], + entitiesEsClient, + }); + const servicesPromise = getFetchEntityPromise({ + index: SERVICE_ENTITIES_LATEST_ALIAS, + shouldFetch: Boolean(serviceName), + shouldMatch: [ + ...(serviceName ? [{ term: { 'service.name': serviceName } }] : []), + ...(shouldFilterForServiceEnvironment + ? [{ term: { 'service.environment': serviceEnvironment } }] + : []), + ], + entitiesEsClient, + }); + + return [containersPromise, hostsPromise, servicesPromise].filter( + (promise) => promise !== null + ) as Array>>; +}; + +const getFetchEntityPromise = ({ + index, + shouldFetch, + shouldMatch, + entitiesEsClient, +}: { + index: string; + shouldFetch: boolean; + shouldMatch: QueryDslQueryContainer[]; + entitiesEsClient: EntitiesESClient; +}): Promise> | null => { + return shouldFetch + ? entitiesEsClient + .search<{ sourceIndex: string[]; entity: EntitiesLatest['entity'] }>(index, { + body: { + query: { + bool: { + should: shouldMatch, + minimum_should_match: 1, + }, + }, + }, + }) + .then((response) => { + return response.hits.hits.map((hit) => { + return { sourceIndex: hit?._source.sourceIndex, entity: hit._source.entity }; + }); + }) + : null; +}; diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_events.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_events.ts index 52eeea7a4cbcc..6b081f51dfee8 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/services/get_events.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_events.ts @@ -6,7 +6,6 @@ */ import datemath from '@elastic/datemath'; -import { estypes } from '@elastic/elasticsearch'; import { GetEventsParams, GetEventsResponse, @@ -21,24 +20,7 @@ import { ALERT_UUID, } from '@kbn/rule-data-utils'; import { AlertsClient } from './get_alerts_client'; - -export function rangeQuery( - start: number, - end: number, - field = '@timestamp' -): estypes.QueryDslQueryContainer[] { - return [ - { - range: { - [field]: { - gte: start, - lte: end, - format: 'epoch_millis', - }, - }, - }, - ]; -} +import { rangeQuery } from '../lib/queries'; export async function getAnnotationEvents( params: GetEventsParams, From 2d305e7a0e488cc7dd60ee7b1fd34797342fb254 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 30 Sep 2024 13:18:06 +0000 Subject: [PATCH 02/19] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- .../observability_solution/investigate_app/tsconfig.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json index cd687f2dcfe70..2f3c840d2e525 100644 --- a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json @@ -61,5 +61,9 @@ "@kbn/observability-plugin", "@kbn/licensing-plugin", "@kbn/rule-data-utils", + "@kbn/entities-schema", + "@kbn/inference-plugin", + "@kbn/core-elasticsearch-server", + "@kbn/datemath", ], } From dc8c47ad511e06c811cd5ac997977fa46c0ce8aa Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Tue, 1 Oct 2024 16:05:08 -0400 Subject: [PATCH 03/19] add initial contextual insight --- .../src/rest_specs/entity.ts | 24 +- .../src/rest_specs/get_entities.ts | 4 +- .../investigate_app/kibana.jsonc | 2 +- .../public/hooks/use_fetch_entities.ts | 2 +- .../assistant_hypothesis.tsx | 105 ++++ .../investigation_items.tsx | 8 +- .../contexts/investigation_context.tsx | 8 + .../investigate_app/public/types.ts | 2 + .../server/lib/get_document_categories.ts | 493 ++++++++++++++++++ .../server/lib/get_sample_documents.ts | 38 ++ ...investigate_app_server_route_repository.ts | 4 +- .../server/services/get_entities.ts | 121 ++--- 12 files changed, 705 insertions(+), 106 deletions(-) create mode 100644 x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx create mode 100644 x-pack/plugins/observability_solution/investigate_app/server/lib/get_document_categories.ts diff --git a/packages/kbn-investigation-shared/src/rest_specs/entity.ts b/packages/kbn-investigation-shared/src/rest_specs/entity.ts index f2477b333ec42..1c29192c2a098 100644 --- a/packages/kbn-investigation-shared/src/rest_specs/entity.ts +++ b/packages/kbn-investigation-shared/src/rest_specs/entity.ts @@ -9,8 +9,6 @@ import { z } from '@kbn/zod'; -const sourceSchema = z.record(z.string(), z.any()); - const metricsSchema = z.object({ failedTransactionRate: z.number().optional(), latency: z.number().optional(), @@ -32,27 +30,19 @@ const entitySchema = z.object({ metrics: metricsSchema, }); -const entityDocumentAnalysisSchema = z.object({ - total: z.number(), - sampled: z.number(), - fields: z.array(z.string()), -}); - -const entitySourcesSchema = z.object({ - index: z.string(), - aliases: sourceSchema.optional(), +const entitySourceSchema = z.object({ dataStream: z.string().optional(), - documentAnalysis: entityDocumentAnalysisSchema, }); -const entityWithSampleDocumentsSchema = z.intersection( +const entityWithSourceSchema = z.intersection( entitySchema, z.object({ - sources: z.array(entitySourcesSchema), + sources: z.array(entitySourceSchema), }) ); -type EntityWithSampledDocuments = z.output; +type EntityWithSource = z.output; +type EntitySource = z.output; -export { entityWithSampleDocumentsSchema }; -export type { EntityWithSampledDocuments }; +export { entitySchema, entityWithSourceSchema }; +export type { EntityWithSource, EntitySource }; diff --git a/packages/kbn-investigation-shared/src/rest_specs/get_entities.ts b/packages/kbn-investigation-shared/src/rest_specs/get_entities.ts index 515e0b753d965..383bc21b58085 100644 --- a/packages/kbn-investigation-shared/src/rest_specs/get_entities.ts +++ b/packages/kbn-investigation-shared/src/rest_specs/get_entities.ts @@ -8,7 +8,7 @@ */ import { z } from '@kbn/zod'; -import { entityWithSampleDocumentsSchema } from './entity'; +import { entityWithSourceSchema } from './entity'; const getEntitiesParamsSchema = z .object({ @@ -24,7 +24,7 @@ const getEntitiesParamsSchema = z .partial(); const getEntitiesResponseSchema = z.object({ - entities: z.array(entityWithSampleDocumentsSchema), + entities: z.array(entityWithSourceSchema), }); type GetEntitiesParams = z.infer; diff --git a/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc b/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc index 2cc904dafac05..1d45fc29c4c1c 100644 --- a/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc +++ b/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc @@ -28,7 +28,7 @@ "kibanaReact", "kibanaUtils", ], - "optionalPlugins": [], + "optionalPlugins": ["observabilityAIAssistant"], "extraPublicDirs": [] } } diff --git a/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts index 007856109bce0..924744e4366d0 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts +++ b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts @@ -34,7 +34,7 @@ export function useFetchEntities({ serviceEnvironment, hostName, containerId, -}: EntityParams): any { +}: EntityParams) { const { core: { http }, } = useKibana(); diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx new file mode 100644 index 0000000000000..48fc3de1a31f5 --- /dev/null +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import dedent from 'dedent'; +import { + ALERT_RULE_PARAMETERS, + ALERT_START, + ALERT_RULE_CATEGORY, + ALERT_REASON, +} from '@kbn/rule-data-utils'; +import { i18n } from '@kbn/i18n'; +import { EntityWithSource } from '@kbn/investigation-shared'; +import React, { useCallback } from 'react'; +import { useKibana } from '../../../../hooks/use_kibana'; +import { useInvestigation } from '../../contexts/investigation_context'; +import { useFetchEntities } from '../../../../hooks/use_fetch_entities'; + +export interface InvestigationContextualInsight { + key: string; + description: string; + data: unknown; +} + +export function AssistantHypothesis({ investigationId }: { investigationId: string }) { + const { alert } = useInvestigation(); + const { + dependencies: { + start: { + observabilityAIAssistant: { + ObservabilityAIAssistantContextualInsight, + getContextualInsightMessages, + }, + }, + }, + } = useKibana(); + const { data: entitiesData } = useFetchEntities({ + investigationId, + serviceName: `${alert?.['service.name']}`, + serviceEnvironment: `${alert?.['service.environment']}`, + hostName: `${alert?.['host.name']}`, + containerId: `${alert?.['container.id']}`, + }); + + const getAlertContextMessages = useCallback(async () => { + if (!getContextualInsightMessages || !alert) { + return []; + } + + const entities = entitiesData?.entities ?? []; + + const entityContext = entities?.length + ? ` + Alerts can optionally be associated with entities. Entities can be services, hosts, containers, or other resources. Entities can have metrics associated with them. + + The alert that triggered this investigation is associated with the following entities: ${entities + .map((entity) => { + return formatEntityMetrics(entity); + }) + .join('/n/n')}` + : ''; + + return getContextualInsightMessages({ + message: `I am investigating a failure in my system. I was made aware of the failure by an alert and I am trying to understand the root cause of the issue.`, + instructions: dedent( + `I'm an SRE. I am investigating a failure in my system. I was made aware of the failure via an alert. Your current task is to help me identify the root cause of the failure in my system. + + The rule that triggered the alert is a ${ + alert[ALERT_RULE_CATEGORY] + } rule. The alert started at ${alert[ALERT_START]}. The alert reason is ${ + alert[ALERT_REASON] + }. The rule parameters are ${JSON.stringify(ALERT_RULE_PARAMETERS)}. + + ${entityContext} + + Based on the alert details, suggest a root cause and next steps to mitigate the issue. I do not have the alert details in front of me, so be sure to repeat the alert reason (${ + alert[ALERT_REASON] + }) and when the alert was triggered (${alert[ALERT_START]}). + ` + ), + }); + }, [alert, getContextualInsightMessages, entitiesData?.entities]); + + if (!ObservabilityAIAssistantContextualInsight) { + return null; + } + + return alert && entitiesData ? ( + + ) : null; +} +const formatEntityMetrics = (entity: EntityWithSource): string => { + const entityMetrics = Object.entries(entity.metrics) + .map(([key, value]) => `${key}: ${value}`) + .join(', '); + return `Entity name: ${entity.displayName}; Entity type: ${entity.type}; Entity metrics: ${entityMetrics}`; +}; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_items/investigation_items.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_items/investigation_items.tsx index c15cdf3d3f7f3..083c384356cd7 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_items/investigation_items.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_items/investigation_items.tsx @@ -12,9 +12,10 @@ import { useInvestigation } from '../../contexts/investigation_context'; import { AddInvestigationItem } from '../add_investigation_item/add_investigation_item'; import { InvestigationItemsList } from '../investigation_items_list/investigation_items_list'; import { InvestigationSearchBar } from '../investigation_search_bar/investigation_search_bar'; +import { AssistantHypothesis } from '../assistant_hypothesis/assistant_hypothesis'; export function InvestigationItems() { - const { globalParams, updateInvestigationParams } = useInvestigation(); + const { globalParams, updateInvestigationParams, investigation } = useInvestigation(); return ( @@ -32,6 +33,11 @@ export function InvestigationItems() { }} /> + {investigation?.id && ( + + + + )} diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/contexts/investigation_context.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/contexts/investigation_context.tsx index ad3e01ccb4e71..4bc9dcc6c2619 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/contexts/investigation_context.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/contexts/investigation_context.tsx @@ -7,6 +7,8 @@ import { i18n } from '@kbn/i18n'; import { type GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; +import { alertOriginSchema } from '@kbn/investigation-shared'; +import { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common'; import { GetInvestigationResponse, InvestigationItem, Item } from '@kbn/investigation-shared'; import { isEqual } from 'lodash'; import React, { createContext, useContext, useEffect, useRef, useState } from 'react'; @@ -15,6 +17,7 @@ import { useAddInvestigationNote } from '../../../hooks/use_add_investigation_no import { useDeleteInvestigationItem } from '../../../hooks/use_delete_investigation_item'; import { useDeleteInvestigationNote } from '../../../hooks/use_delete_investigation_note'; import { useFetchInvestigation } from '../../../hooks/use_fetch_investigation'; +import { useFetchAlert } from '../../../hooks/use_fetch_alert'; import { useKibana } from '../../../hooks/use_kibana'; import { useUpdateInvestigation } from '../../../hooks/use_update_investigation'; import { useUpdateInvestigationNote } from '../../../hooks/use_update_investigation_note'; @@ -26,6 +29,7 @@ export type RenderedInvestigationItem = InvestigationItem & { interface InvestigationContextProps { investigation?: GetInvestigationResponse; + alert?: EcsFieldsResponse; renderableItems: RenderedInvestigationItem[]; globalParams: GlobalWidgetParameters; updateInvestigationParams: (params: GlobalWidgetParameters) => Promise; @@ -81,6 +85,9 @@ export function InvestigationProvider({ id: initialInvestigation.id, initialInvestigation, }); + const alertOriginInvestigation = alertOriginSchema.safeParse(investigation?.origin); + const alertId = alertOriginInvestigation.success ? alertOriginInvestigation.data.id : undefined; + const { data: alert } = useFetchAlert({ id: alertId }); const cache = useRef< Record @@ -211,6 +218,7 @@ export function InvestigationProvider({ renderableItems, updateInvestigationParams, investigation, + alert: alert ?? undefined, globalParams, addItem, deleteItem, diff --git a/x-pack/plugins/observability_solution/investigate_app/public/types.ts b/x-pack/plugins/observability_solution/investigate_app/public/types.ts index 660a9ca9c8d1d..97fe90bb5b6b6 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/types.ts +++ b/x-pack/plugins/observability_solution/investigate_app/public/types.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import type { ObservabilityAIAssistantPublicStart } from '@kbn/observability-ai-assistant-plugin/public'; import type { ContentManagementPublicStart } from '@kbn/content-management-plugin/public'; import type { DataPublicPluginSetup, DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { @@ -51,6 +52,7 @@ export interface InvestigateAppSetupDependencies { export interface InvestigateAppStartDependencies { investigate: InvestigatePublicStart; observabilityShared: ObservabilitySharedPluginStart; + observabilityAIAssistant: ObservabilityAIAssistantPublicStart; lens: LensPublicStart; dataViews: DataViewsPublicPluginStart; data: DataPublicPluginStart; diff --git a/x-pack/plugins/observability_solution/investigate_app/server/lib/get_document_categories.ts b/x-pack/plugins/observability_solution/investigate_app/server/lib/get_document_categories.ts new file mode 100644 index 0000000000000..863872fd6d010 --- /dev/null +++ b/x-pack/plugins/observability_solution/investigate_app/server/lib/get_document_categories.ts @@ -0,0 +1,493 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; +import moment from 'moment'; +import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import { calculateAuto } from '@kbn/calculate-auto'; +import { + type RandomSamplerWrapper, + createRandomSamplerWrapper, +} from '@kbn/ml-random-sampler-utils'; +import { z } from '@kbn/zod'; + +const isoTimestampFormat = "YYYY-MM-DD'T'HH:mm:ss.SSS'Z'"; + +export interface LogCategory { + change: LogCategoryChange; + documentCount: number; + histogram: LogCategoryHistogramBucket[]; + terms: string; +} + +export type LogCategoryChange = + | LogCategoryNoChange + | LogCategoryRareChange + | LogCategorySpikeChange + | LogCategoryDipChange + | LogCategoryStepChange + | LogCategoryDistributionChange + | LogCategoryTrendChange + | LogCategoryOtherChange + | LogCategoryUnknownChange; + +export interface LogCategoryNoChange { + type: 'none'; +} + +export interface LogCategoryRareChange { + type: 'rare'; + timestamp: string; +} + +export interface LogCategorySpikeChange { + type: 'spike'; + timestamp: string; +} + +export interface LogCategoryDipChange { + type: 'dip'; + timestamp: string; +} + +export interface LogCategoryStepChange { + type: 'step'; + timestamp: string; +} + +export interface LogCategoryTrendChange { + type: 'trend'; + timestamp: string; + correlationCoefficient: number; +} + +export interface LogCategoryDistributionChange { + type: 'distribution'; + timestamp: string; +} + +export interface LogCategoryOtherChange { + type: 'other'; + timestamp?: string; +} + +export interface LogCategoryUnknownChange { + type: 'unknown'; + rawChange: string; +} + +export interface LogCategoryHistogramBucket { + documentCount: number; + timestamp: string; +} + +export interface LogCategorizationParams { + documentFilters: QueryDslQueryContainer[]; + endTimestamp: string; + index: string; + messageField: string; + startTimestamp: string; + timeField: string; +} + +// the fraction of a category's histogram below which the category is considered rare +const rarityThreshold = 0.2; + +export const categorizeDocuments = async ({ + esClient, + index, + endTimestamp, + startTimestamp, + timeField, + messageField, + samplingProbability, + ignoredCategoryTerms, + documentFilters = [], + minDocsPerCategory, +}: { + esClient: ElasticsearchClient; + index: string; + endTimestamp: string; + startTimestamp: string; + timeField: string; + messageField: string; + samplingProbability: number; + ignoredCategoryTerms: string[]; + documentFilters?: QueryDslQueryContainer[]; + minDocsPerCategory?: number; +}) => { + const randomSampler = createRandomSamplerWrapper({ + probability: samplingProbability, + seed: 1, + }); + + const requestParams = createCategorizationRequestParams({ + index, + timeField, + messageField, + startTimestamp, + endTimestamp, + randomSampler, + additionalFilters: documentFilters, + ignoredCategoryTerms, + minDocsPerCategory, + maxCategoriesCount: 1000, + }); + + const rawResponse = await esClient.search(requestParams); + + if (rawResponse.aggregations == null) { + throw new Error('No aggregations found in large categories response'); + } + + const logCategoriesAggResult = randomSampler.unwrap(rawResponse.aggregations); + + if (!('categories' in logCategoriesAggResult)) { + throw new Error('No categorization aggregation found in large categories response'); + } + + const logCategories = + (logCategoriesAggResult.categories.buckets as unknown[]).map(mapCategoryBucket) ?? []; + + return { + categories: logCategories, + hasReachedLimit: logCategories.length >= 1000, + }; +}; + +const mapCategoryBucket = (bucket: any): LogCategory => + esCategoryBucketSchema + .transform((parsedBucket) => ({ + change: mapChangePoint(parsedBucket), + documentCount: parsedBucket.doc_count, + histogram: parsedBucket.histogram, + terms: parsedBucket.key, + })) + .parse(bucket); + +const mapChangePoint = ({ change, histogram }: EsCategoryBucket): LogCategoryChange => { + switch (change.type) { + case 'stationary': + if (isRareInHistogram(histogram)) { + return { + type: 'rare', + timestamp: findFirstNonZeroBucket(histogram)?.timestamp ?? histogram[0].timestamp, + }; + } else { + return { + type: 'none', + }; + } + case 'dip': + case 'spike': + return { + type: change.type, + timestamp: change.bucket.key, + }; + case 'step_change': + return { + type: 'step', + timestamp: change.bucket.key, + }; + case 'distribution_change': + return { + type: 'distribution', + timestamp: change.bucket.key, + }; + case 'trend_change': + return { + type: 'trend', + timestamp: change.bucket.key, + correlationCoefficient: change.details.r_value, + }; + case 'unknown': + return { + type: 'unknown', + rawChange: change.rawChange, + }; + case 'non_stationary': + default: + return { + type: 'other', + }; + } +}; + +/** + * The official types are lacking the change_point aggregation + */ +const esChangePointBucketSchema = z.object({ + key: z.string().datetime(), + doc_count: z.number(), +}); + +const esChangePointDetailsSchema = z.object({ + p_value: z.number(), +}); + +const esChangePointCorrelationSchema = esChangePointDetailsSchema.extend({ + r_value: z.number(), +}); + +const esChangePointSchema = z.union([ + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + dip: esChangePointDetailsSchema, + }), + }) + .transform(({ bucket, type: { dip: details } }) => ({ + type: 'dip' as const, + bucket, + details, + })), + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + spike: esChangePointDetailsSchema, + }), + }) + .transform(({ bucket, type: { spike: details } }) => ({ + type: 'spike' as const, + bucket, + details, + })), + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + step_change: esChangePointDetailsSchema, + }), + }) + .transform(({ bucket, type: { step_change: details } }) => ({ + type: 'step_change' as const, + bucket, + details, + })), + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + trend_change: esChangePointCorrelationSchema, + }), + }) + .transform(({ bucket, type: { trend_change: details } }) => ({ + type: 'trend_change' as const, + bucket, + details, + })), + z + .object({ + bucket: esChangePointBucketSchema, + type: z.object({ + distribution_change: esChangePointDetailsSchema, + }), + }) + .transform(({ bucket, type: { distribution_change: details } }) => ({ + type: 'distribution_change' as const, + bucket, + details, + })), + z + .object({ + type: z.object({ + non_stationary: esChangePointCorrelationSchema.extend({ + trend: z.enum(['increasing', 'decreasing']), + }), + }), + }) + .transform(({ type: { non_stationary: details } }) => ({ + type: 'non_stationary' as const, + details, + })), + z + .object({ + type: z.object({ + stationary: z.object({}), + }), + }) + .transform(() => ({ type: 'stationary' as const })), + z + .object({ + type: z.object({}), + }) + .transform((value) => ({ type: 'unknown' as const, rawChange: JSON.stringify(value) })), +]); + +const esHistogramSchema = z + .object({ + buckets: z.array( + z + .object({ + key_as_string: z.string(), + doc_count: z.number(), + }) + .transform((bucket) => ({ + timestamp: bucket.key_as_string, + documentCount: bucket.doc_count, + })) + ), + }) + .transform(({ buckets }) => buckets); + +type EsHistogram = z.output; + +const esCategoryBucketSchema = z.object({ + key: z.string(), + doc_count: z.number(), + change: esChangePointSchema, + histogram: esHistogramSchema, +}); + +type EsCategoryBucket = z.output; + +const isRareInHistogram = (histogram: EsHistogram): boolean => + histogram.filter((bucket) => bucket.documentCount > 0).length < + histogram.length * rarityThreshold; + +const findFirstNonZeroBucket = (histogram: EsHistogram) => + histogram.find((bucket) => bucket.documentCount > 0); + +export const createCategorizationRequestParams = ({ + index, + timeField, + messageField, + startTimestamp, + endTimestamp, + randomSampler, + minDocsPerCategory = 0, + additionalFilters = [], + ignoredCategoryTerms = [], + maxCategoriesCount = 1000, +}: { + startTimestamp: string; + endTimestamp: string; + index: string; + timeField: string; + messageField: string; + randomSampler: RandomSamplerWrapper; + minDocsPerCategory?: number; + additionalFilters?: QueryDslQueryContainer[]; + ignoredCategoryTerms?: string[]; + maxCategoriesCount?: number; +}) => { + const startMoment = moment(startTimestamp, isoTimestampFormat); + const endMoment = moment(endTimestamp, isoTimestampFormat); + const fixedIntervalDuration = calculateAuto.atLeast( + 24, + moment.duration(endMoment.diff(startMoment)) + ); + const fixedIntervalSize = `${Math.ceil(fixedIntervalDuration?.asMinutes() ?? 1)}m`; + + return { + index, + size: 0, + track_total_hits: false, + query: createCategorizationQuery({ + messageField, + timeField, + startTimestamp, + endTimestamp, + additionalFilters, + ignoredCategoryTerms, + }), + aggs: randomSampler.wrap({ + histogram: { + date_histogram: { + field: '@timestamp', + fixed_interval: fixedIntervalSize, + extended_bounds: { + min: startTimestamp, + max: endTimestamp, + }, + }, + }, + categories: { + categorize_text: { + field: messageField, + size: maxCategoriesCount, + categorization_analyzer: { + tokenizer: 'standard', + }, + ...(minDocsPerCategory > 0 ? { min_doc_count: minDocsPerCategory } : {}), + }, + aggs: { + histogram: { + date_histogram: { + field: timeField, + fixed_interval: fixedIntervalSize, + extended_bounds: { + min: startTimestamp, + max: endTimestamp, + }, + }, + }, + change: { + // @ts-expect-error the official types don't support the change_point aggregation + change_point: { + buckets_path: 'histogram>_count', + }, + }, + }, + }, + }), + }; +}; + +export const createCategoryQuery = + (messageField: string) => + (categoryTerms: string): QueryDslQueryContainer => ({ + match: { + [messageField]: { + query: categoryTerms, + operator: 'AND' as const, + fuzziness: 0, + auto_generate_synonyms_phrase_query: false, + }, + }, + }); + +export const createCategorizationQuery = ({ + messageField, + timeField, + startTimestamp, + endTimestamp, + additionalFilters = [], + ignoredCategoryTerms = [], +}: { + messageField: string; + timeField: string; + startTimestamp: string; + endTimestamp: string; + additionalFilters?: QueryDslQueryContainer[]; + ignoredCategoryTerms?: string[]; +}): QueryDslQueryContainer => { + return { + bool: { + filter: [ + { + exists: { + field: messageField, + }, + }, + { + range: { + [timeField]: { + gte: startTimestamp, + lte: endTimestamp, + format: 'strict_date_time', + }, + }, + }, + ...additionalFilters, + ], + must_not: ignoredCategoryTerms.map(createCategoryQuery(messageField)), + }, + }; +}; diff --git a/x-pack/plugins/observability_solution/investigate_app/server/lib/get_sample_documents.ts b/x-pack/plugins/observability_solution/investigate_app/server/lib/get_sample_documents.ts index 1b56d9f7cbd95..75e21526a6506 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/lib/get_sample_documents.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/lib/get_sample_documents.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import pLimit from 'p-limit'; import { estypes } from '@elastic/elasticsearch'; import { castArray, sortBy, uniq, partition, shuffle } from 'lodash'; import { truncateList } from '@kbn/inference-plugin/common/util/truncate_list'; @@ -236,3 +237,40 @@ export function sortAndTruncateAnalyzedFields(analysis: DocumentAnalysis) { ).sort(), }; } + +export async function confirmConstantsInDataset({ + esClient, + constants, + indexPatterns, +}: { + esClient: ElasticsearchClient; + constants: Array<{ field: string }>; + indexPatterns: string[]; +}): Promise> { + const limiter = pLimit(5); + + return Promise.all( + constants.map((constant) => { + return limiter(async () => { + return esClient + .termsEnum({ + index: indexPatterns.join(','), + field: constant.field, + index_filter: { + bool: { + filter: [...excludeFrozenQuery()], + }, + }, + }) + .then((response) => { + const isConstant = response.terms.length === 1; + return { + field: constant.field, + constant: isConstant, + value: isConstant ? response.terms[0] : undefined, + }; + }); + }); + }) + ); +} diff --git a/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts b/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts index b22b38adc05b0..6b214976b00d1 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts @@ -46,7 +46,7 @@ import { updateInvestigationItem } from '../services/update_investigation_item'; import { updateInvestigationNote } from '../services/update_investigation_note'; import { createInvestigateAppServerRoute } from './create_investigate_app_server_route'; import { getAllInvestigationStats } from '../services/get_all_investigation_stats'; -import { getEntitiesWithSampledDocuments } from '../services/get_entities'; +import { getEntitiesWithSource } from '../services/get_entities'; import { createEntitiesESClient } from '../clients/create_entities_es_client'; const createInvestigationRoute = createInvestigateAppServerRoute({ @@ -366,7 +366,7 @@ const getEntitiesRoute = createInvestigateAppServerRoute({ 'host.name': hostName, } = params?.query ?? {}; - const { entities } = await getEntitiesWithSampledDocuments({ + const { entities } = await getEntitiesWithSource({ context, serviceName, serviceEnvironment, diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts index c3c6395df782d..040f88c39dd2b 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts @@ -5,10 +5,8 @@ * 2.0. */ import { z } from '@kbn/zod'; -import datemath from '@kbn/datemath'; import { entityLatestSchema } from '@kbn/entities-schema'; -import { GetEntitiesResponse, EntityWithSampledDocuments } from '@kbn/investigation-shared'; -import { ElasticsearchClient } from '@kbn/core/server'; +import { GetEntitiesResponse, EntityWithSource, EntitySource } from '@kbn/investigation-shared'; import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { IndicesIndexState } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; @@ -19,16 +17,11 @@ import { CONTAINER_ENTITIES_LATEST_ALIAS, HOST_ENTITIES_LATEST_ALIAS, } from '../clients/create_entities_es_client'; -import { - getSampleDocuments, - getKeywordAndNumericalFields, - mergeSampleDocumentsWithFieldCaps, - sortAndTruncateAnalyzedFields, -} from '../lib/get_sample_documents'; -type EntitiesLatest = z.infer; +// the official types do not explicitly define sourceIndex in the schema, but it is present in the data at the time of writing this +type EntitiesLatest = z.infer & { sourceIndex: string[] }; -export async function getEntitiesWithSampledDocuments({ +export async function getEntitiesWithSource({ context, serviceEnvironment, serviceName, @@ -52,87 +45,51 @@ export async function getEntitiesWithSampledDocuments({ hostName, containerId, }); + const entityResponses = await Promise.all(entityCategoryPromises); + const entitiesWithSource: EntityWithSource[] = []; + for (const response of entityResponses) { + const processedEntities = await Promise.all( + response.map(async (entity: EntitiesLatest) => { + const sourceIndex = entity?.sourceIndex; + if (!sourceIndex) return null; - const entityCategory = await Promise.all(entityCategoryPromises); - const discoveredEntities = []; - for (const category of entityCategory) { - for (const entity of category) { - const sourceIndex = entity?.sourceIndex; + const indices = await esClient.indices.get({ index: sourceIndex }); + const sources = await fetchSources(indices); - const sources = []; - const indices = await esClient.indices.get({ - index: sourceIndex, - }); - // for all indices related to the entity - for (const index in indices) { - if (index) { - const indexPattern = index; - const source = await sampleEntitySource({ - indexPattern, - index: indices[index], - esClient, - }); - sources.push(source); - } - } - const formattedEntity: EntityWithSampledDocuments = { - identityFields: entity?.entity.identityFields, - id: entity?.entity.id, - definitionId: entity?.entity.definitionId, - firstSeenTimestamp: entity?.entity.firstSeenTimestamp, - lastSeenTimestamp: entity?.entity.lastSeenTimestamp, - displayName: entity?.entity.displayName, - metrics: entity?.entity.metrics, - schemaVersion: entity?.entity.schemaVersion, - definitionVersion: entity?.entity.definitionVersion, - type: entity?.entity.type, - sources, - }; - discoveredEntities.push(formattedEntity); - } + return { + identityFields: entity?.entity.identityFields, + id: entity?.entity.id, + definitionId: entity?.entity.definitionId, + firstSeenTimestamp: entity?.entity.firstSeenTimestamp, + lastSeenTimestamp: entity?.entity.lastSeenTimestamp, + displayName: entity?.entity.displayName, + metrics: entity?.entity.metrics, + schemaVersion: entity?.entity.schemaVersion, + definitionVersion: entity?.entity.definitionVersion, + type: entity?.entity.type, + sources, + }; + }) + ); + entitiesWithSource.push(...(processedEntities.filter(Boolean) as EntityWithSource[])); } return { - entities: discoveredEntities, + entities: entitiesWithSource, }; } -const sampleEntitySource = async ({ - indexPattern, - index, - esClient, -}: { - indexPattern: string; - index: IndicesIndexState; - esClient: ElasticsearchClient; -}) => { +async function fetchSources(indices: Record): Promise { + return await Promise.all( + Object.values(indices).map(async (index) => { + return await getEntitySource({ index }); + }) + ); +} + +const getEntitySource = async ({ index }: { index: IndicesIndexState }) => { const dataStream = index.data_stream; - const { samples, total } = await getSampleDocuments({ - esClient, - indexPatterns: [indexPattern], - count: 500, - start: datemath.parse('now-24h')!.toDate().getTime(), - end: datemath.parse('now')!.toDate().getTime(), - }); - const fieldCaps = await getKeywordAndNumericalFields({ - indexPatterns: [indexPattern], - esClient, - start: datemath.parse('now-24h')!.toDate().getTime(), - end: datemath.parse('now')!.toDate().getTime(), - }); - const documentAnalysis = mergeSampleDocumentsWithFieldCaps({ - total, - samples, - fieldCaps, - }); - const sortedFields = sortAndTruncateAnalyzedFields({ - ...documentAnalysis, - fields: documentAnalysis.fields.filter((field) => !field.empty), - }); const source = { - index: indexPattern, - aliases: index.aliases, dataStream, - documentAnalysis: sortedFields, }; return source; }; From 16f926d5340c286c74948e2adfcb37e2ffecce79 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Tue, 1 Oct 2024 16:21:32 -0400 Subject: [PATCH 04/19] adjust entities es client --- .../server/clients/create_entities_es_client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts b/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts index 8d451b9afd96c..bfe52cbfedba9 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts @@ -110,13 +110,13 @@ export function createEntitiesESClient({ }, async msearch( - allSearches: TSearchRequest[] + allSearches: Array ): Promise<{ responses: Array> }> { const searches = allSearches .map((params) => { const searchParams: [MsearchMultisearchHeader, MsearchMultisearchBody] = [ { - index: [SERVICE_ENTITIES_LATEST_ALIAS], + index: [params.index], ignore_unavailable: true, }, { From 72914b4c8c3ee571c4bde02286ecd5712e49ef68 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 1 Oct 2024 20:33:15 +0000 Subject: [PATCH 05/19] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- .../observability_solution/investigate_app/tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json index 2f3c840d2e525..ddaf442fbae45 100644 --- a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json @@ -64,6 +64,7 @@ "@kbn/entities-schema", "@kbn/inference-plugin", "@kbn/core-elasticsearch-server", - "@kbn/datemath", + "@kbn/calculate-auto", + "@kbn/ml-random-sampler-utils", ], } From a67953deb0fa0d329d1dc1b9c741d73255c1a11d Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Tue, 1 Oct 2024 16:37:18 -0400 Subject: [PATCH 06/19] add entity sources to the assistant prompt --- .../components/assistant_hypothesis/assistant_hypothesis.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx index 48fc3de1a31f5..4ae1a8ae4243b 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx @@ -101,5 +101,6 @@ const formatEntityMetrics = (entity: EntityWithSource): string => { const entityMetrics = Object.entries(entity.metrics) .map(([key, value]) => `${key}: ${value}`) .join(', '); - return `Entity name: ${entity.displayName}; Entity type: ${entity.type}; Entity metrics: ${entityMetrics}`; + const entitySources = entity.sources.map((source) => source.dataStream).join(', '); + return `Entity name: ${entity.displayName}; Entity type: ${entity.type}; Entity metrics: ${entityMetrics}; Entity data streams: ${entitySources}`; }; From 503b7f861e8b29259232200243ef36f3003d8f09 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Wed, 2 Oct 2024 11:43:46 -0400 Subject: [PATCH 07/19] adjust prompt --- .../assistant_hypothesis.tsx | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx index 4ae1a8ae4243b..2dc76d49c282f 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/assistant_hypothesis/assistant_hypothesis.tsx @@ -38,10 +38,12 @@ export function AssistantHypothesis({ investigationId }: { investigationId: stri } = useKibana(); const { data: entitiesData } = useFetchEntities({ investigationId, - serviceName: `${alert?.['service.name']}`, - serviceEnvironment: `${alert?.['service.environment']}`, - hostName: `${alert?.['host.name']}`, - containerId: `${alert?.['container.id']}`, + serviceName: alert?.['service.name'] ? `${alert?.['service.name']}` : undefined, + serviceEnvironment: alert?.['service.environment'] + ? `${alert?.['service.environment']}` + : undefined, + hostName: alert?.['host.name'] ? `${alert?.['host.name']}` : undefined, + containerId: alert?.['container.id'] ? `${alert?.['container.id']}` : undefined, }); const getAlertContextMessages = useCallback(async () => { @@ -56,8 +58,11 @@ export function AssistantHypothesis({ investigationId }: { investigationId: stri Alerts can optionally be associated with entities. Entities can be services, hosts, containers, or other resources. Entities can have metrics associated with them. The alert that triggered this investigation is associated with the following entities: ${entities - .map((entity) => { - return formatEntityMetrics(entity); + .map((entity, index) => { + return dedent(` + ## Entity ${index + 1}: + ${formatEntityMetrics(entity)}; + `); }) .join('/n/n')}` : ''; @@ -75,9 +80,15 @@ export function AssistantHypothesis({ investigationId }: { investigationId: stri ${entityContext} - Based on the alert details, suggest a root cause and next steps to mitigate the issue. I do not have the alert details in front of me, so be sure to repeat the alert reason (${ + Based on the alert details, suggest a root cause and next steps to mitigate the issue. + + I do not have the alert details or entity details in front of me, so be sure to repeat the alert reason (${ alert[ALERT_REASON] - }) and when the alert was triggered (${alert[ALERT_START]}). + }), when the alert was triggered (${ + alert[ALERT_START] + }), and the entity metrics in your response. + + When displaying the entity metrics, please convert the metrics to a human-readable format. For example, convert "logRate" to "Log Rate" and "errorRate" to "Error Rate". ` ), }); @@ -102,5 +113,10 @@ const formatEntityMetrics = (entity: EntityWithSource): string => { .map(([key, value]) => `${key}: ${value}`) .join(', '); const entitySources = entity.sources.map((source) => source.dataStream).join(', '); - return `Entity name: ${entity.displayName}; Entity type: ${entity.type}; Entity metrics: ${entityMetrics}; Entity data streams: ${entitySources}`; + return dedent(` + Entity name: ${entity.displayName}; + Entity type: ${entity.type}; + Entity metrics: ${entityMetrics}; + Entity data streams: ${entitySources} + `); }; From 1d65aecdf7a68e4008fbee83b30a215d3656bd22 Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Wed, 2 Oct 2024 13:34:15 -0400 Subject: [PATCH 08/19] Update x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts --- .../investigate_app/public/hooks/use_fetch_entities.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts index 924744e4366d0..6febe7946288f 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts +++ b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts @@ -19,14 +19,6 @@ export interface EntityParams { containerId?: string; } -export interface UseFetchAlertResponse { - isInitialLoading: boolean; - isLoading: boolean; - isRefetching: boolean; - isSuccess: boolean; - isError: boolean; - data: EcsFieldsResponse | undefined | null; -} export function useFetchEntities({ investigationId, From 29079905cf2fd0cbe277f7196c36c7f0496830ab Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Wed, 2 Oct 2024 16:54:42 -0400 Subject: [PATCH 09/19] remove unnecessary async keyword --- .../investigate_app/server/services/get_entities.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts index 040f88c39dd2b..44711b9078281 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts @@ -78,15 +78,13 @@ export async function getEntitiesWithSource({ }; } -async function fetchSources(indices: Record): Promise { - return await Promise.all( - Object.values(indices).map(async (index) => { - return await getEntitySource({ index }); - }) - ); +function fetchSources(indices: Record): EntitySource[] { + return Object.values(indices).map((index) => { + return getEntitySource({ index }); + }); } -const getEntitySource = async ({ index }: { index: IndicesIndexState }) => { +const getEntitySource = ({ index }: { index: IndicesIndexState }) => { const dataStream = index.data_stream; const source = { dataStream, From 53655e507c240f2b378a9e171e75d1b80cca3c8e Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Wed, 2 Oct 2024 17:06:07 -0400 Subject: [PATCH 10/19] pass esClient to getEntities --- ...investigate_app_server_route_repository.ts | 2 +- .../server/services/get_entities.ts | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts b/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts index 6b214976b00d1..494e13efcba95 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/routes/get_global_investigate_app_server_route_repository.ts @@ -367,12 +367,12 @@ const getEntitiesRoute = createInvestigateAppServerRoute({ } = params?.query ?? {}; const { entities } = await getEntitiesWithSource({ - context, serviceName, serviceEnvironment, containerId, hostName, entitiesEsClient, + esClient, }); return { diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts index 44711b9078281..faaa53dc921a2 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts @@ -6,12 +6,15 @@ */ import { z } from '@kbn/zod'; import { entityLatestSchema } from '@kbn/entities-schema'; -import { GetEntitiesResponse, EntityWithSource, EntitySource } from '@kbn/investigation-shared'; -import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { IndicesIndexState } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - -import { InvestigateAppRequestHandlerContext } from '../routes/types'; -import { EntitiesESClient } from '../clients/create_entities_es_client'; +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { + GetEntitiesResponse, + EntityWithSource, + EntitySource, +} from '@kbn/investigation-shared'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { IndicesIndexState } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { EntitiesESClient } from '../clients/create_entities_es_client'; import { SERVICE_ENTITIES_LATEST_ALIAS, CONTAINER_ENTITIES_LATEST_ALIAS, @@ -22,22 +25,20 @@ import { type EntitiesLatest = z.infer & { sourceIndex: string[] }; export async function getEntitiesWithSource({ - context, serviceEnvironment, serviceName, containerId, hostName, entitiesEsClient, + esClient, }: { - context: InvestigateAppRequestHandlerContext; serviceName?: string; serviceEnvironment?: string; containerId?: string; hostName?: string; entitiesEsClient: EntitiesESClient; + esClient: ElasticsearchClient; }): Promise { - const core = await context.core; - const esClient = core.elasticsearch.client.asCurrentUser; const entityCategoryPromises = getFetchEntitiesPromises({ entitiesEsClient, serviceName, From 3fb90196179bcd432bb1b61b5f48ffd8eb1755f5 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Wed, 2 Oct 2024 17:11:28 -0400 Subject: [PATCH 11/19] remove entity history references --- .../clients/create_entities_es_client.ts | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts b/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts index bfe52cbfedba9..c11cd3eb9bc02 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/clients/create_entities_es_client.ts @@ -8,7 +8,7 @@ import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types'; import type { KibanaRequest } from '@kbn/core/server'; import { ElasticsearchClient } from '@kbn/core/server'; -import { entitiesAliasPattern, ENTITY_LATEST, ENTITY_HISTORY } from '@kbn/entities-schema'; +import { entitiesAliasPattern, ENTITY_LATEST } from '@kbn/entities-schema'; import { unwrapEsResponse } from '@kbn/observability-plugin/common/utils/unwrap_es_response'; import { MsearchMultisearchBody, @@ -19,36 +19,19 @@ export const SERVICE_ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ type: 'service', dataset: ENTITY_LATEST, }); -export const SERVICE_ENTITIES_HISTORY_ALIAS = entitiesAliasPattern({ - type: 'service', - dataset: ENTITY_HISTORY, -}); export const HOST_ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ type: 'host', dataset: ENTITY_LATEST, }); -export const HOST_ENTITIES_HISTORY_ALIAS = entitiesAliasPattern({ - type: 'host', - dataset: ENTITY_HISTORY, -}); export const CONTAINER_ENTITIES_LATEST_ALIAS = entitiesAliasPattern({ type: 'container', dataset: ENTITY_LATEST, }); -export const CONTAINER_ENTITIES_HISTORY_ALIAS = entitiesAliasPattern({ - type: 'container', - dataset: ENTITY_HISTORY, -}); type LatestAlias = | typeof SERVICE_ENTITIES_LATEST_ALIAS | typeof HOST_ENTITIES_LATEST_ALIAS | typeof CONTAINER_ENTITIES_LATEST_ALIAS; -type HistoryAlias = - | typeof SERVICE_ENTITIES_HISTORY_ALIAS - | typeof HOST_ENTITIES_HISTORY_ALIAS - | typeof CONTAINER_ENTITIES_HISTORY_ALIAS; - export function cancelEsRequestOnAbort>( promise: T, request: KibanaRequest, @@ -103,14 +86,14 @@ export function createEntitiesESClient({ return { async search( - entityIndexAlias: LatestAlias | HistoryAlias, + entityIndexAlias: LatestAlias, searchRequest: TSearchRequest ): Promise> { return search(entityIndexAlias, searchRequest); }, async msearch( - allSearches: Array + allSearches: Array ): Promise<{ responses: Array> }> { const searches = allSearches .map((params) => { From 862e98f31be8b3cba142e6d58e52b5646e21ae37 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 2 Oct 2024 22:07:22 +0000 Subject: [PATCH 12/19] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../investigate_app/public/hooks/use_fetch_entities.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts index 6febe7946288f..a8cee1a9c1857 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts +++ b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts @@ -7,7 +7,6 @@ import { useQuery } from '@tanstack/react-query'; import { GetEntitiesResponse } from '@kbn/investigation-shared'; -import { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common'; import { useKibana } from './use_kibana'; import { investigationKeys } from './query_key_factory'; @@ -19,7 +18,6 @@ export interface EntityParams { containerId?: string; } - export function useFetchEntities({ investigationId, serviceName, From ed126e4668ab0c4271766dccf74d544ffd9997a7 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Thu, 3 Oct 2024 09:10:07 -0400 Subject: [PATCH 13/19] adjust alert hook --- .../public/hooks/use_fetch_alert.tsx | 15 +++++++++------ .../details/contexts/investigation_context.tsx | 5 +---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_alert.tsx b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_alert.tsx index 0c0cda89d3eb8..2a2d5776faee2 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_alert.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_alert.tsx @@ -7,10 +7,11 @@ import { useQuery } from '@tanstack/react-query'; import { BASE_RAC_ALERTS_API_PATH, EcsFieldsResponse } from '@kbn/rule-registry-plugin/common'; +import { type GetInvestigationResponse, alertOriginSchema } from '@kbn/investigation-shared'; import { useKibana } from './use_kibana'; -export interface AlertParams { - id?: string; +export interface UseFetchAlertParams { + investigation?: GetInvestigationResponse; } export interface UseFetchAlertResponse { @@ -22,20 +23,22 @@ export interface UseFetchAlertResponse { data: EcsFieldsResponse | undefined | null; } -export function useFetchAlert({ id }: AlertParams): UseFetchAlertResponse { +export function useFetchAlert({ investigation }: UseFetchAlertParams): UseFetchAlertResponse { const { core: { http, notifications: { toasts }, }, } = useKibana(); + const alertOriginInvestigation = alertOriginSchema.safeParse(investigation?.origin); + const alertId = alertOriginInvestigation.success ? alertOriginInvestigation.data.id : undefined; const { isInitialLoading, isLoading, isError, isSuccess, isRefetching, data } = useQuery({ - queryKey: ['fetchAlert', id], + queryKey: ['fetchAlert', investigation?.id], queryFn: async ({ signal }) => { return await http.get(BASE_RAC_ALERTS_API_PATH, { query: { - id, + id: alertId, }, signal, }); @@ -46,7 +49,7 @@ export function useFetchAlert({ id }: AlertParams): UseFetchAlertResponse { title: 'Something went wrong while fetching alert', }); }, - enabled: Boolean(id), + enabled: Boolean(alertId), }); return { diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/contexts/investigation_context.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/contexts/investigation_context.tsx index 4bc9dcc6c2619..ec571d0e2db80 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/contexts/investigation_context.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/contexts/investigation_context.tsx @@ -7,7 +7,6 @@ import { i18n } from '@kbn/i18n'; import { type GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; -import { alertOriginSchema } from '@kbn/investigation-shared'; import { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common'; import { GetInvestigationResponse, InvestigationItem, Item } from '@kbn/investigation-shared'; import { isEqual } from 'lodash'; @@ -85,9 +84,7 @@ export function InvestigationProvider({ id: initialInvestigation.id, initialInvestigation, }); - const alertOriginInvestigation = alertOriginSchema.safeParse(investigation?.origin); - const alertId = alertOriginInvestigation.success ? alertOriginInvestigation.data.id : undefined; - const { data: alert } = useFetchAlert({ id: alertId }); + const { data: alert } = useFetchAlert({ investigation }); const cache = useRef< Record From 2d119deeeed5112da316a28171e420b57761f480 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Thu, 3 Oct 2024 09:12:26 -0400 Subject: [PATCH 14/19] adjust plugin definition --- .../plugins/observability_solution/investigate_app/kibana.jsonc | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc b/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc index 1d45fc29c4c1c..792e40e92e16e 100644 --- a/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc +++ b/x-pack/plugins/observability_solution/investigate_app/kibana.jsonc @@ -9,7 +9,6 @@ "configPath": ["xpack", "investigateApp"], "requiredPlugins": [ "investigate", - "observabilityAIAssistant", "observabilityShared", "lens", "dataViews", From 1932efe38f0048c952ea2296d6603645b5c67f87 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Thu, 3 Oct 2024 10:46:34 -0400 Subject: [PATCH 15/19] remove imports from ai assistant --- .../public/items/embeddable_item/register_embeddable_item.tsx | 2 +- .../public/items/esql_item/register_esql_item.tsx | 2 +- .../components/add_investigation_item/esql_widget_preview.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx b/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx index 8ebf3829b073d..29b2a1319feff 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx @@ -8,7 +8,7 @@ import { EuiLoadingSpinner, EuiFlexItem } from '@elastic/eui'; import { css } from '@emotion/css'; import { ReactEmbeddableRenderer } from '@kbn/embeddable-plugin/public'; import type { GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; -import { useAbortableAsync } from '@kbn/observability-ai-assistant-plugin/public'; +import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; import React, { useEffect, useMemo, useRef, useState } from 'react'; import { v4 } from 'uuid'; import { ErrorMessage } from '../../components/error_message'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx b/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx index 54d3698a5148b..7b88081ca5503 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx @@ -10,7 +10,7 @@ import type { ESQLSearchResponse } from '@kbn/es-types'; import { i18n } from '@kbn/i18n'; import { type GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; import type { Suggestion } from '@kbn/lens-plugin/public'; -import { useAbortableAsync } from '@kbn/observability-ai-assistant-plugin/public'; +import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; import React, { useMemo } from 'react'; import { ErrorMessage } from '../../components/error_message'; import { useKibana } from '../../hooks/use_kibana'; diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx index 6fdba0224b7d5..8d0056dbd538d 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx @@ -11,7 +11,7 @@ import type { ESQLColumn, ESQLRow } from '@kbn/es-types'; import { GlobalWidgetParameters } from '@kbn/investigate-plugin/public'; import { Item } from '@kbn/investigation-shared'; import type { Suggestion } from '@kbn/lens-plugin/public'; -import { useAbortableAsync } from '@kbn/observability-ai-assistant-plugin/public'; +import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async'; import React, { useEffect, useMemo, useState } from 'react'; import { ErrorMessage } from '../../../../components/error_message'; import { SuggestVisualizationList } from '../../../../components/suggest_visualization_list'; From b70caebf02cad05def96c7966447a21d4f37b3e4 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:57:58 +0000 Subject: [PATCH 16/19] [CI] Auto-commit changed files from 'node scripts/yarn_deduplicate' --- .../plugins/observability_solution/investigate_app/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json index ddaf442fbae45..377db42186f5e 100644 --- a/x-pack/plugins/observability_solution/investigate_app/tsconfig.json +++ b/x-pack/plugins/observability_solution/investigate_app/tsconfig.json @@ -66,5 +66,6 @@ "@kbn/core-elasticsearch-server", "@kbn/calculate-auto", "@kbn/ml-random-sampler-utils", + "@kbn/observability-utils", ], } From c24a620a7afbb25b9502657a56943f082f22b7d8 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Thu, 3 Oct 2024 12:49:34 -0400 Subject: [PATCH 17/19] adjust types --- .../investigation_header/alert_details_button.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_header/alert_details_button.tsx b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_header/alert_details_button.tsx index b9c8729e570f2..391be0abf8081 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_header/alert_details_button.tsx +++ b/x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/investigation_header/alert_details_button.tsx @@ -6,8 +6,10 @@ */ import { EuiButtonEmpty, EuiText } from '@elastic/eui'; -import { alertOriginSchema } from '@kbn/investigation-shared'; -import { ALERT_RULE_CATEGORY } from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names'; +import { + ALERT_RULE_CATEGORY, + ALERT_UUID, +} from '@kbn/rule-registry-plugin/common/technical_rule_data_field_names'; import React from 'react'; import { useKibana } from '../../../../hooks/use_kibana'; import { useFetchAlert } from '../../../../hooks/use_fetch_alert'; @@ -21,9 +23,7 @@ export function AlertDetailsButton() { } = useKibana(); const { investigation } = useInvestigation(); - const alertOriginInvestigation = alertOriginSchema.safeParse(investigation?.origin); - const alertId = alertOriginInvestigation.success ? alertOriginInvestigation.data.id : undefined; - const { data: alertDetails } = useFetchAlert({ id: alertId }); + const { data: alertDetails } = useFetchAlert({ investigation }); if (!alertDetails) { return null; @@ -33,7 +33,7 @@ export function AlertDetailsButton() { data-test-subj="investigationDetailsAlertLink" iconType="arrowLeft" size="xs" - href={basePath.prepend(`/app/observability/alerts/${alertId}`)} + href={basePath.prepend(`/app/observability/alerts/${alertDetails[ALERT_UUID]}`)} > {`[Alert] ${alertDetails?.[ALERT_RULE_CATEGORY]} breached`} From 4ab1a832b23f9217dcb8684d80924870e1b3ecfb Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Thu, 3 Oct 2024 21:51:28 -0400 Subject: [PATCH 18/19] account for missing sources --- .../investigate_app/server/services/get_entities.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts index faaa53dc921a2..8f3a0abb62b67 100644 --- a/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts +++ b/x-pack/plugins/observability_solution/investigate_app/server/services/get_entities.ts @@ -52,7 +52,7 @@ export async function getEntitiesWithSource({ const processedEntities = await Promise.all( response.map(async (entity: EntitiesLatest) => { const sourceIndex = entity?.sourceIndex; - if (!sourceIndex) return null; + if (!sourceIndex || !sourceIndex.length) return null; const indices = await esClient.indices.get({ index: sourceIndex }); const sources = await fetchSources(indices); From f95017d80e5dd4cd097dee93f76fb3b3b82e65f6 Mon Sep 17 00:00:00 2001 From: Dominique Belcher Date: Fri, 4 Oct 2024 09:11:59 -0400 Subject: [PATCH 19/19] adjust types --- .../investigate_app/public/types.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/observability_solution/investigate_app/public/types.ts b/x-pack/plugins/observability_solution/investigate_app/public/types.ts index 677dcd524d8eb..101d6993ab9c5 100644 --- a/x-pack/plugins/observability_solution/investigate_app/public/types.ts +++ b/x-pack/plugins/observability_solution/investigate_app/public/types.ts @@ -4,7 +4,10 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { ObservabilityAIAssistantPublicStart } from '@kbn/observability-ai-assistant-plugin/public'; +import type { + ObservabilityAIAssistantPublicSetup, + ObservabilityAIAssistantPublicStart, +} from '@kbn/observability-ai-assistant-plugin/public'; import type { ContentManagementPublicStart } from '@kbn/content-management-plugin/public'; import type { DataPublicPluginSetup, DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { @@ -28,10 +31,6 @@ import type { import type { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/public'; import type { UiActionsSetup, UiActionsStart } from '@kbn/ui-actions-plugin/public'; import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; -import type { - ObservabilityAIAssistantPublicSetup, - ObservabilityAIAssistantPublicStart, -} from '@kbn/observability-ai-assistant-plugin/public'; /* eslint-disable @typescript-eslint/no-empty-interface*/ @@ -42,6 +41,7 @@ export interface ConfigSchema { export interface InvestigateAppSetupDependencies { investigate: InvestigatePublicSetup; observabilityShared: ObservabilitySharedPluginSetup; + observabilityAIAssistant: ObservabilityAIAssistantPublicSetup; lens: LensPublicSetup; dataViews: DataViewsPublicPluginSetup; data: DataPublicPluginSetup; @@ -51,7 +51,6 @@ export interface InvestigateAppSetupDependencies { unifiedSearch: {}; uiActions: UiActionsSetup; security: SecurityPluginSetup; - observabilityAIAssistant: ObservabilityAIAssistantPublicSetup; } export interface InvestigateAppStartDependencies { @@ -67,7 +66,6 @@ export interface InvestigateAppStartDependencies { unifiedSearch: UnifiedSearchPublicPluginStart; uiActions: UiActionsStart; security: SecurityPluginStart; - observabilityAIAssistant: ObservabilityAIAssistantPublicStart; } export interface InvestigateAppPublicSetup {}