[Investigation app] add entities route and investigation Contextual I…

…nsight (#194432)

## Summary

Adds a route that can be used to fetch entities related to an
investigation.

The route fetches associated entities by service name, host name, or
container id. It then identifies the associated indices and datastreams.

The discovered entities are passed to the contextual insight to inform
the LLM.


![image](https://github.com/user-attachments/assets/855a8d68-b039-4557-ba23-5661cd961021)

This PR represents the first step in developing an AI-informed
hypothesis at the beginning of the investigation. Over time, further
insights will be provided to the LLM to deepen it's investigative
analysis and propose a more helpful root cause hypothesis.

### Testing

1. Create some APM data. I'm using the otel demo and triggering a
failure via the flagd service. Since this is in flux, you can reach out
to me about this workflow. However, you can also create APM data via
`synth-trace`.
2. Create an custom threshold rule that you expect to trigger an alert.
I created mine to using `http.response.status_code: 500 /
http.response.status_code : *` and set a low threshold base on the
amount of failures in my current test data. Be sure to also group the
alert by `service.name`
3. Wait for the alert to fire, then visit the alert details page and
start an investigation
4. notice the contextual insight. Expand it to see more information

---------

Co-authored-by: kibanamachine <[email protected]>

packages/kbn-investigation-shared/src/rest_specs/entity.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { z } from '@kbn/zod';
+
+const metricsSchema = z.object({
+  failedTransactionRate: z.number().optional(),
+  latency: z.number().optional(),
+  throughput: z.number().optional(),
+  logErrorRate: z.number().optional(),
+  logRate: z.number().optional(),
+});
+
+const entitySchema = z.object({
+  id: z.string(),
+  definitionId: z.string(),
+  definitionVersion: z.string(),
+  displayName: z.string(),
+  firstSeenTimestamp: z.string(),
+  lastSeenTimestamp: z.string(),
+  identityFields: z.array(z.string()),
+  schemaVersion: z.string(),
+  type: z.string(),
+  metrics: metricsSchema,
+});
+
+const entitySourceSchema = z.object({
+  dataStream: z.string().optional(),
+});
+
+const entityWithSourceSchema = z.intersection(
+  entitySchema,
+  z.object({
+    sources: z.array(entitySourceSchema),
+  })
+);
+
+type EntityWithSource = z.output<typeof entityWithSourceSchema>;
+type EntitySource = z.output<typeof entitySourceSchema>;
+
+export { entitySchema, entityWithSourceSchema };
+export type { EntityWithSource, EntitySource };

packages/kbn-investigation-shared/src/rest_specs/get_entities.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { z } from '@kbn/zod';
+import { entityWithSourceSchema } from './entity';
+
+const getEntitiesParamsSchema = z
+  .object({
+    query: z
+      .object({
+        'service.name': z.string(),
+        'service.environment': z.string(),
+        'host.name': z.string(),
+        'container.id': z.string(),
+      })
+      .partial(),
+  })
+  .partial();
+
+const getEntitiesResponseSchema = z.object({
+  entities: z.array(entityWithSourceSchema),
+});
+
+type GetEntitiesParams = z.infer<typeof getEntitiesParamsSchema.shape.query>;
+type GetEntitiesResponse = z.output<typeof getEntitiesResponseSchema>;
+
+export { getEntitiesParamsSchema, getEntitiesResponseSchema };
+export type { GetEntitiesParams, GetEntitiesResponse };

packages/kbn-investigation-shared/src/rest_specs/index.ts

@@ -27,6 +27,8 @@ export type * from './update_item';
 export type * from './update_note';
 export type * from './event';
 export type * from './get_events';
+export type * from './entity';
+export type * from './get_entities';
 
 export * from './create';
 export * from './create_item';
@@ -48,3 +50,5 @@ export * from './update_item';
 export * from './update_note';
 export * from './event';
 export * from './get_events';
+export * from './entity';
+export * from './get_entities';

x-pack/plugins/observability_solution/investigate_app/kibana.jsonc

@@ -9,7 +9,6 @@
     "configPath": ["xpack", "investigateApp"],
     "requiredPlugins": [
       "investigate",
-      "observabilityAIAssistant",
       "observabilityShared",
       "lens",
       "dataViews",
@@ -28,7 +27,7 @@
       "kibanaReact",
       "kibanaUtils",
     ],
-    "optionalPlugins": [],
+    "optionalPlugins": ["observabilityAIAssistant"],
     "extraPublicDirs": []
   }
 }

x-pack/plugins/observability_solution/investigate_app/public/hooks/query_key_factory.ts

@@ -22,6 +22,16 @@ export const investigationKeys = {
     [...investigationKeys.detail(investigationId), 'notes'] as const,
   detailItems: (investigationId: string) =>
     [...investigationKeys.detail(investigationId), 'items'] as const,
+  entities: ({
+    investigationId,
+    ...params
+  }: {
+    investigationId: string;
+    serviceName?: string;
+    serviceEnvironment?: string;
+    hostName?: string;
+    containerId?: string;
+  }) => [...investigationKeys.detail(investigationId), 'entities', params] as const,
 };
 
 export type InvestigationKeys = typeof investigationKeys;

x-pack/plugins/observability_solution/investigate_app/public/pages/details/hooks/use_fetch_alert.tsx → x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_alert.tsx

@@ -7,10 +7,11 @@
 
 import { useQuery } from '@tanstack/react-query';
 import { BASE_RAC_ALERTS_API_PATH, EcsFieldsResponse } from '@kbn/rule-registry-plugin/common';
-import { useKibana } from '../../../hooks/use_kibana';
+import { type GetInvestigationResponse, alertOriginSchema } from '@kbn/investigation-shared';
+import { useKibana } from './use_kibana';
 
-export interface AlertParams {
-  id?: string;
+export interface UseFetchAlertParams {
+  investigation?: GetInvestigationResponse;
 }
 
 export interface UseFetchAlertResponse {
@@ -22,20 +23,22 @@ export interface UseFetchAlertResponse {
   data: EcsFieldsResponse | undefined | null;
 }
 
-export function useFetchAlert({ id }: AlertParams): UseFetchAlertResponse {
+export function useFetchAlert({ investigation }: UseFetchAlertParams): UseFetchAlertResponse {
   const {
     core: {
       http,
       notifications: { toasts },
     },
   } = useKibana();
+  const alertOriginInvestigation = alertOriginSchema.safeParse(investigation?.origin);
+  const alertId = alertOriginInvestigation.success ? alertOriginInvestigation.data.id : undefined;
 
   const { isInitialLoading, isLoading, isError, isSuccess, isRefetching, data } = useQuery({
-    queryKey: ['fetchAlert', id],
+    queryKey: ['fetchAlert', investigation?.id],
     queryFn: async ({ signal }) => {
       return await http.get<EcsFieldsResponse>(BASE_RAC_ALERTS_API_PATH, {
         query: {
-          id,
+          id: alertId,
         },
         signal,
       });
@@ -47,7 +50,7 @@ export function useFetchAlert({ id }: AlertParams): UseFetchAlertResponse {
         title: 'Something went wrong while fetching alert',
       });
     },
-    enabled: Boolean(id),
+    enabled: Boolean(alertId),
   });
 
   return {

x-pack/plugins/observability_solution/investigate_app/public/hooks/use_fetch_entities.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useQuery } from '@tanstack/react-query';
+import { GetEntitiesResponse } from '@kbn/investigation-shared';
+import { useKibana } from './use_kibana';
+import { investigationKeys } from './query_key_factory';
+
+export interface EntityParams {
+  investigationId: string;
+  serviceName?: string;
+  serviceEnvironment?: string;
+  hostName?: string;
+  containerId?: string;
+}
+
+export function useFetchEntities({
+  investigationId,
+  serviceName,
+  serviceEnvironment,
+  hostName,
+  containerId,
+}: EntityParams) {
+  const {
+    core: { http },
+  } = useKibana();
+
+  const { isInitialLoading, isLoading, isError, isSuccess, isRefetching, data } = useQuery({
+    queryKey: investigationKeys.entities({
+      investigationId,
+      serviceName,
+      serviceEnvironment,
+      hostName,
+      containerId,
+    }),
+    queryFn: async ({ signal }) => {
+      return await http.get<GetEntitiesResponse>('/api/observability/investigation/entities', {
+        query: {
+          'service.name': serviceName,
+          'service.environment': serviceEnvironment,
+          'host.name': hostName,
+          'container.id': containerId,
+        },
+        version: '2023-10-31',
+        signal,
+      });
+    },
+    refetchOnWindowFocus: false,
+    onError: (error: Error) => {
+      // ignore error
+    },
+    enabled: Boolean(investigationId && (serviceName || hostName || containerId)),
+  });
+
+  return {
+    data,
+    isInitialLoading,
+    isLoading,
+    isRefetching,
+    isSuccess,
+    isError,
+  };
+}

x-pack/plugins/observability_solution/investigate_app/public/pages/details/hooks/use_screen_context.tsx → x-pack/plugins/observability_solution/investigate_app/public/hooks/use_screen_context.tsx

@@ -5,13 +5,12 @@
  * 2.0.
  */
 
-import { alertOriginSchema } from '@kbn/investigation-shared';
 import { ALERT_REASON, ALERT_START, ALERT_STATUS } from '@kbn/rule-data-utils';
 import type { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common';
 import dedent from 'dedent';
 import { useEffect } from 'react';
-import { useKibana } from '../../../hooks/use_kibana';
-import { useInvestigation } from '../contexts/investigation_context';
+import { useKibana } from './use_kibana';
+import { useInvestigation } from '../pages/details/contexts/investigation_context';
 import { useFetchAlert } from './use_fetch_alert';
 
 export function useScreenContext() {
@@ -22,9 +21,7 @@ export function useScreenContext() {
   } = useKibana();
 
   const { investigation } = useInvestigation();
-  const alertOriginInvestigation = alertOriginSchema.safeParse(investigation?.origin);
-  const alertId = alertOriginInvestigation.success ? alertOriginInvestigation.data.id : undefined;
-  const { data: alertDetails, isLoading: isAlertDetailsLoading } = useFetchAlert({ id: alertId });
+  const { data: alertDetails, isLoading: isAlertDetailsLoading } = useFetchAlert({ investigation });
 
   useEffect(() => {
     if (!investigation || isAlertDetailsLoading) {

x-pack/plugins/observability_solution/investigate_app/public/items/embeddable_item/register_embeddable_item.tsx

@@ -8,7 +8,7 @@ import { EuiLoadingSpinner, EuiFlexItem } from '@elastic/eui';
 import { css } from '@emotion/css';
 import { ReactEmbeddableRenderer } from '@kbn/embeddable-plugin/public';
 import type { GlobalWidgetParameters } from '@kbn/investigate-plugin/public';
-import { useAbortableAsync } from '@kbn/observability-ai-assistant-plugin/public';
+import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async';
 import React, { useEffect, useMemo, useRef, useState } from 'react';
 import { v4 } from 'uuid';
 import { ErrorMessage } from '../../components/error_message';

x-pack/plugins/observability_solution/investigate_app/public/items/esql_item/register_esql_item.tsx

@@ -10,7 +10,7 @@ import type { ESQLSearchResponse } from '@kbn/es-types';
 import { i18n } from '@kbn/i18n';
 import { type GlobalWidgetParameters } from '@kbn/investigate-plugin/public';
 import type { Suggestion } from '@kbn/lens-plugin/public';
-import { useAbortableAsync } from '@kbn/observability-ai-assistant-plugin/public';
+import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async';
 import React, { useMemo } from 'react';
 import { ErrorMessage } from '../../components/error_message';
 import { useKibana } from '../../hooks/use_kibana';

x-pack/plugins/observability_solution/investigate_app/public/pages/details/components/add_investigation_item/esql_widget_preview.tsx

@@ -11,7 +11,7 @@ import type { ESQLColumn, ESQLRow } from '@kbn/es-types';
 import { GlobalWidgetParameters } from '@kbn/investigate-plugin/public';
 import { Item } from '@kbn/investigation-shared';
 import type { Suggestion } from '@kbn/lens-plugin/public';
-import { useAbortableAsync } from '@kbn/observability-ai-assistant-plugin/public';
+import { useAbortableAsync } from '@kbn/observability-utils/hooks/use_abortable_async';
 import React, { useEffect, useMemo, useState } from 'react';
 import { ErrorMessage } from '../../../../components/error_message';
 import { SuggestVisualizationList } from '../../../../components/suggest_visualization_list';