From e627b85f6bcaeec33bfc52af862a5ab5842b4369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 22 Aug 2024 06:21:31 +0200 Subject: [PATCH 01/42] [Obs AI Assistant] Add uuid to knowledge base entries to avoid overwriting accidentally --- .../common/types.ts | 4 +- .../server/analytics/recall_ranking.ts | 4 +- .../server/functions/summarize.ts | 5 +- .../server/routes/functions/route.ts | 1 - .../server/routes/knowledge_base/route.ts | 2 - .../server/service/client/index.ts | 2 +- .../service/knowledge_base_service/index.ts | 59 ++++++++++++++++--- .../server/utils/recall/recall_and_score.ts | 32 ++++++---- .../utils/recall/retrieve_suggestions.ts | 24 -------- .../server/utils/recall/score_suggestions.ts | 37 +++--------- .../server/utils/recall/types.ts | 10 ---- 11 files changed, 87 insertions(+), 93 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/retrieve_suggestions.ts delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/types.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts index 68595e457a355..38b7460c29ae1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts @@ -79,9 +79,9 @@ export type ConversationUpdateRequest = ConversationRequestBase & { export interface KnowledgeBaseEntry { '@timestamp': string; - id: string; + id: string; // this is a unique ID generated by the client + doc_id: string; // this is the human readable ID generated by the LLM and used by the LLM to update existing entries text: string; - doc_id: string; confidence: 'low' | 'medium' | 'high'; is_correction: boolean; type?: 'user_instruction' | 'contextual'; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/analytics/recall_ranking.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/analytics/recall_ranking.ts index 4c82f79fcba8d..4371310811edf 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/analytics/recall_ranking.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/analytics/recall_ranking.ts @@ -52,9 +52,9 @@ const schema: RootSchema = { }, }; -export const RecallRankingEventType = 'observability_ai_assistant_recall_ranking'; +export const recallRankingEventType = 'observability_ai_assistant_recall_ranking'; export const recallRankingEvent: EventTypeOpts = { - eventType: RecallRankingEventType, + eventType: recallRankingEventType, schema, }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index 8865861d81f45..74f8e5730a38f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -63,15 +63,14 @@ export function registerSummarizationFunction({ }, }, ( - { arguments: { id, text, is_correction: isCorrection, confidence, public: isPublic } }, + { arguments: { id: docId, text, is_correction: isCorrection, confidence, public: isPublic } }, signal ) => { return client .addKnowledgeBaseEntry({ entry: { - doc_id: id, + doc_id: docId, role: KnowledgeBaseEntryRole.AssistantSummarization, - id, text, is_correction: isCorrection, type: KnowledgeBaseType.Contextual, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index fae7077953699..854aeca17833d 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -139,7 +139,6 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ return client.addKnowledgeBaseEntry({ entry: { confidence, - id, doc_id: id, is_correction: isCorrection, type, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts index 6bb024b913cde..c4b8390fbc0f5 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts @@ -109,7 +109,6 @@ const saveKnowledgeBaseUserInstruction = createObservabilityAIAssistantServerRou const { id, text, public: isPublic } = resources.params.body; return client.addKnowledgeBaseEntry({ entry: { - id, doc_id: id, text, public: isPublic, @@ -195,7 +194,6 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ return client.addKnowledgeBaseEntry({ entry: { - id, text, doc_id: id, confidence: confidence ?? 'high', diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 293d2da9c04b9..86b02e9b9c628 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -734,7 +734,7 @@ export class ObservabilityAIAssistantClient { addKnowledgeBaseEntry = async ({ entry, }: { - entry: Omit; + entry: Omit; }): Promise => { return this.dependencies.knowledgeBaseService.addEntry({ namespace: this.dependencies.namespace, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index 679d59a57dbc8..a7b47a4d5a45f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -14,6 +14,7 @@ import pRetry from 'p-retry'; import { map, orderBy } from 'lodash'; import { encode } from 'gpt-tokenizer'; import { MlTrainedModelDeploymentNodesStats } from '@elastic/elasticsearch/lib/api/types'; +import { v4 } from 'uuid'; import { INDEX_QUEUED_DOCUMENTS_TASK_ID, INDEX_QUEUED_DOCUMENTS_TASK_TYPE, @@ -38,6 +39,7 @@ interface Dependencies { export interface RecalledEntry { id: string; + doc_id: string; text: string; score: number | null; is_correction?: boolean; @@ -368,13 +370,13 @@ export class KnowledgeBaseService { }; const response = await this.dependencies.esClient.asInternalUser.search< - Pick + Pick >({ index: [resourceNames.aliases.kb], query: esQuery, size: 20, _source: { - includes: ['text', 'is_correction', 'labels'], + includes: ['text', 'is_correction', 'labels', 'doc_id'], }, }); @@ -598,27 +600,68 @@ export class KnowledgeBaseService { return res.hits.hits[0]?._source?.doc_id; }; + getUuidFromHumanReadableId = async ({ + docId, + user, + namespace, + }: { + docId: string; + user?: { name: string; id?: string }; + namespace?: string; + }) => { + const query = { + bool: { + filter: [ + { term: { doc_id: docId } }, + + // exclude user instructions + { bool: { must_not: { term: { type: KnowledgeBaseType.UserInstruction } } } }, + + // restrict access to user's own entries + ...getAccessQuery({ user, namespace }), + ], + }, + }; + + const response = await this.dependencies.esClient.asInternalUser.search({ + size: 1, + index: resourceNames.aliases.kb, + query, + _source: false, + }); + + const id = response.hits.hits[0]?._id ?? v4(); + + return id; + }; + addEntry = async ({ - entry: { id, ...document }, + entry, user, namespace, }: { - entry: Omit; + entry: Omit; user?: { name: string; id?: string }; namespace?: string; }): Promise => { + let id = ''; + // for now we want to limit the number of user instructions to 1 per user - if (document.type === KnowledgeBaseType.UserInstruction) { + if (entry.type === KnowledgeBaseType.UserInstruction) { const existingId = await this.getExistingUserInstructionId({ - isPublic: document.public, + isPublic: entry.public, user, namespace, }); if (existingId) { id = existingId; - document.doc_id = existingId; + entry.doc_id = existingId; } + + // override previous id if it exists + } else { + id = await this.getUuidFromHumanReadableId({ docId: entry.doc_id, user, namespace }); } try { @@ -627,7 +670,7 @@ export class KnowledgeBaseService { id, document: { '@timestamp': new Date().toISOString(), - ...document, + ...entry, user, namespace, }, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts index 8885ff7e1d7a2..a285ae85890c4 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts @@ -7,13 +7,18 @@ import type { Logger } from '@kbn/logging'; import { AnalyticsServiceStart } from '@kbn/core/server'; +import { scoreSuggestions } from './score_suggestions'; import type { Message } from '../../../common'; import type { ObservabilityAIAssistantClient } from '../../service/client'; import type { FunctionCallChatFunction } from '../../service/types'; -import { retrieveSuggestions } from './retrieve_suggestions'; -import { scoreSuggestions } from './score_suggestions'; -import type { RetrievedSuggestion } from './types'; -import { RecallRanking, RecallRankingEventType } from '../../analytics/recall_ranking'; +import { RecallRanking, recallRankingEventType } from '../../analytics/recall_ranking'; + +export interface RecalledSuggestion { + id: string; + docId: string; + text: string; + score: number | null; +} export async function recallAndScore({ recall, @@ -34,19 +39,24 @@ export async function recallAndScore({ logger: Logger; signal: AbortSignal; }): Promise<{ - relevantDocuments?: RetrievedSuggestion[]; + relevantDocuments?: RecalledSuggestion[]; scores?: Array<{ id: string; score: number }>; - suggestions: RetrievedSuggestion[]; + suggestions: RecalledSuggestion[]; }> { const queries = [ { text: userPrompt, boost: 3 }, { text: context, boost: 1 }, ].filter((query) => query.text.trim()); - const suggestions = await retrieveSuggestions({ - recall, - queries, - }); + const { entries: recalledEntries } = await recall({ queries }); + const suggestions: RecalledSuggestion[] = recalledEntries.map( + ({ id, doc_id: docId, text, score }) => ({ + id, + docId, + text, + score, + }) + ); if (!suggestions.length) { return { @@ -67,7 +77,7 @@ export async function recallAndScore({ chat, }); - analytics.reportEvent(RecallRankingEventType, { + analytics.reportEvent(recallRankingEventType, { prompt: queries.map((query) => query.text).join('\n\n'), scoredDocuments: suggestions.map((suggestion) => { const llmScore = scores.find((score) => score.id === suggestion.id); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/retrieve_suggestions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/retrieve_suggestions.ts deleted file mode 100644 index 3c680229cd5d2..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/retrieve_suggestions.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { omit } from 'lodash'; -import { ObservabilityAIAssistantClient } from '../../service/client'; -import { RetrievedSuggestion } from './types'; - -export async function retrieveSuggestions({ - queries, - recall, -}: { - queries: Array<{ text: string; boost?: number }>; - recall: ObservabilityAIAssistantClient['recall']; -}): Promise { - const recallResponse = await recall({ - queries, - }); - - return recallResponse.entries.map((entry) => omit(entry, 'labels', 'is_correction')); -} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts index 009b91a7a8c2c..1f35986bae8f0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts @@ -5,16 +5,14 @@ * 2.0. */ import * as t from 'io-ts'; -import { omit } from 'lodash'; import { Logger } from '@kbn/logging'; import dedent from 'dedent'; import { lastValueFrom } from 'rxjs'; import { decodeOrThrow, jsonRt } from '@kbn/io-ts-utils'; import { concatenateChatCompletionChunks, Message, MessageRole } from '../../../common'; import type { FunctionCallChatFunction } from '../../service/types'; -import type { RetrievedSuggestion } from './types'; import { parseSuggestionScores } from './parse_suggestion_scores'; -import { ShortIdTable } from '../../../common/utils/short_id_table'; +import { RecalledSuggestion } from './recall_and_score'; const scoreFunctionRequestRt = t.type({ message: t.type({ @@ -38,7 +36,7 @@ export async function scoreSuggestions({ signal, logger, }: { - suggestions: RetrievedSuggestion[]; + suggestions: RecalledSuggestion[]; messages: Message[]; userPrompt: string; context: string; @@ -46,17 +44,9 @@ export async function scoreSuggestions({ signal: AbortSignal; logger: Logger; }): Promise<{ - relevantDocuments: RetrievedSuggestion[]; + relevantDocuments: RecalledSuggestion[]; scores: Array<{ id: string; score: number }>; }> { - const shortIdTable = new ShortIdTable(); - - const suggestionsWithShortId = suggestions.map((suggestion) => ({ - ...omit(suggestion, 'score', 'id'), // To not bias the LLM - originalId: suggestion.id, - shortId: shortIdTable.take(suggestion.id), - })); - const newUserMessageContent = dedent(`Given the following question, score the documents that are relevant to the question. on a scale from 0 to 7, 0 being completely irrelevant, and 7 being extremely relevant. Information is relevant to the question if it helps in @@ -76,10 +66,7 @@ export async function scoreSuggestions({ Documents: ${JSON.stringify( - suggestionsWithShortId.map((suggestion) => ({ - id: suggestion.shortId, - content: suggestion.text, - })), + suggestions.map(({ id, docId: title, text }) => ({ id, title, text })), null, 2 )}`); @@ -127,15 +114,7 @@ export async function scoreSuggestions({ scoreFunctionRequest.message.function_call.arguments ); - const scores = parseSuggestionScores(scoresAsString).map(({ id, score }) => { - const originalSuggestion = suggestionsWithShortId.find( - (suggestion) => suggestion.shortId === id - ); - return { - originalId: originalSuggestion?.originalId, - score, - }; - }); + const scores = parseSuggestionScores(scoresAsString); if (scores.length === 0) { // seemingly invalid or no scores, return all @@ -145,11 +124,11 @@ export async function scoreSuggestions({ const suggestionIds = suggestions.map((document) => document.id); const relevantDocumentIds = scores - .filter((document) => suggestionIds.includes(document.originalId ?? '')) // Remove hallucinated documents + .filter((document) => suggestionIds.includes(document.id ?? '')) // Remove hallucinated documents .filter((document) => document.score > 4) .sort((a, b) => b.score - a.score) .slice(0, 5) - .map((document) => document.originalId); + .map((document) => document.id); const relevantDocuments = suggestions.filter((suggestion) => relevantDocumentIds.includes(suggestion.id) @@ -159,6 +138,6 @@ export async function scoreSuggestions({ return { relevantDocuments, - scores: scores.map((score) => ({ id: score.originalId!, score: score.score })), + scores: scores.map((score) => ({ id: score.id, score: score.score })), }; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/types.ts deleted file mode 100644 index 3774df64c1ee1..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/types.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { RecalledEntry } from '../../service/knowledge_base_service'; - -export type RetrievedSuggestion = Omit; From ede959371a3af0da70159be17e5cbf2d4279c5b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 22 Aug 2024 16:15:40 +0200 Subject: [PATCH 02/42] Fix issues --- .../common/types.ts | 9 +- .../server/functions/summarize.ts | 10 +- .../server/routes/chat/route.ts | 2 +- .../server/routes/functions/route.ts | 7 +- .../server/routes/knowledge_base/route.ts | 17 +++- .../server/service/client/index.ts | 38 +++++++- .../server/service/index.ts | 2 +- .../server/service/kb_component_template.ts | 1 + .../service/knowledge_base_service/index.ts | 96 +++++++++---------- .../server/service/types.ts | 10 +- ...t_system_message_from_instructions.test.ts | 6 +- .../get_system_message_from_instructions.ts | 9 +- .../server/utils/recall/recall_and_score.ts | 15 +-- .../public/helpers/categorize_entries.ts | 2 +- .../hooks/use_create_knowledge_base_entry.ts | 4 +- ..._create_knowledge_base_user_instruction.ts | 4 +- ...nowledge_base_edit_manual_entry_flyout.tsx | 53 +++++----- ...edge_base_edit_user_instruction_flyout.tsx | 6 +- .../routes/components/knowledge_base_tab.tsx | 6 +- .../knowledge_base/knowledge_base.spec.ts | 9 +- 20 files changed, 171 insertions(+), 135 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts index 38b7460c29ae1..6155e5443b4c1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts @@ -79,8 +79,9 @@ export type ConversationUpdateRequest = ConversationRequestBase & { export interface KnowledgeBaseEntry { '@timestamp': string; - id: string; // this is a unique ID generated by the client - doc_id: string; // this is the human readable ID generated by the LLM and used by the LLM to update existing entries + id: string; // unique ID + doc_id?: string; // human readable ID generated by the LLM and used by the LLM to lookup and update existing entries. TODO: rename `doc_id` to `lookup_id` + title?: string; text: string; confidence: 'low' | 'medium' | 'high'; is_correction: boolean; @@ -94,12 +95,12 @@ export interface KnowledgeBaseEntry { } export interface Instruction { - doc_id: string; + id: string; text: string; } export interface AdHocInstruction { - doc_id?: string; + id?: string; text: string; instruction_type: 'user_instruction' | 'application_instruction'; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index 74f8e5730a38f..951b1fc716730 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { v4 } from 'uuid'; import { KnowledgeBaseType } from '../../common/types'; import type { FunctionRegistrationParameters } from '.'; import { KnowledgeBaseEntryRole } from '../../common'; @@ -31,7 +32,7 @@ export function registerSummarizationFunction({ id: { type: 'string', description: - 'An id for the document. This should be a short human-readable keyword field with only alphabetic characters and underscores, that allow you to update it later.', + 'A lookup id for the document. This should be a short human-readable keyword field with only alphabetic characters and underscores, that allow you to find and update it later.', }, text: { type: 'string', @@ -62,13 +63,18 @@ export function registerSummarizationFunction({ ], }, }, - ( + async ( { arguments: { id: docId, text, is_correction: isCorrection, confidence, public: isPublic } }, signal ) => { + // The LLM should be able to update an existing entry by providing the same doc_id + // if no id is provided, we generate a new one + const id = await client.getUuidFromDocId(docId); + return client .addKnowledgeBaseEntry({ entry: { + id: id ?? v4(), doc_id: docId, role: KnowledgeBaseEntryRole.AssistantSummarization, text, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts index 73d8b0d1354e0..c18c4e74967b4 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts @@ -42,7 +42,7 @@ const chatCompleteBaseRt = t.type({ ]), instructions: t.array( t.intersection([ - t.partial({ doc_id: t.string }), + t.partial({ id: t.string }), t.type({ text: t.string, instruction_type: t.union([ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index 854aeca17833d..ed458c01b5558 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -108,6 +108,7 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ params: t.type({ body: t.type({ id: t.string, + doc_id: t.string, text: nonEmptyStringRt, confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), is_correction: toBooleanRt, @@ -127,8 +128,9 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ } const { - confidence, id, + doc_id: docId, + confidence, is_correction: isCorrection, type, text, @@ -139,7 +141,8 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ return client.addKnowledgeBaseEntry({ entry: { confidence, - doc_id: id, + id, + doc_id: docId, is_correction: isCorrection, type, text, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts index c4b8390fbc0f5..3ed206d68451c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts @@ -12,6 +12,7 @@ import type { import { notImplemented } from '@hapi/boom'; import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; +import { v4 } from 'uuid'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; import { Instruction, @@ -107,9 +108,9 @@ const saveKnowledgeBaseUserInstruction = createObservabilityAIAssistantServerRou } const { id, text, public: isPublic } = resources.params.body; - return client.addKnowledgeBaseEntry({ + return client.addUserInstruction({ entry: { - doc_id: id, + id, text, public: isPublic, confidence: 'high', @@ -157,9 +158,11 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ body: t.intersection([ t.type({ id: t.string, + title: t.string, text: nonEmptyStringRt, }), t.partial({ + doc_id: t.string, confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), is_correction: toBooleanRt, public: toBooleanRt, @@ -184,6 +187,8 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ const { id, + doc_id: docId, + title, text, public: isPublic, confidence, @@ -195,7 +200,9 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ return client.addKnowledgeBaseEntry({ entry: { text, - doc_id: id, + id, + doc_id: docId, + title, confidence: confidence ?? 'high', is_correction: isCorrection ?? false, type: 'contextual', @@ -234,7 +241,7 @@ const importKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ body: t.type({ entries: t.array( t.type({ - id: t.string, + doc_id: t.string, text: nonEmptyStringRt, }) ), @@ -251,7 +258,7 @@ const importKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ } const entries = resources.params.body.entries.map((entry) => ({ - doc_id: entry.id, + id: v4(), confidence: 'high' as KnowledgeBaseEntry['confidence'], is_correction: false, type: 'contextual' as const, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 86b02e9b9c628..db0fb3521bc2a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -731,10 +731,44 @@ export class ObservabilityAIAssistantClient { return this.dependencies.knowledgeBaseService.setup(); }; + getUuidFromDocId = async (docId: string) => { + return this.dependencies.knowledgeBaseService.getUuidFromDocId({ + namespace: this.dependencies.namespace, + user: this.dependencies.user, + docId, + }); + }; + + addUserInstruction = async ({ + entry, + }: { + entry: Omit; + }): Promise => { + // for now we want to limit the number of user instructions to 1 per user + // if a user instruction already exists for the user, we get the id and update it + this.dependencies.logger.debug('Adding user instruction entry'); + const existingId = await this.dependencies.knowledgeBaseService.getPersonalUserInstructionId({ + isPublic: entry.public, + namespace: this.dependencies.namespace, + user: this.dependencies.user, + }); + + if (existingId) { + entry.id = existingId; + this.dependencies.logger.debug(`Updating user instruction with id "${existingId}"`); + } + + return this.dependencies.knowledgeBaseService.addEntry({ + namespace: this.dependencies.namespace, + user: this.dependencies.user, + entry, + }); + }; + addKnowledgeBaseEntry = async ({ entry, }: { - entry: Omit; + entry: Omit; }): Promise => { return this.dependencies.knowledgeBaseService.addEntry({ namespace: this.dependencies.namespace, @@ -753,7 +787,7 @@ export class ObservabilityAIAssistantClient { document: { ...entry, '@timestamp': new Date().toISOString() }, })); - await this.dependencies.knowledgeBaseService.addEntries({ operations }); + await this.dependencies.knowledgeBaseService.importEntries({ operations }); }; getKnowledgeBaseEntries = async ({ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts index c087e5940f0b7..002847e9349e0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts @@ -330,8 +330,8 @@ export class ObservabilityAIAssistantService { entries.flatMap((entry) => { const entryWithSystemProperties = { ...entry, - '@timestamp': new Date().toISOString(), doc_id: entry.id, + '@timestamp': new Date().toISOString(), public: true, confidence: 'high' as const, type: 'contextual' as const, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts index a4c6dc25d2e57..5467556a0e3ab 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts @@ -32,6 +32,7 @@ export const kbComponentTemplate: ClusterComponentTemplate['component_template'] '@timestamp': date, id: keyword, doc_id: { type: 'text', fielddata: true }, + title: { type: 'text', fielddata: true }, user: { properties: { id: keyword, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index a7b47a4d5a45f..c1ade1d4f6be4 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -14,7 +14,6 @@ import pRetry from 'p-retry'; import { map, orderBy } from 'lodash'; import { encode } from 'gpt-tokenizer'; import { MlTrainedModelDeploymentNodesStats } from '@elastic/elasticsearch/lib/api/types'; -import { v4 } from 'uuid'; import { INDEX_QUEUED_DOCUMENTS_TASK_ID, INDEX_QUEUED_DOCUMENTS_TASK_TYPE, @@ -39,7 +38,7 @@ interface Dependencies { export interface RecalledEntry { id: string; - doc_id: string; + doc_id?: string; text: string; score: number | null; is_correction?: boolean; @@ -229,27 +228,39 @@ export class KnowledgeBaseService { private async processOperation(operation: KnowledgeBaseEntryOperation) { if (operation.type === KnowledgeBaseEntryOperationType.Delete) { - await this.dependencies.esClient.asInternalUser.deleteByQuery({ - index: resourceNames.aliases.kb, - query: { - bool: { - filter: [ - ...(operation.doc_id ? [{ term: { _id: operation.doc_id } }] : []), - ...(operation.labels - ? map(operation.labels, (value, key) => { - return { term: { [key]: value } }; - }) - : []), - ], + try { + await this.dependencies.esClient.asInternalUser.deleteByQuery({ + index: resourceNames.aliases.kb, + query: { + bool: { + filter: [ + ...(operation.doc_id ? [{ term: { _id: operation.doc_id } }] : []), + ...(operation.labels + ? map(operation.labels, (value, key) => { + return { term: { [key]: value } }; + }) + : []), + ], + }, }, - }, - }); - return; + }); + return; + } catch (error) { + this.dependencies.logger.error( + `Failed to delete document "${operation?.doc_id}" due to ${error.message}` + ); + this.dependencies.logger.debug(() => JSON.stringify(operation)); + throw error; + } } - await this.addEntry({ - entry: operation.document, - }); + try { + await this.addEntry({ entry: operation.document }); + } catch (error) { + this.dependencies.logger.error(`Failed to index document due to ${error.message}`); + this.dependencies.logger.debug(() => JSON.stringify(operation.document)); + throw error; + } } async processQueue() { @@ -492,11 +503,11 @@ export class KnowledgeBaseService { }, }, size: 500, - _source: ['doc_id', 'text', 'public'], + _source: ['id', 'text', 'public'], }); return response.hits.hits.map((hit) => ({ - doc_id: hit._source?.doc_id ?? '', + id: hit._id!, text: hit._source?.text ?? '', public: hit._source?.public, })); @@ -541,6 +552,7 @@ export class KnowledgeBaseService { size: 500, _source: { includes: [ + 'title', 'doc_id', 'text', 'is_correction', @@ -571,7 +583,7 @@ export class KnowledgeBaseService { } }; - getExistingUserInstructionId = async ({ + getPersonalUserInstructionId = async ({ isPublic, user, namespace, @@ -594,13 +606,13 @@ export class KnowledgeBaseService { }, }, size: 1, - _source: ['doc_id'], + _source: false, }); - return res.hits.hits[0]?._source?.doc_id; + return res.hits.hits[0]?._id; }; - getUuidFromHumanReadableId = async ({ + getUuidFromDocId = async ({ docId, user, namespace, @@ -630,47 +642,25 @@ export class KnowledgeBaseService { _source: false, }); - const id = response.hits.hits[0]?._id ?? v4(); - - return id; + return response.hits.hits[0]?._id; }; addEntry = async ({ - entry, + entry: { id, ...doc }, user, namespace, }: { - entry: Omit; + entry: Omit; user?: { name: string; id?: string }; namespace?: string; }): Promise => { - let id = ''; - - // for now we want to limit the number of user instructions to 1 per user - if (entry.type === KnowledgeBaseType.UserInstruction) { - const existingId = await this.getExistingUserInstructionId({ - isPublic: entry.public, - user, - namespace, - }); - - if (existingId) { - id = existingId; - entry.doc_id = existingId; - } - - // override previous id if it exists - } else { - id = await this.getUuidFromHumanReadableId({ docId: entry.doc_id, user, namespace }); - } - try { await this.dependencies.esClient.asInternalUser.index({ index: resourceNames.aliases.kb, id, document: { '@timestamp': new Date().toISOString(), - ...entry, + ...doc, user, namespace, }, @@ -685,7 +675,7 @@ export class KnowledgeBaseService { } }; - addEntries = async ({ + importEntries = async ({ operations, }: { operations: KnowledgeBaseEntryOperation[]; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts index cd8e25843ca59..f73620257317d 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts @@ -13,11 +13,7 @@ import type { FunctionDefinition, FunctionResponse, } from '../../common/functions/types'; -import type { - Message, - ObservabilityAIAssistantScreenContextRequest, - InstructionOrPlainText, -} from '../../common/types'; +import type { Message, ObservabilityAIAssistantScreenContextRequest } from '../../common/types'; import type { ObservabilityAIAssistantRouteHandlerResources } from '../routes/types'; import { ChatFunctionClient } from './chat_function_client'; import type { ObservabilityAIAssistantClient } from './client'; @@ -63,13 +59,13 @@ export interface FunctionHandler { respond: RespondFunction; } -export type InstructionOrCallback = InstructionOrPlainText | RegisterInstructionCallback; +export type InstructionOrCallback = string | RegisterInstructionCallback; type RegisterInstructionCallback = ({ availableFunctionNames, }: { availableFunctionNames: string[]; -}) => InstructionOrPlainText | InstructionOrPlainText[] | undefined; +}) => string | string[] | undefined; export type RegisterInstruction = (...instructions: InstructionOrCallback[]) => void; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts index 8e4075bed7b9d..82f22e08d1fc7 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts @@ -41,10 +41,10 @@ describe('getSystemMessageFromInstructions', () => { expect( getSystemMessageFromInstructions({ applicationInstructions: ['first'], - userInstructions: [{ doc_id: 'second', text: 'second from kb' }], + userInstructions: [{ id: 'second', text: 'second from kb' }], adHocInstructions: [ { - doc_id: 'second', + id: 'second', text: 'second from adhoc instruction', instruction_type: 'user_instruction', }, @@ -58,7 +58,7 @@ describe('getSystemMessageFromInstructions', () => { expect( getSystemMessageFromInstructions({ applicationInstructions: ['first'], - userInstructions: [{ doc_id: 'second', text: 'second_kb' }], + userInstructions: [{ id: 'second', text: 'second_kb' }], adHocInstructions: [], availableFunctionNames: [], }) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts index b2797577883ba..468479c2ed005 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts @@ -45,7 +45,7 @@ export function getSystemMessageFromInstructions({ const adHocInstructionsWithId = adHocInstructions.map((adHocInstruction) => ({ ...adHocInstruction, - doc_id: adHocInstruction.doc_id ?? v4(), + id: adHocInstruction.id ?? v4(), })); // split ad hoc instructions into user instructions and application instructions @@ -55,15 +55,16 @@ export function getSystemMessageFromInstructions({ ); // all adhoc instructions and KB instructions. - // adhoc instructions will be prioritized over Knowledge Base instructions if the doc_id is the same + // adhoc instructions will be prioritized over Knowledge Base instructions if the id is the same const allUserInstructions = withTokenBudget( - uniqBy([...adHocUserInstructions, ...kbUserInstructions], (i) => i.doc_id), + uniqBy([...adHocUserInstructions, ...kbUserInstructions], (i) => i.id), 1000 ); return [ // application instructions - ...allApplicationInstructions.concat(adHocApplicationInstructions), + ...allApplicationInstructions, + ...adHocApplicationInstructions, // user instructions ...(allUserInstructions.length ? [USER_INSTRUCTIONS_HEADER, ...allUserInstructions] : []), diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts index a285ae85890c4..77b7256e017ea 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts @@ -12,13 +12,9 @@ import type { Message } from '../../../common'; import type { ObservabilityAIAssistantClient } from '../../service/client'; import type { FunctionCallChatFunction } from '../../service/types'; import { RecallRanking, recallRankingEventType } from '../../analytics/recall_ranking'; +import { RecalledEntry } from '../../service/knowledge_base_service'; -export interface RecalledSuggestion { - id: string; - docId: string; - text: string; - score: number | null; -} +export type RecalledSuggestion = Pick; export async function recallAndScore({ recall, @@ -50,12 +46,7 @@ export async function recallAndScore({ const { entries: recalledEntries } = await recall({ queries }); const suggestions: RecalledSuggestion[] = recalledEntries.map( - ({ id, doc_id: docId, text, score }) => ({ - id, - docId, - text, - score, - }) + ({ id, doc_id: docId, text, score }) => ({ id, doc_id: docId, text, score }) ); if (!suggestions.length) { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts index 8d32ea4664f9f..04642910708c3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts @@ -15,7 +15,7 @@ export interface KnowledgeBaseEntryCategory { export function categorizeEntries({ entries }: { entries: KnowledgeBaseEntry[] }) { return entries.reduce((acc, entry) => { - const categoryName = entry.labels?.category ?? entry.id; + const categoryName = entry.labels?.category ?? entry.title ?? entry.doc_id ?? entry.id; const index = acc.findIndex((item) => item.categoryName === categoryName); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts index 459de7be2d528..41126264f9011 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts @@ -30,8 +30,8 @@ export function useCreateKnowledgeBaseEntry() { { entry: Omit< KnowledgeBaseEntry, - '@timestamp' | 'confidence' | 'is_correction' | 'role' | 'doc_id' - >; + '@timestamp' | 'confidence' | 'is_correction' | 'role' | 'title' + > & { title: string }; } >( [REACT_QUERY_KEYS.CREATE_KB_ENTRIES], diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_user_instruction.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_user_instruction.ts index b51e45a3bdd6b..8adf5a7d8cfd0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_user_instruction.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_user_instruction.ts @@ -32,7 +32,7 @@ export function useCreateKnowledgeBaseUserInstruction() { signal: null, params: { body: { - id: entry.doc_id, + id: entry.id, text: entry.text, public: entry.public, }, @@ -62,7 +62,7 @@ export function useCreateKnowledgeBaseUserInstruction() { 'xpack.observabilityAiAssistantManagement.kb.addUserInstruction.errorNotification', { defaultMessage: 'Something went wrong while creating {name}', - values: { name: entry.doc_id }, + values: { name: entry.id }, } ), }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx index d809b6cd96d6d..21d92de190325 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx @@ -27,6 +27,7 @@ import { } from '@elastic/eui'; import moment from 'moment'; import type { KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common/types'; +import { v4 } from 'uuid'; import { useCreateKnowledgeBaseEntry } from '../../hooks/use_create_knowledge_base_entry'; import { useDeleteKnowledgeBaseEntry } from '../../hooks/use_delete_knowledge_base_entry'; import { useKibana } from '../../hooks/use_kibana'; @@ -45,18 +46,19 @@ export function KnowledgeBaseEditManualEntryFlyout({ const { mutateAsync: deleteEntry, isLoading: isDeleting } = useDeleteKnowledgeBaseEntry(); const [isPublic, setIsPublic] = useState(entry?.public ?? false); - - const [newEntryId, setNewEntryId] = useState(entry?.id ?? ''); + const [newEntryTitle, setNewEntryTitle] = useState(entry?.title ?? entry?.doc_id ?? ''); const [newEntryText, setNewEntryText] = useState(entry?.text ?? ''); - const isEntryIdInvalid = newEntryId.trim() === ''; + const isEntryIdInvalid = newEntryTitle.trim() === ''; const isEntryTextInvalid = newEntryText.trim() === ''; const isFormInvalid = isEntryIdInvalid || isEntryTextInvalid; const handleSubmit = async () => { await createEntry({ entry: { - id: newEntryId, + id: entry?.id ?? v4(), + doc_id: entry?.doc_id, + title: newEntryTitle, text: newEntryText, public: isPublic, }, @@ -85,8 +87,8 @@ export function KnowledgeBaseEditManualEntryFlyout({ : i18n.translate( 'xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.editEntryLabel', { - defaultMessage: 'Edit {id}', - values: { id: entry.id }, + defaultMessage: 'Edit {title}', + values: { title: entry?.title ?? entry?.doc_id }, } )} @@ -94,23 +96,7 @@ export function KnowledgeBaseEditManualEntryFlyout({ - {!entry ? ( - - setNewEntryId(e.target.value)} - isInvalid={isEntryIdInvalid} - /> - - ) : ( + {entry ? ( @@ -136,7 +122,26 @@ export function KnowledgeBaseEditManualEntryFlyout({ - )} + ) : null} + + + + + setNewEntryTitle(e.target.value)} + isInvalid={isEntryIdInvalid} + /> + + diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx index 8b12f842bf128..be23f3e550b47 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx @@ -22,7 +22,7 @@ import { EuiText, EuiTitle, } from '@elastic/eui'; -import { v4 as uuidv4 } from 'uuid'; +import { v4 } from 'uuid'; import { useGetUserInstructions } from '../../hooks/use_get_user_instructions'; import { useCreateKnowledgeBaseUserInstruction } from '../../hooks/use_create_knowledge_base_user_instruction'; @@ -30,19 +30,17 @@ export function KnowledgeBaseEditUserInstructionFlyout({ onClose }: { onClose: ( const { userInstructions, isLoading: isFetching } = useGetUserInstructions(); const { mutateAsync: createEntry, isLoading: isSaving } = useCreateKnowledgeBaseUserInstruction(); const [newEntryText, setNewEntryText] = useState(''); - const [newEntryDocId, setNewEntryDocId] = useState(); const isSubmitDisabled = newEntryText.trim() === ''; useEffect(() => { const userInstruction = userInstructions?.find((entry) => !entry.public); - setNewEntryDocId(userInstruction?.doc_id); setNewEntryText(userInstruction?.text ?? ''); }, [userInstructions]); const handleSubmit = async () => { await createEntry({ entry: { - doc_id: newEntryDocId ?? uuidv4(), + id: v4(), text: newEntryText, public: false, // limit user instructions to private (for now) }, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx index de27f26f2561c..567d22ee0d9e6 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx @@ -183,7 +183,7 @@ export function KnowledgeBaseTab() { const [isNewEntryPopoverOpen, setIsNewEntryPopoverOpen] = useState(false); const [isEditUserInstructionFlyoutOpen, setIsEditUserInstructionFlyoutOpen] = useState(false); const [query, setQuery] = useState(''); - const [sortBy, setSortBy] = useState<'doc_id' | '@timestamp'>('doc_id'); + const [sortBy, setSortBy] = useState<'title' | '@timestamp'>('title'); const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('asc'); const { @@ -202,7 +202,7 @@ export function KnowledgeBaseTab() { setSortBy(field); } if (field === 'categoryName') { - setSortBy('doc_id'); + setSortBy('title'); } setSortDirection(direction); } @@ -329,7 +329,7 @@ export function KnowledgeBaseTab() { loading={isLoading} sorting={{ sort: { - field: sortBy === 'doc_id' ? 'categoryName' : sortBy, + field: sortBy === 'title' ? 'categoryName' : sortBy, direction: sortDirection, }, }} diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index c8881e82e43bb..88f05deaf7615 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -35,6 +35,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('when managing a single entry', () => { const knowledgeBaseEntry = { id: 'my-doc-id-1', + title: 'My title', text: 'My content', }; it('returns 200 on create', async () => { @@ -56,6 +57,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); const entry = res.body.entries[0]; expect(entry.id).to.equal(knowledgeBaseEntry.id); + expect(entry.title).to.equal(knowledgeBaseEntry.title); expect(entry.text).to.equal(knowledgeBaseEntry.text); }); @@ -74,6 +76,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); const entry = res.body.entries[0]; expect(entry.id).to.equal(knowledgeBaseEntry.id); + expect(entry.title).to.equal(knowledgeBaseEntry.title); expect(entry.text).to.equal(knowledgeBaseEntry.text); }); @@ -126,15 +129,15 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); const knowledgeBaseEntries = [ { - id: 'my_doc_a', + doc_id: 'my_doc_a', text: 'My content a', }, { - id: 'my_doc_b', + doc_id: 'my_doc_b', text: 'My content b', }, { - id: 'my_doc_c', + doc_id: 'my_doc_c', text: 'My content c', }, ]; From 4f5cb9444ca166e6ca0e35c02fc91d879c6d705c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Mon, 26 Aug 2024 12:24:48 +0200 Subject: [PATCH 03/42] Fix user instruction test --- .../server/routes/functions/route.ts | 12 ++- .../common/config.ts | 9 ++- .../common/users/users.ts | 10 ++- .../knowledge_base/knowledge_base.spec.ts | 79 +++++++++++++++++++ .../knowledge_base_user_instructions.spec.ts | 65 +++++++-------- .../common/config.ts | 8 +- 6 files changed, 131 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index ed458c01b5558..f00a2ddb1f483 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -7,8 +7,9 @@ import { notImplemented } from '@hapi/boom'; import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; +import { v4 } from 'uuid'; import { FunctionDefinition } from '../../../common/functions/types'; -import { KnowledgeBaseEntryRole } from '../../../common/types'; +import { KnowledgeBaseEntryRole, KnowledgeBaseType } from '../../../common/types'; import type { RecalledEntry } from '../../service/knowledge_base_service'; import { getSystemMessageFromInstructions } from '../../service/util/get_system_message_from_instructions'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; @@ -107,12 +108,10 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', params: t.type({ body: t.type({ - id: t.string, doc_id: t.string, text: nonEmptyStringRt, confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), is_correction: toBooleanRt, - type: t.union([t.literal('user_instruction'), t.literal('contextual')]), public: toBooleanRt, labels: t.record(t.string, t.string), }), @@ -128,23 +127,22 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ } const { - id, doc_id: docId, confidence, is_correction: isCorrection, - type, text, public: isPublic, labels, } = resources.params.body; + const id = await client.getUuidFromDocId(docId); return client.addKnowledgeBaseEntry({ entry: { confidence, - id, + id: id ?? v4(), doc_id: docId, is_correction: isCorrection, - type, + type: KnowledgeBaseType.Contextual, text, public: isPublic, labels, diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts index 198fcefdc2bc8..0a2a4c796eb5f 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts @@ -11,7 +11,7 @@ import { ObservabilityAIAssistantFtrConfigName } from '../configs'; import { getApmSynthtraceEsClient } from './create_synthtrace_client'; import { InheritedFtrProviderContext, InheritedServices } from './ftr_provider_context'; import { getScopedApiClient } from './observability_ai_assistant_api_client'; -import { editorUser, viewerUser } from './users/users'; +import { editorUser, secondaryEditorUser, viewerUser } from './users/users'; export interface ObservabilityAIAssistantFtrConfig { name: ObservabilityAIAssistantFtrConfigName; @@ -58,9 +58,10 @@ export function createObservabilityAIAssistantAPIConfig({ getApmSynthtraceEsClient(context, apmSynthtraceKibanaClient), observabilityAIAssistantAPIClient: async () => { return { - adminUser: await getScopedApiClient(kibanaServer, 'elastic'), - viewerUser: await getScopedApiClient(kibanaServer, viewerUser.username), - editorUser: await getScopedApiClient(kibanaServer, editorUser.username), + adminUser: getScopedApiClient(kibanaServer, 'elastic'), + viewerUser: getScopedApiClient(kibanaServer, viewerUser.username), + editorUser: getScopedApiClient(kibanaServer, editorUser.username), + secondaryEditorUser: getScopedApiClient(kibanaServer, secondaryEditorUser.username), }; }, }, diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts b/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts index b6fa38e52e60b..bf76c5f770f69 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts @@ -9,7 +9,7 @@ import { kbnTestConfig } from '@kbn/test'; const password = kbnTestConfig.getUrlParts().password!; export interface User { - username: 'elastic' | 'editor' | 'viewer'; + username: 'elastic' | 'editor' | 'viewer' | 'secondary_editor'; password: string; roles: string[]; } @@ -20,10 +20,16 @@ export const editorUser: User = { roles: ['editor'], }; +export const secondaryEditorUser: User = { + username: 'secondary_editor', + password, + roles: ['editor'], +}; + export const viewerUser: User = { username: 'viewer', password, roles: ['viewer'], }; -export const allUsers = [editorUser, viewerUser]; +export const allUsers = [editorUser, secondaryEditorUser, viewerUser]; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index 88f05deaf7615..b9983e5e5c58c 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -32,6 +32,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); expect(res.body).to.eql({}); }); + describe('when managing a single entry', () => { const knowledgeBaseEntry = { id: 'my-doc-id-1', @@ -120,6 +121,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(500); }); }); + describe('when managing multiple entries', () => { before(async () => { await clearKnowledgeBase(es); @@ -234,5 +236,82 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(res.body.entries[0].id).to.eql('my_doc_a'); }); }); + + describe('When the LLM creates entries', () => { + before(async () => { + await clearKnowledgeBase(es); + }); + afterEach(async () => { + await clearKnowledgeBase(es); + }); + + it('can replace an existing entry using the `doc_id`', async () => { + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', + params: { + body: { + doc_id: 'my_doc_id', + text: 'My content', + confidence: 'high', + is_correction: false, + public: false, + labels: {}, + }, + }, + }) + .expect(200); + + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'doc_id', + sortDirection: 'asc', + }, + }, + }) + .expect(200); + + const id = res.body.entries[0].id; + expect(res.body.entries.length).to.eql(1); + expect(res.body.entries[0].text).to.eql('My content'); + + await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', + params: { + body: { + doc_id: 'my_doc_id', + text: 'My content_2', + confidence: 'high', + is_correction: false, + public: false, + labels: {}, + }, + }, + }) + .expect(200); + + const res2 = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'doc_id', + sortDirection: 'asc', + }, + }, + }) + .expect(200); + + expect(res2.body.entries.length).to.eql(1); + expect(res2.body.entries[0].text).to.eql('My content_2'); + expect(res2.body.entries[0].id).to.eql(id); + }); + }); }); } diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index 4cad8079dc0b2..c1ce378be79c4 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -6,7 +6,6 @@ */ import expect from '@kbn/expect'; -import { kbnTestConfig } from '@kbn/test'; import { sortBy } from 'lodash'; import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { CONTEXT_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/context'; @@ -23,20 +22,13 @@ import { createProxyActionConnector, deleteActionConnector } from '../../common/ export default function ApiTest({ getService }: FtrProviderContext) { const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); - const getScopedApiClientForUsername = getService('getScopedApiClientForUsername'); - const security = getService('security'); const supertest = getService('supertest'); const es = getService('es'); const ml = getService('ml'); const log = getService('log'); describe('Knowledge base user instructions', () => { - const userJohn = 'john'; - before(async () => { - // create user - const password = kbnTestConfig.getUrlParts().password!; - await security.user.create(userJohn, { password, roles: ['editor'] }); await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient @@ -46,7 +38,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { after(async () => { await deleteKnowledgeBaseModel(ml); - await security.user.delete(userJohn); await clearKnowledgeBase(es); await clearConversations(es); }); @@ -57,24 +48,26 @@ export default function ApiTest({ getService }: FtrProviderContext) { const promises = [ { - username: 'editor', + username: 'editorUser' as const, isPublic: true, }, { - username: 'editor', + username: 'editorUser' as const, isPublic: false, }, { - username: userJohn, + username: 'secondaryEditorUser' as const, isPublic: true, }, { - username: userJohn, + username: 'secondaryEditorUser' as const, isPublic: false, }, ].map(async ({ username, isPublic }) => { const visibility = isPublic ? 'Public' : 'Private'; - await getScopedApiClientForUsername(username)({ + + const apiClient = observabilityAIAssistantAPIClient[username]; + await apiClient({ endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions', params: { body: { @@ -89,7 +82,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await Promise.all(promises); }); - it('"editor" can retrieve their own private instructions and the public instruction', async () => { + it('"editorUser" can retrieve their own private instructions and the public instruction', async () => { const res = await observabilityAIAssistantAPIClient.editorUser({ endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions', }); @@ -99,26 +92,26 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(sortByDocId(instructions)).to.eql( sortByDocId([ { - doc_id: 'private-doc-from-editor', + id: 'private-doc-from-editorUser', public: false, - text: 'Private user instruction from "editor"', + text: 'Private user instruction from "editorUser"', }, { - doc_id: 'public-doc-from-editor', + id: 'public-doc-from-editorUser', public: true, - text: 'Public user instruction from "editor"', + text: 'Public user instruction from "editorUser"', }, { - doc_id: 'public-doc-from-john', + id: 'public-doc-from-secondaryEditorUser', public: true, - text: 'Public user instruction from "john"', + text: 'Public user instruction from "secondaryEditorUser"', }, ]) ); }); - it('"john" can retrieve their own private instructions and the public instruction', async () => { - const res = await getScopedApiClientForUsername(userJohn)({ + it('"secondaryEditorUser" can retrieve their own private instructions and the public instruction', async () => { + const res = await observabilityAIAssistantAPIClient.secondaryEditorUser({ endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions', }); const instructions = res.body.userInstructions; @@ -127,19 +120,19 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(sortByDocId(instructions)).to.eql( sortByDocId([ { - doc_id: 'public-doc-from-editor', + id: 'public-doc-from-editorUser', public: true, - text: 'Public user instruction from "editor"', + text: 'Public user instruction from "editorUser"', }, { - doc_id: 'public-doc-from-john', + id: 'public-doc-from-secondaryEditorUser', public: true, - text: 'Public user instruction from "john"', + text: 'Public user instruction from "secondaryEditorUser"', }, { - doc_id: 'private-doc-from-john', + id: 'private-doc-from-secondaryEditorUser', public: false, - text: 'Private user instruction from "john"', + text: 'Private user instruction from "secondaryEditorUser"', }, ]) ); @@ -185,7 +178,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(instructions).to.eql([ { - doc_id: 'doc-to-update', + id: 'doc-to-update', text: 'Updated text', public: false, }, @@ -200,10 +193,10 @@ export default function ApiTest({ getService }: FtrProviderContext) { const userInstructionText = 'Be polite and use language that is easy to understand. Never disagree with the user.'; - async function getConversationForUser(username: string) { - const apiClient = getScopedApiClientForUsername(username); + async function getConversationForUser(username: 'editorUser' | 'secondaryEditorUser') { + const apiClient = observabilityAIAssistantAPIClient[username]; - // the user instruction is always created by "editor" user + // the user instruction is always created by "editorUser" user await observabilityAIAssistantAPIClient .editorUser({ endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions', @@ -284,7 +277,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('adds the instruction to the system prompt', async () => { - const conversation = await getConversationForUser('editor'); + const conversation = await getConversationForUser('editorUser'); const systemMessage = conversation.messages.find( (message) => message.message.role === MessageRole.System )!; @@ -292,7 +285,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('does not add the instruction to the context', async () => { - const conversation = await getConversationForUser('editor'); + const conversation = await getConversationForUser('editorUser'); const contextMessage = conversation.messages.find( (message) => message.message.name === CONTEXT_FUNCTION_NAME ); @@ -306,7 +299,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('does not add the instruction conversation for other users', async () => { - const conversation = await getConversationForUser('john'); + const conversation = await getConversationForUser('secondaryEditorUser'); const systemMessage = conversation.messages.find( (message) => message.message.role === MessageRole.System )!; diff --git a/x-pack/test/observability_ai_assistant_functional/common/config.ts b/x-pack/test/observability_ai_assistant_functional/common/config.ts index c0f649d51d90d..bb320f46e3ac0 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/config.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/config.ts @@ -13,6 +13,7 @@ import { KibanaEBTUIProvider, } from '@kbn/test-suites-src/analytics/services/kibana_ebt'; import { + secondaryEditorUser, editorUser, viewerUser, } from '../../observability_ai_assistant_api_integration/common/users/users'; @@ -61,9 +62,10 @@ async function getTestConfig({ ObservabilityAIAssistantUIProvider(context), observabilityAIAssistantAPIClient: async (context: InheritedFtrProviderContext) => { return { - adminUser: await getScopedApiClient(kibanaServer, 'elastic'), - viewerUser: await getScopedApiClient(kibanaServer, viewerUser.username), - editorUser: await getScopedApiClient(kibanaServer, editorUser.username), + adminUser: getScopedApiClient(kibanaServer, 'elastic'), + viewerUser: getScopedApiClient(kibanaServer, viewerUser.username), + editorUser: getScopedApiClient(kibanaServer, editorUser.username), + secondaryEditorUser: getScopedApiClient(kibanaServer, secondaryEditorUser.username), }; }, kibana_ebt_server: KibanaEBTServerProvider, From 14854d2770202ab6a12240295ecee849f02ed0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 27 Aug 2024 20:37:22 +0200 Subject: [PATCH 04/42] Fix test --- .../server/functions/summarize.ts | 3 +- .../server/routes/knowledge_base/route.ts | 53 ++- .../server/service/index.ts | 57 ++- .../server/service/kb_component_template.ts | 10 +- .../service/knowledge_base_service/index.ts | 75 ++- .../server/service/util/split_kb_text.ts | 4 +- .../server/utils/recall/score_suggestions.ts | 2 +- .../components/knowledge_base_tab.test.tsx | 4 +- .../common/config.ts | 4 + .../complete/functions/summarize.spec.ts | 2 +- .../knowledge_base/knowledge_base.spec.ts | 444 ++++++++++++------ .../knowledge_base_user_instructions.spec.ts | 12 +- 12 files changed, 448 insertions(+), 222 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index 951b1fc716730..7d20cd927b42f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -68,13 +68,14 @@ export function registerSummarizationFunction({ signal ) => { // The LLM should be able to update an existing entry by providing the same doc_id - // if no id is provided, we generate a new one + // if no existing entry is found, we generate a uuid const id = await client.getUuidFromDocId(docId); return client .addKnowledgeBaseEntry({ entry: { id: id ?? v4(), + title: docId, // use doc_id as title for now doc_id: docId, role: KnowledgeBaseEntryRole.AssistantSummarization, text, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts index 3ed206d68451c..e3959ed35250f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts @@ -12,7 +12,6 @@ import type { import { notImplemented } from '@hapi/boom'; import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; -import { v4 } from 'uuid'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; import { Instruction, @@ -240,10 +239,15 @@ const importKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ params: t.type({ body: t.type({ entries: t.array( - t.type({ - doc_id: t.string, - text: nonEmptyStringRt, - }) + t.intersection([ + t.type({ + id: t.string, + text: nonEmptyStringRt, + }), + t.partial({ + title: t.string, + }), + ]) ), }), }), @@ -257,18 +261,48 @@ const importKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ throw notImplemented(); } - const entries = resources.params.body.entries.map((entry) => ({ - id: v4(), + const formattedEntries = resources.params.body.entries.map((entry) => ({ + id: entry.id, + title: entry.title, + text: entry.text, confidence: 'high' as KnowledgeBaseEntry['confidence'], is_correction: false, type: 'contextual' as const, public: true, labels: {}, role: KnowledgeBaseEntryRole.UserEntry, - ...entry, })); - return await client.importKnowledgeBaseEntries({ entries }); + return await client.importKnowledgeBaseEntries({ entries: formattedEntries }); + }, +}); + +const importKnowledgeBaseCategoryEntries = createObservabilityAIAssistantServerRoute({ + endpoint: 'POST /internal/observability_ai_assistant/kb/entries/category/import', + params: t.type({ + body: t.type({ + category: t.string, + entries: t.array( + t.type({ + id: t.string, + texts: t.array(t.string), + }) + ), + }), + }), + options: { + tags: ['access:ai_assistant'], + }, + handler: async (resources): Promise => { + const client = await resources.service.getClient({ request: resources.request }); + + if (!client) { + throw notImplemented(); + } + + const { entries, category } = resources.params.body; + + return resources.service.addCategoryToKnowledgeBase(category, entries); }, }); @@ -279,6 +313,7 @@ export const knowledgeBaseRoutes = { ...saveKnowledgeBaseUserInstruction, ...getKnowledgeBaseUserInstructions, ...importKnowledgeBaseEntries, + ...importKnowledgeBaseCategoryEntries, ...saveKnowledgeBaseEntry, ...deleteKnowledgeBaseEntry, }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts index 002847e9349e0..21126748c3372 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts @@ -326,35 +326,34 @@ export class ObservabilityAIAssistantService { addToKnowledgeBaseQueue(entries: KnowledgeBaseEntryRequest[]): void { this.init() .then(() => { - this.kbService!.queue( - entries.flatMap((entry) => { - const entryWithSystemProperties = { - ...entry, - doc_id: entry.id, - '@timestamp': new Date().toISOString(), - public: true, - confidence: 'high' as const, - type: 'contextual' as const, - is_correction: false, - labels: { - ...entry.labels, - }, - role: KnowledgeBaseEntryRole.Elastic, - }; - - const operations = - 'texts' in entryWithSystemProperties - ? splitKbText(entryWithSystemProperties) - : [ - { - type: KnowledgeBaseEntryOperationType.Index, - document: entryWithSystemProperties, - }, - ]; - - return operations; - }) - ); + const operations = entries.flatMap((entry) => { + const entryWithSystemProperties = { + ...entry, + doc_id: entry.id, + '@timestamp': new Date().toISOString(), + public: true, + confidence: 'high' as const, + type: 'contextual' as const, + is_correction: false, + labels: { + ...entry.labels, + }, + role: KnowledgeBaseEntryRole.Elastic, + }; + + return 'texts' in entryWithSystemProperties + ? splitKbText(entryWithSystemProperties) + : [ + { + type: KnowledgeBaseEntryOperationType.Index, + document: entryWithSystemProperties, + }, + ]; + }); + + this.logger.debug(`Queuing ${operations.length} operations (${entries.length} entries)`); + + this.kbService!.queue(operations); }) .catch((error) => { this.logger.error( diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts index 5467556a0e3ab..0c654aef5db39 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts @@ -32,7 +32,15 @@ export const kbComponentTemplate: ClusterComponentTemplate['component_template'] '@timestamp': date, id: keyword, doc_id: { type: 'text', fielddata: true }, - title: { type: 'text', fielddata: true }, + title: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, user: { properties: { id: keyword, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index c1ade1d4f6be4..d5f55307cb727 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -48,8 +48,8 @@ export interface RecalledEntry { function isModelMissingOrUnavailableError(error: Error) { return ( error instanceof errors.ResponseError && - (error.body.error.type === 'resource_not_found_exception' || - error.body.error.type === 'status_exception') + (error.body.error?.type === 'resource_not_found_exception' || + error.body.error?.type === 'status_exception') ); } function isCreateModelValidationError(error: Error) { @@ -70,7 +70,7 @@ export enum KnowledgeBaseEntryOperationType { interface KnowledgeBaseDeleteOperation { type: KnowledgeBaseEntryOperationType.Delete; - doc_id?: string; + groupId?: string; labels?: Record; } @@ -84,7 +84,7 @@ export type KnowledgeBaseEntryOperation = | KnowledgeBaseIndexOperation; export class KnowledgeBaseService { - private hasSetup: boolean = false; + private isModelReady: boolean = false; private _queue: KnowledgeBaseEntryOperation[] = []; @@ -93,6 +93,7 @@ export class KnowledgeBaseService { } setup = async () => { + this.dependencies.logger.debug('Setting up knowledge base'); const elserModelId = await this.dependencies.getModelId(); const retryOptions = { factor: 1, minTimeout: 10000, retries: 12 }; @@ -189,7 +190,7 @@ export class KnowledgeBaseService { ); if (isReady) { - return Promise.resolve(); + return; } this.dependencies.logger.debug('Model is not allocated yet'); @@ -234,7 +235,7 @@ export class KnowledgeBaseService { query: { bool: { filter: [ - ...(operation.doc_id ? [{ term: { _id: operation.doc_id } }] : []), + ...(operation.groupId ? [{ term: { doc_id: operation.groupId } }] : []), ...(operation.labels ? map(operation.labels, (value, key) => { return { term: { [key]: value } }; @@ -247,7 +248,7 @@ export class KnowledgeBaseService { return; } catch (error) { this.dependencies.logger.error( - `Failed to delete document "${operation?.doc_id}" due to ${error.message}` + `Failed to delete document "${operation?.groupId}" due to ${error.message}` ); this.dependencies.logger.debug(() => JSON.stringify(operation)); throw error; @@ -275,7 +276,7 @@ export class KnowledgeBaseService { this.dependencies.logger.debug(`Processing queue`); - this.hasSetup = true; + this.isModelReady = true; this.dependencies.logger.info(`Processing ${this._queue.length} queue operations`); @@ -292,7 +293,7 @@ export class KnowledgeBaseService { ) ); - this.dependencies.logger.info('Processed all queued operations'); + this.dependencies.logger.info(`Finished processing ${operations.length} queued operations`); } queue(operations: KnowledgeBaseEntryOperation[]): void { @@ -300,8 +301,15 @@ export class KnowledgeBaseService { return; } - if (!this.hasSetup) { - this._queue.push(...operations); + this.dependencies.logger.debug( + `Adding ${operations.length} operations to queue. Queue size now: ${this._queue.length})` + ); + this._queue.push(...operations); + + if (!this.isModelReady) { + this.dependencies.logger.debug( + `Delay processing ${operations.length} operations until knowledge base is ready` + ); return; } @@ -311,13 +319,18 @@ export class KnowledgeBaseService { limiter(() => this.processOperation(operation)) ); - Promise.all(limitedFunctions).catch((err) => { - this.dependencies.logger.error(`Failed to process all queued operations`); - this.dependencies.logger.error(err); - }); + Promise.all(limitedFunctions) + .then(() => { + this.dependencies.logger.debug(`Processed all queued operations`); + }) + .catch((err) => { + this.dependencies.logger.error(`Failed to process all queued operations`); + this.dependencies.logger.error(err); + }); } status = async () => { + this.dependencies.logger.debug('Checking model status'); const elserModelId = await this.dependencies.getModelId(); try { @@ -327,14 +340,23 @@ export class KnowledgeBaseService { const elserModelStats = modelStats.trained_model_stats[0]; const deploymentState = elserModelStats.deployment_stats?.state; const allocationState = elserModelStats.deployment_stats?.allocation_status.state; + const ready = deploymentState === 'started' && allocationState === 'fully_allocated'; + + this.dependencies.logger.debug( + `Model deployment state: ${deploymentState}, allocation state: ${allocationState}, ready: ${ready}` + ); return { - ready: deploymentState === 'started' && allocationState === 'fully_allocated', + ready, deployment_state: deploymentState, allocation_state: allocationState, model_name: elserModelId, }; } catch (error) { + this.dependencies.logger.debug( + `Failed to get status for model "${elserModelId}" due to ${error.message}` + ); + return { error: error instanceof errors.ResponseError ? error.body.error : String(error), ready: false, @@ -533,8 +555,10 @@ export class KnowledgeBaseService { query: { bool: { filter: [ - // filter title by query - ...(query ? [{ wildcard: { doc_id: { value: `${query}*` } } }] : []), + // filter by search query + ...(query + ? [{ query_string: { query: `${query}*`, fields: ['doc_id', 'title'] } }] + : []), { // exclude user instructions bool: { must_not: { term: { type: KnowledgeBaseType.UserInstruction } } }, @@ -542,13 +566,13 @@ export class KnowledgeBaseService { ], }, }, - sort: [ - { - [String(sortBy)]: { - order: sortDirection, - }, - }, - ], + sort: + sortBy === 'title' + ? [ + { ['title.keyword']: { order: sortDirection } }, + { doc_id: { order: sortDirection } }, // sort by doc_id for backwards compatibility + ] + : [{ [String(sortBy)]: { order: sortDirection } }], size: 500, _source: { includes: [ @@ -661,6 +685,7 @@ export class KnowledgeBaseService { document: { '@timestamp': new Date().toISOString(), ...doc, + title: doc.title ?? doc.doc_id, user, namespace, }, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts index 9a2f047b60f9b..4b3dd559df3eb 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts @@ -20,14 +20,14 @@ export function splitKbText({ return [ { type: KnowledgeBaseEntryOperationType.Delete, - doc_id: id, + groupId: id, // delete all entries with the same groupId labels: {}, }, ...texts.map((text, index) => ({ type: KnowledgeBaseEntryOperationType.Index, document: merge({}, rest, { id: [id, index].join('_'), - doc_id: id, + doc_id: id, // group_id is used to group entries together labels: {}, text, }), diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts index 1f35986bae8f0..c71676c60a6c4 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts @@ -66,7 +66,7 @@ export async function scoreSuggestions({ Documents: ${JSON.stringify( - suggestions.map(({ id, docId: title, text }) => ({ id, title, text })), + suggestions.map(({ id, doc_id: title, text }) => ({ id, title, text })), null, 2 )}`); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx index d8e2897c6878c..94ae564ac7f96 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx @@ -81,7 +81,9 @@ describe('KnowledgeBaseTab', () => { getByTestId('knowledgeBaseEditManualEntryFlyoutSaveButton').click(); - expect(createMock).toHaveBeenCalledWith({ entry: { id: 'foo', public: false, text: 'bar' } }); + expect(createMock).toHaveBeenCalledWith({ + entry: { id: expect.any(String), title: 'foo', public: false, text: 'bar' }, + }); }); it('should require an id', () => { diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts index 0a2a4c796eb5f..50d660ae1a0dd 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts @@ -23,6 +23,10 @@ export type CreateTestConfig = ReturnType; export type CreateTest = ReturnType; +export type ObservabilityAIAssistantApiClients = Awaited< + ReturnType +>; + export type ObservabilityAIAssistantAPIClient = Awaited< ReturnType >; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index 238be31220aa9..ea929c34526b8 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -64,7 +64,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { params: { query: { query: '', - sortBy: 'doc_id', + sortBy: 'title', sortDirection: 'asc', }, }, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index b9983e5e5c58c..a2ebee43b54dc 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -6,31 +6,35 @@ */ import expect from '@kbn/expect'; +import { type KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; +import pRetry from 'p-retry'; +import { ToolingLog } from '@kbn/tooling-log'; +import { uniq } from 'lodash'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { clearKnowledgeBase, createKnowledgeBaseModel, deleteKnowledgeBaseModel } from './helpers'; +import { ObservabilityAIAssistantApiClients } from '../../common/config'; +import { ObservabilityAIAssistantApiClient } from '../../common/observability_ai_assistant_api_client'; export default function ApiTest({ getService }: FtrProviderContext) { const ml = getService('ml'); const es = getService('es'); - + const log = getService('log'); const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); describe('Knowledge base', () => { before(async () => { await createKnowledgeBaseModel(ml); - }); - after(async () => { - await deleteKnowledgeBaseModel(ml); - }); - - it('returns 200 on knowledge base setup', async () => { - const res = await observabilityAIAssistantAPIClient + await observabilityAIAssistantAPIClient .editorUser({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(200); - expect(res.body).to.eql({}); + }); + + after(async () => { + await deleteKnowledgeBaseModel(ml); + await clearKnowledgeBase(es); }); describe('when managing a single entry', () => { @@ -51,7 +55,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { params: { query: { query: '', - sortBy: 'doc_id', + sortBy: 'title', sortDirection: 'asc', }, }, @@ -69,7 +73,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { params: { query: { query: '', - sortBy: 'doc_id', + sortBy: 'title', sortDirection: 'asc', }, }, @@ -98,7 +102,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { params: { query: { query: '', - sortBy: 'doc_id', + sortBy: 'title', sortDirection: 'asc', }, }, @@ -123,195 +127,343 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('when managing multiple entries', () => { - before(async () => { - await clearKnowledgeBase(es); - }); - afterEach(async () => { - await clearKnowledgeBase(es); - }); - const knowledgeBaseEntries = [ - { - doc_id: 'my_doc_a', - text: 'My content a', - }, - { - doc_id: 'my_doc_b', - text: 'My content b', - }, - { - doc_id: 'my_doc_c', - text: 'My content c', - }, - ]; - it('returns 200 on create', async () => { - await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', - params: { body: { entries: knowledgeBaseEntries } }, - }) - .expect(200); - + async function getEntries({ + query = '', + sortBy = 'title', + sortDirection = 'asc', + }: { query?: string; sortBy?: string; sortDirection?: 'asc' | 'desc' } = {}) { const res = await observabilityAIAssistantAPIClient .editorUser({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { - query: { - query: '', - sortBy: 'doc_id', - sortDirection: 'asc', - }, + query: { query, sortBy, sortDirection }, }, }) .expect(200); - expect(res.body.entries.filter((entry) => entry.id.startsWith('my_doc')).length).to.eql(3); - }); - it('allows sorting', async () => { + return omitCategories(res.body.entries); + } + + beforeEach(async () => { + await clearKnowledgeBase(es); + await observabilityAIAssistantAPIClient .editorUser({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', - params: { body: { entries: knowledgeBaseEntries } }, - }) - .expect(200); - - const res = await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { - query: { - query: '', - sortBy: 'doc_id', - sortDirection: 'desc', + body: { + entries: [ + { + id: 'my_doc_a', + title: 'My title a', + text: 'My content a', + }, + { + id: 'my_doc_b', + title: 'My title b', + text: 'My content b', + }, + { + id: 'my_doc_c', + title: 'My title c', + text: 'My content c', + }, + ], }, }, }) .expect(200); + }); - const entries = res.body.entries.filter((entry) => entry.id.startsWith('my_doc')); - expect(entries[0].id).to.eql('my_doc_c'); - expect(entries[1].id).to.eql('my_doc_b'); - expect(entries[2].id).to.eql('my_doc_a'); - - // asc - const resAsc = await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', - params: { - query: { - query: '', - sortBy: 'doc_id', - sortDirection: 'asc', - }, - }, - }) - .expect(200); + afterEach(async () => { + await clearKnowledgeBase(es); + }); - const entriesAsc = resAsc.body.entries.filter((entry) => entry.id.startsWith('my_doc')); - expect(entriesAsc[0].id).to.eql('my_doc_a'); - expect(entriesAsc[1].id).to.eql('my_doc_b'); - expect(entriesAsc[2].id).to.eql('my_doc_c'); + it('returns 200 on create', async () => { + const entries = await getEntries(); + expect(omitCategories(entries).length).to.eql(3); }); - it('allows searching', async () => { - await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', - params: { body: { entries: knowledgeBaseEntries } }, - }) - .expect(200); - const res = await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', - params: { - query: { - query: 'my_doc_a', - sortBy: 'doc_id', - sortDirection: 'asc', - }, - }, - }) - .expect(200); + describe('when sorting ', () => { + const ascendingOrder = ['my_doc_a', 'my_doc_b', 'my_doc_c']; + + it('allows sorting ascending', async () => { + const entries = await getEntries({ sortBy: 'title', sortDirection: 'asc' }); + expect(entries.map(({ id }) => id)).to.eql(ascendingOrder); + }); - expect(res.body.entries.length).to.eql(1); - expect(res.body.entries[0].id).to.eql('my_doc_a'); + it('allows sorting descending', async () => { + const entries = await getEntries({ sortBy: 'title', sortDirection: 'desc' }); + expect(entries.map(({ id }) => id)).to.eql([...ascendingOrder].reverse()); + }); + }); + + it('allows searching by title', async () => { + const entries = await getEntries({ query: 'b' }); + expect(entries.length).to.eql(1); + expect(entries[0].title).to.eql('My title b'); }); }); - describe('When the LLM creates entries', () => { - before(async () => { + describe('when importing categories', () => { + beforeEach(async () => { await clearKnowledgeBase(es); }); + afterEach(async () => { await clearKnowledgeBase(es); }); - it('can replace an existing entry using the `doc_id`', async () => { - await observabilityAIAssistantAPIClient + const importCategories = () => + observabilityAIAssistantAPIClient .editorUser({ - endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', + endpoint: 'POST /internal/observability_ai_assistant/kb/entries/category/import', params: { body: { - doc_id: 'my_doc_id', - text: 'My content', - confidence: 'high', - is_correction: false, - public: false, - labels: {}, + category: 'my_new_category', + entries: [ + { + id: 'my_new_category_a', + texts: [ + 'My first category content a', + 'My second category content a', + 'my third category content a', + ], + }, + { + id: 'my_new_category_b', + texts: [ + 'My first category content b', + 'My second category content b', + 'my third category content b', + ], + }, + { + id: 'my_new_category_c', + texts: [ + 'My first category content c', + 'My second category content c', + 'my third category content c', + ], + }, + ], }, }, }) .expect(200); + it('overwrites existing entries on subsequent import', async () => { + await waitForModelReady(observabilityAIAssistantAPIClient, log); + await importCategories(); + await importCategories(); + + await pRetry( + async () => { + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'title', + sortDirection: 'asc', + }, + }, + }) + .expect(200); + + const categoryEntries = res.body.entries.filter( + (entry) => entry.labels?.category === 'my_new_category' + ); + + const entryGroups = uniq(categoryEntries.map((entry) => entry.doc_id)); + + log.debug( + `Waiting for entries to be created. Found ${categoryEntries.length} entries and ${entryGroups.length} groups` + ); + + if (categoryEntries.length !== 9 || entryGroups.length !== 3) { + throw new Error( + `Expected 9 entries, found ${categoryEntries.length} and ${entryGroups.length} groups` + ); + } + + expect(categoryEntries.length).to.eql(9); + expect(entryGroups.length).to.eql(3); + }, + { + retries: 100, + factor: 1, + } + ); + }); + }); + + describe('When the LLM creates entries', () => { + async function addEntryWithDocId({ + apiClient, + docId, + text, + }: { + apiClient: ObservabilityAIAssistantApiClient; + docId: string; + text: string; + }) { + return apiClient({ + endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', + params: { + body: { + doc_id: docId, + text, + confidence: 'high', + is_correction: false, + public: false, + labels: {}, + }, + }, + }).expect(200); + } + + async function getEntriesWithDocId(docId: string) { const res = await observabilityAIAssistantAPIClient .editorUser({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { query: '', - sortBy: 'doc_id', + sortBy: 'title', sortDirection: 'asc', }, }, }) .expect(200); - const id = res.body.entries[0].id; - expect(res.body.entries.length).to.eql(1); - expect(res.body.entries[0].text).to.eql('My content'); + return res.body.entries.filter((entry) => entry.doc_id === docId); + } - await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', - params: { - body: { - doc_id: 'my_doc_id', - text: 'My content_2', - confidence: 'high', - is_correction: false, - public: false, - labels: {}, - }, - }, - }) - .expect(200); + describe('when the LLM uses the same doc_id for two entries created by the same user', () => { + let entries1: KnowledgeBaseEntry[]; + let entries2: KnowledgeBaseEntry[]; - const res2 = await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', - params: { - query: { - query: '', - sortBy: 'doc_id', - sortDirection: 'asc', + before(async () => { + const docId = 'my_favourite_color'; + + await addEntryWithDocId({ + apiClient: observabilityAIAssistantAPIClient.editorUser, + docId, + text: 'My favourite color is blue', + }); + entries1 = await getEntriesWithDocId(docId); + + await addEntryWithDocId({ + apiClient: observabilityAIAssistantAPIClient.editorUser, + docId, + text: 'My favourite color is green', + }); + entries2 = await getEntriesWithDocId(docId); + }); + + after(async () => { + await clearKnowledgeBase(es); + }); + + it('overwrites the first entry so there is only one', async () => { + expect(entries1.length).to.eql(1); + expect(entries2.length).to.eql(1); + }); + + it('replaces the text content of the first entry with the new text content', async () => { + expect(entries1[0].text).to.eql('My favourite color is blue'); + expect(entries2[0].text).to.eql('My favourite color is green'); + }); + + it('updates the timestamp', async () => { + const getAsMs = (timestamp: string) => new Date(timestamp).getTime(); + expect(getAsMs(entries1[0]['@timestamp'])).to.be.lessThan( + getAsMs(entries2[0]['@timestamp']) + ); + }); + + it('does not change the _id', () => { + expect(entries1[0].id).to.eql(entries2[0].id); + }); + }); + + describe('when the LLM uses same doc_id for two entries created by different users', () => { + let entries: KnowledgeBaseEntry[]; + + before(async () => { + await addEntryWithDocId({ + apiClient: observabilityAIAssistantAPIClient.editorUser, + docId: 'users_favorite_animal', + text: "The user's favourite animal is a dog", + }); + await addEntryWithDocId({ + apiClient: observabilityAIAssistantAPIClient.secondaryEditorUser, + docId: 'users_favorite_animal', + text: "The user's favourite animal is a cat", + }); + + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'title', + sortDirection: 'asc', + }, }, - }, - }) - .expect(200); + }) + .expect(200); + + entries = omitCategories(res.body.entries); + }); + + after(async () => { + await clearKnowledgeBase(es); + }); + + it('creates two separate entries with the same doc_id', async () => { + expect(entries.map(({ doc_id: docId }) => docId)).to.eql([ + 'users_favorite_animal', + 'users_favorite_animal', + ]); + }); + + it('creates two entries with different text content', async () => { + expect(entries.map(({ text }) => text)).to.eql([ + "The user's favourite animal is a cat", + "The user's favourite animal is a dog", + ]); + }); - expect(res2.body.entries.length).to.eql(1); - expect(res2.body.entries[0].text).to.eql('My content_2'); - expect(res2.body.entries[0].id).to.eql(id); + it('creates two entries by different users', async () => { + expect(entries.map(({ user }) => user?.name)).to.eql(['secondary_editor', 'editor']); + }); }); }); }); } + +function omitCategories(entries: KnowledgeBaseEntry[]) { + return entries.filter((entry) => entry.labels?.category === undefined); +} + +async function waitForModelReady( + observabilityAIAssistantAPIClient: ObservabilityAIAssistantApiClients, + log: ToolingLog +) { + return pRetry(async () => { + const res = await observabilityAIAssistantAPIClient + .editorUser({ endpoint: 'GET /internal/observability_ai_assistant/kb/status' }) + .expect(200); + + const isModelReady = res.body.ready; + log.debug(`Model status: ${isModelReady ? 'ready' : 'not ready'}`); + + if (!isModelReady) { + throw new Error('Model not ready'); + } + }); +} diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index c1ce378be79c4..3083374c309a0 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -88,9 +88,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); const instructions = res.body.userInstructions; - const sortByDocId = (data: any) => sortBy(data, 'doc_id'); - expect(sortByDocId(instructions)).to.eql( - sortByDocId([ + const sortById = (data: any) => sortBy(data, 'id'); + expect(sortById(instructions)).to.eql( + sortById([ { id: 'private-doc-from-editorUser', public: false, @@ -116,9 +116,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); const instructions = res.body.userInstructions; - const sortByDocId = (data: any) => sortBy(data, 'doc_id'); - expect(sortByDocId(instructions)).to.eql( - sortByDocId([ + const sortById = (data: any) => sortBy(data, 'id'); + expect(sortById(instructions)).to.eql( + sortById([ { id: 'public-doc-from-editorUser', public: true, From f6b8303e80409871f9160b907b11221ed3477a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 27 Aug 2024 22:57:20 +0200 Subject: [PATCH 05/42] Cleanup --- .../server/functions/summarize.ts | 1 - .../server/routes/functions/route.ts | 1 - .../server/routes/knowledge_base/route.ts | 19 ++------------- .../server/service/client/index.ts | 23 +++++++++++++++---- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index 7d20cd927b42f..c46aadccec728 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -80,7 +80,6 @@ export function registerSummarizationFunction({ role: KnowledgeBaseEntryRole.AssistantSummarization, text, is_correction: isCorrection, - type: KnowledgeBaseType.Contextual, confidence, public: isPublic, labels: {}, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index f00a2ddb1f483..271da81d093e4 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -142,7 +142,6 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ id: id ?? v4(), doc_id: docId, is_correction: isCorrection, - type: KnowledgeBaseType.Contextual, text, public: isPublic, labels, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts index e3959ed35250f..793ed755ad95e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts @@ -13,12 +13,7 @@ import { notImplemented } from '@hapi/boom'; import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; -import { - Instruction, - KnowledgeBaseEntry, - KnowledgeBaseEntryRole, - KnowledgeBaseType, -} from '../../../common/types'; +import { Instruction, KnowledgeBaseEntry, KnowledgeBaseEntryRole } from '../../../common/types'; const getKnowledgeBaseStatus = createObservabilityAIAssistantServerRoute({ endpoint: 'GET /internal/observability_ai_assistant/kb/status', @@ -108,16 +103,7 @@ const saveKnowledgeBaseUserInstruction = createObservabilityAIAssistantServerRou const { id, text, public: isPublic } = resources.params.body; return client.addUserInstruction({ - entry: { - id, - text, - public: isPublic, - confidence: 'high', - is_correction: false, - type: KnowledgeBaseType.UserInstruction, - labels: {}, - role: KnowledgeBaseEntryRole.UserEntry, - }, + entry: { id, text, public: isPublic }, }); }, }); @@ -204,7 +190,6 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ title, confidence: confidence ?? 'high', is_correction: isCorrection ?? false, - type: 'contextual', public: isPublic ?? true, labels: labels ?? {}, role: (role as KnowledgeBaseEntryRole) ?? KnowledgeBaseEntryRole.UserEntry, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index db0fb3521bc2a..1e0f9ce833263 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -52,6 +52,8 @@ import { type KnowledgeBaseEntry, type Message, type AdHocInstruction, + KnowledgeBaseType, + KnowledgeBaseEntryRole, } from '../../../common/types'; import { withoutTokenCountEvents } from '../../../common/utils/without_token_count_events'; import { CONTEXT_FUNCTION_NAME } from '../../functions/context'; @@ -742,7 +744,10 @@ export class ObservabilityAIAssistantClient { addUserInstruction = async ({ entry, }: { - entry: Omit; + entry: Omit< + KnowledgeBaseEntry, + '@timestamp' | 'confidence' | 'is_correction' | 'type' | 'role' + >; }): Promise => { // for now we want to limit the number of user instructions to 1 per user // if a user instruction already exists for the user, we get the id and update it @@ -761,19 +766,29 @@ export class ObservabilityAIAssistantClient { return this.dependencies.knowledgeBaseService.addEntry({ namespace: this.dependencies.namespace, user: this.dependencies.user, - entry, + entry: { + ...entry, + confidence: 'high', + is_correction: false, + type: KnowledgeBaseType.UserInstruction, + labels: {}, + role: KnowledgeBaseEntryRole.UserEntry, + }, }); }; addKnowledgeBaseEntry = async ({ entry, }: { - entry: Omit; + entry: Omit; }): Promise => { return this.dependencies.knowledgeBaseService.addEntry({ namespace: this.dependencies.namespace, user: this.dependencies.user, - entry, + entry: { + ...entry, + type: KnowledgeBaseType.Contextual, + }, }); }; From f88d826a8a73413d627b7ac38b6efaca6e194e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 00:05:55 +0200 Subject: [PATCH 06/42] Improve types --- .../public/hooks/use_create_knowledge_base_entry.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts index 41126264f9011..ffd4ea8e0106e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts @@ -28,10 +28,9 @@ export function useCreateKnowledgeBaseEntry() { void, ServerError, { - entry: Omit< - KnowledgeBaseEntry, - '@timestamp' | 'confidence' | 'is_correction' | 'role' | 'title' - > & { title: string }; + entry: Omit & { + title: string; + }; } >( [REACT_QUERY_KEYS.CREATE_KB_ENTRIES], From f42f1ec46fdf845940eea5346e8baac36beaf4c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 00:53:07 +0200 Subject: [PATCH 07/42] i18n --- .../translations/translations/fr-FR.json | 27 +++++++++---------- .../translations/translations/ja-JP.json | 27 +++++++++---------- .../translations/translations/zh-CN.json | 27 +++++++++---------- 3 files changed, 39 insertions(+), 42 deletions(-) diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index f0ef033d63236..5f4ca8c55cc74 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -7035,6 +7035,14 @@ "securitySolutionPackages.features.featureRegistry.subFeatures.trustedApplications": "Applications de confiance", "securitySolutionPackages.features.featureRegistry.subFeatures.trustedApplications.description": "Aide à atténuer les conflits avec d'autres logiciels, généralement d'autres applications d'antivirus ou de sécurité des points de terminaison.", "securitySolutionPackages.features.featureRegistry.subFeatures.trustedApplications.privilegesTooltip": "\"Tous les espaces\" est requis pour l'accès aux applications de confiance.", + "securitySolutionPackages.flyout.right.header.collapseDetailButtonAriaLabel": "Réduire les détails", + "securitySolutionPackages.flyout.right.header.collapseDetailButtonLabel": "Réduire les détails", + "securitySolutionPackages.flyout.right.header.expandDetailButtonAriaLabel": "Développer les détails", + "securitySolutionPackages.flyout.right.header.expandDetailButtonLabel": "Développer les détails", + "securitySolutionPackages.flyout.shared.errorDescription": "Une erreur est survenue lors de l'affichage de {message}.", + "securitySolutionPackages.flyout.shared.errorTitle": "Impossible d'afficher {title}.", + "securitySolutionPackages.flyout.shared.ExpandablePanelButtonIconAriaLabel": "Activer/Désactiver le panneau extensible", + "securitySolutionPackages.flyout.shared.expandablePanelLoadingAriaLabel": "panneau extensible", "securitySolutionPackages.markdown.insight.upsell": "Passez au niveau {requiredLicense} pour pouvoir utiliser les informations des guides d'investigation", "securitySolutionPackages.markdown.investigationGuideInteractions.upsell": "Passez au niveau {requiredLicense} pour pouvoir utiliser les interactions des guides d'investigation", "securitySolutionPackages.navigation.landingLinks": "Vues de sécurité", @@ -11106,8 +11114,8 @@ "xpack.apm.serviceIcons.service": "Service", "xpack.apm.serviceIcons.serviceDetails.cloud.architecture": "Architecture", "xpack.apm.serviceIcons.serviceDetails.cloud.availabilityZoneLabel": "{zones, plural, =0 {Zone de disponibilité} one {Zone de disponibilité} other {Zones de disponibilité}} ", - "xpack.apm.serviceIcons.serviceDetails.cloud.faasTriggerTypeLabel": "{triggerTypes, plural, =0 {Type de déclencheur} one {Type de déclencheur} other {Types de déclencheurs}} ", "xpack.apm.serviceIcons.serviceDetails.cloud.functionNameLabel": "{functionNames, plural, =0 {Nom de fonction} one {Nom de fonction} other {Noms de fonction}} ", + "xpack.apm.serviceIcons.serviceDetails.cloud.faasTriggerTypeLabel": "{triggerTypes, plural, =0 {Type de déclencheur} one {Type de déclencheur} other {Types de déclencheurs}} ", "xpack.apm.serviceIcons.serviceDetails.cloud.machineTypesLabel": "{machineTypes, plural, =0{Type de machine} one {Type de machine} other {Types de machines}} ", "xpack.apm.serviceIcons.serviceDetails.cloud.projectIdLabel": "ID de projet", "xpack.apm.serviceIcons.serviceDetails.cloud.providerLabel": "Fournisseur cloud", @@ -26983,8 +26991,8 @@ "xpack.maps.source.esSearch.descendingLabel": "décroissant", "xpack.maps.source.esSearch.extentFilterLabel": "Filtre dynamique pour les données de la zone de carte visible", "xpack.maps.source.esSearch.fieldNotFoundMsg": "Impossible de trouver \"{fieldName}\" dans le modèle d'indexation \"{indexPatternName}\".", - "xpack.maps.source.esSearch.geofieldLabel": "Champ géospatial", "xpack.maps.source.esSearch.geoFieldLabel": "Champ géospatial", + "xpack.maps.source.esSearch.geofieldLabel": "Champ géospatial", "xpack.maps.source.esSearch.geoFieldTypeLabel": "Type de champ géospatial", "xpack.maps.source.esSearch.indexOverOneLengthEditError": "Votre vue de données pointe vers plusieurs index. Un seul index est autorisé par vue de données.", "xpack.maps.source.esSearch.indexZeroLengthEditError": "Votre vue de données ne pointe vers aucun index.", @@ -33073,7 +33081,6 @@ "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.contentsLabel": "Contenu", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.idLabel": "Nom", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiMarkdownEditor.enterContentsLabel": "Entrer du contenu", - "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.editEntryLabel": "Modifier {id}", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.newEntryLabel": "Nouvelle entrée", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.cancelButtonEmptyLabel": "Annuler", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.euiMarkdownEditor.observabilityAiAssistantKnowledgeBaseViewMarkdownEditorLabel": "observabilityAiAssistantKnowledgeBaseViewMarkdownEditor", @@ -36447,8 +36454,8 @@ "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.maxAlertsFieldLessThanWarning": "Kibana ne permet qu'un maximum de {maxNumber} {maxNumber, plural, =1 {alerte} other {alertes}} par exécution de règle.", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.nameFieldRequiredError": "Nom obligatoire.", "xpack.securitySolution.detectionEngine.createRule.stepAboutrule.noteHelpText": "Ajouter un guide d'investigation sur les règles...", - "xpack.securitySolution.detectionEngine.createRule.stepAboutrule.setupHelpText": "Ajouter le guide de configuration de règle...", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.setupHelpText": "Fournissez des instructions sur les conditions préalables à la règle, telles que les intégrations requises, les étapes de configuration et tout ce qui est nécessaire au bon fonctionnement de la règle.", + "xpack.securitySolution.detectionEngine.createRule.stepAboutrule.setupHelpText": "Ajouter le guide de configuration de règle...", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.setupLabel": "Guide de configuration", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.tagFieldEmptyError": "Une balise ne doit pas être vide", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.threatIndicatorPathFieldEmptyError": "Le remplacement du préfixe d'indicateur ne peut pas être vide.", @@ -39260,10 +39267,6 @@ "xpack.securitySolution.flyout.right.alertPreview.ariaLabel": "Voir un aperçu de l'alerte avec l'id {id}", "xpack.securitySolution.flyout.right.eventCategoryText": "Catégorie d'événement", "xpack.securitySolution.flyout.right.header.assignedTitle": "Utilisateurs affectés", - "securitySolutionPackages.flyout.right.header.collapseDetailButtonAriaLabel": "Réduire les détails", - "securitySolutionPackages.flyout.right.header.collapseDetailButtonLabel": "Réduire les détails", - "securitySolutionPackages.flyout.right.header.expandDetailButtonAriaLabel": "Développer les détails", - "securitySolutionPackages.flyout.right.header.expandDetailButtonLabel": "Développer les détails", "xpack.securitySolution.flyout.right.header.headerTitle": "Détails des documents", "xpack.securitySolution.flyout.right.header.jsonTabLabel": "JSON", "xpack.securitySolution.flyout.right.header.overviewTabLabel": "Aperçu", @@ -39350,10 +39353,6 @@ "xpack.securitySolution.flyout.right.visualizations.sessionPreview.timeDescription": "à", "xpack.securitySolution.flyout.right.visualizations.sessionPreview.upsellDescription": "Cette fonctionnalité requiert un {subscription}", "xpack.securitySolution.flyout.right.visualizations.sessionPreview.upsellLinkText": "Abonnement Enterprise", - "securitySolutionPackages.flyout.shared.errorDescription": "Une erreur est survenue lors de l'affichage de {message}.", - "securitySolutionPackages.flyout.shared.errorTitle": "Impossible d'afficher {title}.", - "securitySolutionPackages.flyout.shared.ExpandablePanelButtonIconAriaLabel": "Activer/Désactiver le panneau extensible", - "securitySolutionPackages.flyout.shared.expandablePanelLoadingAriaLabel": "panneau extensible", "xpack.securitySolution.flyout.tour.entities.description": "Consultez la vue {entities} étendue pour en savoir plus sur les hôtes et les utilisateurs liés à l'alerte.", "xpack.securitySolution.flyout.tour.entities.text": "Entités", "xpack.securitySolution.flyout.tour.entities.title": "De nouvelles informations sur les hôtes et les utilisateurs sont disponibles", @@ -42061,8 +42060,8 @@ "xpack.slo.sloEmbeddable.config.sloSelector.placeholder": "Sélectionner un SLO", "xpack.slo.sloEmbeddable.displayName": "Aperçu du SLO", "xpack.slo.sloEmbeddable.overview.sloNotFoundText": "Le SLO a été supprimé. Vous pouvez supprimer sans risque le widget du tableau de bord.", - "xpack.slo.sloGridItem.targetFlexItemLabel": "Cible {target}", "xpack.slo.sLOGridItem.targetFlexItemLabel": "Cible {target}", + "xpack.slo.sloGridItem.targetFlexItemLabel": "Cible {target}", "xpack.slo.sloGroupConfiguration.customFiltersLabel": "Personnaliser le filtre", "xpack.slo.sloGroupConfiguration.customFiltersOptional": "Facultatif", "xpack.slo.sloGroupConfiguration.customFilterText": "Personnaliser le filtre", @@ -43508,8 +43507,8 @@ "xpack.stackConnectors.components.casesWebhookxpack.stackConnectors.components.casesWebhook.connectorTypeTitle": "Webhook - Données de gestion des cas", "xpack.stackConnectors.components.d3security.bodyCodeEditorAriaLabel": "Éditeur de code", "xpack.stackConnectors.components.d3security.bodyFieldLabel": "Corps", - "xpack.stackConnectors.components.d3security.connectorTypeTitle": "Données D3", "xpack.stackConnectors.components.d3Security.connectorTypeTitle": "D3 Security", + "xpack.stackConnectors.components.d3security.connectorTypeTitle": "Données D3", "xpack.stackConnectors.components.d3security.eventTypeFieldLabel": "Type d'événement", "xpack.stackConnectors.components.d3security.invalidActionText": "Nom d'action non valide.", "xpack.stackConnectors.components.d3security.requiredActionText": "L'action est requise.", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index b3be85aaa7bd1..2950d528b5b0b 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -7031,6 +7031,14 @@ "securitySolutionPackages.features.featureRegistry.subFeatures.trustedApplications": "信頼できるアプリケーション", "securitySolutionPackages.features.featureRegistry.subFeatures.trustedApplications.description": "他のソフトウェア（通常は他のウイルス対策またはエンドポイントセキュリティアプリケーション）との競合を軽減することができます。", "securitySolutionPackages.features.featureRegistry.subFeatures.trustedApplications.privilegesTooltip": "信頼できるアプリケーションのアクセスには、すべてのスペースが必要です。", + "securitySolutionPackages.flyout.right.header.collapseDetailButtonAriaLabel": "詳細を折りたたむ", + "securitySolutionPackages.flyout.right.header.collapseDetailButtonLabel": "詳細を折りたたむ", + "securitySolutionPackages.flyout.right.header.expandDetailButtonAriaLabel": "詳細を展開", + "securitySolutionPackages.flyout.right.header.expandDetailButtonLabel": "詳細を展開", + "securitySolutionPackages.flyout.shared.errorDescription": "{message}の表示中にエラーが発生しました。", + "securitySolutionPackages.flyout.shared.errorTitle": "{title}を表示できません。", + "securitySolutionPackages.flyout.shared.ExpandablePanelButtonIconAriaLabel": "展開可能なパネルトグル", + "securitySolutionPackages.flyout.shared.expandablePanelLoadingAriaLabel": "展開可能なパネル", "securitySolutionPackages.markdown.investigationGuideInteractions.upsell": "{requiredLicense}にアップグレードして、調査ガイドのインタラクションを利用", "securitySolutionPackages.navigation.landingLinks": "セキュリティビュー", "securitySolutionPackages.sideNav.betaBadge.label": "ベータ", @@ -11096,8 +11104,8 @@ "xpack.apm.serviceIcons.service": "サービス", "xpack.apm.serviceIcons.serviceDetails.cloud.architecture": "アーキテクチャー", "xpack.apm.serviceIcons.serviceDetails.cloud.availabilityZoneLabel": "{zones, plural, other {可用性ゾーン}} ", - "xpack.apm.serviceIcons.serviceDetails.cloud.faasTriggerTypeLabel": "{triggerTypes, plural, other {トリガータイプ}} ", "xpack.apm.serviceIcons.serviceDetails.cloud.functionNameLabel": "{functionNames, plural, other {関数名}} ", + "xpack.apm.serviceIcons.serviceDetails.cloud.faasTriggerTypeLabel": "{triggerTypes, plural, other {トリガータイプ}} ", "xpack.apm.serviceIcons.serviceDetails.cloud.machineTypesLabel": "{machineTypes, plural, other {コンピュータータイプ} }\n ", "xpack.apm.serviceIcons.serviceDetails.cloud.projectIdLabel": "プロジェクト ID", "xpack.apm.serviceIcons.serviceDetails.cloud.providerLabel": "クラウドプロバイダー", @@ -26972,8 +26980,8 @@ "xpack.maps.source.esSearch.descendingLabel": "降順", "xpack.maps.source.esSearch.extentFilterLabel": "マップの表示範囲でデータを動的にフィルタリング", "xpack.maps.source.esSearch.fieldNotFoundMsg": "インデックスパターン''{indexPatternName}''に''{fieldName}''が見つかりません。", - "xpack.maps.source.esSearch.geofieldLabel": "地理空間フィールド", "xpack.maps.source.esSearch.geoFieldLabel": "地理空間フィールド", + "xpack.maps.source.esSearch.geofieldLabel": "地理空間フィールド", "xpack.maps.source.esSearch.geoFieldTypeLabel": "地理空間フィールドタイプ", "xpack.maps.source.esSearch.indexOverOneLengthEditError": "データビューは複数のインデックスを参照しています。データビューごとに1つのインデックスのみが許可されています。", "xpack.maps.source.esSearch.indexZeroLengthEditError": "データビューはどのインデックスも参照していません。", @@ -33060,7 +33068,6 @@ "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.contentsLabel": "目次", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.idLabel": "名前", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiMarkdownEditor.enterContentsLabel": "コンテンツを入力", - "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.editEntryLabel": "{id}を編集", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.newEntryLabel": "新しいエントリー", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.cancelButtonEmptyLabel": "キャンセル", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.euiMarkdownEditor.observabilityAiAssistantKnowledgeBaseViewMarkdownEditorLabel": "observabilityAiAssistantKnowledgeBaseViewMarkdownEditor", @@ -36431,8 +36438,8 @@ "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.maxAlertsFieldLessThanWarning": "Kibanaで許可される最大数は、1回の実行につき、{maxNumber} {maxNumber, plural, other {アラート}}です。", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.nameFieldRequiredError": "名前が必要です。", "xpack.securitySolution.detectionEngine.createRule.stepAboutrule.noteHelpText": "ルール調査ガイドを追加...", - "xpack.securitySolution.detectionEngine.createRule.stepAboutrule.setupHelpText": "ルールセットアップガイドを追加...", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.setupHelpText": "必要な統合、構成ステップ、ルールが正常に動作するために必要な他のすべての項目といった、ルール前提条件に関する指示を入力します。", + "xpack.securitySolution.detectionEngine.createRule.stepAboutrule.setupHelpText": "ルールセットアップガイドを追加...", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.setupLabel": "セットアップガイド", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.tagFieldEmptyError": "タグを空にすることはできません", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.threatIndicatorPathFieldEmptyError": "インジケータープレフィックスの無効化を空にすることはできません", @@ -39243,10 +39250,6 @@ "xpack.securitySolution.flyout.right.alertPreview.ariaLabel": "ID {id}のアラートをプレビュー", "xpack.securitySolution.flyout.right.eventCategoryText": "イベントカテゴリ", "xpack.securitySolution.flyout.right.header.assignedTitle": "担当者", - "securitySolutionPackages.flyout.right.header.collapseDetailButtonAriaLabel": "詳細を折りたたむ", - "securitySolutionPackages.flyout.right.header.collapseDetailButtonLabel": "詳細を折りたたむ", - "securitySolutionPackages.flyout.right.header.expandDetailButtonAriaLabel": "詳細を展開", - "securitySolutionPackages.flyout.right.header.expandDetailButtonLabel": "詳細を展開", "xpack.securitySolution.flyout.right.header.headerTitle": "ドキュメント詳細", "xpack.securitySolution.flyout.right.header.jsonTabLabel": "JSON", "xpack.securitySolution.flyout.right.header.overviewTabLabel": "概要", @@ -39333,10 +39336,6 @@ "xpack.securitySolution.flyout.right.visualizations.sessionPreview.timeDescription": "に", "xpack.securitySolution.flyout.right.visualizations.sessionPreview.upsellDescription": "この機能には{subscription}が必要です", "xpack.securitySolution.flyout.right.visualizations.sessionPreview.upsellLinkText": "エンタープライズサブスクリプション", - "securitySolutionPackages.flyout.shared.errorDescription": "{message}の表示中にエラーが発生しました。", - "securitySolutionPackages.flyout.shared.errorTitle": "{title}を表示できません。", - "securitySolutionPackages.flyout.shared.ExpandablePanelButtonIconAriaLabel": "展開可能なパネルトグル", - "securitySolutionPackages.flyout.shared.expandablePanelLoadingAriaLabel": "展開可能なパネル", "xpack.securitySolution.flyout.tour.entities.description": "アラートに関連付けられたホストとユーザーの詳細については、展開された{entities}ビューを確認してください。", "xpack.securitySolution.flyout.tour.entities.text": "エンティティ", "xpack.securitySolution.flyout.tour.entities.title": "新しいホストとユーザーのインサイトがあります", @@ -42045,8 +42044,8 @@ "xpack.slo.sloEmbeddable.config.sloSelector.placeholder": "SLOを選択", "xpack.slo.sloEmbeddable.displayName": "SLO概要", "xpack.slo.sloEmbeddable.overview.sloNotFoundText": "SLOが削除されました。ウィジェットをダッシュボードから安全に削除できます。", - "xpack.slo.sloGridItem.targetFlexItemLabel": "目標{target}", "xpack.slo.sLOGridItem.targetFlexItemLabel": "目標{target}", + "xpack.slo.sloGridItem.targetFlexItemLabel": "目標{target}", "xpack.slo.sloGroupConfiguration.customFiltersLabel": "カスタムフィルター", "xpack.slo.sloGroupConfiguration.customFiltersOptional": "オプション", "xpack.slo.sloGroupConfiguration.customFilterText": "カスタムフィルター", @@ -43488,8 +43487,8 @@ "xpack.stackConnectors.components.casesWebhookxpack.stackConnectors.components.casesWebhook.connectorTypeTitle": "Webフック - ケース管理データ", "xpack.stackConnectors.components.d3security.bodyCodeEditorAriaLabel": "コードエディター", "xpack.stackConnectors.components.d3security.bodyFieldLabel": "本文", - "xpack.stackConnectors.components.d3security.connectorTypeTitle": "D3データ", "xpack.stackConnectors.components.d3Security.connectorTypeTitle": "D3セキュリティ", + "xpack.stackConnectors.components.d3security.connectorTypeTitle": "D3データ", "xpack.stackConnectors.components.d3security.eventTypeFieldLabel": "イベントタイプ", "xpack.stackConnectors.components.d3security.invalidActionText": "無効なアクション名です。", "xpack.stackConnectors.components.d3security.requiredActionText": "アクションが必要です。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 6ffabdf583e73..ec877d355b31c 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -7042,6 +7042,14 @@ "securitySolutionPackages.features.featureRegistry.subFeatures.trustedApplications": "受信任的应用程序", "securitySolutionPackages.features.featureRegistry.subFeatures.trustedApplications.description": "帮助减少与其他软件（通常指其他防病毒或终端安全应用程序）的冲突。", "securitySolutionPackages.features.featureRegistry.subFeatures.trustedApplications.privilegesTooltip": "访问受信任的应用程序需要所有工作区。", + "securitySolutionPackages.flyout.right.header.collapseDetailButtonAriaLabel": "折叠详情", + "securitySolutionPackages.flyout.right.header.collapseDetailButtonLabel": "折叠详情", + "securitySolutionPackages.flyout.right.header.expandDetailButtonAriaLabel": "展开详情", + "securitySolutionPackages.flyout.right.header.expandDetailButtonLabel": "展开详情", + "securitySolutionPackages.flyout.shared.errorDescription": "显示 {message} 时出现错误。", + "securitySolutionPackages.flyout.shared.errorTitle": "无法显示 {title}。", + "securitySolutionPackages.flyout.shared.ExpandablePanelButtonIconAriaLabel": "可展开面板切换按钮", + "securitySolutionPackages.flyout.shared.expandablePanelLoadingAriaLabel": "可展开面板", "securitySolutionPackages.markdown.insight.upsell": "升级到{requiredLicense}以利用调查指南中的洞见", "securitySolutionPackages.markdown.investigationGuideInteractions.upsell": "升级到 {requiredLicense} 以利用调查指南交互", "securitySolutionPackages.navigation.landingLinks": "安全视图", @@ -11115,8 +11123,8 @@ "xpack.apm.serviceIcons.service": "服务", "xpack.apm.serviceIcons.serviceDetails.cloud.architecture": "架构", "xpack.apm.serviceIcons.serviceDetails.cloud.availabilityZoneLabel": "{zones, plural, other {可用性区域}} ", - "xpack.apm.serviceIcons.serviceDetails.cloud.faasTriggerTypeLabel": "{triggerTypes, plural, other {触发类型}} ", "xpack.apm.serviceIcons.serviceDetails.cloud.functionNameLabel": "{functionNames, plural, other {功能名称}} ", + "xpack.apm.serviceIcons.serviceDetails.cloud.faasTriggerTypeLabel": "{triggerTypes, plural, other {触发类型}} ", "xpack.apm.serviceIcons.serviceDetails.cloud.machineTypesLabel": "{machineTypes, plural, other {机器类型}} ", "xpack.apm.serviceIcons.serviceDetails.cloud.projectIdLabel": "项目 ID", "xpack.apm.serviceIcons.serviceDetails.cloud.providerLabel": "云服务提供商", @@ -27004,8 +27012,8 @@ "xpack.maps.source.esSearch.descendingLabel": "降序", "xpack.maps.source.esSearch.extentFilterLabel": "在可见地图区域中动态筛留数据", "xpack.maps.source.esSearch.fieldNotFoundMsg": "在索引模式“{indexPatternName}”中找不到“{fieldName}”。", - "xpack.maps.source.esSearch.geofieldLabel": "地理空间字段", "xpack.maps.source.esSearch.geoFieldLabel": "地理空间字段", + "xpack.maps.source.esSearch.geofieldLabel": "地理空间字段", "xpack.maps.source.esSearch.geoFieldTypeLabel": "地理空间字段类型", "xpack.maps.source.esSearch.indexOverOneLengthEditError": "您的数据视图指向多个索引。每个数据视图只允许一个索引。", "xpack.maps.source.esSearch.indexZeroLengthEditError": "您的数据视图未指向任何索引。", @@ -33100,7 +33108,6 @@ "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.contentsLabel": "内容", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.idLabel": "名称", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiMarkdownEditor.enterContentsLabel": "输入内容", - "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.editEntryLabel": "编辑 {id}", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.newEntryLabel": "新条目", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.cancelButtonEmptyLabel": "取消", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.euiMarkdownEditor.observabilityAiAssistantKnowledgeBaseViewMarkdownEditorLabel": "observabilityAiAssistantKnowledgeBaseViewMarkdownEditor", @@ -36473,8 +36480,8 @@ "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.maxAlertsFieldLessThanWarning": "每次规则运行时，Kibana 最多只允许 {maxNumber} 个{maxNumber, plural, other {告警}}。", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.nameFieldRequiredError": "名称必填。", "xpack.securitySolution.detectionEngine.createRule.stepAboutrule.noteHelpText": "添加规则调查指南......", - "xpack.securitySolution.detectionEngine.createRule.stepAboutrule.setupHelpText": "添加规则设置指南......", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.setupHelpText": "提供有关规则先决条件的说明，如所需集成、配置步骤，以及规则正常运行所需的任何其他内容。", + "xpack.securitySolution.detectionEngine.createRule.stepAboutrule.setupHelpText": "添加规则设置指南......", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.setupLabel": "设置指南", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.tagFieldEmptyError": "标签不得为空", "xpack.securitySolution.detectionEngine.createRule.stepAboutRule.threatIndicatorPathFieldEmptyError": "指标前缀覆盖不得为空", @@ -39287,10 +39294,6 @@ "xpack.securitySolution.flyout.right.alertPreview.ariaLabel": "预览 ID 为 {id} 的告警", "xpack.securitySolution.flyout.right.eventCategoryText": "事件类别", "xpack.securitySolution.flyout.right.header.assignedTitle": "被分配人", - "securitySolutionPackages.flyout.right.header.collapseDetailButtonAriaLabel": "折叠详情", - "securitySolutionPackages.flyout.right.header.collapseDetailButtonLabel": "折叠详情", - "securitySolutionPackages.flyout.right.header.expandDetailButtonAriaLabel": "展开详情", - "securitySolutionPackages.flyout.right.header.expandDetailButtonLabel": "展开详情", "xpack.securitySolution.flyout.right.header.headerTitle": "文档详情", "xpack.securitySolution.flyout.right.header.jsonTabLabel": "JSON", "xpack.securitySolution.flyout.right.header.overviewTabLabel": "概览", @@ -39376,10 +39379,6 @@ "xpack.securitySolution.flyout.right.visualizations.sessionPreview.timeDescription": "处于", "xpack.securitySolution.flyout.right.visualizations.sessionPreview.upsellDescription": "此功能需要{subscription}", "xpack.securitySolution.flyout.right.visualizations.sessionPreview.upsellLinkText": "企业级订阅", - "securitySolutionPackages.flyout.shared.errorDescription": "显示 {message} 时出现错误。", - "securitySolutionPackages.flyout.shared.errorTitle": "无法显示 {title}。", - "securitySolutionPackages.flyout.shared.ExpandablePanelButtonIconAriaLabel": "可展开面板切换按钮", - "securitySolutionPackages.flyout.shared.expandablePanelLoadingAriaLabel": "可展开面板", "xpack.securitySolution.flyout.tour.entities.description": "请查阅展开的 {entities} 视图以了解与该告警有关的主机和用户的更多信息。", "xpack.securitySolution.flyout.tour.entities.text": "实体", "xpack.securitySolution.flyout.tour.entities.title": "有新主机和用户洞见可用", @@ -42089,8 +42088,8 @@ "xpack.slo.sloEmbeddable.config.sloSelector.placeholder": "选择 SLO", "xpack.slo.sloEmbeddable.displayName": "SLO 概览", "xpack.slo.sloEmbeddable.overview.sloNotFoundText": "SLO 已删除。您可以放心从仪表板中删除小组件。", - "xpack.slo.sloGridItem.targetFlexItemLabel": "目标 {target}", "xpack.slo.sLOGridItem.targetFlexItemLabel": "目标 {target}", + "xpack.slo.sloGridItem.targetFlexItemLabel": "目标 {target}", "xpack.slo.sloGroupConfiguration.customFiltersLabel": "定制筛选", "xpack.slo.sloGroupConfiguration.customFiltersOptional": "可选", "xpack.slo.sloGroupConfiguration.customFilterText": "定制筛选", @@ -43536,8 +43535,8 @@ "xpack.stackConnectors.components.casesWebhookxpack.stackConnectors.components.casesWebhook.connectorTypeTitle": "Webhook - 案例管理数据", "xpack.stackConnectors.components.d3security.bodyCodeEditorAriaLabel": "代码编辑器", "xpack.stackConnectors.components.d3security.bodyFieldLabel": "正文", - "xpack.stackConnectors.components.d3security.connectorTypeTitle": "D3 数据", "xpack.stackConnectors.components.d3Security.connectorTypeTitle": "D3 Security", + "xpack.stackConnectors.components.d3security.connectorTypeTitle": "D3 数据", "xpack.stackConnectors.components.d3security.eventTypeFieldLabel": "事件类型", "xpack.stackConnectors.components.d3security.invalidActionText": "操作名称无效。", "xpack.stackConnectors.components.d3security.requiredActionText": "“操作”必填。", From b718ae509bc8abe0ddb794382c9f1ea23022619e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 10:56:55 +0200 Subject: [PATCH 08/42] Remove unused imports --- .../observability_ai_assistant/server/functions/summarize.ts | 1 - .../observability_ai_assistant/server/routes/functions/route.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index c46aadccec728..01b03cfc5bdca 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -6,7 +6,6 @@ */ import { v4 } from 'uuid'; -import { KnowledgeBaseType } from '../../common/types'; import type { FunctionRegistrationParameters } from '.'; import { KnowledgeBaseEntryRole } from '../../common'; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index 271da81d093e4..5939552cab36a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -9,7 +9,7 @@ import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { v4 } from 'uuid'; import { FunctionDefinition } from '../../../common/functions/types'; -import { KnowledgeBaseEntryRole, KnowledgeBaseType } from '../../../common/types'; +import { KnowledgeBaseEntryRole } from '../../../common/types'; import type { RecalledEntry } from '../../service/knowledge_base_service'; import { getSystemMessageFromInstructions } from '../../service/util/get_system_message_from_instructions'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; From 59a93624dc96b571c9a847c0724f1c30b0b54140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 13:19:27 +0200 Subject: [PATCH 09/42] Re-add `ShortIdTable` --- .../service/knowledge_base_service/index.ts | 13 ++++++++---- .../server/utils/recall/recall_and_score.ts | 4 ++-- .../server/utils/recall/score_suggestions.ts | 20 ++++++++++++++----- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index d5f55307cb727..caf4b9fc1125b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -38,7 +38,8 @@ interface Dependencies { export interface RecalledEntry { id: string; - doc_id?: string; + title?: string; + docId?: string; text: string; score: number | null; is_correction?: boolean; @@ -403,18 +404,22 @@ export class KnowledgeBaseService { }; const response = await this.dependencies.esClient.asInternalUser.search< - Pick + Pick >({ index: [resourceNames.aliases.kb], query: esQuery, size: 20, _source: { - includes: ['text', 'is_correction', 'labels', 'doc_id'], + includes: ['text', 'is_correction', 'labels', 'doc_id', 'title'], }, }); return response.hits.hits.map((hit) => ({ - ...hit._source!, + text: hit._source?.text!, + is_correction: hit._source?.is_correction, + labels: hit._source?.labels, + title: hit._source?.title, + docId: hit._source?.doc_id, score: hit._score!, id: hit._id!, })); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts index 77b7256e017ea..bb1164b1cf4b3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts @@ -14,7 +14,7 @@ import type { FunctionCallChatFunction } from '../../service/types'; import { RecallRanking, recallRankingEventType } from '../../analytics/recall_ranking'; import { RecalledEntry } from '../../service/knowledge_base_service'; -export type RecalledSuggestion = Pick; +export type RecalledSuggestion = Pick; export async function recallAndScore({ recall, @@ -46,7 +46,7 @@ export async function recallAndScore({ const { entries: recalledEntries } = await recall({ queries }); const suggestions: RecalledSuggestion[] = recalledEntries.map( - ({ id, doc_id: docId, text, score }) => ({ id, doc_id: docId, text, score }) + ({ id, title, docId, text, score }) => ({ id, title, docId, text, score }) ); if (!suggestions.length) { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts index c71676c60a6c4..d5ed19d6ec34c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts @@ -9,10 +9,12 @@ import { Logger } from '@kbn/logging'; import dedent from 'dedent'; import { lastValueFrom } from 'rxjs'; import { decodeOrThrow, jsonRt } from '@kbn/io-ts-utils'; +import { omit } from 'lodash'; import { concatenateChatCompletionChunks, Message, MessageRole } from '../../../common'; import type { FunctionCallChatFunction } from '../../service/types'; import { parseSuggestionScores } from './parse_suggestion_scores'; import { RecalledSuggestion } from './recall_and_score'; +import { ShortIdTable } from '../../../common/utils/short_id_table'; const scoreFunctionRequestRt = t.type({ message: t.type({ @@ -47,6 +49,8 @@ export async function scoreSuggestions({ relevantDocuments: RecalledSuggestion[]; scores: Array<{ id: string; score: number }>; }> { + const shortIdTable = new ShortIdTable(); + const newUserMessageContent = dedent(`Given the following question, score the documents that are relevant to the question. on a scale from 0 to 7, 0 being completely irrelevant, and 7 being extremely relevant. Information is relevant to the question if it helps in @@ -66,7 +70,10 @@ export async function scoreSuggestions({ Documents: ${JSON.stringify( - suggestions.map(({ id, doc_id: title, text }) => ({ id, title, text })), + suggestions.map((suggestion) => ({ + ...omit(suggestion, 'score'), // Omit score to not bias the LLM + id: shortIdTable.take(suggestion.id), // Shorten id to save tokens + })), null, 2 )}`); @@ -114,7 +121,9 @@ export async function scoreSuggestions({ scoreFunctionRequest.message.function_call.arguments ); - const scores = parseSuggestionScores(scoresAsString); + const scores = parseSuggestionScores(scoresAsString) + // Restore original IDs + .map(({ id, score }) => ({ id: shortIdTable.lookup(id)!, score })); if (scores.length === 0) { // seemingly invalid or no scores, return all @@ -123,12 +132,13 @@ export async function scoreSuggestions({ const suggestionIds = suggestions.map((document) => document.id); + // get top 5 documents ids with scores > 4 const relevantDocumentIds = scores - .filter((document) => suggestionIds.includes(document.id ?? '')) // Remove hallucinated documents - .filter((document) => document.score > 4) + .filter(({ score }) => score > 4) .sort((a, b) => b.score - a.score) .slice(0, 5) - .map((document) => document.id); + .filter(({ id }) => suggestionIds.includes(id ?? '')) // Remove hallucinated documents + .map(({ id }) => id); const relevantDocuments = suggestions.filter((suggestion) => relevantDocumentIds.includes(suggestion.id) From 51cb6884e4cdd46c9c75c54fdc69ab6807164eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 13:28:59 +0200 Subject: [PATCH 10/42] Change `recall` to not return entries as nested object --- .../server/routes/functions/route.ts | 3 ++- .../server/service/client/index.ts | 2 +- .../server/service/knowledge_base_service/index.ts | 8 ++------ .../server/utils/recall/recall_and_score.ts | 3 +-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index 5939552cab36a..4eae7e0b2d6ae 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -100,7 +100,8 @@ const functionRecallRoute = createObservabilityAIAssistantServerRoute({ throw notImplemented(); } - return client.recall({ queries, categories }); + const entries = await client.recall({ queries, categories }); + return { entries }; }, }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 1e0f9ce833263..a88686140172e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -714,7 +714,7 @@ export class ObservabilityAIAssistantClient { }: { queries: Array<{ text: string; boost?: number }>; categories?: string[]; - }): Promise<{ entries: RecalledEntry[] }> => { + }): Promise => { return this.dependencies.knowledgeBaseService.recall({ namespace: this.dependencies.namespace, user: this.dependencies.user, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index caf4b9fc1125b..2d2058af9d33c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -439,9 +439,7 @@ export class KnowledgeBaseService { namespace: string; esClient: { asCurrentUser: ElasticsearchClient; asInternalUser: ElasticsearchClient }; uiSettingsClient: IUiSettingsClient; - }): Promise<{ - entries: RecalledEntry[]; - }> => { + }): Promise => { this.dependencies.logger.debug( () => `Recalling entries from KB for queries: "${JSON.stringify(queries)}"` ); @@ -505,9 +503,7 @@ export class KnowledgeBaseService { this.dependencies.logger.info(`Dropped ${droppedEntries} entries because of token limit`); } - return { - entries: returnedEntries, - }; + return returnedEntries; }; getUserInstructions = async ( diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts index bb1164b1cf4b3..b71cf68f026e5 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts @@ -44,8 +44,7 @@ export async function recallAndScore({ { text: context, boost: 1 }, ].filter((query) => query.text.trim()); - const { entries: recalledEntries } = await recall({ queries }); - const suggestions: RecalledSuggestion[] = recalledEntries.map( + const suggestions: RecalledSuggestion[] = (await recall({ queries })).map( ({ id, title, docId, text, score }) => ({ id, title, docId, text, score }) ); From 51495d22871e8b9970e804e4003eeed9a5ead21a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 13:36:08 +0200 Subject: [PATCH 11/42] Rename `groupId` back to `docId` to reduce diff --- .../server/service/knowledge_base_service/index.ts | 6 +++--- .../server/service/util/split_kb_text.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index 2d2058af9d33c..6524fafea26be 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -71,7 +71,7 @@ export enum KnowledgeBaseEntryOperationType { interface KnowledgeBaseDeleteOperation { type: KnowledgeBaseEntryOperationType.Delete; - groupId?: string; + docId?: string; labels?: Record; } @@ -236,7 +236,7 @@ export class KnowledgeBaseService { query: { bool: { filter: [ - ...(operation.groupId ? [{ term: { doc_id: operation.groupId } }] : []), + ...(operation.docId ? [{ term: { doc_id: operation.docId } }] : []), ...(operation.labels ? map(operation.labels, (value, key) => { return { term: { [key]: value } }; @@ -249,7 +249,7 @@ export class KnowledgeBaseService { return; } catch (error) { this.dependencies.logger.error( - `Failed to delete document "${operation?.groupId}" due to ${error.message}` + `Failed to delete document "${operation?.docId}" due to ${error.message}` ); this.dependencies.logger.debug(() => JSON.stringify(operation)); throw error; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts index 4b3dd559df3eb..7d1b178129c6a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts @@ -20,14 +20,14 @@ export function splitKbText({ return [ { type: KnowledgeBaseEntryOperationType.Delete, - groupId: id, // delete all entries with the same groupId + docId: id, // delete all entries with the same docId labels: {}, }, ...texts.map((text, index) => ({ type: KnowledgeBaseEntryOperationType.Index, document: merge({}, rest, { id: [id, index].join('_'), - doc_id: id, // group_id is used to group entries together + doc_id: id, // doc_id is used to group entries together. labels: {}, text, }), From 5fdcc4db054cf083e321229e21dfa1c83aa6f1a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 14:27:03 +0200 Subject: [PATCH 12/42] Add test for shortIdTable --- .../common/utils/short_id_table.test.ts | 10 ++++++++++ .../knowledge_base_edit_manual_entry_flyout.tsx | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/short_id_table.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/short_id_table.test.ts index 784cf67530652..03d1cb177826e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/short_id_table.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/short_id_table.test.ts @@ -7,6 +7,16 @@ import { ShortIdTable } from './short_id_table'; describe('shortIdTable', () => { + it('generates a short id from a uuid', () => { + const table = new ShortIdTable(); + + const uuid = 'd877f65c-4036-42c4-b105-19e2f1a1c045'; + const shortId = table.take(uuid); + + expect(shortId.length).toBe(4); + expect(table.lookup(shortId)).toBe(uuid); + }); + it('generates at least 10k unique ids consistently', () => { const ids = new Set(); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx index 21d92de190325..075495783c8d1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx @@ -49,9 +49,9 @@ export function KnowledgeBaseEditManualEntryFlyout({ const [newEntryTitle, setNewEntryTitle] = useState(entry?.title ?? entry?.doc_id ?? ''); const [newEntryText, setNewEntryText] = useState(entry?.text ?? ''); - const isEntryIdInvalid = newEntryTitle.trim() === ''; + const isEntryTitleInvalid = newEntryTitle.trim() === ''; const isEntryTextInvalid = newEntryText.trim() === ''; - const isFormInvalid = isEntryIdInvalid || isEntryTextInvalid; + const isFormInvalid = isEntryTitleInvalid || isEntryTextInvalid; const handleSubmit = async () => { await createEntry({ @@ -138,7 +138,7 @@ export function KnowledgeBaseEditManualEntryFlyout({ fullWidth value={newEntryTitle} onChange={(e) => setNewEntryTitle(e.target.value)} - isInvalid={isEntryIdInvalid} + isInvalid={isEntryTitleInvalid} /> From d470e397ed71a022f0d3c1f98ea2f2d011bd487d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 14:27:52 +0200 Subject: [PATCH 13/42] Revert change to reduce diff --- .../knowledge_base_edit_user_instruction_flyout.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx index be23f3e550b47..b95cf4418ff87 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx @@ -22,7 +22,7 @@ import { EuiText, EuiTitle, } from '@elastic/eui'; -import { v4 } from 'uuid'; +import { v4 as uuidv4 } from 'uuid'; import { useGetUserInstructions } from '../../hooks/use_get_user_instructions'; import { useCreateKnowledgeBaseUserInstruction } from '../../hooks/use_create_knowledge_base_user_instruction'; @@ -40,7 +40,7 @@ export function KnowledgeBaseEditUserInstructionFlyout({ onClose }: { onClose: ( const handleSubmit = async () => { await createEntry({ entry: { - id: v4(), + id: uuidv4(), text: newEntryText, public: false, // limit user instructions to private (for now) }, From fb7370b6f708470dc73960d16c9d771de233728c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 15:14:16 +0200 Subject: [PATCH 14/42] Keep original id --- .../knowledge_base_edit_user_instruction_flyout.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx index b95cf4418ff87..aa4fe3d5fbdaf 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx @@ -30,17 +30,19 @@ export function KnowledgeBaseEditUserInstructionFlyout({ onClose }: { onClose: ( const { userInstructions, isLoading: isFetching } = useGetUserInstructions(); const { mutateAsync: createEntry, isLoading: isSaving } = useCreateKnowledgeBaseUserInstruction(); const [newEntryText, setNewEntryText] = useState(''); + const [newEntryId, setNewEntryId] = useState(); const isSubmitDisabled = newEntryText.trim() === ''; useEffect(() => { const userInstruction = userInstructions?.find((entry) => !entry.public); setNewEntryText(userInstruction?.text ?? ''); + setNewEntryId(userInstruction?.id); }, [userInstructions]); const handleSubmit = async () => { await createEntry({ entry: { - id: uuidv4(), + id: newEntryId ?? uuidv4(), text: newEntryText, public: false, // limit user instructions to private (for now) }, From c63a8f9af9ce40e823be88d7c6ab1d068a3850a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 15:15:19 +0200 Subject: [PATCH 15/42] =?UTF-8?q?Omit=20=E2=80=9CUser=E2=80=9D=20suffix=20?= =?UTF-8?q?from=20API=20client=20methods?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/config.ts | 10 ++-- .../common/users/users.ts | 8 +-- .../tests/complete/complete.spec.ts | 12 ++-- .../tests/complete/functions/helpers.ts | 2 +- .../complete/functions/summarize.spec.ts | 2 +- .../tests/connectors/connectors.spec.ts | 6 +- .../tests/conversations/conversations.spec.ts | 24 ++++---- .../knowledge_base/knowledge_base.spec.ts | 36 +++++------ .../knowledge_base_setup.spec.ts | 4 +- .../knowledge_base_status.spec.ts | 6 +- .../knowledge_base_user_instructions.spec.ts | 60 +++++++++---------- .../public_complete/public_complete.spec.ts | 2 +- .../common/config.ts | 14 ++--- .../tests/conversations/index.spec.ts | 16 ++--- 14 files changed, 101 insertions(+), 101 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts index 50d660ae1a0dd..b5e4bde29b11e 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts @@ -11,7 +11,7 @@ import { ObservabilityAIAssistantFtrConfigName } from '../configs'; import { getApmSynthtraceEsClient } from './create_synthtrace_client'; import { InheritedFtrProviderContext, InheritedServices } from './ftr_provider_context'; import { getScopedApiClient } from './observability_ai_assistant_api_client'; -import { editorUser, secondaryEditorUser, viewerUser } from './users/users'; +import { editor, secondaryEditor, viewer } from './users/users'; export interface ObservabilityAIAssistantFtrConfig { name: ObservabilityAIAssistantFtrConfigName; @@ -62,10 +62,10 @@ export function createObservabilityAIAssistantAPIConfig({ getApmSynthtraceEsClient(context, apmSynthtraceKibanaClient), observabilityAIAssistantAPIClient: async () => { return { - adminUser: getScopedApiClient(kibanaServer, 'elastic'), - viewerUser: getScopedApiClient(kibanaServer, viewerUser.username), - editorUser: getScopedApiClient(kibanaServer, editorUser.username), - secondaryEditorUser: getScopedApiClient(kibanaServer, secondaryEditorUser.username), + admin: getScopedApiClient(kibanaServer, 'elastic'), + viewer: getScopedApiClient(kibanaServer, viewer.username), + editor: getScopedApiClient(kibanaServer, editor.username), + secondaryEditor: getScopedApiClient(kibanaServer, secondaryEditor.username), }; }, }, diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts b/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts index bf76c5f770f69..898954a9bfb97 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts @@ -14,22 +14,22 @@ export interface User { roles: string[]; } -export const editorUser: User = { +export const editor: User = { username: 'editor', password, roles: ['editor'], }; -export const secondaryEditorUser: User = { +export const secondaryEditor: User = { username: 'secondary_editor', password, roles: ['editor'], }; -export const viewerUser: User = { +export const viewer: User = { username: 'viewer', password, roles: ['viewer'], }; -export const allUsers = [editorUser, secondaryEditorUser, viewerUser]; +export const allUsers = [editor, secondaryEditor, viewer]; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts index d2cab3a697cf0..2b62cc64b2519 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts @@ -286,7 +286,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { )[0]?.conversation.id; await observabilityAIAssistantAPIClient - .adminUser({ + .admin({ endpoint: 'DELETE /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -362,7 +362,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { ).to.eql(0); const conversations = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }) .expect(200); @@ -392,7 +392,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .completeAfterIntercept(); const createResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', params: { body: { @@ -410,7 +410,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { conversationCreatedEvent = getConversationCreatedEvent(createResponse.body); const conversationId = conversationCreatedEvent.conversation.id; - const fullConversation = await observabilityAIAssistantAPIClient.editorUser({ + const fullConversation = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -424,7 +424,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .completeAfterIntercept(); const updatedResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', params: { body: { @@ -454,7 +454,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { after(async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'DELETE /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts index a32c22abcf7aa..3e50a4cee7b7f 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts @@ -40,7 +40,7 @@ export async function invokeChatCompleteWithFunctionRequest({ functionCall: Message['message']['function_call']; }) { const { body } = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', params: { body: { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index ea929c34526b8..efcc196fb6e8d 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -59,7 +59,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('persists entry in knowledge base', async () => { - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts index e8363ba41513b..41700b21555fa 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts @@ -26,14 +26,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('Returns a 2xx for enterprise license', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }) .expect(200); }); it('returns an empty list of connectors', async () => { - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }); @@ -43,7 +43,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it("returns the gen ai connector if it's been created", async () => { const connectorId = await createProxyActionConnector({ supertest, log, port: 1234 }); - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts index 91a418b3000ee..71eb37d357696 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts @@ -48,7 +48,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('without conversations', () => { it('returns no conversations when listing', async () => { const response = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }) .expect(200); @@ -58,7 +58,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns a 404 for updating conversations', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -74,7 +74,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns a 404 for retrieving a conversation', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -92,7 +92,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { >; before(async () => { createResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/conversation', params: { body: { @@ -105,7 +105,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { after(async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'DELETE /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -116,7 +116,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -148,7 +148,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns a 404 for updating a non-existing conversation', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -164,7 +164,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns a 404 for retrieving a non-existing conversation', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -177,7 +177,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns the conversation that was created', async () => { const response = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -192,7 +192,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns the created conversation when listing', async () => { const response = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }) .expect(200); @@ -210,7 +210,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { updateResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -234,7 +234,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns the updated conversation after get', async () => { const updateAfterCreateResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index a2ebee43b54dc..b019d46c1e8f2 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -26,7 +26,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(200); @@ -45,12 +45,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { }; it('returns 200 on create', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/save', params: { body: knowledgeBaseEntry }, }) .expect(200); - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { @@ -68,7 +68,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns 200 on get entries and entry exists', async () => { const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { @@ -88,7 +88,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns 200 on delete', async () => { const entryId = 'my-doc-id-1'; await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'DELETE /internal/observability_ai_assistant/kb/entries/{entryId}', params: { path: { entryId }, @@ -97,7 +97,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { @@ -116,7 +116,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns 500 on delete not found', async () => { const entryId = 'my-doc-id-1'; await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'DELETE /internal/observability_ai_assistant/kb/entries/{entryId}', params: { path: { entryId }, @@ -133,7 +133,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { sortDirection = 'asc', }: { query?: string; sortBy?: string; sortDirection?: 'asc' | 'desc' } = {}) { const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { query, sortBy, sortDirection }, @@ -148,7 +148,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await clearKnowledgeBase(es); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', params: { body: { @@ -216,7 +216,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const importCategories = () => observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/category/import', params: { body: { @@ -260,7 +260,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await pRetry( async () => { const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { @@ -326,7 +326,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { async function getEntriesWithDocId(docId: string) { const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { @@ -349,14 +349,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { const docId = 'my_favourite_color'; await addEntryWithDocId({ - apiClient: observabilityAIAssistantAPIClient.editorUser, + apiClient: observabilityAIAssistantAPIClient.editor, docId, text: 'My favourite color is blue', }); entries1 = await getEntriesWithDocId(docId); await addEntryWithDocId({ - apiClient: observabilityAIAssistantAPIClient.editorUser, + apiClient: observabilityAIAssistantAPIClient.editor, docId, text: 'My favourite color is green', }); @@ -394,18 +394,18 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { await addEntryWithDocId({ - apiClient: observabilityAIAssistantAPIClient.editorUser, + apiClient: observabilityAIAssistantAPIClient.editor, docId: 'users_favorite_animal', text: "The user's favourite animal is a dog", }); await addEntryWithDocId({ - apiClient: observabilityAIAssistantAPIClient.secondaryEditorUser, + apiClient: observabilityAIAssistantAPIClient.secondaryEditor, docId: 'users_favorite_animal', text: "The user's favourite animal is a cat", }); const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { @@ -456,7 +456,7 @@ async function waitForModelReady( ) { return pRetry(async () => { const res = await observabilityAIAssistantAPIClient - .editorUser({ endpoint: 'GET /internal/observability_ai_assistant/kb/status' }) + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/status' }) .expect(200); const isModelReady = res.body.ready; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts index 9099eff540d35..77f010d851f3c 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts @@ -17,7 +17,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns empty object when successful', async () => { await createKnowledgeBaseModel(ml); const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(200); @@ -27,7 +27,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns bad request if model cannot be installed', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(400); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts index 62d2ab7cae785..65d3e0f45ac57 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts @@ -17,7 +17,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(200); @@ -29,7 +29,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns correct status after knowledge base is setup', async () => { const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/status', }) .expect(200); @@ -41,7 +41,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await ml.api.stopTrainedModelDeploymentES(TINY_ELSER.id, true); const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/status', }) .expect(200); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index 3083374c309a0..a30e07c85b7c4 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -32,7 +32,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient - .editorUser({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup' }) + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup' }) .expect(200); }); @@ -48,19 +48,19 @@ export default function ApiTest({ getService }: FtrProviderContext) { const promises = [ { - username: 'editorUser' as const, + username: 'editor' as const, isPublic: true, }, { - username: 'editorUser' as const, + username: 'editor' as const, isPublic: false, }, { - username: 'secondaryEditorUser' as const, + username: 'secondaryEditor' as const, isPublic: true, }, { - username: 'secondaryEditorUser' as const, + username: 'secondaryEditor' as const, isPublic: false, }, ].map(async ({ username, isPublic }) => { @@ -82,8 +82,8 @@ export default function ApiTest({ getService }: FtrProviderContext) { await Promise.all(promises); }); - it('"editorUser" can retrieve their own private instructions and the public instruction', async () => { - const res = await observabilityAIAssistantAPIClient.editorUser({ + it('"editor" can retrieve their own private instructions and the public instruction', async () => { + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions', }); const instructions = res.body.userInstructions; @@ -92,26 +92,26 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(sortById(instructions)).to.eql( sortById([ { - id: 'private-doc-from-editorUser', + id: 'private-doc-from-editor', public: false, - text: 'Private user instruction from "editorUser"', + text: 'Private user instruction from "editor"', }, { - id: 'public-doc-from-editorUser', + id: 'public-doc-from-editor', public: true, - text: 'Public user instruction from "editorUser"', + text: 'Public user instruction from "editor"', }, { - id: 'public-doc-from-secondaryEditorUser', + id: 'public-doc-from-secondaryEditor', public: true, - text: 'Public user instruction from "secondaryEditorUser"', + text: 'Public user instruction from "secondaryEditor"', }, ]) ); }); - it('"secondaryEditorUser" can retrieve their own private instructions and the public instruction', async () => { - const res = await observabilityAIAssistantAPIClient.secondaryEditorUser({ + it('"secondaryEditor" can retrieve their own private instructions and the public instruction', async () => { + const res = await observabilityAIAssistantAPIClient.secondaryEditor({ endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions', }); const instructions = res.body.userInstructions; @@ -120,19 +120,19 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(sortById(instructions)).to.eql( sortById([ { - id: 'public-doc-from-editorUser', + id: 'public-doc-from-editor', public: true, - text: 'Public user instruction from "editorUser"', + text: 'Public user instruction from "editor"', }, { - id: 'public-doc-from-secondaryEditorUser', + id: 'public-doc-from-secondaryEditor', public: true, - text: 'Public user instruction from "secondaryEditorUser"', + text: 'Public user instruction from "secondaryEditor"', }, { - id: 'private-doc-from-secondaryEditorUser', + id: 'private-doc-from-secondaryEditor', public: false, - text: 'Private user instruction from "secondaryEditorUser"', + text: 'Private user instruction from "secondaryEditor"', }, ]) ); @@ -144,7 +144,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await clearKnowledgeBase(es); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions', params: { body: { @@ -157,7 +157,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions', params: { body: { @@ -171,7 +171,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('updates the user instruction', async () => { - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions', }); const instructions = res.body.userInstructions; @@ -193,12 +193,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { const userInstructionText = 'Be polite and use language that is easy to understand. Never disagree with the user.'; - async function getConversationForUser(username: 'editorUser' | 'secondaryEditorUser') { + async function getConversationForUser(username: 'editor' | 'secondaryEditor') { const apiClient = observabilityAIAssistantAPIClient[username]; - // the user instruction is always created by "editorUser" user + // the user instruction is always created by "editor" user await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions', params: { body: { @@ -277,7 +277,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('adds the instruction to the system prompt', async () => { - const conversation = await getConversationForUser('editorUser'); + const conversation = await getConversationForUser('editor'); const systemMessage = conversation.messages.find( (message) => message.message.role === MessageRole.System )!; @@ -285,7 +285,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('does not add the instruction to the context', async () => { - const conversation = await getConversationForUser('editorUser'); + const conversation = await getConversationForUser('editor'); const contextMessage = conversation.messages.find( (message) => message.message.name === CONTEXT_FUNCTION_NAME ); @@ -299,7 +299,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('does not add the instruction conversation for other users', async () => { - const conversation = await getConversationForUser('secondaryEditorUser'); + const conversation = await getConversationForUser('secondaryEditor'); const systemMessage = conversation.messages.find( (message) => message.message.role === MessageRole.System )!; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts index 344b387bdde38..935eabf3f5a9d 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts @@ -70,7 +70,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { (body) => !isFunctionTitleRequest(body) ); - const responsePromise = observabilityAIAssistantAPIClient.adminUser({ + const responsePromise = observabilityAIAssistantAPIClient.admin({ endpoint: 'POST /api/observability_ai_assistant/chat/complete 2023-10-31', params: { query: { format }, diff --git a/x-pack/test/observability_ai_assistant_functional/common/config.ts b/x-pack/test/observability_ai_assistant_functional/common/config.ts index bb320f46e3ac0..99213e629e0e3 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/config.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/config.ts @@ -13,9 +13,9 @@ import { KibanaEBTUIProvider, } from '@kbn/test-suites-src/analytics/services/kibana_ebt'; import { - secondaryEditorUser, - editorUser, - viewerUser, + secondaryEditor, + editor, + viewer, } from '../../observability_ai_assistant_api_integration/common/users/users'; import { ObservabilityAIAssistantFtrConfig, @@ -62,10 +62,10 @@ async function getTestConfig({ ObservabilityAIAssistantUIProvider(context), observabilityAIAssistantAPIClient: async (context: InheritedFtrProviderContext) => { return { - adminUser: getScopedApiClient(kibanaServer, 'elastic'), - viewerUser: getScopedApiClient(kibanaServer, viewerUser.username), - editorUser: getScopedApiClient(kibanaServer, editorUser.username), - secondaryEditorUser: getScopedApiClient(kibanaServer, secondaryEditorUser.username), + admin: getScopedApiClient(kibanaServer, 'elastic'), + viewer: getScopedApiClient(kibanaServer, viewer.username), + editor: getScopedApiClient(kibanaServer, editor.username), + secondaryEditor: getScopedApiClient(kibanaServer, secondaryEditor.username), }; }, kibana_ebt_server: KibanaEBTServerProvider, diff --git a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts index ff20297efdeca..e43d8c15a75e7 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts @@ -36,12 +36,12 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte const flyoutService = getService('flyout'); async function deleteConversations() { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }); for (const conversation of response.body.conversations) { - await observabilityAIAssistantAPIClient.editorUser({ + await observabilityAIAssistantAPIClient.editor({ endpoint: `DELETE /internal/observability_ai_assistant/conversation/{conversationId}`, params: { path: { @@ -53,7 +53,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte } async function deleteConnectors() { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }); @@ -66,7 +66,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte } async function createOldConversation() { - await observabilityAIAssistantAPIClient.editorUser({ + await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversation', params: { body: { @@ -204,7 +204,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); it('creates a connector', async () => { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }); @@ -259,7 +259,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); it('creates a conversation and updates the URL', async () => { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }); @@ -325,7 +325,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); it('does not create another conversation', async () => { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }); @@ -333,7 +333,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); it('appends to the existing one', async () => { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }); From a6d55ed47ba4343110f3d2b3134aa097d8abd255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 28 Aug 2024 15:35:41 +0200 Subject: [PATCH 16/42] Type fix --- .../knowledge_base/knowledge_base_user_instructions.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index a30e07c85b7c4..1375e7d51b721 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -19,6 +19,7 @@ import { import { getConversationCreatedEvent } from '../conversations/helpers'; import { LlmProxy, createLlmProxy } from '../../common/create_llm_proxy'; import { createProxyActionConnector, deleteActionConnector } from '../../common/action_connectors'; +import { ObservabilityAIAssistantApiClients } from '../../common/config'; export default function ApiTest({ getService }: FtrProviderContext) { const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); @@ -193,7 +194,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const userInstructionText = 'Be polite and use language that is easy to understand. Never disagree with the user.'; - async function getConversationForUser(username: 'editor' | 'secondaryEditor') { + async function getConversationForUser(username: keyof ObservabilityAIAssistantApiClients) { const apiClient = observabilityAIAssistantAPIClient[username]; // the user instruction is always created by "editor" user From b3f7d3a1769b0012450f4ce3605f6f37cc255365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 29 Aug 2024 00:29:58 +0200 Subject: [PATCH 17/42] Fix categorization bug --- .../server/functions/summarize.ts | 15 +++++++--- .../server/routes/knowledge_base/route.ts | 8 ++--- .../public/helpers/categorize_entries.ts | 29 ++++++++++++------- .../hooks/use_create_knowledge_base_entry.ts | 7 ++--- ...nowledge_base_edit_manual_entry_flyout.tsx | 7 ++++- .../routes/components/knowledge_base_tab.tsx | 21 +++++--------- 6 files changed, 49 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index 01b03cfc5bdca..1f3c82d0fb368 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -14,6 +14,7 @@ export const SUMMARIZE_FUNCTION_NAME = 'summarize'; export function registerSummarizationFunction({ client, functions, + resources, }: FunctionRegistrationParameters) { functions.registerFunction( { @@ -70,17 +71,23 @@ export function registerSummarizationFunction({ // if no existing entry is found, we generate a uuid const id = await client.getUuidFromDocId(docId); + resources.logger.debug( + id + ? `Updating knowledge base entry with id: ${id}, doc_id: ${docId}` + : `Creating new knowledge base entry with doc_id: ${docId}` + ); + return client .addKnowledgeBaseEntry({ entry: { id: id ?? v4(), - title: docId, // use doc_id as title for now doc_id: docId, - role: KnowledgeBaseEntryRole.AssistantSummarization, + title: docId, // use doc_id as title for now text, - is_correction: isCorrection, - confidence, public: isPublic, + role: KnowledgeBaseEntryRole.AssistantSummarization, + confidence, + is_correction: isCorrection, labels: {}, }, // signal, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts index 793ed755ad95e..30b765149a56e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts @@ -153,9 +153,9 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ public: toBooleanRt, labels: t.record(t.string, t.string), role: t.union([ - t.literal('assistant_summarization'), - t.literal('user_entry'), - t.literal('elastic'), + t.literal(KnowledgeBaseEntryRole.AssistantSummarization), + t.literal(KnowledgeBaseEntryRole.UserEntry), + t.literal(KnowledgeBaseEntryRole.Elastic), ]), }), ]), @@ -192,7 +192,7 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ is_correction: isCorrection ?? false, public: isPublic ?? true, labels: labels ?? {}, - role: (role as KnowledgeBaseEntryRole) ?? KnowledgeBaseEntryRole.UserEntry, + role: role ?? KnowledgeBaseEntryRole.UserEntry, }, }); }, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts index 04642910708c3..1df1c3584a5b3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts @@ -9,21 +9,30 @@ import type { KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/ export interface KnowledgeBaseEntryCategory { '@timestamp': string; - categoryName: string; + categoryKey: string; + title: string; entries: KnowledgeBaseEntry[]; } -export function categorizeEntries({ entries }: { entries: KnowledgeBaseEntry[] }) { +export function categorizeEntries({ + entries, +}: { + entries: KnowledgeBaseEntry[]; +}): KnowledgeBaseEntryCategory[] { return entries.reduce((acc, entry) => { - const categoryName = entry.labels?.category ?? entry.title ?? entry.doc_id ?? entry.id; - - const index = acc.findIndex((item) => item.categoryName === categoryName); + const categoryKey = entry.labels?.category ?? entry.id; - if (index > -1) { - acc[index].entries.push(entry); + const existingEntry = acc.find((item) => item.categoryKey === categoryKey); + if (existingEntry) { + existingEntry.entries.push(entry); return acc; - } else { - return acc.concat({ categoryName, entries: [entry], '@timestamp': entry['@timestamp'] }); } - }, [] as Array<{ categoryName: string; entries: KnowledgeBaseEntry[]; '@timestamp': string }>); + + return acc.concat({ + categoryKey, + title: entry.labels?.category ?? entry.title ?? entry.doc_id ?? 'No title', + entries: [entry], + '@timestamp': entry['@timestamp'], + }); + }, [] as Array<{ categoryKey: string; title: string; entries: KnowledgeBaseEntry[]; '@timestamp': string }>); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts index ffd4ea8e0106e..4cc6c8e2b9bf1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts @@ -28,7 +28,7 @@ export function useCreateKnowledgeBaseEntry() { void, ServerError, { - entry: Omit & { + entry: Omit & { title: string; }; } @@ -40,10 +40,7 @@ export function useCreateKnowledgeBaseEntry() { { signal: null, params: { - body: { - ...entry, - role: 'user_entry', - }, + body: entry, }, } ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx index 075495783c8d1..5bb198d4f6a44 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx @@ -26,7 +26,8 @@ import { EuiTitle, } from '@elastic/eui'; import moment from 'moment'; -import type { KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common/types'; +import { KnowledgeBaseEntryRole } from '@kbn/observability-ai-assistant-plugin/public'; +import { type KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; import { v4 } from 'uuid'; import { useCreateKnowledgeBaseEntry } from '../../hooks/use_create_knowledge_base_entry'; import { useDeleteKnowledgeBaseEntry } from '../../hooks/use_delete_knowledge_base_entry'; @@ -61,6 +62,10 @@ export function KnowledgeBaseEditManualEntryFlyout({ title: newEntryTitle, text: newEntryText, public: isPublic, + role: KnowledgeBaseEntryRole.UserEntry, + confidence: 'high', + is_correction: false, + labels: {}, }, }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx index 567d22ee0d9e6..60db7d0d81702 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx @@ -57,10 +57,10 @@ export function KnowledgeBaseTab() { data-test-subj="pluginsColumnsButton" onClick={() => setSelectedCategory(category)} aria-label={ - category.categoryName === selectedCategory?.categoryName ? 'Collapse' : 'Expand' + category.categoryKey === selectedCategory?.categoryKey ? 'Collapse' : 'Expand' } iconType={ - category.categoryName === selectedCategory?.categoryName ? 'minimize' : 'expand' + category.categoryKey === selectedCategory?.categoryKey ? 'minimize' : 'expand' } /> ); @@ -85,7 +85,7 @@ export function KnowledgeBaseTab() { width: '40px', }, { - field: 'categoryName', + field: 'title', name: i18n.translate('xpack.observabilityAiAssistantManagement.kbTab.columns.name', { defaultMessage: 'Name', }), @@ -183,7 +183,7 @@ export function KnowledgeBaseTab() { const [isNewEntryPopoverOpen, setIsNewEntryPopoverOpen] = useState(false); const [isEditUserInstructionFlyoutOpen, setIsEditUserInstructionFlyoutOpen] = useState(false); const [query, setQuery] = useState(''); - const [sortBy, setSortBy] = useState<'title' | '@timestamp'>('title'); + const [sortBy, setSortBy] = useState('title'); const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('asc'); const { @@ -193,17 +193,10 @@ export function KnowledgeBaseTab() { } = useGetKnowledgeBaseEntries({ query, sortBy, sortDirection }); const categorizedEntries = categorizeEntries({ entries }); - const handleChangeSort = ({ - sort, - }: Criteria) => { + const handleChangeSort = ({ sort }: Criteria) => { if (sort) { const { field, direction } = sort; - if (field === '@timestamp') { - setSortBy(field); - } - if (field === 'categoryName') { - setSortBy('title'); - } + setSortBy(field); setSortDirection(direction); } }; @@ -329,7 +322,7 @@ export function KnowledgeBaseTab() { loading={isLoading} sorting={{ sort: { - field: sortBy === 'title' ? 'categoryName' : sortBy, + field: sortBy, direction: sortDirection, }, }} From b20bd1b2c0629f8eaa9edcd65b9cad8ca2e7ad16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 29 Aug 2024 01:26:35 +0200 Subject: [PATCH 18/42] Revert changes to files --- .github/CODEOWNERS | 2 +- tsconfig.base.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 79a8000fe2f68..ef858bd7304b9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,7 +5,6 @@ ## `node scripts/generate codeowners`. #### -x-pack/test/alerting_api_integration/common/plugins/aad @elastic/response-ops packages/kbn-ace @elastic/kibana-management x-pack/plugins/actions @elastic/response-ops x-pack/test/alerting_api_integration/common/plugins/actions_simulators @elastic/response-ops @@ -982,6 +981,7 @@ packages/kbn-xstate-utils @elastic/obs-ux-logs-team packages/kbn-yarn-lock-validator @elastic/kibana-operations packages/kbn-zod @elastic/kibana-core packages/kbn-zod-helpers @elastic/security-detection-rule-management +x-pack/test/alerting_api_integration/common/plugins/aad @elastic/response-ops #### ## Everything below this line overrides the default assignments for each package. ## Items lower in the file have higher precedence: diff --git a/tsconfig.base.json b/tsconfig.base.json index 860e8dae3d523..c9c5232b094d0 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -4,8 +4,6 @@ "rootDir": ".", "paths": { // START AUTOMATED PACKAGE LISTING - "@kbn/aad-fixtures-plugin": ["x-pack/test/alerting_api_integration/common/plugins/aad"], - "@kbn/aad-fixtures-plugin/*": ["x-pack/test/alerting_api_integration/common/plugins/aad/*"], "@kbn/ace": ["packages/kbn-ace"], "@kbn/ace/*": ["packages/kbn-ace/*"], "@kbn/actions-plugin": ["x-pack/plugins/actions"], @@ -1958,6 +1956,8 @@ "@kbn/zod/*": ["packages/kbn-zod/*"], "@kbn/zod-helpers": ["packages/kbn-zod-helpers"], "@kbn/zod-helpers/*": ["packages/kbn-zod-helpers/*"], + "@kbn/aad-fixtures-plugin": ["x-pack/test/alerting_api_integration/common/plugins/aad"], + "@kbn/aad-fixtures-plugin/*": ["x-pack/test/alerting_api_integration/common/plugins/aad/*"], // END AUTOMATED PACKAGE LISTING // Allows for importing from `kibana` package for the exported types. "@emotion/core": ["typings/@emotion"] From 0f531e4e191b1483198857cf26201fbd99bafbc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 29 Aug 2024 16:06:32 +0200 Subject: [PATCH 19/42] Add functional test --- .../routes/components/knowledge_base_tab.tsx | 2 + .../knowledge_base/knowledge_base.spec.ts | 14 +- .../common/ui/index.ts | 5 + .../knowledge_base_management/index.spec.ts | 125 ++++++++++++++++++ 4 files changed, 138 insertions(+), 8 deletions(-) create mode 100644 x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx index 60db7d0d81702..c8424aada44d6 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx @@ -85,6 +85,7 @@ export function KnowledgeBaseTab() { width: '40px', }, { + 'data-test-subj': 'knowledgeBaseTableTitleCell', field: 'title', name: i18n.translate('xpack.observabilityAiAssistantManagement.kbTab.columns.name', { defaultMessage: 'Name', @@ -107,6 +108,7 @@ export function KnowledgeBaseTab() { }, }, { + 'data-test-subj': 'knowledgeBaseTableAuthorCell', name: i18n.translate('xpack.observabilityAiAssistantManagement.kbTab.columns.author', { defaultMessage: 'Author', }), diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index b019d46c1e8f2..ced38977a35f9 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -26,9 +26,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient - .editor({ - endpoint: 'POST /internal/observability_ai_assistant/kb/setup', - }) + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup' }) .expect(200); }); @@ -300,7 +298,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); describe('When the LLM creates entries', () => { - async function addEntryWithDocId({ + async function saveKbEntry({ apiClient, docId, text, @@ -348,14 +346,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { const docId = 'my_favourite_color'; - await addEntryWithDocId({ + await saveKbEntry({ apiClient: observabilityAIAssistantAPIClient.editor, docId, text: 'My favourite color is blue', }); entries1 = await getEntriesWithDocId(docId); - await addEntryWithDocId({ + await saveKbEntry({ apiClient: observabilityAIAssistantAPIClient.editor, docId, text: 'My favourite color is green', @@ -393,12 +391,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { let entries: KnowledgeBaseEntry[]; before(async () => { - await addEntryWithDocId({ + await saveKbEntry({ apiClient: observabilityAIAssistantAPIClient.editor, docId: 'users_favorite_animal', text: "The user's favourite animal is a dog", }); - await addEntryWithDocId({ + await saveKbEntry({ apiClient: observabilityAIAssistantAPIClient.secondaryEditor, docId: 'users_favorite_animal', text: "The user's favourite animal is a cat", diff --git a/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts b/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts index a5d2802dfbcc5..4680cf03d3b97 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts @@ -27,6 +27,11 @@ export interface ObservabilityAIAssistantUIService { } const pages = { + kbManagementTab: { + table: 'knowledgeBaseTable', + tableTitleCell: 'knowledgeBaseTableTitleCell', + tableAuthorCell: 'knowledgeBaseTableAuthorCell', + }, conversations: { setupGenAiConnectorsButtonSelector: `observabilityAiAssistantInitialSetupPanelSetUpGenerativeAiConnectorButton`, chatInput: 'observabilityAiAssistantChatPromptEditorTextArea', diff --git a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts new file mode 100644 index 0000000000000..0dc100ee7b7c0 --- /dev/null +++ b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { subj as testSubjSelector } from '@kbn/test-subj-selector'; +import { + createKnowledgeBaseModel, + deleteKnowledgeBaseModel, +} from '../../../observability_ai_assistant_api_integration/tests/knowledge_base/helpers'; +import { ObservabilityAIAssistantApiClient } from '../../../observability_ai_assistant_api_integration/common/observability_ai_assistant_api_client'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ApiTest({ getService, getPageObjects }: FtrProviderContext) { + const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); + const ui = getService('observabilityAIAssistantUI'); + const testSubjects = getService('testSubjects'); + const log = getService('log'); + const ml = getService('ml'); + const { common } = getPageObjects(['common']); + + async function saveKbEntry({ + apiClient, + docId, + text, + }: { + apiClient: ObservabilityAIAssistantApiClient; + docId: string; + text: string; + }) { + return apiClient({ + endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', + params: { + body: { + doc_id: docId, + text, + confidence: 'high', + is_correction: false, + public: false, + labels: {}, + }, + }, + }).expect(200); + } + + describe('Knowledge management tab', () => { + before(async () => { + await createKnowledgeBaseModel(ml); + + await Promise.all([ + observabilityAIAssistantAPIClient + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup' }) + .expect(200), + ui.auth.login('editor'), + ]); + + await common.navigateToUrlWithBrowserHistory( + 'management', + '/kibana/observabilityAiAssistantManagement', + 'tab=knowledge_base' + ); + }); + + after(async () => { + await Promise.all([deleteKnowledgeBaseModel(ml), ui.auth.logout()]); + }); + + describe('when the LLM calls the "summarize" function for two different users', () => { + let rowsWithFavColor: Array<{ + rowText: string[]; + authorText: string; + entryTitleText: string; + }>; + + before(async () => { + await saveKbEntry({ + apiClient: observabilityAIAssistantAPIClient.editor, + docId: 'my_fav_color', + text: 'My favourite color is red', + }); + + await saveKbEntry({ + apiClient: observabilityAIAssistantAPIClient.secondaryEditor, + docId: 'my_fav_color', + text: 'My favourite color is blue', + }); + + const entryTitleCells = await testSubjects.findAll(ui.pages.kbManagementTab.tableTitleCell); + + const rows = await Promise.all( + entryTitleCells.map(async (cell) => { + const entryTitleText = await cell.getVisibleText(); + const parentRow = await cell.findByXpath('ancestor::tr'); + + const author = await parentRow.findByCssSelector( + testSubjSelector(ui.pages.kbManagementTab.tableAuthorCell) + ); + const authorText = await author.getVisibleText(); + const rowText = (await parentRow.getVisibleText()).split('\n'); + + return { rowText, authorText, entryTitleText }; + }) + ); + + log.debug(`Found ${rows.length} rows in the KB management table: ${JSON.stringify(rows)}`); + + rowsWithFavColor = rows.filter(({ entryTitleText }) => entryTitleText === 'my_fav_color'); + }); + + it('shows two entries', async () => { + expect(rowsWithFavColor.length).to.eql(2); + }); + + it('shows two different authors', async () => { + expect(rowsWithFavColor.map(({ authorText }) => authorText)).to.eql([ + 'secondary_editor', + 'editor', + ]); + }); + }); + }); +} From 08aa578d06489b996cb64cb1ec391f8a2cbe0eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 29 Aug 2024 16:12:26 +0200 Subject: [PATCH 20/42] i18n --- .../translations/translations/fr-FR.json | 18 +++++++++--------- .../translations/translations/ja-JP.json | 18 +++++++++--------- .../translations/translations/zh-CN.json | 18 +++++++++--------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index f4726a6ec1c04..4e503d37ed875 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -9473,7 +9473,16 @@ "visualizationUiComponents.filterQueryInput.clickToEdit": "Cliquer pour modifier", "visualizationUiComponents.filterQueryInput.emptyFilterQuery": "(vide)", "visualizationUiComponents.filterQueryInput.label": "Filtrer par", + "visualizationUiComponents.iconSelect.alertIconLabel": "Alerte", + "visualizationUiComponents.iconSelect.asteriskIconLabel": "Astérisque", + "visualizationUiComponents.iconSelect.bellIconLabel": "Cloche", + "visualizationUiComponents.iconSelect.boltIconLabel": "Éclair", + "visualizationUiComponents.iconSelect.bugIconLabel": "Bug", + "visualizationUiComponents.iconSelect.commentIconLabel": "Commentaire", + "visualizationUiComponents.iconSelect.flagIconLabel": "Drapeau", "visualizationUiComponents.iconSelect.label": "Décoration de l’icône", + "visualizationUiComponents.iconSelect.noIconLabel": "Aucun", + "visualizationUiComponents.iconSelect.tagIconLabel": "Balise", "visualizationUiComponents.lineMarker.textVisibility": "Décoration du texte", "visualizationUiComponents.nameInput.columnLabel": "Nom", "visualizationUiComponents.queryInput.queryPlaceholderKql": "{example}", @@ -26030,15 +26039,6 @@ "xpack.lens.xyChart.horizontalAxisLabel": "Axe horizontal", "xpack.lens.xyChart.horizontalLeftAxisLabel": "Axe supérieur horizontal", "xpack.lens.xyChart.horizontalRightAxisLabel": "Axe inférieur horizontal", - "visualizationUiComponents.iconSelect.alertIconLabel": "Alerte", - "visualizationUiComponents.iconSelect.asteriskIconLabel": "Astérisque", - "visualizationUiComponents.iconSelect.bellIconLabel": "Cloche", - "visualizationUiComponents.iconSelect.boltIconLabel": "Éclair", - "visualizationUiComponents.iconSelect.bugIconLabel": "Bug", - "visualizationUiComponents.iconSelect.commentIconLabel": "Commentaire", - "visualizationUiComponents.iconSelect.flagIconLabel": "Drapeau", - "visualizationUiComponents.iconSelect.noIconLabel": "Aucun", - "visualizationUiComponents.iconSelect.tagIconLabel": "Balise", "xpack.lens.xyChart.layerAnnotation": "Annotation", "xpack.lens.xyChart.layerAnnotationsIgnoreTitle": "Calques ignorant les filtres globaux", "xpack.lens.xyChart.layerAnnotationsLabel": "Annotations", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index f9ab6a827ed37..349f3c3945136 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -9468,7 +9468,16 @@ "visualizationUiComponents.filterQueryInput.clickToEdit": "クリックして編集", "visualizationUiComponents.filterQueryInput.emptyFilterQuery": "（空）", "visualizationUiComponents.filterQueryInput.label": "フィルタリング条件", + "visualizationUiComponents.iconSelect.alertIconLabel": "アラート", + "visualizationUiComponents.iconSelect.asteriskIconLabel": "アスタリスク", + "visualizationUiComponents.iconSelect.bellIconLabel": "ベル", + "visualizationUiComponents.iconSelect.boltIconLabel": "ボルト", + "visualizationUiComponents.iconSelect.bugIconLabel": "バグ", + "visualizationUiComponents.iconSelect.commentIconLabel": "コメント", + "visualizationUiComponents.iconSelect.flagIconLabel": "旗", "visualizationUiComponents.iconSelect.label": "アイコン装飾", + "visualizationUiComponents.iconSelect.noIconLabel": "なし", + "visualizationUiComponents.iconSelect.tagIconLabel": "タグ", "visualizationUiComponents.lineMarker.textVisibility": "テキスト装飾", "visualizationUiComponents.nameInput.columnLabel": "名前", "visualizationUiComponents.queryInput.queryPlaceholderKql": "{example}", @@ -26019,15 +26028,6 @@ "xpack.lens.xyChart.horizontalAxisLabel": "横軸", "xpack.lens.xyChart.horizontalLeftAxisLabel": "横上軸", "xpack.lens.xyChart.horizontalRightAxisLabel": "横下軸", - "visualizationUiComponents.iconSelect.alertIconLabel": "アラート", - "visualizationUiComponents.iconSelect.asteriskIconLabel": "アスタリスク", - "visualizationUiComponents.iconSelect.bellIconLabel": "ベル", - "visualizationUiComponents.iconSelect.boltIconLabel": "ボルト", - "visualizationUiComponents.iconSelect.bugIconLabel": "バグ", - "visualizationUiComponents.iconSelect.commentIconLabel": "コメント", - "visualizationUiComponents.iconSelect.flagIconLabel": "旗", - "visualizationUiComponents.iconSelect.noIconLabel": "なし", - "visualizationUiComponents.iconSelect.tagIconLabel": "タグ", "xpack.lens.xyChart.layerAnnotation": "注釈", "xpack.lens.xyChart.layerAnnotationsIgnoreTitle": "グローバルフィルターを無視するレイヤー", "xpack.lens.xyChart.layerAnnotationsLabel": "注釈", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 917dcaaac1c51..ef827f7157c05 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9482,7 +9482,16 @@ "visualizationUiComponents.filterQueryInput.clickToEdit": "单击以编辑", "visualizationUiComponents.filterQueryInput.emptyFilterQuery": "（空）", "visualizationUiComponents.filterQueryInput.label": "筛选依据", + "visualizationUiComponents.iconSelect.alertIconLabel": "告警", + "visualizationUiComponents.iconSelect.asteriskIconLabel": "星号", + "visualizationUiComponents.iconSelect.bellIconLabel": "钟铃", + "visualizationUiComponents.iconSelect.boltIconLabel": "闪电", + "visualizationUiComponents.iconSelect.bugIconLabel": "昆虫", + "visualizationUiComponents.iconSelect.commentIconLabel": "注释", + "visualizationUiComponents.iconSelect.flagIconLabel": "旗帜", "visualizationUiComponents.iconSelect.label": "图标装饰", + "visualizationUiComponents.iconSelect.noIconLabel": "无", + "visualizationUiComponents.iconSelect.tagIconLabel": "标签", "visualizationUiComponents.lineMarker.textVisibility": "文本装饰", "visualizationUiComponents.nameInput.columnLabel": "名称", "visualizationUiComponents.queryInput.queryPlaceholderKql": "{example}", @@ -26051,15 +26060,6 @@ "xpack.lens.xyChart.horizontalAxisLabel": "水平轴", "xpack.lens.xyChart.horizontalLeftAxisLabel": "水平顶轴", "xpack.lens.xyChart.horizontalRightAxisLabel": "水平底轴", - "visualizationUiComponents.iconSelect.alertIconLabel": "告警", - "visualizationUiComponents.iconSelect.asteriskIconLabel": "星号", - "visualizationUiComponents.iconSelect.bellIconLabel": "钟铃", - "visualizationUiComponents.iconSelect.boltIconLabel": "闪电", - "visualizationUiComponents.iconSelect.bugIconLabel": "昆虫", - "visualizationUiComponents.iconSelect.commentIconLabel": "注释", - "visualizationUiComponents.iconSelect.flagIconLabel": "旗帜", - "visualizationUiComponents.iconSelect.noIconLabel": "无", - "visualizationUiComponents.iconSelect.tagIconLabel": "标签", "xpack.lens.xyChart.layerAnnotation": "标注", "xpack.lens.xyChart.layerAnnotationsIgnoreTitle": "忽略全局筛选的图层", "xpack.lens.xyChart.layerAnnotationsLabel": "标注", From ad425beca4319af78a458321daa92eacf3048499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Fri, 30 Aug 2024 00:42:37 +0200 Subject: [PATCH 21/42] Revert tsconfig changes --- tsconfig.base.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tsconfig.base.json b/tsconfig.base.json index 089364b52779f..44bb897bea1c2 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -4,6 +4,8 @@ "rootDir": ".", "paths": { // START AUTOMATED PACKAGE LISTING + "@kbn/aad-fixtures-plugin": ["x-pack/test/alerting_api_integration/common/plugins/aad"], + "@kbn/aad-fixtures-plugin/*": ["x-pack/test/alerting_api_integration/common/plugins/aad/*"], "@kbn/ace": ["packages/kbn-ace"], "@kbn/ace/*": ["packages/kbn-ace/*"], "@kbn/actions-plugin": ["x-pack/plugins/actions"], @@ -1960,8 +1962,6 @@ "@kbn/zod/*": ["packages/kbn-zod/*"], "@kbn/zod-helpers": ["packages/kbn-zod-helpers"], "@kbn/zod-helpers/*": ["packages/kbn-zod-helpers/*"], - "@kbn/aad-fixtures-plugin": ["x-pack/test/alerting_api_integration/common/plugins/aad"], - "@kbn/aad-fixtures-plugin/*": ["x-pack/test/alerting_api_integration/common/plugins/aad/*"], // END AUTOMATED PACKAGE LISTING // Allows for importing from `kibana` package for the exported types. "@emotion/core": ["typings/@emotion"] From 90b5484e8b0ac47643bd7bedcd2605c630c4294e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Fri, 30 Aug 2024 00:44:27 +0200 Subject: [PATCH 22/42] Include KB entries when they contain contradicting info --- .../server/utils/recall/score_suggestions.ts | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts index d5ed19d6ec34c..52d8c52ec5af8 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts @@ -52,16 +52,20 @@ export async function scoreSuggestions({ const shortIdTable = new ShortIdTable(); const newUserMessageContent = - dedent(`Given the following question, score the documents that are relevant to the question. on a scale from 0 to 7, - 0 being completely irrelevant, and 7 being extremely relevant. Information is relevant to the question if it helps in - answering the question. Judge it according to the following criteria: - - - The document is relevant to the question, and the rest of the conversation - - The document has information relevant to the question that is not mentioned, - or more detailed than what is available in the conversation - - The document has a high amount of information relevant to the question compared to other documents - - The document contains new information not mentioned before in the conversation - + dedent(`Given the following prompt, score the documents that are relevant to the prompt on a scale from 0 to 7, + 0 being completely irrelevant, and 7 being extremely relevant. Information is relevant to the prompt if it helps in + answering the prompt. Judge the document according to the following criteria : + + If the prompt is a statement that should be stored in the knowledge base: + - The document contains information that directly contradicts the user's prompt or previous statements, indicating that it may need to be updated or corrected. + - The document contains outdated user preferences or information that the user indicate they want corrected or replaced. + + If the prompt is a question: + - The document is relevant to the prompt, and the rest of the conversation + - The document has information relevant to the prompt that is not mentioned, or more detailed than what is available in the conversation + - The document has a high amount of information relevant to the prompt compared to other documents + - The document contains new information not mentioned before in the conversation or provides a correction to previously stated information. + User prompt: ${userPrompt} From 923681c9d8c5919eb88143ac5d2482e051063ca7 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 29 Aug 2024 22:53:49 +0000 Subject: [PATCH 23/42] [CI] Auto-commit changed files from 'node scripts/build_plugin_list_docs' --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 49f5b8c59dadf..cbffef0027017 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,6 +5,7 @@ ## `node scripts/generate codeowners`. #### +x-pack/test/alerting_api_integration/common/plugins/aad @elastic/response-ops packages/kbn-ace @elastic/kibana-management x-pack/plugins/actions @elastic/response-ops x-pack/test/alerting_api_integration/common/plugins/actions_simulators @elastic/response-ops @@ -983,7 +984,6 @@ packages/kbn-xstate-utils @elastic/obs-ux-logs-team packages/kbn-yarn-lock-validator @elastic/kibana-operations packages/kbn-zod @elastic/kibana-core packages/kbn-zod-helpers @elastic/security-detection-rule-management -x-pack/test/alerting_api_integration/common/plugins/aad @elastic/response-ops #### ## Everything below this line overrides the default assignments for each package. ## Items lower in the file have higher precedence: From 03e8fe6bcd1e7a8e64dcb45b134242975c130e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Fri, 30 Aug 2024 22:31:16 +0200 Subject: [PATCH 24/42] Fix tsc and jest --- .../components/knowledge_base_category_flyout.tsx | 4 ++-- .../routes/components/knowledge_base_tab.test.tsx | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_category_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_category_flyout.tsx index fab3b34809c19..8dcf76e4bc56e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_category_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_category_flyout.tsx @@ -104,13 +104,13 @@ export function KnowledgeBaseCategoryFlyout({ ]; const hasDescription = - CATEGORY_MAP[category.categoryName as unknown as keyof typeof CATEGORY_MAP]?.description; + CATEGORY_MAP[category.categoryKey as unknown as keyof typeof CATEGORY_MAP]?.description; return ( -

{capitalize(category.categoryName)}

+

{capitalize(category.categoryKey)}

 diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx index 94ae564ac7f96..91e5b21feb4f0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx @@ -82,7 +82,17 @@ describe('KnowledgeBaseTab', () => { getByTestId('knowledgeBaseEditManualEntryFlyoutSaveButton').click(); expect(createMock).toHaveBeenCalledWith({ - entry: { id: expect.any(String), title: 'foo', public: false, text: 'bar' }, + entry: { + id: expect.any(String), + title: 'foo', + public: false, + text: 'bar', + role: 'user_entry', + confidence: 'high', + doc_id: undefined, + is_correction: false, + labels: expect.any(Object), + }, }); }); From 9b2cf9b1b689bf88c1e6dbb7da7149002e1e1a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Sat, 31 Aug 2024 01:09:53 +0200 Subject: [PATCH 25/42] Improve functional test --- .../knowledge_base_management/index.spec.ts | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts index 0dc100ee7b7c0..e59f875350112 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts @@ -48,20 +48,18 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte describe('Knowledge management tab', () => { before(async () => { + // create a knowledge base model await createKnowledgeBaseModel(ml); await Promise.all([ + // setup the knowledge base observabilityAIAssistantAPIClient .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup' }) .expect(200), + + // login as editor ui.auth.login('editor'), ]); - - await common.navigateToUrlWithBrowserHistory( - 'management', - '/kibana/observabilityAiAssistantManagement', - 'tab=knowledge_base' - ); }); after(async () => { @@ -69,24 +67,12 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); describe('when the LLM calls the "summarize" function for two different users', () => { - let rowsWithFavColor: Array<{ - rowText: string[]; - authorText: string; - entryTitleText: string; - }>; - - before(async () => { - await saveKbEntry({ - apiClient: observabilityAIAssistantAPIClient.editor, - docId: 'my_fav_color', - text: 'My favourite color is red', - }); - - await saveKbEntry({ - apiClient: observabilityAIAssistantAPIClient.secondaryEditor, - docId: 'my_fav_color', - text: 'My favourite color is blue', - }); + async function getKnowledgeBaseEntries() { + await common.navigateToUrlWithBrowserHistory( + 'management', + '/kibana/observabilityAiAssistantManagement', + 'tab=knowledge_base' + ); const entryTitleCells = await testSubjects.findAll(ui.pages.kbManagementTab.tableTitleCell); @@ -107,18 +93,31 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte log.debug(`Found ${rows.length} rows in the KB management table: ${JSON.stringify(rows)}`); - rowsWithFavColor = rows.filter(({ entryTitleText }) => entryTitleText === 'my_fav_color'); + return rows.filter(({ entryTitleText }) => entryTitleText === 'my_fav_color'); + } + + before(async () => { + await saveKbEntry({ + apiClient: observabilityAIAssistantAPIClient.editor, + docId: 'my_fav_color', + text: 'My favourite color is red', + }); + + await saveKbEntry({ + apiClient: observabilityAIAssistantAPIClient.secondaryEditor, + docId: 'my_fav_color', + text: 'My favourite color is blue', + }); }); it('shows two entries', async () => { - expect(rowsWithFavColor.length).to.eql(2); + const entries = await getKnowledgeBaseEntries(); + expect(entries.length).to.eql(2); }); it('shows two different authors', async () => { - expect(rowsWithFavColor.map(({ authorText }) => authorText)).to.eql([ - 'secondary_editor', - 'editor', - ]); + const entries = await getKnowledgeBaseEntries(); + expect(entries.map(({ authorText }) => authorText)).to.eql(['secondary_editor', 'editor']); }); }); }); From fc0a970d3d413ce704b2b7014475cb22f1622ea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 31 Oct 2024 10:43:55 +0100 Subject: [PATCH 26/42] Fix tsc --- .../observability_ai_assistant/server/service/client/index.ts | 3 +-- .../observability_ai_assistant/server/service/types.ts | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 77ad887049ba9..055909d83a83f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -47,13 +47,12 @@ import { } from '../../../common/conversation_complete'; import { CompatibleJSONSchema } from '../../../common/functions/types'; import { - AdHocInstruction, + type AdHocInstruction, type Conversation, type ConversationCreateRequest, type ConversationUpdateRequest, type KnowledgeBaseEntry, type Message, - type AdHocInstruction, KnowledgeBaseType, KnowledgeBaseEntryRole, } from '../../../common/types'; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts index affe0cb6ec1f2..2e24cf25902e0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts @@ -17,7 +17,6 @@ import type { import type { Message, ObservabilityAIAssistantScreenContextRequest, - InstructionOrPlainText, AdHocInstruction, } from '../../common/types'; import type { ObservabilityAIAssistantRouteHandlerResources } from '../routes/types'; From 0aaf657de211fdb5627605252609bcce33151d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 31 Oct 2024 11:24:36 +0100 Subject: [PATCH 27/42] Fix serverless test --- .../ai_assistant/tests/knowledge_base/knowledge_base.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts index b540ee5829e59..b32a48d1a9c2e 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts @@ -52,6 +52,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('when managing a single entry', () => { const knowledgeBaseEntry = { id: 'my-doc-id-1', + title: 'My title', text: 'My content', }; it('returns 200 on create', async () => { From b0e2781acc1e43de8c4e9babd44ca0c24d5c7696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 5 Nov 2024 09:56:58 +0100 Subject: [PATCH 28/42] Remove `doc_id` --- .../common/types.ts | 3 +- .../server/functions/summarize.ts | 24 +- .../server/plugin.ts | 5 - .../server/routes/functions/route.ts | 6 +- .../server/routes/knowledge_base/route.ts | 3 - .../server/service/client/index.ts | 8 - .../server/service/index.ts | 3 +- .../server/service/kb_component_template.ts | 2 +- .../service/knowledge_base_service/index.ts | 20 +- .../knowledge_base_service/kb_docs/lens.ts | 596 ------------------ .../server/service/util/split_kb_text.ts | 4 +- .../server/utils/recall/recall_and_score.ts | 4 +- .../server/utils/recall/score_suggestions.ts | 7 +- .../public/helpers/categorize_entries.ts | 2 +- ...nowledge_base_edit_manual_entry_flyout.tsx | 5 +- .../components/knowledge_base_tab.test.tsx | 7 +- .../knowledge_base/knowledge_base.spec.ts | 148 +---- .../knowledge_base_management/index.spec.ts | 5 - 18 files changed, 32 insertions(+), 820 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/kb_docs/lens.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts index 0d415520ff4b4..c04af74adbff6 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts @@ -81,8 +81,7 @@ export type ConversationUpdateRequest = ConversationRequestBase & { export interface KnowledgeBaseEntry { '@timestamp': string; - id: string; // unique ID - doc_id?: string; // human readable ID generated by the LLM and used by the LLM to lookup and update existing entries. TODO: rename `doc_id` to `lookup_id` + id: string; title?: string; text: string; confidence: 'low' | 'medium' | 'high'; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index 1f3c82d0fb368..1f4afdbdd56bb 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -29,10 +29,10 @@ export function registerSummarizationFunction({ parameters: { type: 'object', properties: { - id: { + title: { type: 'string', description: - 'A lookup id for the document. This should be a short human-readable keyword field with only alphabetic characters and underscores, that allow you to find and update it later.', + 'A human readable title that can be used to identify the document later. This should be no longer than 255 characters', }, text: { type: 'string', @@ -55,7 +55,7 @@ export function registerSummarizationFunction({ }, }, required: [ - 'id' as const, + 'title' as const, 'text' as const, 'is_correction' as const, 'confidence' as const, @@ -64,25 +64,17 @@ export function registerSummarizationFunction({ }, }, async ( - { arguments: { id: docId, text, is_correction: isCorrection, confidence, public: isPublic } }, + { arguments: { title, text, is_correction: isCorrection, confidence, public: isPublic } }, signal ) => { - // The LLM should be able to update an existing entry by providing the same doc_id - // if no existing entry is found, we generate a uuid - const id = await client.getUuidFromDocId(docId); - - resources.logger.debug( - id - ? `Updating knowledge base entry with id: ${id}, doc_id: ${docId}` - : `Creating new knowledge base entry with doc_id: ${docId}` - ); + const id = v4(); + resources.logger.debug(`Creating new knowledge base entry with id: ${id}`); return client .addKnowledgeBaseEntry({ entry: { - id: id ?? v4(), - doc_id: docId, - title: docId, // use doc_id as title for now + id, + title, text, public: isPublic, role: KnowledgeBaseEntryRole.AssistantSummarization, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts index 81f4e24d4d21f..3bdff9eb17606 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/plugin.ts @@ -32,7 +32,6 @@ import { ObservabilityAIAssistantPluginSetupDependencies, ObservabilityAIAssistantPluginStartDependencies, } from './types'; -import { addLensDocsToKb } from './service/knowledge_base_service/kb_docs/lens'; import { registerFunctions } from './functions'; import { recallRankingEvent } from './analytics/recall_ranking'; import { initLangtrace } from './service/client/instrumentation/init_langtrace'; @@ -164,10 +163,6 @@ export class ObservabilityAIAssistantPlugin service.register(registerFunctions); - if (this.config.enableKnowledgeBase) { - addLensDocsToKb({ service, logger: this.logger.get('kb').get('lens') }); - } - registerServerRoutes({ core, logger: this.logger, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index eca4409e763b2..e71d40b51aef7 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -124,7 +124,6 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', params: t.type({ body: t.type({ - doc_id: t.string, text: nonEmptyStringRt, confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), is_correction: toBooleanRt, @@ -143,7 +142,6 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ } const { - doc_id: docId, confidence, is_correction: isCorrection, text, @@ -151,12 +149,10 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ labels, } = resources.params.body; - const id = await client.getUuidFromDocId(docId); return client.addKnowledgeBaseEntry({ entry: { confidence, - id: id ?? v4(), - doc_id: docId, + id: v4(), is_correction: isCorrection, text, public: isPublic, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts index 50525d334b1f9..8f7e7988d6d79 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts @@ -148,7 +148,6 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ text: nonEmptyStringRt, }), t.partial({ - doc_id: t.string, confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), is_correction: toBooleanRt, public: toBooleanRt, @@ -173,7 +172,6 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ const { id, - doc_id: docId, title, text, public: isPublic, @@ -187,7 +185,6 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ entry: { text, id, - doc_id: docId, title, confidence: confidence ?? 'high', is_correction: isCorrection ?? false, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 055909d83a83f..935043d289609 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -732,14 +732,6 @@ export class ObservabilityAIAssistantClient { return this.dependencies.knowledgeBaseService.setup(); }; - getUuidFromDocId = async (docId: string) => { - return this.dependencies.knowledgeBaseService.getUuidFromDocId({ - namespace: this.dependencies.namespace, - user: this.dependencies.user, - docId, - }); - }; - addUserInstruction = async ({ entry, }: { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts index d1aba4f232b0d..6d11120ee7dda 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts @@ -69,7 +69,7 @@ export class ObservabilityAIAssistantService { private readonly core: CoreSetup; private readonly logger: Logger; private readonly getModelId: () => Promise; - private kbService?: KnowledgeBaseService; + public kbService?: KnowledgeBaseService; private enableKnowledgeBase: boolean; private readonly registrations: RegistrationCallback[] = []; @@ -345,7 +345,6 @@ export class ObservabilityAIAssistantService { const entryWithSystemProperties = { ...entry, '@timestamp': new Date().toISOString(), - doc_id: entry.id, public: true, confidence: 'high' as const, type: 'contextual' as const, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts index 0c654aef5db39..b1b2d3293a234 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts @@ -31,7 +31,7 @@ export const kbComponentTemplate: ClusterComponentTemplate['component_template'] properties: { '@timestamp': date, id: keyword, - doc_id: { type: 'text', fielddata: true }, + doc_id: { type: 'text', fielddata: true }, // deprecated but kept for backwards compatibility title: { type: 'text', fields: { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index 6ff6f57cb6bdb..21e7ce364aee9 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -39,8 +39,6 @@ interface Dependencies { export interface RecalledEntry { id: string; - title?: string; - docId?: string; text: string; score: number | null; is_correction?: boolean; @@ -72,7 +70,7 @@ export enum KnowledgeBaseEntryOperationType { interface KnowledgeBaseDeleteOperation { type: KnowledgeBaseEntryOperationType.Delete; - docId?: string; + operationDocId?: string; labels?: Record; } @@ -243,7 +241,9 @@ export class KnowledgeBaseService { query: { bool: { filter: [ - ...(operation.docId ? [{ term: { doc_id: operation.docId } }] : []), + ...(operation.operationDocId + ? [{ term: { doc_id: operation.operationDocId } }] + : []), ...(operation.labels ? map(operation.labels, (value, key) => { return { term: { [key]: value } }; @@ -256,7 +256,7 @@ export class KnowledgeBaseService { return; } catch (error) { this.dependencies.logger.error( - `Failed to delete document "${operation?.docId}" due to ${error.message}` + `Failed to delete document "${operation?.operationDocId}" due to ${error.message}` ); this.dependencies.logger.debug(() => JSON.stringify(operation)); throw error; @@ -416,7 +416,7 @@ export class KnowledgeBaseService { }; const response = await this.dependencies.esClient.asInternalUser.search< - Pick + Pick & { doc_id?: string } >({ index: [resourceNames.aliases.kb], query: esQuery, @@ -430,8 +430,7 @@ export class KnowledgeBaseService { text: hit._source?.text!, is_correction: hit._source?.is_correction, labels: hit._source?.labels, - title: hit._source?.title, - docId: hit._source?.doc_id, + title: hit._source?.title ?? hit._source?.doc_id, // use `doc_id` as fallback title for backwards compatibility score: hit._score!, id: hit._id!, })); @@ -642,9 +641,7 @@ export class KnowledgeBaseService { if (!this.dependencies.enabled) { return null; } - const res = await this.dependencies.esClient.asInternalUser.search< - Pick - >({ + const res = await this.dependencies.esClient.asInternalUser.search({ index: resourceNames.aliases.kb, query: { bool: { @@ -715,7 +712,6 @@ export class KnowledgeBaseService { document: { '@timestamp': new Date().toISOString(), ...doc, - title: doc.title ?? doc.doc_id, user, namespace, }, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/kb_docs/lens.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/kb_docs/lens.ts deleted file mode 100644 index 9baf75f6ff552..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/kb_docs/lens.ts +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import dedent from 'dedent'; -import type { Logger } from '@kbn/logging'; -import type { ObservabilityAIAssistantService } from '../..'; - -export function addLensDocsToKb({ - service, -}: { - service: ObservabilityAIAssistantService; - logger: Logger; -}) { - service.addCategoryToKnowledgeBase('lens', [ - { - id: 'lens_formulas_how_it_works', - texts: [ - `Lens formulas let you do math using a combination of Elasticsearch aggregations and - math functions. There are three main types of functions: - - * Elasticsearch metrics, like \`sum(bytes)\` - * Time series functions use Elasticsearch metrics as input, like \`cumulative_sum()\` - * Math functions like \`round()\` - - An example formula that uses all of these: - - \`\`\` - round(100 * moving_average( - average(cpu.load.pct), - window=10, - kql='datacenter.name: east*' - )) - \`\`\` - `, - `Elasticsearch functions take a field name, which can be in quotes. \`sum(bytes)\` is the same - as \`sum('bytes')\`. - - Some functions take named arguments, like \`moving_average(count(), window=5)\`. - - Elasticsearch metrics can be filtered using KQL or Lucene syntax. To add a filter, use the named - parameter \`kql='field: value'\` or \`lucene=''\`. Always use single quotes when writing KQL or Lucene - queries. If your search has a single quote in it, use a backslash to escape, like: \`kql='Women's'\' - - Math functions can take positional arguments, like pow(count(), 3) is the same as count() * count() * count() - - Use the symbols +, -, /, and * to perform basic math.`, - ], - }, - { - id: 'lens_common_formulas', - texts: [ - `The most common formulas are dividing two values to produce a percent. To display accurately, set - "value format" to "percent"`, - `### Filter ratio: - - Use \`kql=''\` to filter one set of documents and compare it to other documents within the same grouping. - For example, to see how the error rate changes over time: - - \`\`\` - count(kql='response.status_code > 400') / count() - \`\`\``, - `### Week over week: - - Use \`shift='1w'\` to get the value of each grouping from - the previous week. Time shift should not be used with the *Top values* function. - - \`\`\` - percentile(system.network.in.bytes, percentile=99) / - percentile(system.network.in.bytes, percentile=99, shift='1w') - \`\`\``, - - `### Percent of total - - Formulas can calculate \`overall_sum\` for all the groupings, - which lets you convert each grouping into a percent of total: - - \`\`\` - sum(products.base_price) / overall_sum(sum(products.base_price)) - \`\`\``, - - `### Recent change - - Use \`reducedTimeRange='30m'\` to add an additional filter on the - time range of a metric aligned with the end of the global time range. - This can be used to calculate how much a value changed recently. - - \`\`\` - max(system.network.in.bytes, reducedTimeRange="30m") - - min(system.network.in.bytes, reducedTimeRange="30m") - \`\`\` - `, - ], - }, - { - id: 'lens_formulas_elasticsearch_functions', - texts: [ - `## Elasticsearch functions - - These functions will be executed on the raw documents for each row of the - resulting table, aggregating all documents matching the break down - dimensions into a single value.`, - - `#### average(field: string) - Returns the average of a field. This function only works for number fields. - - Example: Get the average of price: \`average(price)\` - - Example: Get the average of price for orders from the UK: \`average(price, - kql='location:UK')\``, - - `#### count([field: string]) - The total number of documents. When you provide a field, the total number of - field values is counted. When you use the Count function for fields that have - multiple values in a single document, all values are counted. - - To calculate the total number of documents, use \`count().\` - - To calculate the number of products in all orders, use \`count(products.id)\`. - - To calculate the number of documents that match a specific filter, use - \`count(kql='price > 500')\`.`, - - `#### last_value(field: string) - Returns the value of a field from the last document, ordered by the default - time field of the data view. - - This function is usefull the retrieve the latest state of an entity. - - Example: Get the current status of server A: \`last_value(server.status, - kql='server.name="A"')\``, - - `#### max(field: string) - Returns the max of a field. This function only works for number fields. - - Example: Get the max of price: \`max(price)\` - - Example: Get the max of price for orders from the UK: \`max(price, - kql='location:UK')\``, - - `#### median(field: string) - Returns the median of a field. This function only works for number fields. - - Example: Get the median of price: \`median(price)\` - - Example: Get the median of price for orders from the UK: \`median(price, - kql='location:UK')\``, - - `#### min(field: string) - Returns the min of a field. This function only works for number fields. - - Example: Get the min of price: \`min(price)\` - - Example: Get the min of price for orders from the UK: \`min(price, - kql='location:UK')\``, - - `#### percentile(field: string, [percentile]: number) - Returns the specified percentile of the values of a field. This is the value n - percent of the values occuring in documents are smaller. - - Example: Get the number of bytes larger than 95 % of values: - \`percentile(bytes, percentile=95)\``, - - `#### percentile_rank(field: string, [value]: number) - Returns the percentage of values which are below a certain value. For example, - if a value is greater than or equal to 95% of the observed values it is said to - be at the 95th percentile rank - - Example: Get the percentage of values which are below of 100: - \`percentile_rank(bytes, value=100)\``, - - `#### standard_deviation(field: string) - Returns the amount of variation or dispersion of the field. The function works - only for number fields. - - Example: To get the standard deviation of price, use - \`standard_deviation(price).\` - - Example: To get the variance of price for orders from the UK, use - \`square(standard_deviation(price, kql='location:UK'))\`.`, - - `#### sum(field: string) - Returns the sum of a field. This function only works for number fields. - - Example: Get the sum of price: sum(price) - - Example: Get the sum of price for orders from the UK: \`sum(price, - kql='location:UK')\``, - - `#### unique_count(field: string) - Calculates the number of unique values of a specified field. Works for number, - string, date and boolean values. - - Example: Calculate the number of different products: - \`unique_count(product.name)\` - - Example: Calculate the number of different products from the "clothes" group: - \`unique_count(product.name, kql='product.group=clothes')\` - - `, - ], - }, - { - id: 'lens_formulas_column_functions', - texts: [ - `## Column calculations - These functions are executed for each row, but are provided with the whole - column as context. This is also known as a window function.`, - - `#### counter_rate(metric: number) - Calculates the rate of an ever increasing counter. This function will only - yield helpful results on counter metric fields which contain a measurement of - some kind monotonically growing over time. If the value does get smaller, it - will interpret this as a counter reset. To get most precise results, - counter_rate should be calculated on the max of a field. - - This calculation will be done separately for separate series defined by filters - or top values dimensions. It uses the current interval when used in Formula. - - Example: Visualize the rate of bytes received over time by a memcached server: - counter_rate(max(memcached.stats.read.bytes))`, - - `cumulative_sum(metric: number) - Calculates the cumulative sum of a metric over time, adding all previous values - of a series to each value. To use this function, you need to configure a date - histogram dimension as well. - - This calculation will be done separately for separate series defined by filters - or top values dimensions. - - Example: Visualize the received bytes accumulated over time: - cumulative_sum(sum(bytes))`, - - `differences(metric: number) - Calculates the difference to the last value of a metric over time. To use this - function, you need to configure a date histogram dimension as well. Differences - requires the data to be sequential. If your data is empty when using - differences, try increasing the date histogram interval. - - This calculation will be done separately for separate series defined by filters - or top values dimensions. - - Example: Visualize the change in bytes received over time: - differences(sum(bytes))`, - - `moving_average(metric: number, [window]: number) - Calculates the moving average of a metric over time, averaging the last n-th - values to calculate the current value. To use this function, you need to - configure a date histogram dimension as well. The default window value is 5. - - This calculation will be done separately for separate series defined by filters - or top values dimensions. - - Takes a named parameter window which specifies how many last values to include - in the average calculation for the current value. - - Example: Smooth a line of measurements: moving_average(sum(bytes), window=5)`, - - `normalize_by_unit(metric: number, unit: s|m|h|d|w|M|y) - This advanced function is useful for normalizing counts and sums to a specific - time interval. It allows for integration with metrics that are stored already - normalized to a specific time interval. - - This function can only be used if there's a date histogram function used in the - current chart. - - Example: A ratio comparing an already normalized metric to another metric that - needs to be normalized. - normalize_by_unit(counter_rate(max(system.diskio.write.bytes)), unit='s') / - last_value(apache.status.bytes_per_second)`, - - `overall_average(metric: number) - Calculates the average of a metric for all data points of a series in the - current chart. A series is defined by a dimension using a date histogram or - interval function. Other dimensions breaking down the data like top values or - filter are treated as separate series. - - If no date histograms or interval functions are used in the current chart, - overall_average is calculating the average over all dimensions no matter the - used function - - Example: Divergence from the mean: sum(bytes) - overall_average(sum(bytes))`, - - `overall_max(metric: number) - Calculates the maximum of a metric for all data points of a series in the - current chart. A series is defined by a dimension using a date histogram or - interval function. Other dimensions breaking down the data like top values or - filter are treated as separate series. - - If no date histograms or interval functions are used in the current chart, - overall_max is calculating the maximum over all dimensions no matter the used - function - - Example: Percentage of range (sum(bytes) - overall_min(sum(bytes))) / - (overall_max(sum(bytes)) - overall_min(sum(bytes)))`, - - `overall_min(metric: number) - Calculates the minimum of a metric for all data points of a series in the - current chart. A series is defined by a dimension using a date histogram or - interval function. Other dimensions breaking down the data like top values or - filter are treated as separate series. - - If no date histograms or interval functions are used in the current chart, - overall_min is calculating the minimum over all dimensions no matter the used - function - - Example: Percentage of range (sum(bytes) - overall_min(sum(bytes)) / - (overall_max(sum(bytes)) - overall_min(sum(bytes)))`, - - `overall_sum(metric: number) - Calculates the sum of a metric of all data points of a series in the current - chart. A series is defined by a dimension using a date histogram or interval - function. Other dimensions breaking down the data like top values or filter are - treated as separate series. - - If no date histograms or interval functions are used in the current chart, - overall_sum is calculating the sum over all dimensions no matter the used - function. - - Example: Percentage of total sum(bytes) / overall_sum(sum(bytes))`, - ], - }, - { - id: 'lens_formulas_math_functions', - texts: [ - `Math - These functions will be executed for reach row of the resulting table using single values from the same row calculated using other functions.`, - - `abs([value]: number) - Calculates absolute value. A negative value is multiplied by -1, a positive value stays the same. - - Example: Calculate average distance to sea level abs(average(altitude))`, - - `add([left]: number, [right]: number) - Adds up two numbers. - - Also works with + symbol. - - Example: Calculate the sum of two fields - - sum(price) + sum(tax) - - Example: Offset count by a static value - - add(count(), 5)`, - - `cbrt([value]: number) - Cube root of value. - - Example: Calculate side length from volume - - cbrt(last_value(volume)) - - ceil([value]: number) - Ceiling of value, rounds up. - - Example: Round up price to the next dollar - - ceil(sum(price))`, - - `clamp([value]: number, [min]: number, [max]: number) - Limits the value from a minimum to maximum. - - Example: Make sure to catch outliers - - clamp( - average(bytes), - percentile(bytes, percentile=5), - percentile(bytes, percentile=95) - )`, - `cube([value]: number) - Calculates the cube of a number. - - Example: Calculate volume from side length - - cube(last_value(length))`, - - `defaults([value]: number, [default]: number) - Returns a default numeric value when value is null. - - Example: Return -1 when a field has no data - - defaults(average(bytes), -1)`, - - `divide([left]: number, [right]: number) - Divides the first number by the second number. - - Also works with / symbol - - Example: Calculate profit margin - - sum(profit) / sum(revenue) - - Example: divide(sum(bytes), 2)`, - - `exp([value]: number) - Raises e to the nth power. - - Example: Calculate the natural exponential function - - exp(last_value(duration))`, - - `fix([value]: number) - For positive values, takes the floor. For negative values, takes the ceiling. - - Example: Rounding towards zero - - fix(sum(profit))`, - - `floor([value]: number) - Round down to nearest integer value - - Example: Round down a price - - floor(sum(price))`, - - `log([value]: number, [base]?: number) - Logarithm with optional base. The natural base e is used as default. - - Example: Calculate number of bits required to store values - - log(sum(bytes)) - log(sum(bytes), 2)`, - `mod([value]: number, [base]: number) - Remainder after dividing the function by a number - - Example: Calculate last three digits of a value - - mod(sum(price), 1000)`, - - `multiply([left]: number, [right]: number) - Multiplies two numbers. - - Also works with * symbol. - - Example: Calculate price after current tax rate - - sum(bytes) * last_value(tax_rate) - - Example: Calculate price after constant tax rate - - multiply(sum(price), 1.2)`, - - `pick_max([left]: number, [right]: number) - Finds the maximum value between two numbers. - - Example: Find the maximum between two fields averages - - pick_max(average(bytes), average(memory))`, - - `pick_min([left]: number, [right]: number) - Finds the minimum value between two numbers. - - Example: Find the minimum between two fields averages - - pick_min(average(bytes), average(memory))`, - - `pow([value]: number, [base]: number) - Raises the value to a certain power. The second argument is required - - Example: Calculate volume based on side length - - pow(last_value(length), 3)`, - - `round([value]: number, [decimals]?: number) - Rounds to a specific number of decimal places, default of 0 - - Examples: Round to the cent - - round(sum(bytes)) - round(sum(bytes), 2)`, - `sqrt([value]: number) - Square root of a positive value only - - Example: Calculate side length based on area - - sqrt(last_value(area))`, - - `square([value]: number) - Raise the value to the 2nd power - - Example: Calculate area based on side length - - square(last_value(length))`, - - `subtract([left]: number, [right]: number) - Subtracts the first number from the second number. - - Also works with - symbol. - - Example: Calculate the range of a field - - subtract(max(bytes), min(bytes))`, - ], - }, - { - id: 'lens_formulas_comparison_functions', - texts: [ - `Comparison - These functions are used to perform value comparison.`, - - `eq([left]: number, [right]: number) - Performs an equality comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with == symbol. - - Example: Returns true if the average of bytes is exactly the same amount of average memory - - average(bytes) == average(memory) - - Example: eq(sum(bytes), 1000000)`, - - `gt([left]: number, [right]: number) - Performs a greater than comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with > symbol. - - Example: Returns true if the average of bytes is greater than the average amount of memory - - average(bytes) > average(memory) - - Example: gt(average(bytes), 1000)`, - - `gte([left]: number, [right]: number) - Performs a greater than comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with >= symbol. - - Example: Returns true if the average of bytes is greater than or equal to the average amount of memory - - average(bytes) >= average(memory) - - Example: gte(average(bytes), 1000)`, - - `ifelse([condition]: boolean, [left]: number, [right]: number) - Returns a value depending on whether the element of condition is true or false. - - Example: Average revenue per customer but in some cases customer id is not provided which counts as additional customer - - sum(total)/(unique_count(customer_id) + ifelse( count() > count(kql='customer_id:*'), 1, 0))`, - - `lt([left]: number, [right]: number) - Performs a lower than comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with < symbol. - - Example: Returns true if the average of bytes is lower than the average amount of memory - - average(bytes) <= average(memory) - - Example: lt(average(bytes), 1000)`, - - `lte([left]: number, [right]: number) - Performs a lower than or equal comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with <= symbol. - - Example: Returns true if the average of bytes is lower than or equal to the average amount of memory - - average(bytes) <= average(memory) - - Example: lte(average(bytes), 1000)`, - ], - }, - { - id: 'lens_formulas_kibana_context', - text: dedent(`Kibana context - - These functions are used to retrieve Kibana context variables, which are the - date histogram \`interval\`, the current \`now\` and the selected \`time_range\` - and help you to compute date math operations. - - interval() - The specified minimum interval for the date histogram, in milliseconds (ms). - - now() - The current now moment used in Kibana expressed in milliseconds (ms). - - time_range() - The specified time range, in milliseconds (ms).`), - }, - ]); -} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts index 7d1b178129c6a..1e1a67a20acf3 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts @@ -20,14 +20,14 @@ export function splitKbText({ return [ { type: KnowledgeBaseEntryOperationType.Delete, - docId: id, // delete all entries with the same docId + operationDocId: id, // delete all entries with the same operationDocId labels: {}, }, ...texts.map((text, index) => ({ type: KnowledgeBaseEntryOperationType.Index, document: merge({}, rest, { id: [id, index].join('_'), - doc_id: id, // doc_id is used to group entries together. + title: id, // title is used to group entries together. labels: {}, text, }), diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts index b71cf68f026e5..fefe324805e59 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts @@ -14,7 +14,7 @@ import type { FunctionCallChatFunction } from '../../service/types'; import { RecallRanking, recallRankingEventType } from '../../analytics/recall_ranking'; import { RecalledEntry } from '../../service/knowledge_base_service'; -export type RecalledSuggestion = Pick; +export type RecalledSuggestion = Pick; export async function recallAndScore({ recall, @@ -45,7 +45,7 @@ export async function recallAndScore({ ].filter((query) => query.text.trim()); const suggestions: RecalledSuggestion[] = (await recall({ queries })).map( - ({ id, title, docId, text, score }) => ({ id, title, docId, text, score }) + ({ id, text, score }) => ({ id, text, score }) ); if (!suggestions.length) { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts index 52d8c52ec5af8..7d1a19463cceb 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts @@ -54,13 +54,8 @@ export async function scoreSuggestions({ const newUserMessageContent = dedent(`Given the following prompt, score the documents that are relevant to the prompt on a scale from 0 to 7, 0 being completely irrelevant, and 7 being extremely relevant. Information is relevant to the prompt if it helps in - answering the prompt. Judge the document according to the following criteria : + answering the prompt. Judge the document according to the following criteria: - If the prompt is a statement that should be stored in the knowledge base: - - The document contains information that directly contradicts the user's prompt or previous statements, indicating that it may need to be updated or corrected. - - The document contains outdated user preferences or information that the user indicate they want corrected or replaced. - - If the prompt is a question: - The document is relevant to the prompt, and the rest of the conversation - The document has information relevant to the prompt that is not mentioned, or more detailed than what is available in the conversation - The document has a high amount of information relevant to the prompt compared to other documents diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts index 1df1c3584a5b3..7ea25eab8c661 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts @@ -30,7 +30,7 @@ export function categorizeEntries({ return acc.concat({ categoryKey, - title: entry.labels?.category ?? entry.title ?? entry.doc_id ?? 'No title', + title: entry.labels?.category ?? entry.title ?? 'No title', entries: [entry], '@timestamp': entry['@timestamp'], }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx index 5bb198d4f6a44..20c7a75a401a8 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx @@ -47,7 +47,7 @@ export function KnowledgeBaseEditManualEntryFlyout({ const { mutateAsync: deleteEntry, isLoading: isDeleting } = useDeleteKnowledgeBaseEntry(); const [isPublic, setIsPublic] = useState(entry?.public ?? false); - const [newEntryTitle, setNewEntryTitle] = useState(entry?.title ?? entry?.doc_id ?? ''); + const [newEntryTitle, setNewEntryTitle] = useState(entry?.title ?? ''); const [newEntryText, setNewEntryText] = useState(entry?.text ?? ''); const isEntryTitleInvalid = newEntryTitle.trim() === ''; @@ -58,7 +58,6 @@ export function KnowledgeBaseEditManualEntryFlyout({ await createEntry({ entry: { id: entry?.id ?? v4(), - doc_id: entry?.doc_id, title: newEntryTitle, text: newEntryText, public: isPublic, @@ -93,7 +92,7 @@ export function KnowledgeBaseEditManualEntryFlyout({ 'xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.editEntryLabel', { defaultMessage: 'Edit {title}', - values: { title: entry?.title ?? entry?.doc_id }, + values: { title: entry?.title }, } )} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx index 91e5b21feb4f0..50d0cb8ba47c8 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx @@ -89,7 +89,6 @@ describe('KnowledgeBaseTab', () => { text: 'bar', role: 'user_entry', confidence: 'high', - doc_id: undefined, is_correction: false, labels: expect.any(Object), }, @@ -138,7 +137,7 @@ describe('KnowledgeBaseTab', () => { entries: [ { id: 'test', - doc_id: 'test', + title: 'test', text: 'test', '@timestamp': 1638340456, labels: {}, @@ -146,7 +145,7 @@ describe('KnowledgeBaseTab', () => { }, { id: 'test2', - doc_id: 'test2', + title: 'test2', text: 'test', '@timestamp': 1638340456, labels: { @@ -156,7 +155,7 @@ describe('KnowledgeBaseTab', () => { }, { id: 'test3', - doc_id: 'test3', + title: 'test3', text: 'test', '@timestamp': 1638340456, labels: { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index ced38977a35f9..4101846c3cacd 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -13,7 +13,6 @@ import { uniq } from 'lodash'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { clearKnowledgeBase, createKnowledgeBaseModel, deleteKnowledgeBaseModel } from './helpers'; import { ObservabilityAIAssistantApiClients } from '../../common/config'; -import { ObservabilityAIAssistantApiClient } from '../../common/observability_ai_assistant_api_client'; export default function ApiTest({ getService }: FtrProviderContext) { const ml = getService('ml'); @@ -274,7 +273,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { (entry) => entry.labels?.category === 'my_new_category' ); - const entryGroups = uniq(categoryEntries.map((entry) => entry.doc_id)); + const entryGroups = uniq(categoryEntries.map((entry) => entry.title)); log.debug( `Waiting for entries to be created. Found ${categoryEntries.length} entries and ${entryGroups.length} groups` @@ -296,151 +295,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); }); }); - - describe('When the LLM creates entries', () => { - async function saveKbEntry({ - apiClient, - docId, - text, - }: { - apiClient: ObservabilityAIAssistantApiClient; - docId: string; - text: string; - }) { - return apiClient({ - endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', - params: { - body: { - doc_id: docId, - text, - confidence: 'high', - is_correction: false, - public: false, - labels: {}, - }, - }, - }).expect(200); - } - - async function getEntriesWithDocId(docId: string) { - const res = await observabilityAIAssistantAPIClient - .editor({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', - params: { - query: { - query: '', - sortBy: 'title', - sortDirection: 'asc', - }, - }, - }) - .expect(200); - - return res.body.entries.filter((entry) => entry.doc_id === docId); - } - - describe('when the LLM uses the same doc_id for two entries created by the same user', () => { - let entries1: KnowledgeBaseEntry[]; - let entries2: KnowledgeBaseEntry[]; - - before(async () => { - const docId = 'my_favourite_color'; - - await saveKbEntry({ - apiClient: observabilityAIAssistantAPIClient.editor, - docId, - text: 'My favourite color is blue', - }); - entries1 = await getEntriesWithDocId(docId); - - await saveKbEntry({ - apiClient: observabilityAIAssistantAPIClient.editor, - docId, - text: 'My favourite color is green', - }); - entries2 = await getEntriesWithDocId(docId); - }); - - after(async () => { - await clearKnowledgeBase(es); - }); - - it('overwrites the first entry so there is only one', async () => { - expect(entries1.length).to.eql(1); - expect(entries2.length).to.eql(1); - }); - - it('replaces the text content of the first entry with the new text content', async () => { - expect(entries1[0].text).to.eql('My favourite color is blue'); - expect(entries2[0].text).to.eql('My favourite color is green'); - }); - - it('updates the timestamp', async () => { - const getAsMs = (timestamp: string) => new Date(timestamp).getTime(); - expect(getAsMs(entries1[0]['@timestamp'])).to.be.lessThan( - getAsMs(entries2[0]['@timestamp']) - ); - }); - - it('does not change the _id', () => { - expect(entries1[0].id).to.eql(entries2[0].id); - }); - }); - - describe('when the LLM uses same doc_id for two entries created by different users', () => { - let entries: KnowledgeBaseEntry[]; - - before(async () => { - await saveKbEntry({ - apiClient: observabilityAIAssistantAPIClient.editor, - docId: 'users_favorite_animal', - text: "The user's favourite animal is a dog", - }); - await saveKbEntry({ - apiClient: observabilityAIAssistantAPIClient.secondaryEditor, - docId: 'users_favorite_animal', - text: "The user's favourite animal is a cat", - }); - - const res = await observabilityAIAssistantAPIClient - .editor({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', - params: { - query: { - query: '', - sortBy: 'title', - sortDirection: 'asc', - }, - }, - }) - .expect(200); - - entries = omitCategories(res.body.entries); - }); - - after(async () => { - await clearKnowledgeBase(es); - }); - - it('creates two separate entries with the same doc_id', async () => { - expect(entries.map(({ doc_id: docId }) => docId)).to.eql([ - 'users_favorite_animal', - 'users_favorite_animal', - ]); - }); - - it('creates two entries with different text content', async () => { - expect(entries.map(({ text }) => text)).to.eql([ - "The user's favourite animal is a cat", - "The user's favourite animal is a dog", - ]); - }); - - it('creates two entries by different users', async () => { - expect(entries.map(({ user }) => user?.name)).to.eql(['secondary_editor', 'editor']); - }); - }); - }); }); } diff --git a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts index e59f875350112..e7f7690a0f524 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts @@ -24,18 +24,15 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte async function saveKbEntry({ apiClient, - docId, text, }: { apiClient: ObservabilityAIAssistantApiClient; - docId: string; text: string; }) { return apiClient({ endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', params: { body: { - doc_id: docId, text, confidence: 'high', is_correction: false, @@ -99,13 +96,11 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte before(async () => { await saveKbEntry({ apiClient: observabilityAIAssistantAPIClient.editor, - docId: 'my_fav_color', text: 'My favourite color is red', }); await saveKbEntry({ apiClient: observabilityAIAssistantAPIClient.secondaryEditor, - docId: 'my_fav_color', text: 'My favourite color is blue', }); }); From 035a05403efea5371b9e25ffb22ba75604442962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 5 Nov 2024 11:23:55 +0100 Subject: [PATCH 29/42] Remove queue logic --- .../server/routes/knowledge_base/route.ts | 139 +++++-------- .../server/service/client/index.ts | 19 +- .../server/service/index.ts | 99 +-------- .../service/knowledge_base_service/index.ts | 192 +----------------- .../server/service/util/split_kb_text.ts | 36 ---- .../use_import_knowledge_base_entries.ts | 2 +- .../knowledge_base_bulk_import_flyout.tsx | 6 +- .../knowledge_base/knowledge_base.spec.ts | 95 --------- 8 files changed, 59 insertions(+), 529 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts index 8f7e7988d6d79..0f1852c0e396c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts @@ -9,6 +9,7 @@ import type { MlDeploymentAllocationState, MlDeploymentState, } from '@elastic/elasticsearch/lib/api/types'; +import pLimit from 'p-limit'; import { notImplemented } from '@hapi/boom'; import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; @@ -138,27 +139,29 @@ const getKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ }, }); +const knowledgeBaseEntryRt = t.intersection([ + t.type({ + id: t.string, + title: t.string, + text: nonEmptyStringRt, + }), + t.partial({ + confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), + is_correction: toBooleanRt, + public: toBooleanRt, + labels: t.record(t.string, t.string), + role: t.union([ + t.literal(KnowledgeBaseEntryRole.AssistantSummarization), + t.literal(KnowledgeBaseEntryRole.UserEntry), + t.literal(KnowledgeBaseEntryRole.Elastic), + ]), + }), +]); + const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/save', params: t.type({ - body: t.intersection([ - t.type({ - id: t.string, - title: t.string, - text: nonEmptyStringRt, - }), - t.partial({ - confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), - is_correction: toBooleanRt, - public: toBooleanRt, - labels: t.record(t.string, t.string), - role: t.union([ - t.literal(KnowledgeBaseEntryRole.AssistantSummarization), - t.literal(KnowledgeBaseEntryRole.UserEntry), - t.literal(KnowledgeBaseEntryRole.Elastic), - ]), - }), - ]), + body: knowledgeBaseEntryRt, }), options: { tags: ['access:ai_assistant'], @@ -170,27 +173,15 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ throw notImplemented(); } - const { - id, - title, - text, - public: isPublic, - confidence, - is_correction: isCorrection, - labels, - role, - } = resources.params.body; - + const entry = resources.params.body; return client.addKnowledgeBaseEntry({ entry: { - text, - id, - title, - confidence: confidence ?? 'high', - is_correction: isCorrection ?? false, - public: isPublic ?? true, - labels: labels ?? {}, - role: role ?? KnowledgeBaseEntryRole.UserEntry, + confidence: 'high', + is_correction: false, + public: true, + labels: {}, + role: KnowledgeBaseEntryRole.UserEntry, + ...entry, }, }); }, @@ -221,17 +212,7 @@ const importKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', params: t.type({ body: t.type({ - entries: t.array( - t.intersection([ - t.type({ - id: t.string, - text: nonEmptyStringRt, - }), - t.partial({ - title: t.string, - }), - ]) - ), + entries: t.array(knowledgeBaseEntryRt), }), }), options: { @@ -244,48 +225,29 @@ const importKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ throw notImplemented(); } - const formattedEntries = resources.params.body.entries.map((entry) => ({ - id: entry.id, - title: entry.title, - text: entry.text, - confidence: 'high' as KnowledgeBaseEntry['confidence'], - is_correction: false, - type: 'contextual' as const, - public: true, - labels: {}, - role: KnowledgeBaseEntryRole.UserEntry, - })); - - return await client.importKnowledgeBaseEntries({ entries: formattedEntries }); - }, -}); - -const importKnowledgeBaseCategoryEntries = createObservabilityAIAssistantServerRoute({ - endpoint: 'POST /internal/observability_ai_assistant/kb/entries/category/import', - params: t.type({ - body: t.type({ - category: t.string, - entries: t.array( - t.type({ - id: t.string, - texts: t.array(t.string), - }) - ), - }), - }), - options: { - tags: ['access:ai_assistant'], - }, - handler: async (resources): Promise => { - const client = await resources.service.getClient({ request: resources.request }); - - if (!client) { - throw notImplemented(); + const status = await client.getKnowledgeBaseStatus(); + if (!status.ready) { + throw new Error('Knowledge base is not ready'); } - const { entries, category } = resources.params.body; + const limiter = pLimit(5); + + const promises = resources.params.body.entries.map(async (entry) => { + return limiter(async () => { + return client.addKnowledgeBaseEntry({ + entry: { + confidence: 'high', + is_correction: false, + public: true, + labels: {}, + role: KnowledgeBaseEntryRole.UserEntry, + ...entry, + }, + }); + }); + }); - return resources.service.addCategoryToKnowledgeBase(category, entries); + await Promise.all(promises); }, }); @@ -294,9 +256,8 @@ export const knowledgeBaseRoutes = { ...getKnowledgeBaseStatus, ...getKnowledgeBaseEntries, ...saveKnowledgeBaseUserInstruction, - ...getKnowledgeBaseUserInstructions, ...importKnowledgeBaseEntries, - ...importKnowledgeBaseCategoryEntries, + ...getKnowledgeBaseUserInstructions, ...saveKnowledgeBaseEntry, ...deleteKnowledgeBaseEntry, }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 935043d289609..048bbd2d362c2 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -59,11 +59,7 @@ import { import { withoutTokenCountEvents } from '../../../common/utils/without_token_count_events'; import { CONTEXT_FUNCTION_NAME } from '../../functions/context'; import type { ChatFunctionClient } from '../chat_function_client'; -import { - KnowledgeBaseEntryOperationType, - KnowledgeBaseService, - RecalledEntry, -} from '../knowledge_base_service'; +import { KnowledgeBaseService, RecalledEntry } from '../knowledge_base_service'; import { getAccessQuery } from '../util/get_access_query'; import { getSystemMessageFromInstructions } from '../util/get_system_message_from_instructions'; import { replaceSystemMessage } from '../util/replace_system_message'; @@ -783,19 +779,6 @@ export class ObservabilityAIAssistantClient { }); }; - importKnowledgeBaseEntries = async ({ - entries, - }: { - entries: Array>; - }): Promise => { - const operations = entries.map((entry) => ({ - type: KnowledgeBaseEntryOperationType.Index, - document: { ...entry, '@timestamp': new Date().toISOString() }, - })); - - await this.dependencies.knowledgeBaseService.importEntries({ operations }); - }; - getKnowledgeBaseEntries = async ({ query, sortBy, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts index 6d11120ee7dda..eb7eab19340ce 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts @@ -13,18 +13,14 @@ import { getSpaceIdFromPath } from '@kbn/spaces-plugin/common'; import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; import { once } from 'lodash'; import type { AssistantScope } from '@kbn/ai-assistant-common'; -import { - KnowledgeBaseEntryRole, - ObservabilityAIAssistantScreenContextRequest, -} from '../../common/types'; +import { ObservabilityAIAssistantScreenContextRequest } from '../../common/types'; import type { ObservabilityAIAssistantPluginStartDependencies } from '../types'; import { ChatFunctionClient } from './chat_function_client'; import { ObservabilityAIAssistantClient } from './client'; import { conversationComponentTemplate } from './conversation_component_template'; import { kbComponentTemplate } from './kb_component_template'; -import { KnowledgeBaseEntryOperationType, KnowledgeBaseService } from './knowledge_base_service'; +import { KnowledgeBaseService } from './knowledge_base_service'; import type { RegistrationCallback, RespondFunctionResources } from './types'; -import { splitKbText } from './util/split_kb_text'; function getResourceName(resource: string) { return `.kibana-observability-ai-assistant-${resource}`; @@ -52,19 +48,6 @@ export const resourceNames = { }, }; -export const INDEX_QUEUED_DOCUMENTS_TASK_ID = 'observabilityAIAssistant:indexQueuedDocumentsTask'; - -export const INDEX_QUEUED_DOCUMENTS_TASK_TYPE = INDEX_QUEUED_DOCUMENTS_TASK_ID + 'Type'; - -type KnowledgeBaseEntryRequest = { id: string; labels?: Record } & ( - | { - text: string; - } - | { - texts: string[]; - } -); - export class ObservabilityAIAssistantService { private readonly core: CoreSetup; private readonly logger: Logger; @@ -93,26 +76,6 @@ export class ObservabilityAIAssistantService { this.enableKnowledgeBase = enableKnowledgeBase; this.allowInit(); - if (enableKnowledgeBase) { - taskManager.registerTaskDefinitions({ - [INDEX_QUEUED_DOCUMENTS_TASK_TYPE]: { - title: 'Index queued KB articles', - description: - 'Indexes previously registered entries into the knowledge base when it is ready', - timeout: '30m', - maxAttempts: 2, - createTaskRunner: (context) => { - return { - run: async () => { - if (this.kbService) { - await this.kbService.processQueue(); - } - }, - }; - }, - }, - }); - } } getKnowledgeBaseStatus() { @@ -336,64 +299,6 @@ export class ObservabilityAIAssistantService { return fnClient; } - addToKnowledgeBaseQueue(entries: KnowledgeBaseEntryRequest[]): void { - if (this.enableKnowledgeBase) { - this.init() - .then(() => { - this.kbService!.queue( - entries.flatMap((entry) => { - const entryWithSystemProperties = { - ...entry, - '@timestamp': new Date().toISOString(), - public: true, - confidence: 'high' as const, - type: 'contextual' as const, - is_correction: false, - labels: { - ...entry.labels, - }, - role: KnowledgeBaseEntryRole.Elastic, - }; - - const operations = - 'texts' in entryWithSystemProperties - ? splitKbText(entryWithSystemProperties) - : [ - { - type: KnowledgeBaseEntryOperationType.Index, - document: entryWithSystemProperties, - }, - ]; - - return operations; - }) - ); - }) - .catch((error) => { - this.logger.error( - `Could not index ${entries.length} entries because of an initialisation error` - ); - this.logger.error(error); - }); - } - } - - addCategoryToKnowledgeBase(categoryId: string, entries: KnowledgeBaseEntryRequest[]) { - if (this.enableKnowledgeBase) { - this.addToKnowledgeBaseQueue( - entries.map((entry) => { - return { - ...entry, - labels: { - ...entry.labels, - category: categoryId, - }, - }; - }) - ); - } - } - register(cb: RegistrationCallback) { this.registrations.push(cb); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index 21e7ce364aee9..98d2078c81cd7 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -9,16 +9,11 @@ import { serverUnavailable, gatewayTimeout, badRequest } from '@hapi/boom'; import type { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; -import pLimit from 'p-limit'; import pRetry from 'p-retry'; -import { map, orderBy } from 'lodash'; +import { orderBy } from 'lodash'; import { encode } from 'gpt-tokenizer'; import { MlTrainedModelDeploymentNodesStats } from '@elastic/elasticsearch/lib/api/types'; -import { - INDEX_QUEUED_DOCUMENTS_TASK_ID, - INDEX_QUEUED_DOCUMENTS_TASK_TYPE, - resourceNames, -} from '..'; +import { resourceNames } from '..'; import { Instruction, KnowledgeBaseEntry, @@ -63,34 +58,8 @@ function throwKnowledgeBaseNotReady(body: any) { throw serverUnavailable(`Knowledge base is not ready yet`, body); } -export enum KnowledgeBaseEntryOperationType { - Index = 'index', - Delete = 'delete', -} - -interface KnowledgeBaseDeleteOperation { - type: KnowledgeBaseEntryOperationType.Delete; - operationDocId?: string; - labels?: Record; -} - -interface KnowledgeBaseIndexOperation { - type: KnowledgeBaseEntryOperationType.Index; - document: KnowledgeBaseEntry; -} - -export type KnowledgeBaseEntryOperation = - | KnowledgeBaseDeleteOperation - | KnowledgeBaseIndexOperation; - export class KnowledgeBaseService { - private isModelReady: boolean = false; - - private _queue: KnowledgeBaseEntryOperation[] = []; - - constructor(private readonly dependencies: Dependencies) { - this.ensureTaskScheduled(); - } + constructor(private readonly dependencies: Dependencies) {} setup = async () => { this.dependencies.logger.debug('Setting up knowledge base'); @@ -203,140 +172,8 @@ export class KnowledgeBaseService { }, retryOptions); this.dependencies.logger.info(`${elserModelId} model is ready`); - this.ensureTaskScheduled(); }; - private ensureTaskScheduled() { - if (!this.dependencies.enabled) { - return; - } - this.dependencies.taskManagerStart - .ensureScheduled({ - taskType: INDEX_QUEUED_DOCUMENTS_TASK_TYPE, - id: INDEX_QUEUED_DOCUMENTS_TASK_ID, - state: {}, - params: {}, - schedule: { - interval: '1h', - }, - }) - .then(() => { - this.dependencies.logger.debug('Scheduled queue task'); - return this.dependencies.taskManagerStart.runSoon(INDEX_QUEUED_DOCUMENTS_TASK_ID); - }) - .then(() => { - this.dependencies.logger.debug('Queue task ran'); - }) - .catch((err) => { - this.dependencies.logger.error(`Failed to schedule queue task`); - this.dependencies.logger.error(err); - }); - } - - private async processOperation(operation: KnowledgeBaseEntryOperation) { - if (operation.type === KnowledgeBaseEntryOperationType.Delete) { - try { - await this.dependencies.esClient.asInternalUser.deleteByQuery({ - index: resourceNames.aliases.kb, - query: { - bool: { - filter: [ - ...(operation.operationDocId - ? [{ term: { doc_id: operation.operationDocId } }] - : []), - ...(operation.labels - ? map(operation.labels, (value, key) => { - return { term: { [key]: value } }; - }) - : []), - ], - }, - }, - }); - return; - } catch (error) { - this.dependencies.logger.error( - `Failed to delete document "${operation?.operationDocId}" due to ${error.message}` - ); - this.dependencies.logger.debug(() => JSON.stringify(operation)); - throw error; - } - } - - try { - await this.addEntry({ entry: operation.document }); - } catch (error) { - this.dependencies.logger.error(`Failed to index document due to ${error.message}`); - this.dependencies.logger.debug(() => JSON.stringify(operation.document)); - throw error; - } - } - - async processQueue() { - if (!this._queue.length || !this.dependencies.enabled) { - return; - } - - if (!(await this.status()).ready) { - this.dependencies.logger.debug(`Bailing on queue task: KB is not ready yet`); - return; - } - - this.dependencies.logger.debug(`Processing queue`); - - this.isModelReady = true; - - this.dependencies.logger.info(`Processing ${this._queue.length} queue operations`); - - const limiter = pLimit(5); - - const operations = this._queue.concat(); - - await Promise.all( - operations.map((operation) => - limiter(async () => { - this._queue.splice(operations.indexOf(operation), 1); - await this.processOperation(operation); - }) - ) - ); - - this.dependencies.logger.info(`Finished processing ${operations.length} queued operations`); - } - - queue(operations: KnowledgeBaseEntryOperation[]): void { - if (!operations.length) { - return; - } - - this.dependencies.logger.debug( - `Adding ${operations.length} operations to queue. Queue size now: ${this._queue.length})` - ); - this._queue.push(...operations); - - if (!this.isModelReady) { - this.dependencies.logger.debug( - `Delay processing ${operations.length} operations until knowledge base is ready` - ); - return; - } - - const limiter = pLimit(5); - - const limitedFunctions = this._queue.map((operation) => - limiter(() => this.processOperation(operation)) - ); - - Promise.all(limitedFunctions) - .then(() => { - this.dependencies.logger.debug(`Processed all queued operations`); - }) - .catch((err) => { - this.dependencies.logger.error(`Failed to process all queued operations`); - this.dependencies.logger.error(err); - }); - } - status = async () => { this.dependencies.logger.debug('Checking model status'); if (!this.dependencies.enabled) { @@ -726,29 +563,6 @@ export class KnowledgeBaseService { } }; - importEntries = async ({ - operations, - }: { - operations: KnowledgeBaseEntryOperation[]; - }): Promise => { - if (!this.dependencies.enabled) { - return; - } - this.dependencies.logger.info(`Starting import of ${operations.length} entries`); - - const limiter = pLimit(5); - - await Promise.all( - operations.map((operation) => - limiter(async () => { - await this.processOperation(operation); - }) - ) - ); - - this.dependencies.logger.info(`Completed import of ${operations.length} entries`); - }; - deleteEntry = async ({ id }: { id: string }): Promise => { try { await this.dependencies.esClient.asInternalUser.delete({ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts deleted file mode 100644 index 1e1a67a20acf3..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { merge } from 'lodash'; -import type { KnowledgeBaseEntry } from '../../../common/types'; -import { - type KnowledgeBaseEntryOperation, - KnowledgeBaseEntryOperationType, -} from '../knowledge_base_service'; - -export function splitKbText({ - id, - texts, - ...rest -}: Omit & { texts: string[] }): KnowledgeBaseEntryOperation[] { - return [ - { - type: KnowledgeBaseEntryOperationType.Delete, - operationDocId: id, // delete all entries with the same operationDocId - labels: {}, - }, - ...texts.map((text, index) => ({ - type: KnowledgeBaseEntryOperationType.Index, - document: merge({}, rest, { - id: [id, index].join('_'), - title: id, // title is used to group entries together. - labels: {}, - text, - }), - })), - ]; -} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_import_knowledge_base_entries.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_import_knowledge_base_entries.ts index 9ff0748793bc8..239e72d99109e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_import_knowledge_base_entries.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_import_knowledge_base_entries.ts @@ -30,7 +30,7 @@ export function useImportKnowledgeBaseEntries() { Omit< KnowledgeBaseEntry, '@timestamp' | 'confidence' | 'is_correction' | 'public' | 'labels' - > + > & { title: string } >; } >( diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_bulk_import_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_bulk_import_flyout.tsx index ac7632243a2a0..23d5ad00af12a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_bulk_import_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_bulk_import_flyout.tsx @@ -47,15 +47,13 @@ export function KnowledgeBaseBulkImportFlyout({ onClose }: { onClose: () => void }; const handleSubmitNewEntryClick = async () => { - let entries: Array> = []; + let entries: Array & { title: string }> = []; const text = await files[0].text(); const elements = text.split('\n').filter(Boolean); try { - entries = elements.map((el) => JSON.parse(el)) as Array< - Omit - >; + entries = elements.map((el) => JSON.parse(el)); } catch (_) { toasts.addError( new Error( diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index 4101846c3cacd..181ae1547d115 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -9,7 +9,6 @@ import expect from '@kbn/expect'; import { type KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; import pRetry from 'p-retry'; import { ToolingLog } from '@kbn/tooling-log'; -import { uniq } from 'lodash'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { clearKnowledgeBase, createKnowledgeBaseModel, deleteKnowledgeBaseModel } from './helpers'; import { ObservabilityAIAssistantApiClients } from '../../common/config'; @@ -201,100 +200,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(entries[0].title).to.eql('My title b'); }); }); - - describe('when importing categories', () => { - beforeEach(async () => { - await clearKnowledgeBase(es); - }); - - afterEach(async () => { - await clearKnowledgeBase(es); - }); - - const importCategories = () => - observabilityAIAssistantAPIClient - .editor({ - endpoint: 'POST /internal/observability_ai_assistant/kb/entries/category/import', - params: { - body: { - category: 'my_new_category', - entries: [ - { - id: 'my_new_category_a', - texts: [ - 'My first category content a', - 'My second category content a', - 'my third category content a', - ], - }, - { - id: 'my_new_category_b', - texts: [ - 'My first category content b', - 'My second category content b', - 'my third category content b', - ], - }, - { - id: 'my_new_category_c', - texts: [ - 'My first category content c', - 'My second category content c', - 'my third category content c', - ], - }, - ], - }, - }, - }) - .expect(200); - - it('overwrites existing entries on subsequent import', async () => { - await waitForModelReady(observabilityAIAssistantAPIClient, log); - await importCategories(); - await importCategories(); - - await pRetry( - async () => { - const res = await observabilityAIAssistantAPIClient - .editor({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', - params: { - query: { - query: '', - sortBy: 'title', - sortDirection: 'asc', - }, - }, - }) - .expect(200); - - const categoryEntries = res.body.entries.filter( - (entry) => entry.labels?.category === 'my_new_category' - ); - - const entryGroups = uniq(categoryEntries.map((entry) => entry.title)); - - log.debug( - `Waiting for entries to be created. Found ${categoryEntries.length} entries and ${entryGroups.length} groups` - ); - - if (categoryEntries.length !== 9 || entryGroups.length !== 3) { - throw new Error( - `Expected 9 entries, found ${categoryEntries.length} and ${entryGroups.length} groups` - ); - } - - expect(categoryEntries.length).to.eql(9); - expect(entryGroups.length).to.eql(3); - }, - { - retries: 100, - factor: 1, - } - ); - }); - }); }); } From 996e3d8be488a4b36160cd217382908a1c148967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 5 Nov 2024 13:29:58 +0100 Subject: [PATCH 30/42] Fall back to doc_id --- .../server/service/knowledge_base_service/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index 98d2078c81cd7..92ce3a4a7e03b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -409,7 +409,9 @@ export class KnowledgeBaseService { return { entries: [] }; } try { - const response = await this.dependencies.esClient.asInternalUser.search({ + const response = await this.dependencies.esClient.asInternalUser.search< + KnowledgeBaseEntry & { doc_id?: string } + >({ index: resourceNames.aliases.kb, query: { bool: { @@ -453,6 +455,7 @@ export class KnowledgeBaseService { return { entries: response.hits.hits.map((hit) => ({ ...hit._source!, + title: hit._source!.title ?? hit._source!.doc_id, // use `doc_id` as fallback title for backwards compatibility role: hit._source!.role ?? KnowledgeBaseEntryRole.UserEntry, score: hit._score, id: hit._id!, From 94996c8286cd4104f2032d57dabbec0beb8b5483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 5 Nov 2024 13:40:37 +0100 Subject: [PATCH 31/42] Remove log --- .../tests/knowledge_base/knowledge_base.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index 181ae1547d115..dc6ce83c2a1aa 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -16,7 +16,6 @@ import { ObservabilityAIAssistantApiClients } from '../../common/config'; export default function ApiTest({ getService }: FtrProviderContext) { const ml = getService('ml'); const es = getService('es'); - const log = getService('log'); const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); describe('Knowledge base', () => { From e1ae033cbbdbafb0452c5dace6e2c3a3765f5aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 5 Nov 2024 15:02:13 +0100 Subject: [PATCH 32/42] Remove unused knowledge_base/knowledge_base.spec.ts --- .../knowledge_base/knowledge_base.spec.ts | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index dc6ce83c2a1aa..89a9a4a97193b 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -205,21 +205,3 @@ export default function ApiTest({ getService }: FtrProviderContext) { function omitCategories(entries: KnowledgeBaseEntry[]) { return entries.filter((entry) => entry.labels?.category === undefined); } - -async function waitForModelReady( - observabilityAIAssistantAPIClient: ObservabilityAIAssistantApiClients, - log: ToolingLog -) { - return pRetry(async () => { - const res = await observabilityAIAssistantAPIClient - .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/status' }) - .expect(200); - - const isModelReady = res.body.ready; - log.debug(`Model status: ${isModelReady ? 'ready' : 'not ready'}`); - - if (!isModelReady) { - throw new Error('Model not ready'); - } - }); -} From 993745e6d7f836685109513624e9e45b26b9e72a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 5 Nov 2024 21:27:36 +0100 Subject: [PATCH 33/42] Fix lint issues --- .../tests/knowledge_base/knowledge_base.spec.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index 89a9a4a97193b..27659f62ad579 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -7,11 +7,8 @@ import expect from '@kbn/expect'; import { type KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; -import pRetry from 'p-retry'; -import { ToolingLog } from '@kbn/tooling-log'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { clearKnowledgeBase, createKnowledgeBaseModel, deleteKnowledgeBaseModel } from './helpers'; -import { ObservabilityAIAssistantApiClients } from '../../common/config'; export default function ApiTest({ getService }: FtrProviderContext) { const ml = getService('ml'); From 4022dab5419446fade83ee87c3a94d1ec1fa3be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 5 Nov 2024 22:07:09 +0100 Subject: [PATCH 34/42] Fix serverless test --- .../ai_assistant/tests/knowledge_base/knowledge_base.spec.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts index b32a48d1a9c2e..ce46939c365be 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts @@ -157,14 +157,17 @@ export default function ApiTest({ getService }: FtrProviderContext) { const knowledgeBaseEntries = [ { id: 'my_doc_a', + title: 'My title a', text: 'My content a', }, { id: 'my_doc_b', + title: 'My title b', text: 'My content b', }, { id: 'my_doc_c', + title: 'My title c', text: 'My content c', }, ]; From 3671085f3bee873610ec92c14af52a806f2ceba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Tue, 5 Nov 2024 23:44:23 +0100 Subject: [PATCH 35/42] Fix broken api test --- .../knowledge_base/knowledge_base_user_instructions.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index 63826637c02db..952fecbbff1ae 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -199,7 +199,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { 'Be polite and use language that is easy to understand. Never disagree with the user.'; async function getConversationForUser(username: keyof ObservabilityAIAssistantApiClients) { - const apiClient = observabilityAIAssistantAPIClient[username]; + const apiClient = getScopedApiClientForUsername(username); // the user instruction is always created by "editor" user await observabilityAIAssistantAPIClient From 5708359ef6a5a97d27a6745b13e79fd4df6936d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 6 Nov 2024 00:29:12 +0100 Subject: [PATCH 36/42] fix tests --- .../common/config.ts | 15 ++++++++------- .../knowledge_base_user_instructions.spec.ts | 19 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts index b5e4bde29b11e..427258a6e2910 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts @@ -50,22 +50,23 @@ export function createObservabilityAIAssistantAPIConfig({ const apmSynthtraceKibanaClient = services.apmSynthtraceKibanaClient(); const allConfigs = config.getAll() as Record; + const getScopedApiClientForUsername = (username: string) => + getScopedApiClient(kibanaServer, username); + return { ...allConfigs, servers, services: { ...services, - getScopedApiClientForUsername: () => { - return (username: string) => getScopedApiClient(kibanaServer, username); - }, + getScopedApiClientForUsername: () => getScopedApiClientForUsername, apmSynthtraceEsClient: (context: InheritedFtrProviderContext) => getApmSynthtraceEsClient(context, apmSynthtraceKibanaClient), observabilityAIAssistantAPIClient: async () => { return { - admin: getScopedApiClient(kibanaServer, 'elastic'), - viewer: getScopedApiClient(kibanaServer, viewer.username), - editor: getScopedApiClient(kibanaServer, editor.username), - secondaryEditor: getScopedApiClient(kibanaServer, secondaryEditor.username), + admin: getScopedApiClientForUsername('elastic'), + viewer: getScopedApiClientForUsername(viewer.username), + editor: getScopedApiClientForUsername(editor.username), + secondaryEditor: getScopedApiClientForUsername(secondaryEditor.username), }; }, }, diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index 952fecbbff1ae..8d5baa46cf9cc 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -20,7 +20,6 @@ import { import { getConversationCreatedEvent } from '../conversations/helpers'; import { LlmProxy, createLlmProxy } from '../../common/create_llm_proxy'; import { createProxyActionConnector, deleteActionConnector } from '../../common/action_connectors'; -import { ObservabilityAIAssistantApiClients } from '../../common/config'; export default function ApiTest({ getService }: FtrProviderContext) { const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); @@ -59,11 +58,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { isPublic: false, }, { - username: 'secondaryEditor' as const, + username: 'secondary_editor' as const, isPublic: true, }, { - username: 'secondaryEditor' as const, + username: 'secondary_editor' as const, isPublic: false, }, ].map(async ({ username, isPublic }) => { @@ -106,9 +105,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { text: 'Public user instruction from "editor"', }, { - id: 'public-doc-from-secondaryEditor', + id: 'public-doc-from-secondary_editor', public: true, - text: 'Public user instruction from "secondaryEditor"', + text: 'Public user instruction from "secondary_editor"', }, ]) ); @@ -130,14 +129,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { text: 'Public user instruction from "editor"', }, { - id: 'public-doc-from-secondaryEditor', + id: 'public-doc-from-secondary_editor', public: true, - text: 'Public user instruction from "secondaryEditor"', + text: 'Public user instruction from "secondary_editor"', }, { - id: 'private-doc-from-secondaryEditor', + id: 'private-doc-from-secondary_editor', public: false, - text: 'Private user instruction from "secondaryEditor"', + text: 'Private user instruction from "secondary_editor"', }, ]) ); @@ -198,7 +197,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const userInstructionText = 'Be polite and use language that is easy to understand. Never disagree with the user.'; - async function getConversationForUser(username: keyof ObservabilityAIAssistantApiClients) { + async function getConversationForUser(username: string) { const apiClient = getScopedApiClientForUsername(username); // the user instruction is always created by "editor" user From 6e448b80b5c327848b316f53774bcccd93f455a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 6 Nov 2024 07:35:24 +0100 Subject: [PATCH 37/42] Fix test --- .../knowledge_base/knowledge_base_user_instructions.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index 8d5baa46cf9cc..44a072b0100b4 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -304,7 +304,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('does not add the instruction conversation for other users', async () => { - const conversation = await getConversationForUser('secondaryEditor'); + const conversation = await getConversationForUser('secondary_editor'); const systemMessage = conversation.messages.find( (message) => message.message.role === MessageRole.System )!; From 3114e7d67dba6eac0853222d90952acd9611aa4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 6 Nov 2024 07:59:12 +0100 Subject: [PATCH 38/42] Improve type --- .../knowledge_base/knowledge_base_user_instructions.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index 44a072b0100b4..a9c1f245a1ac3 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -20,6 +20,7 @@ import { import { getConversationCreatedEvent } from '../conversations/helpers'; import { LlmProxy, createLlmProxy } from '../../common/create_llm_proxy'; import { createProxyActionConnector, deleteActionConnector } from '../../common/action_connectors'; +import { User } from '../../common/users/users'; export default function ApiTest({ getService }: FtrProviderContext) { const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); @@ -197,7 +198,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const userInstructionText = 'Be polite and use language that is easy to understand. Never disagree with the user.'; - async function getConversationForUser(username: string) { + async function getConversationForUser(username: User['username']) { const apiClient = getScopedApiClientForUsername(username); // the user instruction is always created by "editor" user From 094f94ce1606e6e1a984e7a069de758b7fb1a77b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 6 Nov 2024 10:10:12 +0100 Subject: [PATCH 39/42] Fix functional test --- .../server/routes/functions/route.ts | 3 +++ .../knowledge_base_management/index.spec.ts | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index e71d40b51aef7..1571487765c09 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -124,6 +124,7 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', params: t.type({ body: t.type({ + title: t.string, text: nonEmptyStringRt, confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), is_correction: toBooleanRt, @@ -142,6 +143,7 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ } const { + title, confidence, is_correction: isCorrection, text, @@ -151,6 +153,7 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ return client.addKnowledgeBaseEntry({ entry: { + title, confidence, id: v4(), is_correction: isCorrection, diff --git a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts index e7f7690a0f524..7a5a51ae58b6a 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts @@ -8,6 +8,7 @@ import expect from '@kbn/expect'; import { subj as testSubjSelector } from '@kbn/test-subj-selector'; import { + clearKnowledgeBase, createKnowledgeBaseModel, deleteKnowledgeBaseModel, } from '../../../observability_ai_assistant_api_integration/tests/knowledge_base/helpers'; @@ -20,6 +21,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte const testSubjects = getService('testSubjects'); const log = getService('log'); const ml = getService('ml'); + const es = getService('es'); const { common } = getPageObjects(['common']); async function saveKbEntry({ @@ -33,6 +35,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', params: { body: { + title: 'Favourite color', text, confidence: 'high', is_correction: false, @@ -45,6 +48,8 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte describe('Knowledge management tab', () => { before(async () => { + await clearKnowledgeBase(es); + // create a knowledge base model await createKnowledgeBaseModel(ml); @@ -60,7 +65,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); after(async () => { - await Promise.all([deleteKnowledgeBaseModel(ml), ui.auth.logout()]); + await Promise.all([deleteKnowledgeBaseModel(ml), clearKnowledgeBase(es), ui.auth.logout()]); }); describe('when the LLM calls the "summarize" function for two different users', () => { @@ -75,22 +80,22 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte const rows = await Promise.all( entryTitleCells.map(async (cell) => { - const entryTitleText = await cell.getVisibleText(); + const title = await cell.getVisibleText(); const parentRow = await cell.findByXpath('ancestor::tr'); - const author = await parentRow.findByCssSelector( + const authorElm = await parentRow.findByCssSelector( testSubjSelector(ui.pages.kbManagementTab.tableAuthorCell) ); - const authorText = await author.getVisibleText(); + const author = await authorElm.getVisibleText(); const rowText = (await parentRow.getVisibleText()).split('\n'); - return { rowText, authorText, entryTitleText }; + return { rowText, author, title }; }) ); log.debug(`Found ${rows.length} rows in the KB management table: ${JSON.stringify(rows)}`); - return rows.filter(({ entryTitleText }) => entryTitleText === 'my_fav_color'); + return rows.filter(({ title }) => title === 'Favourite color'); } before(async () => { @@ -112,7 +117,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte it('shows two different authors', async () => { const entries = await getKnowledgeBaseEntries(); - expect(entries.map(({ authorText }) => authorText)).to.eql(['secondary_editor', 'editor']); + expect(entries.map(({ author }) => author)).to.eql(['secondary_editor', 'editor']); }); }); }); From 55e580099a27538d7f57ff51590fd96da6bcd9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 7 Nov 2024 00:25:16 +0100 Subject: [PATCH 40/42] Remove queue for task manager types --- .../test_suites/task_manager/check_registered_task_types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 79bdd3ad7df09..091e0fe01e415 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -151,7 +151,6 @@ export default function ({ getService }: FtrProviderContext) { 'fleet:update_agent_tags:retry', 'fleet:upgrade_action:retry', 'logs-data-telemetry', - 'observabilityAIAssistant:indexQueuedDocumentsTaskType', 'osquery:telemetry-configs', 'osquery:telemetry-packs', 'osquery:telemetry-saved-queries', From efa7bf20c873fbc22f9d07d999825694304811a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 7 Nov 2024 01:14:53 +0100 Subject: [PATCH 41/42] editorUser -> editor --- .../tests/complete/functions/summarize.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index 724c8c6378d11..fd1565ec78e5e 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -35,7 +35,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await clearKnowledgeBase(es); await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(200); From 732f4ae5430dab1dd35a4aeebf61b65ea899b887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Thu, 7 Nov 2024 08:15:25 +0100 Subject: [PATCH 42/42] Fix summarisation test --- .../tests/complete/functions/summarize.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index fd1565ec78e5e..34da4270f7721 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -32,7 +32,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { let connectorId: string; before(async () => { - await clearKnowledgeBase(es); await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient .editor({ @@ -55,7 +54,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { name: 'summarize', trigger: MessageRole.User, arguments: JSON.stringify({ - id: 'my-id', + title: 'My Title', text: 'Hello world', is_correction: false, confidence: 'high', @@ -72,6 +71,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await deleteActionConnector({ supertest, connectorId, log }); await deleteKnowledgeBaseModel(ml); + await clearKnowledgeBase(es); }); it('persists entry in knowledge base', async () => { @@ -86,14 +86,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { }, }); - const { role, public: isPublic, text, type, user, id } = res.body.entries[0]; + const { role, public: isPublic, text, type, user, title } = res.body.entries[0]; expect(role).to.eql('assistant_summarization'); expect(isPublic).to.eql(false); expect(text).to.eql('Hello world'); expect(type).to.eql('contextual'); expect(user?.name).to.eql('editor'); - expect(id).to.eql('my-id'); + expect(title).to.eql('My Title'); expect(res.body.entries).to.have.length(1); }); });