From 37a52d7a2fbfc8757ca757b2580e2d1e6b5a2265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 3 Jul 2024 13:56:33 +0200 Subject: [PATCH 1/9] [Obs AI Assistant] Add ES function API test --- .../public/utils/builders.ts | 4 +- .../get_context_function_request_if_needed.ts | 9 +- .../server/service/client/index.ts | 7 +- .../client/operators/continue_conversation.ts | 10 +- .../server/functions/query/index.ts | 9 +- .../common/config.ts | 29 +-- .../common/create_llm_proxy.ts | 4 +- .../tests/complete/complete.spec.ts | 6 +- .../complete/functions/elasticsearch.spec.ts | 173 ++++++++++++++++++ .../tests/contextual_insights/index.spec.ts | 2 +- 10 files changed, 202 insertions(+), 51 deletions(-) create mode 100644 x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/public/utils/builders.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/public/utils/builders.ts index e233bf6da5d11..3a18b53cef669 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/public/utils/builders.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/public/utils/builders.ts @@ -7,7 +7,7 @@ import type { FunctionDefinition } from '../../common/functions/types'; -export function buildFunction(): FunctionDefinition { +export function buildFunctionElasticsearch(): FunctionDefinition { return { name: 'elasticsearch', description: 'Call Elasticsearch APIs on behalf of the user', @@ -30,8 +30,6 @@ export function buildFunction(): FunctionDefinition { }; } -export const buildFunctionElasticsearch = buildFunction; - export function buildFunctionServiceSummary(): FunctionDefinition { return { name: 'get_service_summary', diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/get_context_function_request_if_needed.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/get_context_function_request_if_needed.ts index e5ea0ad0ff829..8e0da55ff8c76 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/get_context_function_request_if_needed.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/get_context_function_request_if_needed.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { findLastIndex } from 'lodash'; +import { findLastIndex, last } from 'lodash'; import { Message, MessageAddEvent, MessageRole } from '../../../common'; import { createFunctionRequestMessage } from '../../../common/utils/create_function_request_message'; import { CONTEXT_FUNCTION_NAME } from '../../functions/context'; @@ -22,11 +22,10 @@ export function getContextFunctionRequestIfNeeded( .slice(indexOfLastUserMessage) .some((message) => message.message.name === CONTEXT_FUNCTION_NAME); - if (hasContextSinceLastUserMessage) { + const isLastMessageFunctionRequest = !!last(messages)?.message.function_call?.name; + if (hasContextSinceLastUserMessage || isLastMessageFunctionRequest) { return undefined; } - return createFunctionRequestMessage({ - name: CONTEXT_FUNCTION_NAME, - }); + return createFunctionRequestMessage({ name: CONTEXT_FUNCTION_NAME }); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 139955742ef6e..27a06e7951df0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -11,7 +11,7 @@ import type { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { SpanKind, context } from '@opentelemetry/api'; -import { merge, omit } from 'lodash'; +import { last, merge, omit } from 'lodash'; import { catchError, combineLatest, @@ -335,13 +335,12 @@ export class ObservabilityAIAssistantClient { const initialMessagesWithAddedMessages = messagesWithUpdatedSystemMessage.concat(addedMessages); - const lastMessage = - initialMessagesWithAddedMessages[initialMessagesWithAddedMessages.length - 1]; + const lastMessage = last(initialMessagesWithAddedMessages); // if a function request is at the very end, close the stream to consumer // without persisting or updating the conversation. we need to wait // on the function response to have a valid conversation - const isFunctionRequest = lastMessage.message.function_call?.name; + const isFunctionRequest = !!lastMessage?.message.function_call?.name; if (!persist || isFunctionRequest) { return of(); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts index 83d9bf37e7efb..237eea9411b23 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/operators/continue_conversation.ts @@ -7,7 +7,7 @@ import { Logger } from '@kbn/logging'; import { decode, encode } from 'gpt-tokenizer'; -import { pick, take } from 'lodash'; +import { last, pick, take } from 'lodash'; import { catchError, concat, @@ -212,10 +212,8 @@ export function continueConversation({ initialMessages ); - const lastMessage = - messagesWithUpdatedSystemMessage[messagesWithUpdatedSystemMessage.length - 1].message; - - const isUserMessage = lastMessage.role === MessageRole.User; + const lastMessage = last(messagesWithUpdatedSystemMessage)?.message; + const isUserMessage = lastMessage?.role === MessageRole.User; return executeNextStep().pipe(handleEvents()); @@ -233,7 +231,7 @@ export function continueConversation({ }).pipe(emitWithConcatenatedMessage(), catchFunctionNotFoundError(functionLimitExceeded)); } - const functionCallName = lastMessage.function_call?.name; + const functionCallName = lastMessage?.function_call?.name; if (!functionCallName) { // reply from the LLM without a function request, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts index 69a6e5967f746..cd1aa806b9f31 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/index.ts @@ -28,6 +28,7 @@ import { runAndValidateEsqlQuery } from './validate_esql_query'; import { INLINE_ESQL_QUERY_REGEX } from './constants'; export const QUERY_FUNCTION_NAME = 'query'; +export const EXECUTE_QUERY_NAME = 'execute_query'; const readFile = promisify(Fs.readFile); const readdir = promisify(Fs.readdir); @@ -89,13 +90,13 @@ export function registerQueryFunction({ functions, resources }: FunctionRegistra even if it has been called before. When the "visualize_query" function has been called, a visualization has been displayed to the user. DO NOT UNDER ANY CIRCUMSTANCES follow up a "visualize_query" function call with your own visualization attempt. - If the "execute_query" function has been called, summarize these results for the user. The user does not see a visualization in this case.` + If the "${EXECUTE_QUERY_NAME}" function has been called, summarize these results for the user. The user does not see a visualization in this case.` : undefined ); functions.registerFunction( { - name: 'execute_query', + name: EXECUTE_QUERY_NAME, visibility: FunctionVisibility.UserOnly, description: 'Display the results of an ES|QL query.', parameters: { @@ -365,7 +366,7 @@ export function registerQueryFunction({ functions, resources }: FunctionRegistra '@timestamp': new Date().toISOString(), message: { role: MessageRole.User, - content: `Answer the user's question that was previously asked ("${abbreviatedUserQuestion}...") using the attached documentation. Take into account any previous errors from the \`execute_query\` or \`visualize_query\` function. + content: `Answer the user's question that was previously asked ("${abbreviatedUserQuestion}...") using the attached documentation. Take into account any previous errors from the \`${EXECUTE_QUERY_NAME}\` or \`visualize_query\` function. Format any ES|QL query as follows: \`\`\`esql @@ -449,7 +450,7 @@ export function registerQueryFunction({ functions, resources }: FunctionRegistra functionCall = undefined; } else if (args.intention === VisualizeESQLUserIntention.executeAndReturnResults) { functionCall = { - name: 'execute_query', + name: EXECUTE_QUERY_NAME, arguments: JSON.stringify({ query: esqlQuery }), trigger: MessageRole.Assistant as const, }; diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts index 559bb5d65dd2a..0fd49c3c724bb 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts @@ -10,10 +10,7 @@ import { UrlObject } from 'url'; import { ObservabilityAIAssistantFtrConfigName } from '../configs'; import { getApmSynthtraceEsClient } from './create_synthtrace_client'; import { InheritedFtrProviderContext, InheritedServices } from './ftr_provider_context'; -import { - getScopedApiClient, - ObservabilityAIAssistantAPIClient, -} from './observability_ai_assistant_api_client'; +import { getScopedApiClient } from './observability_ai_assistant_api_client'; import { editorUser, viewerUser } from './users/users'; export interface ObservabilityAIAssistantFtrConfig { @@ -24,20 +21,7 @@ export interface ObservabilityAIAssistantFtrConfig { export type CreateTestConfig = ReturnType; -export interface CreateTest { - testFiles: string[]; - servers: any; - services: InheritedServices & { - observabilityAIAssistantAPIClient: () => Promise<{ - adminUser: ObservabilityAIAssistantAPIClient; - viewerUser: ObservabilityAIAssistantAPIClient; - editorUser: ObservabilityAIAssistantAPIClient; - }>; - }; - junit: { reportName: string }; - esTestCluster: any; - kbnTestServer: any; -} +export type CreateTest = ReturnType; export function createObservabilityAIAssistantAPIConfig({ config, @@ -49,14 +33,15 @@ export function createObservabilityAIAssistantAPIConfig({ license: 'basic' | 'trial'; name: string; kibanaConfig?: Record; -}): Omit { +}) { const services = config.get('services') as InheritedServices; const servers = config.get('servers'); const kibanaServer = servers.kibana as UrlObject; const apmSynthtraceKibanaClient = services.apmSynthtraceKibanaClient(); + const allConfigs = config.getAll() as Record; - const createTest: Omit = { - ...config.getAll(), + return { + ...allConfigs, servers, services: { ...services, @@ -89,8 +74,6 @@ export function createObservabilityAIAssistantAPIConfig({ ], }, }; - - return createTest; } export function createTestConfig( diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts b/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts index cce12d2c1b958..2d01c7692e54f 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/create_llm_proxy.ts @@ -98,7 +98,7 @@ export class LlmProxy { waitForIntercept: () => Promise; } : { - waitAndComplete: () => Promise; + completeAfterIntercept: () => Promise; } { const waitForInterceptPromise = Promise.race([ new Promise((outerResolve) => { @@ -162,7 +162,7 @@ export class LlmProxy { : responseChunks.split(' ').map((token, i) => (i === 0 ? token : ` ${token}`)); return { - waitAndComplete: async () => { + completeAfterIntercept: async () => { const simulator = await waitForInterceptPromise; for (const chunk of parsedChunks) { await simulator.next(chunk); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts index 6be64a1daf3ff..e159475f50523 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts @@ -414,11 +414,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { }, }, ]) - .waitAndComplete(); + .completeAfterIntercept(); proxy .intercept('conversation', (body) => !isFunctionTitleRequest(body), 'Good morning, sir!') - .waitAndComplete(); + .completeAfterIntercept(); const createResponse = await observabilityAIAssistantAPIClient .editorUser({ @@ -450,7 +450,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { proxy .intercept('conversation', (body) => !isFunctionTitleRequest(body), 'Good night, sir!') - .waitAndComplete(); + .completeAfterIntercept(); const updatedResponse = await observabilityAIAssistantAPIClient .editorUser({ diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts new file mode 100644 index 0000000000000..04a38a7921349 --- /dev/null +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + MessageAddEvent, + MessageRole, + StreamingChatResponseEvent, +} from '@kbn/observability-ai-assistant-plugin/common'; +import { ToolingLog } from '@kbn/tooling-log'; +import { Agent } from 'supertest'; +import expect from '@kbn/expect'; +import { Readable } from 'stream'; +import { apm, timerange } from '@kbn/apm-synthtrace-client'; +import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { createLlmProxy, LlmProxy } from '../../../common/create_llm_proxy'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const log = getService('log'); + const apmSynthtraceEsClient = getService('apmSynthtraceEsClient'); + const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); + + describe('when calling elasticsearch', () => { + let proxy: LlmProxy; + let connectorId: string; + let events: MessageAddEvent[]; + + before(async () => { + ({ connectorId, proxy } = await createLLMProxyConnector({ log, supertest })); + await generateApmData(apmSynthtraceEsClient); + + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/chat/complete', + params: { + body: { + messages: [ + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.Assistant, + content: '', + function_call: { + name: 'elasticsearch', + trigger: MessageRole.User, + arguments: JSON.stringify({ + method: 'POST', + path: 'traces*/_search', + body: { + size: 0, + aggs: { + services: { + terms: { + field: 'service.name', + }, + }, + }, + }, + }), + }, + }, + }, + ], + connectorId, + persist: false, + screenContexts: [], + }, + }, + }) + .expect(200); + + events = getMessageAddedEvents(res.body); + }); + + after(async () => { + await deleteLLMProxyConnector({ supertest, connectorId, proxy }); + await apmSynthtraceEsClient.clean(); + }); + + it('returns elasticsearch function response', async () => { + await proxy.waitForAllInterceptorsSettled(); + + const esFunctionResponse = events[0]; + const parsedEsResponse = JSON.parse(esFunctionResponse.message.message.content!).response; + + expect(esFunctionResponse.message.message.name).to.be('elasticsearch'); + expect(parsedEsResponse.hits.total.value).to.be(15); + expect(parsedEsResponse.aggregations.services.buckets).to.eql([ + { key: 'foo', doc_count: 15 }, + ]); + expect(events.length).to.be(2); + }); + }); +} + +function decodeEvents(body: Readable | string) { + return String(body) + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as StreamingChatResponseEvent); +} + +function getMessageAddedEvents(body: Readable | string) { + return decodeEvents(body).filter( + (event): event is MessageAddEvent => event.type === 'messageAdd' + ); +} + +async function createLLMProxyConnector({ log, supertest }: { log: ToolingLog; supertest: Agent }) { + const proxy = await createLlmProxy(log); + + // intercept the LLM request and return a fixed response + proxy.intercept('conversation', () => true, 'Hello from LLM Proxy').completeAfterIntercept(); + + const response = await supertest + .post('/api/actions/connector') + .set('kbn-xsrf', 'foo') + .send({ + name: 'OpenAI Proxy', + connector_type_id: '.gen-ai', + config: { + apiProvider: 'OpenAI', + apiUrl: `http://localhost:${proxy.getPort()}`, + }, + secrets: { + apiKey: 'my-api-key', + }, + }) + .expect(200); + + return { + proxy, + connectorId: response.body.id, + }; +} + +async function deleteLLMProxyConnector({ + supertest, + connectorId, + proxy, +}: { + supertest: Agent; + connectorId: string; + proxy: LlmProxy; +}) { + await supertest + .delete(`/api/actions/connector/${connectorId}`) + .set('kbn-xsrf', 'foo') + .expect(204); + + proxy.close(); +} + +async function generateApmData(apmSynthtraceEsClient: ApmSynthtraceEsClient) { + const serviceA = apm + .service({ name: 'foo', environment: 'production', agentName: 'java' }) + .instance('a'); + + const events = timerange('now-15m', 'now') + .interval('1m') + .rate(1) + .generator((timestamp) => { + return serviceA.transaction({ transactionName: 'tx' }).timestamp(timestamp).duration(1000); + }); + + return apmSynthtraceEsClient.index(events); +} diff --git a/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts index aff0d91173dd3..41ad6f793ee93 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/contextual_insights/index.spec.ts @@ -159,7 +159,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte await openContextualInsights(); - await interceptor.waitAndComplete(); + await interceptor.completeAfterIntercept(); await retry.tryForTime(5 * 1000, async () => { const llmResponse = await testSubjects.getVisibleText(ui.pages.contextualInsights.text); From 0ade4f3bd48b1dc040708771bbc406d3a7e9a5eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 3 Jul 2024 14:12:52 +0200 Subject: [PATCH 2/9] Use constant --- .../server/functions/elasticsearch.ts | 4 +++- .../tests/complete/functions/elasticsearch.spec.ts | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts index 8c255b885fd4c..6008b53dd42c5 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/elasticsearch.ts @@ -7,13 +7,15 @@ import type { FunctionRegistrationParameters } from '.'; +export const ELASTICSEARCH_FUNCTION_NAME = 'elasticsearch'; + export function registerElasticsearchFunction({ functions, resources, }: FunctionRegistrationParameters) { functions.registerFunction( { - name: 'elasticsearch', + name: ELASTICSEARCH_FUNCTION_NAME, description: 'Call Elasticsearch APIs on behalf of the user. Make sure the request body is valid for the API that you are using. Only call this function when the user has explicitly requested it.', descriptionForUser: 'Call Elasticsearch APIs on behalf of the user', diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts index 04a38a7921349..6b9ff5c89623e 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts @@ -16,6 +16,7 @@ import expect from '@kbn/expect'; import { Readable } from 'stream'; import { apm, timerange } from '@kbn/apm-synthtrace-client'; import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; +import { ELASTICSEARCH_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/elasticsearch'; import { createLlmProxy, LlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; @@ -46,7 +47,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { role: MessageRole.Assistant, content: '', function_call: { - name: 'elasticsearch', + name: ELASTICSEARCH_FUNCTION_NAME, trigger: MessageRole.User, arguments: JSON.stringify({ method: 'POST', From 838a5ac9e3320976e658980f0835659a59ffe37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 3 Jul 2024 14:15:00 +0200 Subject: [PATCH 3/9] Move wait for call --- .../complete/functions/elasticsearch.spec.ts | 4 +- .../complete/functions/summarize.spec.ts | 145 ++++++++++++++++++ 2 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts index 6b9ff5c89623e..4739262e65548 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts @@ -75,6 +75,8 @@ export default function ApiTest({ getService }: FtrProviderContext) { }) .expect(200); + await proxy.waitForAllInterceptorsSettled(); + events = getMessageAddedEvents(res.body); }); @@ -84,8 +86,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('returns elasticsearch function response', async () => { - await proxy.waitForAllInterceptorsSettled(); - const esFunctionResponse = events[0]; const parsedEsResponse = JSON.parse(esFunctionResponse.message.message.content!).response; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts new file mode 100644 index 0000000000000..2dc741247371c --- /dev/null +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + MessageAddEvent, + MessageRole, + StreamingChatResponseEvent, +} from '@kbn/observability-ai-assistant-plugin/common'; +import { ToolingLog } from '@kbn/tooling-log'; +import { Agent } from 'supertest'; +import expect from '@kbn/expect'; +import { Readable } from 'stream'; +import { createLlmProxy, LlmProxy } from '../../../common/create_llm_proxy'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const log = getService('log'); + const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); + + describe('when calling summarize function', () => { + let proxy: LlmProxy; + let connectorId: string; + let events: MessageAddEvent[]; + + before(async () => { + ({ connectorId, proxy } = await createLLMProxyConnector({ log, supertest })); + + const res = await observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/chat/complete', + params: { + body: { + messages: [ + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.Assistant, + content: '', + function_call: { + name: 'summarize', + trigger: MessageRole.User, + arguments: JSON.stringify({ + id: 'my-id', + text: 'Hello world', + is_correction: false, + confidence: 1, + public: false, + }), + }, + }, + }, + ], + connectorId, + persist: false, + screenContexts: [], + }, + }, + }) + .expect(200); + + await proxy.waitForAllInterceptorsSettled(); + events = getMessageAddedEvents(res.body); + }); + + after(async () => { + await deleteLLMProxyConnector({ supertest, connectorId, proxy }); + }); + + it('persists entry in knowledge base', async () => { + const esFunctionResponse = events[0]; + const parsedEsResponse = JSON.parse(esFunctionResponse.message.message.content!).response; + + expect(esFunctionResponse.message.message.name).to.be('elasticsearch'); + expect(parsedEsResponse.hits.total.value).to.be(15); + expect(parsedEsResponse.aggregations.services.buckets).to.eql([ + { key: 'foo', doc_count: 15 }, + ]); + expect(events.length).to.be(2); + }); + }); +} + +function decodeEvents(body: Readable | string) { + return String(body) + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as StreamingChatResponseEvent); +} + +function getMessageAddedEvents(body: Readable | string) { + return decodeEvents(body).filter( + (event): event is MessageAddEvent => event.type === 'messageAdd' + ); +} + +async function createLLMProxyConnector({ log, supertest }: { log: ToolingLog; supertest: Agent }) { + const proxy = await createLlmProxy(log); + + // intercept the LLM request and return a fixed response + proxy.intercept('conversation', () => true, 'Hello from LLM Proxy').completeAfterIntercept(); + + const response = await supertest + .post('/api/actions/connector') + .set('kbn-xsrf', 'foo') + .send({ + name: 'OpenAI Proxy', + connector_type_id: '.gen-ai', + config: { + apiProvider: 'OpenAI', + apiUrl: `http://localhost:${proxy.getPort()}`, + }, + secrets: { + apiKey: 'my-api-key', + }, + }) + .expect(200); + + return { + proxy, + connectorId: response.body.id, + }; +} + +async function deleteLLMProxyConnector({ + supertest, + connectorId, + proxy, +}: { + supertest: Agent; + connectorId: string; + proxy: LlmProxy; +}) { + await supertest + .delete(`/api/actions/connector/${connectorId}`) + .set('kbn-xsrf', 'foo') + .expect(204); + + proxy.close(); +} From 6e72a291bffa22e4ffc772e5eaa85831ce542296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 3 Jul 2024 14:45:16 +0200 Subject: [PATCH 4/9] Add test for summarize --- .../complete/functions/elasticsearch.spec.ts | 73 +------------- .../tests/complete/functions/helpers.ts | 80 +++++++++++++++ .../complete/functions/summarize.spec.ts | 97 ++++--------------- 3 files changed, 102 insertions(+), 148 deletions(-) create mode 100644 x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts index 4739262e65548..c90e7a4ce390d 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts @@ -5,20 +5,14 @@ * 2.0. */ -import { - MessageAddEvent, - MessageRole, - StreamingChatResponseEvent, -} from '@kbn/observability-ai-assistant-plugin/common'; -import { ToolingLog } from '@kbn/tooling-log'; -import { Agent } from 'supertest'; +import { MessageAddEvent, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import expect from '@kbn/expect'; -import { Readable } from 'stream'; import { apm, timerange } from '@kbn/apm-synthtrace-client'; import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { ELASTICSEARCH_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/elasticsearch'; -import { createLlmProxy, LlmProxy } from '../../../common/create_llm_proxy'; +import { LlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { createLLMProxyConnector, deleteLLMProxyConnector, getMessageAddedEvents } from './helpers'; export default function ApiTest({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -99,66 +93,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); } -function decodeEvents(body: Readable | string) { - return String(body) - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) - .map((line) => JSON.parse(line) as StreamingChatResponseEvent); -} - -function getMessageAddedEvents(body: Readable | string) { - return decodeEvents(body).filter( - (event): event is MessageAddEvent => event.type === 'messageAdd' - ); -} - -async function createLLMProxyConnector({ log, supertest }: { log: ToolingLog; supertest: Agent }) { - const proxy = await createLlmProxy(log); - - // intercept the LLM request and return a fixed response - proxy.intercept('conversation', () => true, 'Hello from LLM Proxy').completeAfterIntercept(); - - const response = await supertest - .post('/api/actions/connector') - .set('kbn-xsrf', 'foo') - .send({ - name: 'OpenAI Proxy', - connector_type_id: '.gen-ai', - config: { - apiProvider: 'OpenAI', - apiUrl: `http://localhost:${proxy.getPort()}`, - }, - secrets: { - apiKey: 'my-api-key', - }, - }) - .expect(200); - - return { - proxy, - connectorId: response.body.id, - }; -} - -async function deleteLLMProxyConnector({ - supertest, - connectorId, - proxy, -}: { - supertest: Agent; - connectorId: string; - proxy: LlmProxy; -}) { - await supertest - .delete(`/api/actions/connector/${connectorId}`) - .set('kbn-xsrf', 'foo') - .expect(204); - - proxy.close(); -} - -async function generateApmData(apmSynthtraceEsClient: ApmSynthtraceEsClient) { +export async function generateApmData(apmSynthtraceEsClient: ApmSynthtraceEsClient) { const serviceA = apm .service({ name: 'foo', environment: 'production', agentName: 'java' }) .instance('a'); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts new file mode 100644 index 0000000000000..6c7b8eac5eef5 --- /dev/null +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + MessageAddEvent, + StreamingChatResponseEvent, +} from '@kbn/observability-ai-assistant-plugin/common'; +import { ToolingLog } from '@kbn/tooling-log'; +import { Agent } from 'supertest'; +import { Readable } from 'stream'; +import { createLlmProxy, LlmProxy } from '../../../common/create_llm_proxy'; + +function decodeEvents(body: Readable | string) { + return String(body) + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as StreamingChatResponseEvent); +} + +export function getMessageAddedEvents(body: Readable | string) { + return decodeEvents(body).filter( + (event): event is MessageAddEvent => event.type === 'messageAdd' + ); +} + +export async function createLLMProxyConnector({ + log, + supertest, +}: { + log: ToolingLog; + supertest: Agent; +}) { + const proxy = await createLlmProxy(log); + + // intercept the LLM request and return a fixed response + proxy.intercept('conversation', () => true, 'Hello from LLM Proxy').completeAfterIntercept(); + + const response = await supertest + .post('/api/actions/connector') + .set('kbn-xsrf', 'foo') + .send({ + name: 'OpenAI Proxy', + connector_type_id: '.gen-ai', + config: { + apiProvider: 'OpenAI', + apiUrl: `http://localhost:${proxy.getPort()}`, + }, + secrets: { + apiKey: 'my-api-key', + }, + }) + .expect(200); + + return { + proxy, + connectorId: response.body.id, + }; +} + +export async function deleteLLMProxyConnector({ + supertest, + connectorId, + proxy, +}: { + supertest: Agent; + connectorId: string; + proxy: LlmProxy; +}) { + await supertest + .delete(`/api/actions/connector/${connectorId}`) + .set('kbn-xsrf', 'foo') + .expect(204); + + proxy.close(); +} diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index 2dc741247371c..e566810697b9f 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -5,24 +5,19 @@ * 2.0. */ -import { - MessageAddEvent, - MessageRole, - StreamingChatResponseEvent, -} from '@kbn/observability-ai-assistant-plugin/common'; -import { ToolingLog } from '@kbn/tooling-log'; -import { Agent } from 'supertest'; +import { MessageAddEvent, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import expect from '@kbn/expect'; -import { Readable } from 'stream'; -import { createLlmProxy, LlmProxy } from '../../../common/create_llm_proxy'; +import { LlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; +import { createLLMProxyConnector, deleteLLMProxyConnector, getMessageAddedEvents } from './helpers'; export default function ApiTest({ getService }: FtrProviderContext) { const supertest = getService('supertest'); const log = getService('log'); const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); - describe('when calling summarize function', () => { + // Skipped until Elser is available in tests + describe.skip('when calling summarize function', () => { let proxy: LlmProxy; let connectorId: string; let events: MessageAddEvent[]; @@ -30,7 +25,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { ({ connectorId, proxy } = await createLLMProxyConnector({ log, supertest })); - const res = await observabilityAIAssistantAPIClient + const chatResponse = await observabilityAIAssistantAPIClient .editorUser({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', params: { @@ -64,7 +59,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); await proxy.waitForAllInterceptorsSettled(); - events = getMessageAddedEvents(res.body); + events = getMessageAddedEvents(chatResponse.body); }); after(async () => { @@ -72,74 +67,18 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('persists entry in knowledge base', async () => { - const esFunctionResponse = events[0]; - const parsedEsResponse = JSON.parse(esFunctionResponse.message.message.content!).response; + const res = await observabilityAIAssistantAPIClient.editorUser({ + endpoint: 'GET /internal/observability_ai_assistant/kb/entries', + params: { + query: { + query: '', + sortBy: 'doc_id', + sortDirection: 'asc', + }, + }, + }); - expect(esFunctionResponse.message.message.name).to.be('elasticsearch'); - expect(parsedEsResponse.hits.total.value).to.be(15); - expect(parsedEsResponse.aggregations.services.buckets).to.eql([ - { key: 'foo', doc_count: 15 }, - ]); - expect(events.length).to.be(2); + expect(res.body.entries).to.have.length(1); }); }); } - -function decodeEvents(body: Readable | string) { - return String(body) - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) - .map((line) => JSON.parse(line) as StreamingChatResponseEvent); -} - -function getMessageAddedEvents(body: Readable | string) { - return decodeEvents(body).filter( - (event): event is MessageAddEvent => event.type === 'messageAdd' - ); -} - -async function createLLMProxyConnector({ log, supertest }: { log: ToolingLog; supertest: Agent }) { - const proxy = await createLlmProxy(log); - - // intercept the LLM request and return a fixed response - proxy.intercept('conversation', () => true, 'Hello from LLM Proxy').completeAfterIntercept(); - - const response = await supertest - .post('/api/actions/connector') - .set('kbn-xsrf', 'foo') - .send({ - name: 'OpenAI Proxy', - connector_type_id: '.gen-ai', - config: { - apiProvider: 'OpenAI', - apiUrl: `http://localhost:${proxy.getPort()}`, - }, - secrets: { - apiKey: 'my-api-key', - }, - }) - .expect(200); - - return { - proxy, - connectorId: response.body.id, - }; -} - -async function deleteLLMProxyConnector({ - supertest, - connectorId, - proxy, -}: { - supertest: Agent; - connectorId: string; - proxy: LlmProxy; -}) { - await supertest - .delete(`/api/actions/connector/${connectorId}`) - .set('kbn-xsrf', 'foo') - .expect(204); - - proxy.close(); -} From 7e078bdbd3d493ede8d27ef8d263d688940facb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Wed, 3 Jul 2024 15:00:15 +0200 Subject: [PATCH 5/9] minor refactor --- .../common/config.ts | 12 +++-- .../tests/complete/functions/helpers.ts | 50 +++++++++++++++++++ .../complete/functions/summarize.spec.ts | 8 ++- .../common/config.ts | 35 ++----------- 4 files changed, 64 insertions(+), 41 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts index 0fd49c3c724bb..acd6a6287d446 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts @@ -23,6 +23,12 @@ export type CreateTestConfig = ReturnType; export type CreateTest = ReturnType; +export type ObservabilityAIAssistantAPIClient = Awaited< + ReturnType +>; + +export type ObservabilityAIAssistantServices = Awaited>['services']; + export function createObservabilityAIAssistantAPIConfig({ config, license, @@ -76,9 +82,7 @@ export function createObservabilityAIAssistantAPIConfig({ }; } -export function createTestConfig( - config: ObservabilityAIAssistantFtrConfig -): ({ readConfigFile }: FtrConfigProviderContext) => Promise { +export function createTestConfig(config: ObservabilityAIAssistantFtrConfig) { const { license, name, kibanaConfig } = config; return async ({ readConfigFile }: FtrConfigProviderContext) => { @@ -97,5 +101,3 @@ export function createTestConfig( }; }; } - -export type ObservabilityAIAssistantServices = Awaited>['services']; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts index 6c7b8eac5eef5..18e0533b0e062 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts @@ -7,11 +7,14 @@ import { MessageAddEvent, + MessageRole, StreamingChatResponseEvent, } from '@kbn/observability-ai-assistant-plugin/common'; import { ToolingLog } from '@kbn/tooling-log'; import { Agent } from 'supertest'; import { Readable } from 'stream'; +import { ELASTICSEARCH_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/elasticsearch'; +import { CreateTest } from '../../../common/config'; import { createLlmProxy, LlmProxy } from '../../../common/create_llm_proxy'; function decodeEvents(body: Readable | string) { @@ -78,3 +81,50 @@ export async function deleteLLMProxyConnector({ proxy.close(); } + +export function foo( + connectorId: string, + observabilityAIAssistantAPIClient: Awaited< + ReturnType + > +) { + return observabilityAIAssistantAPIClient + .editorUser({ + endpoint: 'POST /internal/observability_ai_assistant/chat/complete', + params: { + body: { + messages: [ + { + '@timestamp': new Date().toISOString(), + message: { + role: MessageRole.Assistant, + content: '', + function_call: { + name: ELASTICSEARCH_FUNCTION_NAME, + trigger: MessageRole.User, + arguments: JSON.stringify({ + method: 'POST', + path: 'traces*/_search', + body: { + size: 0, + aggs: { + services: { + terms: { + field: 'service.name', + }, + }, + }, + }, + }), + }, + }, + }, + ], + connectorId, + persist: false, + screenContexts: [], + }, + }, + }) + .expect(200); +} diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index e566810697b9f..1145be2249db8 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { MessageAddEvent, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; +import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import expect from '@kbn/expect'; import { LlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; -import { createLLMProxyConnector, deleteLLMProxyConnector, getMessageAddedEvents } from './helpers'; +import { createLLMProxyConnector, deleteLLMProxyConnector } from './helpers'; export default function ApiTest({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -20,12 +20,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe.skip('when calling summarize function', () => { let proxy: LlmProxy; let connectorId: string; - let events: MessageAddEvent[]; before(async () => { ({ connectorId, proxy } = await createLLMProxyConnector({ log, supertest })); - const chatResponse = await observabilityAIAssistantAPIClient + await observabilityAIAssistantAPIClient .editorUser({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', params: { @@ -59,7 +58,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); await proxy.waitForAllInterceptorsSettled(); - events = getMessageAddedEvents(chatResponse.body); }); after(async () => { diff --git a/x-pack/test/observability_ai_assistant_functional/common/config.ts b/x-pack/test/observability_ai_assistant_functional/common/config.ts index e92bf3729cb40..61558991b5ad1 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/config.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/config.ts @@ -8,8 +8,6 @@ import { FtrConfigProviderContext } from '@kbn/test'; import { merge } from 'lodash'; import { UrlObject } from 'url'; -import type { EBTHelpersContract } from '@kbn/analytics-ftr-helpers-plugin/common/types'; -import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { editorUser, viewerUser, @@ -20,40 +18,15 @@ import { } from '../../../../test/analytics/services/kibana_ebt'; import { ObservabilityAIAssistantFtrConfig, - CreateTest as CreateTestAPI, createObservabilityAIAssistantAPIConfig, } from '../../observability_ai_assistant_api_integration/common/config'; -import { - getScopedApiClient, - ObservabilityAIAssistantAPIClient, -} from '../../observability_ai_assistant_api_integration/common/observability_ai_assistant_api_client'; -import { InheritedFtrProviderContext, InheritedServices } from '../ftr_provider_context'; -import { ObservabilityAIAssistantUIProvider, ObservabilityAIAssistantUIService } from './ui'; - -export interface TestConfig extends CreateTestAPI { - services: Omit & - InheritedServices & { - observabilityAIAssistantUI: ( - context: InheritedFtrProviderContext - ) => Promise; - observabilityAIAssistantAPIClient: () => Promise<{ - adminUser: ObservabilityAIAssistantAPIClient; - viewerUser: ObservabilityAIAssistantAPIClient; - editorUser: ObservabilityAIAssistantAPIClient; - }>; - kibana_ebt_server: (context: InheritedFtrProviderContext) => EBTHelpersContract; - kibana_ebt_ui: (context: InheritedFtrProviderContext) => EBTHelpersContract; - apmSynthtraceEsClient: ( - context: InheritedFtrProviderContext - ) => Promise; - }; -} +import { getScopedApiClient } from '../../observability_ai_assistant_api_integration/common/observability_ai_assistant_api_client'; +import { InheritedFtrProviderContext } from '../ftr_provider_context'; +import { ObservabilityAIAssistantUIProvider } from './ui'; export type CreateTestConfig = ReturnType; -export function createTestConfig( - config: ObservabilityAIAssistantFtrConfig -): ({ readConfigFile }: FtrConfigProviderContext) => Promise { +export function createTestConfig(config: ObservabilityAIAssistantFtrConfig) { const { license, name, kibanaConfig } = config; return async ({ readConfigFile, log, esVersion }: FtrConfigProviderContext) => { From 29d17da28406e24f9d3929390ae642ae626cee34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Fri, 5 Jul 2024 13:39:19 +0200 Subject: [PATCH 6/9] Address feedback --- .../complete/functions/elasticsearch.spec.ts | 64 ++++++++----------- .../tests/complete/functions/helpers.ts | 34 ++++------ .../complete/functions/summarize.spec.ts | 53 ++++++--------- 3 files changed, 57 insertions(+), 94 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts index c90e7a4ce390d..717db2973a816 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts @@ -12,7 +12,12 @@ import { ApmSynthtraceEsClient } from '@kbn/apm-synthtrace'; import { ELASTICSEARCH_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/elasticsearch'; import { LlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; -import { createLLMProxyConnector, deleteLLMProxyConnector, getMessageAddedEvents } from './helpers'; +import { + createLLMProxyConnector, + deleteLLMProxyConnector, + getMessageAddedEvents, + invokeChatCompleteWithFunctionRequest, +} from './helpers'; export default function ApiTest({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -29,45 +34,28 @@ export default function ApiTest({ getService }: FtrProviderContext) { ({ connectorId, proxy } = await createLLMProxyConnector({ log, supertest })); await generateApmData(apmSynthtraceEsClient); - const res = await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'POST /internal/observability_ai_assistant/chat/complete', - params: { + const res = await invokeChatCompleteWithFunctionRequest({ + connectorId, + observabilityAIAssistantAPIClient, + functionCall: { + name: ELASTICSEARCH_FUNCTION_NAME, + trigger: MessageRole.User, + arguments: JSON.stringify({ + method: 'POST', + path: 'traces*/_search', body: { - messages: [ - { - '@timestamp': new Date().toISOString(), - message: { - role: MessageRole.Assistant, - content: '', - function_call: { - name: ELASTICSEARCH_FUNCTION_NAME, - trigger: MessageRole.User, - arguments: JSON.stringify({ - method: 'POST', - path: 'traces*/_search', - body: { - size: 0, - aggs: { - services: { - terms: { - field: 'service.name', - }, - }, - }, - }, - }), - }, + size: 0, + aggs: { + services: { + terms: { + field: 'service.name', }, }, - ], - connectorId, - persist: false, - screenContexts: [], + }, }, - }, - }) - .expect(200); + }), + }, + }); await proxy.waitForAllInterceptorsSettled(); @@ -86,7 +74,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(esFunctionResponse.message.message.name).to.be('elasticsearch'); expect(parsedEsResponse.hits.total.value).to.be(15); expect(parsedEsResponse.aggregations.services.buckets).to.eql([ - { key: 'foo', doc_count: 15 }, + { key: 'java-backend', doc_count: 15 }, ]); expect(events.length).to.be(2); }); @@ -95,7 +83,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { export async function generateApmData(apmSynthtraceEsClient: ApmSynthtraceEsClient) { const serviceA = apm - .service({ name: 'foo', environment: 'production', agentName: 'java' }) + .service({ name: 'java-backend', environment: 'production', agentName: 'java' }) .instance('a'); const events = timerange('now-15m', 'now') diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts index 18e0533b0e062..0bed1d9026fc2 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts @@ -6,6 +6,7 @@ */ import { + Message, MessageAddEvent, MessageRole, StreamingChatResponseEvent, @@ -13,7 +14,6 @@ import { import { ToolingLog } from '@kbn/tooling-log'; import { Agent } from 'supertest'; import { Readable } from 'stream'; -import { ELASTICSEARCH_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/elasticsearch'; import { CreateTest } from '../../../common/config'; import { createLlmProxy, LlmProxy } from '../../../common/create_llm_proxy'; @@ -82,12 +82,17 @@ export async function deleteLLMProxyConnector({ proxy.close(); } -export function foo( - connectorId: string, +export function invokeChatCompleteWithFunctionRequest({ + connectorId, + observabilityAIAssistantAPIClient, + functionCall, +}: { + connectorId: string; observabilityAIAssistantAPIClient: Awaited< ReturnType - > -) { + >; + functionCall: Message['message']['function_call']; +}) { return observabilityAIAssistantAPIClient .editorUser({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', @@ -99,24 +104,7 @@ export function foo( message: { role: MessageRole.Assistant, content: '', - function_call: { - name: ELASTICSEARCH_FUNCTION_NAME, - trigger: MessageRole.User, - arguments: JSON.stringify({ - method: 'POST', - path: 'traces*/_search', - body: { - size: 0, - aggs: { - services: { - terms: { - field: 'service.name', - }, - }, - }, - }, - }), - }, + function_call: functionCall, }, }, ], diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index 1145be2249db8..0cda45a1a4253 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -9,7 +9,11 @@ import { MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import expect from '@kbn/expect'; import { LlmProxy } from '../../../common/create_llm_proxy'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; -import { createLLMProxyConnector, deleteLLMProxyConnector } from './helpers'; +import { + createLLMProxyConnector, + deleteLLMProxyConnector, + invokeChatCompleteWithFunctionRequest, +} from './helpers'; export default function ApiTest({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -24,38 +28,21 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { ({ connectorId, proxy } = await createLLMProxyConnector({ log, supertest })); - await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'POST /internal/observability_ai_assistant/chat/complete', - params: { - body: { - messages: [ - { - '@timestamp': new Date().toISOString(), - message: { - role: MessageRole.Assistant, - content: '', - function_call: { - name: 'summarize', - trigger: MessageRole.User, - arguments: JSON.stringify({ - id: 'my-id', - text: 'Hello world', - is_correction: false, - confidence: 1, - public: false, - }), - }, - }, - }, - ], - connectorId, - persist: false, - screenContexts: [], - }, - }, - }) - .expect(200); + await invokeChatCompleteWithFunctionRequest({ + connectorId, + observabilityAIAssistantAPIClient, + functionCall: { + name: 'summarize', + trigger: MessageRole.User, + arguments: JSON.stringify({ + id: 'my-id', + text: 'Hello world', + is_correction: false, + confidence: 1, + public: false, + }), + }, + }); await proxy.waitForAllInterceptorsSettled(); }); From f396cf0be7270efd1563b53acbc7037b4627bdd4 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 5 Jul 2024 11:50:55 +0000 Subject: [PATCH 7/9] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- x-pack/test/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 13e3d0634cd40..cda6e59087262 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -168,7 +168,6 @@ "@kbn/apm-data-view", "@kbn/core-saved-objects-api-server", "@kbn/search-types", - "@kbn/analytics-ftr-helpers-plugin", "@kbn/alerting-comparators", "@kbn/alerting-state-types", "@kbn/reporting-server", From efbd404815a38c1a4d17122cab91d5fd1f1cbdf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Fri, 5 Jul 2024 21:21:51 +0200 Subject: [PATCH 8/9] Fix tsc --- .../common/config.ts | 83 +++++++++++-------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_functional/common/config.ts b/x-pack/test/observability_ai_assistant_functional/common/config.ts index 61558991b5ad1..e80597cc2b2b0 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/config.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/config.ts @@ -21,47 +21,62 @@ import { createObservabilityAIAssistantAPIConfig, } from '../../observability_ai_assistant_api_integration/common/config'; import { getScopedApiClient } from '../../observability_ai_assistant_api_integration/common/observability_ai_assistant_api_client'; -import { InheritedFtrProviderContext } from '../ftr_provider_context'; +import { InheritedFtrProviderContext, InheritedServices } from '../ftr_provider_context'; import { ObservabilityAIAssistantUIProvider } from './ui'; export type CreateTestConfig = ReturnType; +export type TestConfig = Awaited>; -export function createTestConfig(config: ObservabilityAIAssistantFtrConfig) { - const { license, name, kibanaConfig } = config; - - return async ({ readConfigFile, log, esVersion }: FtrConfigProviderContext) => { - const testConfig = await readConfigFile(require.resolve('../../functional/config.base.js')); +async function getTestConfig({ + license, + name, + kibanaConfig, + readConfigFile, +}: { + license: 'basic' | 'trial'; + name: string; + kibanaConfig: Record | undefined; + readConfigFile: FtrConfigProviderContext['readConfigFile']; +}) { + const testConfig = await readConfigFile(require.resolve('../../functional/config.base.js')); - const baseConfig = createObservabilityAIAssistantAPIConfig({ - config: testConfig, - license, - name, - kibanaConfig, - }); + const baseConfig = createObservabilityAIAssistantAPIConfig({ + config: testConfig, + license, + name, + kibanaConfig, + }); - const kibanaServer = baseConfig.servers.kibana as UrlObject; + const kibanaServer = baseConfig.servers.kibana as UrlObject; - return merge( - { - services: testConfig.get('services'), - }, - baseConfig, - { - testFiles: [require.resolve('../tests')], - services: { - observabilityAIAssistantUI: (context: InheritedFtrProviderContext) => - ObservabilityAIAssistantUIProvider(context), - observabilityAIAssistantAPIClient: async (context: InheritedFtrProviderContext) => { - return { - adminUser: await getScopedApiClient(kibanaServer, 'elastic'), - viewerUser: await getScopedApiClient(kibanaServer, viewerUser.username), - editorUser: await getScopedApiClient(kibanaServer, editorUser.username), - }; - }, - kibana_ebt_server: KibanaEBTServerProvider, - kibana_ebt_ui: KibanaEBTUIProvider, + return merge( + { + services: testConfig.get('services') as InheritedServices, + }, + baseConfig, + { + testFiles: [require.resolve('../tests')], + services: { + observabilityAIAssistantUI: (context: InheritedFtrProviderContext) => + ObservabilityAIAssistantUIProvider(context), + observabilityAIAssistantAPIClient: async (context: InheritedFtrProviderContext) => { + return { + adminUser: await getScopedApiClient(kibanaServer, 'elastic'), + viewerUser: await getScopedApiClient(kibanaServer, viewerUser.username), + editorUser: await getScopedApiClient(kibanaServer, editorUser.username), + }; }, - } - ); + kibana_ebt_server: KibanaEBTServerProvider, + kibana_ebt_ui: KibanaEBTUIProvider, + }, + } + ); +} + +export function createTestConfig(config: ObservabilityAIAssistantFtrConfig) { + const { license, name, kibanaConfig } = config; + + return async ({ readConfigFile }: FtrConfigProviderContext) => { + return getTestConfig({ license, name, kibanaConfig, readConfigFile }); }; } From bc1039e74cc788fb109169e0bb3bce8fa4500ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Sat, 6 Jul 2024 08:53:30 +0200 Subject: [PATCH 9/9] Fix typescript error --- .../tests/complete/functions/elasticsearch.spec.ts | 4 ++-- .../tests/complete/functions/helpers.ts | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts index 717db2973a816..559f21e944011 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/elasticsearch.spec.ts @@ -34,7 +34,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { ({ connectorId, proxy } = await createLLMProxyConnector({ log, supertest })); await generateApmData(apmSynthtraceEsClient); - const res = await invokeChatCompleteWithFunctionRequest({ + const responseBody = await invokeChatCompleteWithFunctionRequest({ connectorId, observabilityAIAssistantAPIClient, functionCall: { @@ -59,7 +59,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await proxy.waitForAllInterceptorsSettled(); - events = getMessageAddedEvents(res.body); + events = getMessageAddedEvents(responseBody); }); after(async () => { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts index 0bed1d9026fc2..f4323c96e3eff 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts @@ -82,7 +82,7 @@ export async function deleteLLMProxyConnector({ proxy.close(); } -export function invokeChatCompleteWithFunctionRequest({ +export async function invokeChatCompleteWithFunctionRequest({ connectorId, observabilityAIAssistantAPIClient, functionCall, @@ -93,7 +93,7 @@ export function invokeChatCompleteWithFunctionRequest({ >; functionCall: Message['message']['function_call']; }) { - return observabilityAIAssistantAPIClient + const { body } = await observabilityAIAssistantAPIClient .editorUser({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', params: { @@ -115,4 +115,6 @@ export function invokeChatCompleteWithFunctionRequest({ }, }) .expect(200); + + return body; }