Observability AI Assistant Tests Deployment Agnostic (elastic#205194)

Closes elastic#192718 ## Summary This PR add a deployment-agnostic testing environment for Observability AI Assistant tests by unifying the duplicated tests for stateful and serverless environments. It create the ObservabilityAIAssistantApiClient to work seamlessly in both environments, enabling a single test to run across stateful, CI, and MKI. Initial efforts focus on deduplicating the `conversations.spec.ts` and `connectors.spec.ts` files, as these already run in all environments. Move / dedup the tests that exist in stateful and serverless. They run in serverless CI but not MKI and add the skipMki tag. `chat.spec.ts` `complete.spec.ts` `elasticsearch.spec.ts` `public_complete.spec.ts` `alerts.spec.ts` `knowledge_base_setup.spec.ts` `knowledge_base_status.spec.ts` `knowledge_base.spec.ts` `summarize.ts` `knowledge_base_user_instructions.spec.ts`
delanni · Jan 13, 2025 · 68611d9 · 68611d9
1 parent 8e49a4e
commit 68611d9
Show file tree

Hide file tree

Showing 43 changed files with 1,183 additions and 4,135 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1260,7 +1260,7 @@ packages/kbn-monaco/src/esql @elastic/kibana-esql
 /x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant
 /x-pack/test_serverless/**/test_suites/observability/ai_assistant @elastic/obs-ai-assistant
 /x-pack/test/functional/es_archives/observability/ai_assistant @elastic/obs-ai-assistant
-
+/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant @elastic/obs-ai-assistant
 # Infra Obs
 ## This plugin mostly contains the codebase for the infra services, but also includes some code for the Logs UI app.
 ## To keep @elastic/obs-ux-logs-team as codeowner of the plugin manifest without requiring a review for all the other code changes

diff --git a/...t_api_integration/tests/chat/chat.spec.ts → ...ervability/ai_assistant/chat/chat.spec.ts b/...t_api_integration/tests/chat/chat.spec.ts → ...ervability/ai_assistant/chat/chat.spec.ts
@@ -8,17 +8,16 @@
 import expect from '@kbn/expect';
 import { MessageRole, type Message } from '@kbn/observability-ai-assistant-plugin/common';
 import { PassThrough } from 'stream';
-import { createLlmProxy, LlmProxy } from '../../common/create_llm_proxy';
-import { FtrProviderContext } from '../../common/ftr_provider_context';
-import { createProxyActionConnector, deleteActionConnector } from '../../common/action_connectors';
-import { ForbiddenApiError } from '../../common/config';
-
-export default function ApiTest({ getService }: FtrProviderContext) {
-  const supertest = getService('supertest');
+import {
+  LlmProxy,
+  createLlmProxy,
+} from '../../../../../../observability_ai_assistant_api_integration/common/create_llm_proxy';
+import { SupertestWithRoleScope } from '../../../../services/role_scoped_supertest';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
   const log = getService('log');
-  const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient');
-
-  const CHAT_API_URL = `/internal/observability_ai_assistant/chat`;
+  const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi');
 
   const messages: Message[] = [
     {
@@ -37,37 +36,50 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     },
   ];
 
-  describe('/internal/observability_ai_assistant/chat', () => {
+  describe('/internal/observability_ai_assistant/chat', function () {
+    // Fails on MKI: https://github.com/elastic/kibana/issues/205581
+    this.tags(['failsOnMKI']);
     let proxy: LlmProxy;
 
     let connectorId: string;
 
     before(async () => {
       proxy = await createLlmProxy(log);
-      connectorId = await createProxyActionConnector({ supertest, log, port: proxy.getPort() });
+      connectorId = await observabilityAIAssistantAPIClient.createProxyActionConnector({
+        port: proxy.getPort(),
+      });
     });
 
     after(async () => {
       proxy.close();
-      await deleteActionConnector({ supertest, connectorId, log });
+      await observabilityAIAssistantAPIClient.deleteActionConnector({
+        actionId: connectorId,
+      });
     });
 
     it("returns a 4xx if the connector doesn't exist", async () => {
-      await supertest
-        .post(CHAT_API_URL)
-        .set('kbn-xsrf', 'foo')
-        .send({
-          name: 'my_api_call',
-          messages,
-          connectorId: 'does not exist',
-          functions: [],
-          scopes: ['all'],
-        })
-        .expect(404);
+      const { status } = await observabilityAIAssistantAPIClient.editor({
+        endpoint: 'POST /internal/observability_ai_assistant/chat',
+        params: {
+          body: {
+            name: 'my_api_call',
+            messages,
+            connectorId: 'does not exist',
+            functions: [],
+            scopes: ['all'],
+          },
+        },
+      });
+      expect(status).to.be(404);
     });
-
     it('returns a streaming response from the server', async () => {
       const NUM_RESPONSES = 5;
+      const roleScopedSupertest = getService('roleScopedSupertest');
+      const supertestEditorWithCookieCredentials: SupertestWithRoleScope =
+        await roleScopedSupertest.getSupertestWithRoleScope('editor', {
+          useCookieHeader: true,
+          withInternalHeaders: true,
+        });
 
       await Promise.race([
         new Promise((resolve, reject) => {
@@ -81,9 +93,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
             const receivedChunks: Array<Record<string, any>> = [];
 
             const passThrough = new PassThrough();
-            supertest
-              .post(CHAT_API_URL)
-              .set('kbn-xsrf', 'foo')
+            supertestEditorWithCookieCredentials
+              .post('/internal/observability_ai_assistant/chat')
               .on('error', reject)
               .send({
                 name: 'my_api_call',
@@ -136,26 +147,21 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         }),
       ]);
     });
-
     describe('security roles and access privileges', () => {
       it('should deny access for users without the ai_assistant privilege', async () => {
-        try {
-          await observabilityAIAssistantAPIClient.unauthorizedUser({
-            endpoint: `POST ${CHAT_API_URL}`,
-            params: {
-              body: {
-                name: 'my_api_call',
-                messages,
-                connectorId,
-                functions: [],
-                scopes: ['all'],
-              },
+        const { status } = await observabilityAIAssistantAPIClient.viewer({
+          endpoint: 'POST /internal/observability_ai_assistant/chat',
+          params: {
+            body: {
+              name: 'my_api_call',
+              messages,
+              connectorId,
+              functions: [],
+              scopes: ['all'],
             },
-          });
-          throw new ForbiddenApiError('Expected unauthorizedUser() to throw a 403 Forbidden error');
-        } catch (e) {
-          expect(e.status).to.be(403);
-        }
+          },
+        });
+        expect(status).to.be(403);
       });
     });
   });