[8.x] Add base FTR test coverage for inference APIs (#198000)

.buildkite/ftr_platform_stateful_configs.yml

@@ -42,6 +42,9 @@ disabled:
   # Default http2 config to use for performance journeys
   - x-pack/performance/configs/http2_config.ts
 
+  # Gen AI suites, running with their own pipeline
+  - x-pack/test/functional_gen_ai/inference/config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - test/accessibility/config.ts

.buildkite/pipelines/on_merge.yml

@@ -169,6 +169,28 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/test/ftr_configs.sh
+    env:
+      FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts"
+      FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api'
+      FTR_GEN_AI: "1"
+    label: AppEx AI-Infra Inference APIs FTR tests
+    key: ai-infra-gen-ai-inference-api
+    timeout_in_minutes: 50
+    parallelism: 1
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
     label: 'Serverless Explore - Security Solution Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/ai_infra_gen_ai.yml

@@ -0,0 +1,30 @@
+steps:
+  - group: AppEx AI-Infra genAI tests
+    key: ai-infra-gen-ai
+    depends_on:
+      - build
+      - quick_checks
+      - checks
+      - linting
+      - linting_with_types
+      - check_types
+      - check_oas_snapshot
+    steps:
+      - command: .buildkite/scripts/steps/test/ftr_configs.sh
+        env:
+          FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts"
+          FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api'
+          FTR_GEN_AI: "1"
+        label: AppEx AI-Infra Inference APIs FTR tests
+        key: ai-infra-gen-ai-inference-api
+        timeout_in_minutes: 50
+        parallelism: 1
+        agents:
+          machineType: n2-standard-4
+          preemptible: true
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+            - exit_status: '*'
+              limit: 1

.buildkite/scripts/common/setup_job_env.sh

@@ -132,6 +132,14 @@ EOF
   export ELASTIC_APM_API_KEY
 }
 
+# Set up GenAI keys
+{
+  if [[ "${FTR_GEN_AI:-}" =~ ^(1|true)$ ]]; then
+    echo "FTR_GEN_AI was set - exposing LLM connectors"
+    export KIBANA_TESTING_AI_CONNECTORS="$(vault_get ai-infra-ci-connectors connectors-config)"
+  fi
+}
+
 # Set up GCS Service Account for CDN
 {
   GCS_SA_CDN_KEY="$(vault_get gcs-sa-cdn-prod key)"

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -140,6 +140,20 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/slo_plugin_e2e.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^x-pack\/packages\/ai-infra/,
+        /^x-pack\/plugins\/ai_infra/,
+        /^x-pack\/plugins\/inference/,
+        /^x-pack\/plugins\/stack_connectors\/server\/connector_types\/bedrock/,
+        /^x-pack\/plugins\/stack_connectors\/server\/connector_types\/gemini/,
+        /^x-pack\/plugins\/stack_connectors\/server\/connector_types\/openai/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:all-gen-ai-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ai_infra_gen_ai.yml'));
+    }
+
     if (
       GITHUB_PR_LABELS.includes('ci:deploy-cloud') ||
       GITHUB_PR_LABELS.includes('ci:cloud-deploy') ||

.github/CODEOWNERS

@@ -493,6 +493,7 @@ packages/kbn-ftr-common-functional-services @elastic/kibana-operations @elastic/
 packages/kbn-ftr-common-functional-ui-services @elastic/appex-qa
 packages/kbn-ftr-screenshot-filename @elastic/kibana-operations @elastic/appex-qa
 x-pack/test/functional_with_es_ssl/plugins/cases @elastic/response-ops
+packages/kbn-gen-ai-functional-testing @elastic/appex-ai-infra
 x-pack/examples/gen_ai_streaming_response_example @elastic/response-ops
 packages/kbn-generate @elastic/kibana-operations
 packages/kbn-generate-console-definitions @elastic/kibana-management
@@ -1414,6 +1415,7 @@ x-pack/test/**/deployment_agnostic/ @elastic/appex-qa #temporarily to monitor te
 
 # AppEx AI Infra
 /x-pack/plugins/inference @elastic/appex-ai-infra @elastic/obs-ai-assistant @elastic/security-generative-ai
+/x-pack/test/functional_gen_ai/inference @elastic/appex-ai-infra
 
 # AppEx Platform Services Security
 x-pack/test_serverless/api_integration/test_suites/common/security_response_headers.ts  @elastic/kibana-security

package.json

@@ -1452,6 +1452,7 @@
     "@kbn/ftr-common-functional-services": "link:packages/kbn-ftr-common-functional-services",
     "@kbn/ftr-common-functional-ui-services": "link:packages/kbn-ftr-common-functional-ui-services",
     "@kbn/ftr-screenshot-filename": "link:packages/kbn-ftr-screenshot-filename",
+    "@kbn/gen-ai-functional-testing": "link:packages/kbn-gen-ai-functional-testing",
     "@kbn/generate": "link:packages/kbn-generate",
     "@kbn/get-repo-files": "link:packages/kbn-get-repo-files",
     "@kbn/import-locator": "link:packages/kbn-import-locator",

packages/kbn-gen-ai-functional-testing/.gitignore

@@ -0,0 +1,2 @@
+## local version of the connector config
+connector_config.json

packages/kbn-gen-ai-functional-testing/README.md

@@ -0,0 +1,49 @@
+# @kbn/gen-ai-functional-testing
+
+Package exposing various utilities for GenAI/LLM related functional testing.
+
+## Features
+
+### LLM connectors
+
+Utilizing LLM connectors on FTR tests can be done via the `getPreconfiguredConnectorConfig` and `getAvailableConnectors` tools.
+
+`getPreconfiguredConnectorConfig` should be used to define the list of connectors when creating the FTR test's configuration.
+
+```ts
+import { getPreconfiguredConnectorConfig } from '@kbn/gen-ai-functional-testing'
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const xpackFunctionalConfig = {...};
+  const preconfiguredConnectors = getPreconfiguredConnectorConfig();
+
+  return {
+    ...xpackFunctionalConfig.getAll(),
+    kbnTestServer: {
+      ...xpackFunctionalConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
+        `--xpack.actions.preconfigured=${JSON.stringify(preconfiguredConnectors)}`,
+      ],
+    },
+  };
+}
+```
+
+then the `getAvailableConnectors` can be used during the test suite to retrieve the list of LLM connectors.
+
+For example to run some predefined test suite against all exposed LLM connectors:
+
+```ts
+import { getAvailableConnectors } from '@kbn/gen-ai-functional-testing';
+
+export default function (providerContext: FtrProviderContext) {
+  describe('Some GenAI FTR test suite', async () => {
+    getAvailableConnectors().forEach((connector) => {
+      describe(`Using connector ${connector.id}`, () => {
+        myTestSuite(connector, providerContext);
+      });
+    });
+  });
+}
+```

packages/kbn-gen-ai-functional-testing/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export {
+  AI_CONNECTORS_VAR_ENV,
+  getPreconfiguredConnectorConfig,
+  getAvailableConnectors,
+  type AvailableConnector,
+  type AvailableConnectorWithId,
+} from './src/connectors';

packages/kbn-gen-ai-functional-testing/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-gen-ai-functional-testing'],
+};

packages/kbn-gen-ai-functional-testing/kibana.jsonc

@@ -0,0 +1,6 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/gen-ai-functional-testing",
+  "owner": "@elastic/appex-ai-infra",
+  "devOnly": true
+}

packages/kbn-gen-ai-functional-testing/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/gen-ai-functional-testing",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
+}

packages/kbn-gen-ai-functional-testing/scripts/format_connector_config.js

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+require('@kbn/babel-register').install();
+require('../src/manage_connector_config').formatCurrentConfig();

packages/kbn-gen-ai-functional-testing/scripts/retrieve_connector_config.js

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+require('@kbn/babel-register').install();
+require('../src/manage_connector_config').retrieveFromVault();

packages/kbn-gen-ai-functional-testing/scripts/upload_connector_config.js

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+require('@kbn/babel-register').install();
+require('../src/manage_connector_config').uploadToVault();

packages/kbn-gen-ai-functional-testing/src/connectors.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { schema } from '@kbn/config-schema';
+
+/**
+ * The environment variable that is used by the CI to load the connectors configuration
+ */
+export const AI_CONNECTORS_VAR_ENV = 'KIBANA_TESTING_AI_CONNECTORS';
+
+const connectorsSchema = schema.recordOf(
+  schema.string(),
+  schema.object({
+    name: schema.string(),
+    actionTypeId: schema.string(),
+    config: schema.recordOf(schema.string(), schema.any()),
+    secrets: schema.recordOf(schema.string(), schema.any()),
+  })
+);
+
+export interface AvailableConnector {
+  name: string;
+  actionTypeId: string;
+  config: Record<string, unknown>;
+  secrets: Record<string, unknown>;
+}
+
+export interface AvailableConnectorWithId extends AvailableConnector {
+  id: string;
+}
+
+const loadConnectors = (): Record<string, AvailableConnector> => {
+  const envValue = process.env[AI_CONNECTORS_VAR_ENV];
+  if (!envValue) {
+    return {};
+  }
+
+  let connectors: Record<string, AvailableConnector>;
+  try {
+    connectors = JSON.parse(Buffer.from(envValue, 'base64').toString('utf-8'));
+  } catch (e) {
+    throw new Error(
+      `Error trying to parse value from KIBANA_AI_CONNECTORS environment variable: ${e.message}`
+    );
+  }
+  return connectorsSchema.validate(connectors);
+};
+
+/**
+ * Retrieve the list of preconfigured connectors that should be used when defining the
+ * FTR configuration of suites using the connectors.
+ *
+ * @example
+ * ```ts
+ * import { getPreconfiguredConnectorConfig } from '@kbn/gen-ai-functional-testing'
+ *
+ * export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+ *   const xpackFunctionalConfig = {...};
+ *   const preconfiguredConnectors = getPreconfiguredConnectorConfig();
+ *
+ *   return {
+ *     ...xpackFunctionalConfig.getAll(),
+ *     kbnTestServer: {
+ *       ...xpackFunctionalConfig.get('kbnTestServer'),
+ *       serverArgs: [
+ *         ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
+ *         `--xpack.actions.preconfigured=${JSON.stringify(preconfiguredConnectors)}`,
+ *       ],
+ *     },
+ *   };
+ * }
+ * ```
+ */
+export const getPreconfiguredConnectorConfig = () => {
+  return loadConnectors();
+};
+
+export const getAvailableConnectors = (): AvailableConnectorWithId[] => {
+  return Object.entries(loadConnectors()).map(([id, connector]) => {
+    return {
+      id,
+      ...connector,
+    };
+  });
+};