GenAI FTR tests: switch to daily job (elastic#203369)

## Summary Part of elastic/kibana-team#1271 Follow-up of elastic#198000 Fix elastic#203205 Fix elastic#203062 - Fix the problem with the Gemini token count events - Unskip the test suite - Remove the suite from the `on-merge` pipeline - Add a dedicated pipeline to run the tests daily
jesuswr · Dec 9, 2024 · 1f2a76a · 1f2a76a
1 parent 6bf02e4
commit 1f2a76a
Show file tree

Hide file tree

Showing 6 changed files with 107 additions and 26 deletions.
diff --git a/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml b/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml
@@ -0,0 +1,54 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-gen-ai-daily
+  description: Runs the GenAI FTR tests daily
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-gen-ai-tests-daily'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:appex-ai-infra'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / gen-ai-tests / daily
+      description: Runs the GenAI FTR tests daily
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#appex-ai-infra-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main
+      cancel_intermediate_builds: true
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/gen_ai_testing.yml
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        appex-ai-infra:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+      schedules:
+        Daily build:
+          cronline: 0 0 * * * America/New_York
+          message: Daily build
+          branch: main
+      tags:
+        - kibana
diff --git a/.buildkite/pipeline-resource-definitions/locations.yml b/.buildkite/pipeline-resource-definitions/locations.yml
@@ -50,3 +50,4 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml
diff --git a/.buildkite/pipelines/gen_ai_testing.yml b/.buildkite/pipelines/gen_ai_testing.yml
@@ -0,0 +1,48 @@
+env:
+  FTR_GEN_AI: "1"
+steps:
+  - label: '👨‍🔧 Pre-Build'
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+
+  - wait
+
+  - label: '🧑‍🏭 Build Kibana Distribution'
+    command: .buildkite/scripts/steps/build_kibana.sh
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+
+  - wait
+
+  - command: .buildkite/scripts/steps/test/ftr_configs.sh
+    env:
+      FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts"
+      FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api'
+      FTR_GEN_AI: "1"
+    label: AppEx AI-Infra Inference APIs FTR tests
+    key: ai-infra-gen-ai-inference-api
+    timeout_in_minutes: 50
+    parallelism: 1
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+
diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
@@ -169,28 +169,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: .buildkite/scripts/steps/test/ftr_configs.sh
-    env:
-      FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts"
-      FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api'
-      FTR_GEN_AI: "1"
-    label: AppEx AI-Infra Inference APIs FTR tests
-    key: ai-infra-gen-ai-inference-api
-    timeout_in_minutes: 50
-    parallelism: 1
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-4
-      preemptible: true
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh
     label: 'Serverless Entity Analytics - Security Cypress Tests'
     agents:

diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts b/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts
@@ -38,8 +38,9 @@ export function processVertexStream() {
           });
         }
 
-        // completion: only present on last chunk
-        if (value.usageMetadata) {
+        // 'usageMetadata' can be present as an empty object on chunks
+        // only the last chunk will have its fields populated
+        if (value.usageMetadata?.totalTokenCount) {
           subscriber.next({
             type: ChatCompletionEventType.ChatCompletionTokenCount,
             tokens: {

diff --git a/x-pack/test/functional_gen_ai/inference/tests/index.ts b/x-pack/test/functional_gen_ai/inference/tests/index.ts
@@ -11,8 +11,7 @@ import { chatCompleteSuite } from './chat_complete';
 
 // eslint-disable-next-line import/no-default-export
 export default function (providerContext: FtrProviderContext) {
-  // Failing: See https://github.com/elastic/kibana/issues/203205
-  describe.skip('Inference plugin - API integration tests', async () => {
+  describe('Inference plugin - API integration tests', async () => {
     getAvailableConnectors().forEach((connector) => {
       describe(`Connector ${connector.id}`, () => {
         chatCompleteSuite(connector, providerContext);