From 1f2a76a44bb6e339441e1ba10c861d5a2b7b9667 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Mon, 9 Dec 2024 16:09:29 +0100 Subject: [PATCH] GenAI FTR tests: switch to daily job (#203369) ## Summary Part of https://github.com/elastic/kibana-team/issues/1271 Follow-up of https://github.com/elastic/kibana/pull/198000 Fix https://github.com/elastic/kibana/issues/203205 Fix https://github.com/elastic/kibana/issues/203062 - Fix the problem with the Gemini token count events - Unskip the test suite - Remove the suite from the `on-merge` pipeline - Add a dedicated pipeline to run the tests daily --- .../kibana-gen-ai-daily.yml | 54 +++++++++++++++++++ .../locations.yml | 1 + .buildkite/pipelines/gen_ai_testing.yml | 48 +++++++++++++++++ .buildkite/pipelines/on_merge.yml | 22 -------- .../adapters/gemini/process_vertex_stream.ts | 5 +- .../inference/tests/index.ts | 3 +- 6 files changed, 107 insertions(+), 26 deletions(-) create mode 100644 .buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml create mode 100644 .buildkite/pipelines/gen_ai_testing.yml diff --git a/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml b/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml new file mode 100644 index 0000000000000..b4142cac87ec1 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml @@ -0,0 +1,54 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-gen-ai-daily + description: Runs the GenAI FTR tests daily + links: + - url: 'https://buildkite.com/elastic/kibana-gen-ai-tests-daily' + title: Pipeline link +spec: + type: buildkite-pipeline + owner: 'group:appex-ai-infra' + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / gen-ai-tests / daily + description: Runs the GenAI FTR tests daily + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#appex-ai-infra-alerts' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: main + cancel_intermediate_builds: true + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/gen_ai_testing.yml + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + appex-ai-infra: + access_level: MANAGE_BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + schedules: + Daily build: + cronline: 0 0 * * * America/New_York + message: Daily build + branch: main + tags: + - kibana diff --git a/.buildkite/pipeline-resource-definitions/locations.yml b/.buildkite/pipeline-resource-definitions/locations.yml index ca454f64c2696..11c0195902eec 100644 --- a/.buildkite/pipeline-resource-definitions/locations.yml +++ b/.buildkite/pipeline-resource-definitions/locations.yml @@ -50,3 +50,4 @@ spec: - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-gen-ai-daily.yml diff --git a/.buildkite/pipelines/gen_ai_testing.yml b/.buildkite/pipelines/gen_ai_testing.yml new file mode 100644 index 0000000000000..e6771d209deea --- /dev/null +++ b/.buildkite/pipelines/gen_ai_testing.yml @@ -0,0 +1,48 @@ +env: + FTR_GEN_AI: "1" +steps: + - label: '👨‍🔧 Pre-Build' + command: .buildkite/scripts/lifecycle/pre_build.sh + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + + - wait + + - label: '🧑‍🏭 Build Kibana Distribution' + command: .buildkite/scripts/steps/build_kibana.sh + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + key: build + if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" + + - wait + + - command: .buildkite/scripts/steps/test/ftr_configs.sh + env: + FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts" + FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api' + FTR_GEN_AI: "1" + label: AppEx AI-Infra Inference APIs FTR tests + key: ai-infra-gen-ai-inference-api + timeout_in_minutes: 50 + parallelism: 1 + agents: + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true + retry: + automatic: + - exit_status: '-1' + limit: 3 + - exit_status: '*' + limit: 1 + + diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 2f1562ef1d741..66cc3f9f33042 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -169,28 +169,6 @@ steps: - exit_status: '*' limit: 1 - - command: .buildkite/scripts/steps/test/ftr_configs.sh - env: - FTR_CONFIG: "x-pack/test/functional_gen_ai/inference/config.ts" - FTR_CONFIG_GROUP_KEY: 'ftr-ai-infra-gen-ai-inference-api' - FTR_GEN_AI: "1" - label: AppEx AI-Infra Inference APIs FTR tests - key: ai-infra-gen-ai-inference-api - timeout_in_minutes: 50 - parallelism: 1 - agents: - image: family/kibana-ubuntu-2004 - imageProject: elastic-images-prod - provider: gcp - machineType: n2-standard-4 - preemptible: true - retry: - automatic: - - exit_status: '-1' - limit: 3 - - exit_status: '*' - limit: 1 - - command: .buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh label: 'Serverless Entity Analytics - Security Cypress Tests' agents: diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts b/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts index 7b2ed2869c21d..fd497441f137c 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/gemini/process_vertex_stream.ts @@ -38,8 +38,9 @@ export function processVertexStream() { }); } - // completion: only present on last chunk - if (value.usageMetadata) { + // 'usageMetadata' can be present as an empty object on chunks + // only the last chunk will have its fields populated + if (value.usageMetadata?.totalTokenCount) { subscriber.next({ type: ChatCompletionEventType.ChatCompletionTokenCount, tokens: { diff --git a/x-pack/test/functional_gen_ai/inference/tests/index.ts b/x-pack/test/functional_gen_ai/inference/tests/index.ts index 65d93647511a1..36cf2bbaffa14 100644 --- a/x-pack/test/functional_gen_ai/inference/tests/index.ts +++ b/x-pack/test/functional_gen_ai/inference/tests/index.ts @@ -11,8 +11,7 @@ import { chatCompleteSuite } from './chat_complete'; // eslint-disable-next-line import/no-default-export export default function (providerContext: FtrProviderContext) { - // Failing: See https://github.com/elastic/kibana/issues/203205 - describe.skip('Inference plugin - API integration tests', async () => { + describe('Inference plugin - API integration tests', async () => { getAvailableConnectors().forEach((connector) => { describe(`Connector ${connector.id}`, () => { chatCompleteSuite(connector, providerContext);