From 65d84cf27678af1c1edd693a6c6a77bafc7cb0b2 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 20:50:35 +1100 Subject: [PATCH 01/12] [8.x] [ResponseOps][Rules] Allow users to create rules with predefined non random IDs (#199119) (#199259) # Backport This will backport the following commits from `main` to `8.x`: - [[ResponseOps][Rules] Allow users to create rules with predefined non random IDs (#199119)](https://github.com/elastic/kibana/pull/199119) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Christos Nasikas --- .../alerting/server/saved_objects/index.ts | 5 +++++ .../tests/alerting/group1/create.ts | 20 +++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/alerting/server/saved_objects/index.ts b/x-pack/plugins/alerting/server/saved_objects/index.ts index 8e76f28ff7fb8..8f11020ee6285 100644 --- a/x-pack/plugins/alerting/server/saved_objects/index.ts +++ b/x-pack/plugins/alerting/server/saved_objects/index.ts @@ -217,6 +217,11 @@ export function setupSavedObjects( // Encrypted attributes encryptedSavedObjects.registerType({ type: RULE_SAVED_OBJECT_TYPE, + /** + * We disable enforcing random SO IDs for the rule SO + * to allow users creating rules with a predefined ID. + */ + enforceRandomId: false, attributesToEncrypt: new Set(RuleAttributesToEncrypt), attributesToIncludeInAAD: new Set(RuleAttributesIncludedInAAD), }); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/create.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/create.ts index 57d41424186b3..5a6385a3895d2 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/create.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/create.ts @@ -395,20 +395,18 @@ export default function createAlertTests({ getService }: FtrProviderContext) { }); }); - it('should not allow providing simple custom ids (non uuid)', async () => { - const customId = '1'; + it('should create a rule with a predefined non random ID', async () => { + const ruleId = 'my_id'; + const response = await supertest - .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule/${customId}`) + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule/${ruleId}`) .set('kbn-xsrf', 'foo') - .send(getTestRuleData()); + .send(getTestRuleData()) + .expect(200); - expect(response.status).to.eql(400); - expect(response.body).to.eql({ - statusCode: 400, - error: 'Bad Request', - message: - 'Predefined IDs are not allowed for saved objects with encrypted attributes unless the ID is a UUID.: Bad Request', - }); + objectRemover.add(Spaces.space1.id, response.body.id, 'rule', 'alerting'); + + expect(response.body.id).to.eql(ruleId); }); it('should return 409 when document with id already exists', async () => { From feab4ef51b8f2ce7daf46ea5064427dbbf59fa23 Mon Sep 17 00:00:00 2001 From: Ash <1849116+ashokaditya@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:16:34 +0100 Subject: [PATCH 02/12] [8.x] [Security Solution] Removing cypress folder (#197273) (#199260) > [!Warning] > `.github/CODEOWNERS` and `.buildkite/pipelines/pull_request/security_solution/defend_workflows.yml` were updated as part of merge conflicts so would need a thorough review. # Backport This will backport the following commits from `main` to `8.x`: - [[Security Solution] Removing cypress folder (#197273)](https://github.com/elastic/kibana/pull/197273) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --------- Co-authored-by: Gloria Hornero Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../ftr_security_serverless_configs.yml | 6 +- .../security_solution/defend_workflows.yml | 17 + .../security_serverless_defend_workflows.sh | 12 - .github/CODEOWNERS | 2022 +++++++++++++++++ .../osquery/cypress/cypress_base.config.ts | 2 +- x-pack/plugins/osquery/cypress/support/e2e.ts | 11 + .../support/setup_data_loader_tasks.ts | 17 +- .../serverless_config.base.ts} | 2 +- .../serverless_config.ts | 4 +- .../osquery_cypress/serverless_cli_config.ts | 4 +- .../osquery_cypress/serverless_config.base.ts | 35 + .../security/cypress/.eslintrc.json | 13 - .../test_suites/security/cypress/.gitignore | 3 - .../test_suites/security/cypress/README.md | 65 - .../security/cypress/cypress.config.ts | 40 - .../test_suites/security/cypress/cypress.d.ts | 207 -- .../security/cypress/e2e/serverless.cy.ts | 22 - .../test_suites/security/cypress/package.json | 13 - .../security/cypress/reporter_config.json | 10 - .../test_suites/security/cypress/runner.ts | 24 - .../security/cypress/screens/index.ts | 8 - .../security/cypress/screens/landing_page.ts | 8 - .../security/cypress/security_config.ts | 31 - .../security/cypress/support/commands.js | 32 - .../security/cypress/support/e2e.js | 29 - .../security/cypress/support/index.d.ts | 52 - .../index_endpoint_hosts.ts | 35 - .../security/cypress/tasks/login.ts | 87 - .../security/cypress/tasks/navigation.ts | 10 - x-pack/test_serverless/tsconfig.json | 1 - 30 files changed, 2099 insertions(+), 723 deletions(-) delete mode 100644 .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh create mode 100644 .github/CODEOWNERS rename x-pack/{test_serverless/functional/test_suites/security => plugins/osquery}/cypress/support/setup_data_loader_tasks.ts (77%) rename x-pack/{test_serverless/functional/test_suites/security/cypress/security_config.base.ts => test/defend_workflows_cypress/serverless_config.base.ts} (93%) create mode 100644 x-pack/test/osquery_cypress/serverless_config.base.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/.eslintrc.json delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/.gitignore delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/README.md delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/cypress.d.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/e2e/serverless.cy.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/package.json delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/reporter_config.json delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/runner.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/screens/index.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/screens/landing_page.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/security_config.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/support/commands.js delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/support/e2e.js delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/support/index.d.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/tasks/endpoint_management/index_endpoint_hosts.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts delete mode 100644 x-pack/test_serverless/functional/test_suites/security/cypress/tasks/navigation.ts diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml index b642b2c680bb1..834db3ce9849e 100644 --- a/.buildkite/ftr_security_serverless_configs.yml +++ b/.buildkite/ftr_security_serverless_configs.yml @@ -1,17 +1,17 @@ disabled: # Base config files, only necessary to inform config finding script - - x-pack/test_serverless/functional/test_suites/security/cypress/security_config.base.ts - - x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts - x-pack/test/security_solution_api_integration/config/serverless/config.base.edr_workflows.ts + - x-pack/test/defend_workflows_cypress/serverless_config.base.ts + - x-pack/test/osquery_cypress/serverless_config.base.ts # Cypress configs, for now these are still run manually - x-pack/test/defend_workflows_cypress/serverless_config.ts - x-pack/test/osquery_cypress/serverless_cli_config.ts - - x-pack/test_serverless/functional/test_suites/security/cypress/security_config.ts - x-pack/test/security_solution_cypress/serverless_config.ts + # Playwright - x-pack/test/security_solution_playwright/serverless_config.ts diff --git a/.buildkite/pipelines/pull_request/security_solution/defend_workflows.yml b/.buildkite/pipelines/pull_request/security_solution/defend_workflows.yml index 28cc4f2812b5a..104853c27b112 100644 --- a/.buildkite/pipelines/pull_request/security_solution/defend_workflows.yml +++ b/.buildkite/pipelines/pull_request/security_solution/defend_workflows.yml @@ -18,3 +18,20 @@ steps: automatic: - exit_status: '-1' limit: 1 + + - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh + label: 'Defend Workflows Cypress Tests on Serverless' + agents: + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 + depends_on: + - build + - quick_checks + timeout_in_minutes: 60 + parallelism: 14 + retry: + automatic: + - exit_status: '-1' + limit: 1 diff --git a/.buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh b/.buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh deleted file mode 100644 index 7b16afa214fed..0000000000000 --- a/.buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/scripts/steps/functional/common.sh - -export JOB=kibana-serverless-security-cypress -export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION} - -echo "--- Security Defend Workflows Serverless Cypress" - -yarn --cwd x-pack/test_serverless/functional/test_suites/security/cypress cypress:run diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000000..dda83bdf40615 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2022 @@ +#### +## Everything at the top of the codeowners file is auto generated based on the +## "owner" fields in the kibana.jsonc files at the root of each package. This +## file is automatically updated by CI or can be updated locally by running +## `node scripts/generate codeowners`. +#### + +x-pack/test/alerting_api_integration/common/plugins/aad @elastic/response-ops +x-pack/plugins/actions @elastic/response-ops +x-pack/test/alerting_api_integration/common/plugins/actions_simulators @elastic/response-ops +packages/kbn-actions-types @elastic/response-ops +src/plugins/advanced_settings @elastic/appex-sharedux @elastic/kibana-management +x-pack/packages/kbn-ai-assistant @elastic/search-kibana +x-pack/packages/kbn-ai-assistant-common @elastic/search-kibana +src/plugins/ai_assistant_management/selection @elastic/obs-knowledge-team +x-pack/packages/ml/aiops_change_point_detection @elastic/ml-ui +x-pack/packages/ml/aiops_common @elastic/ml-ui +x-pack/packages/ml/aiops_components @elastic/ml-ui +x-pack/packages/ml/aiops_log_pattern_analysis @elastic/ml-ui +x-pack/packages/ml/aiops_log_rate_analysis @elastic/ml-ui +x-pack/plugins/aiops @elastic/ml-ui +x-pack/packages/ml/aiops_test_utils @elastic/ml-ui +x-pack/test/alerting_api_integration/packages/helpers @elastic/response-ops +x-pack/test/alerting_api_integration/common/plugins/alerts @elastic/response-ops +x-pack/packages/kbn-alerting-comparators @elastic/response-ops +x-pack/examples/alerting_example @elastic/response-ops +x-pack/test/functional_with_es_ssl/plugins/alerts @elastic/response-ops +x-pack/plugins/alerting @elastic/response-ops +x-pack/packages/kbn-alerting-state-types @elastic/response-ops +packages/kbn-alerting-types @elastic/response-ops +packages/kbn-alerts-as-data-utils @elastic/response-ops +packages/kbn-alerts-grouping @elastic/response-ops +x-pack/test/alerting_api_integration/common/plugins/alerts_restricted @elastic/response-ops +packages/kbn-alerts-ui-shared @elastic/response-ops +packages/kbn-ambient-common-types @elastic/kibana-operations +packages/kbn-ambient-ftr-types @elastic/kibana-operations @elastic/appex-qa +packages/kbn-ambient-storybook-types @elastic/kibana-operations +packages/kbn-ambient-ui-types @elastic/kibana-operations +packages/kbn-analytics @elastic/kibana-core +packages/analytics/utils/analytics_collection_utils @elastic/kibana-core +test/analytics/plugins/analytics_ftr_helpers @elastic/kibana-core +test/analytics/plugins/analytics_plugin_a @elastic/kibana-core +packages/kbn-apm-config-loader @elastic/kibana-core @vigneshshanmugam +x-pack/plugins/observability_solution/apm_data_access @elastic/obs-knowledge-team @elastic/obs-ux-infra_services-team +packages/kbn-apm-data-view @elastic/obs-ux-infra_services-team +x-pack/plugins/observability_solution/apm/ftr_e2e @elastic/obs-ux-infra_services-team +x-pack/plugins/observability_solution/apm @elastic/obs-ux-infra_services-team +packages/kbn-apm-synthtrace @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team +packages/kbn-apm-synthtrace-client @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team +packages/kbn-apm-types @elastic/obs-ux-infra_services-team +packages/kbn-apm-utils @elastic/obs-ux-infra_services-team +test/plugin_functional/plugins/app_link_test @elastic/kibana-core +x-pack/test/usage_collection/plugins/application_usage_test @elastic/kibana-core +x-pack/test/security_api_integration/plugins/audit_log @elastic/kibana-security +packages/kbn-avc-banner @elastic/security-defend-workflows +packages/kbn-axe-config @elastic/kibana-qa +packages/kbn-babel-preset @elastic/kibana-operations +packages/kbn-babel-register @elastic/kibana-operations +packages/kbn-babel-transform @elastic/kibana-operations +x-pack/plugins/banners @elastic/appex-sharedux +packages/kbn-bazel-runner @elastic/kibana-operations +packages/kbn-bfetch-error @elastic/appex-sharedux +examples/bfetch_explorer @elastic/appex-sharedux +src/plugins/bfetch @elastic/appex-sharedux +packages/kbn-calculate-auto @elastic/obs-ux-management-team +packages/kbn-calculate-width-from-char-count @elastic/kibana-visualizations +x-pack/plugins/canvas @elastic/kibana-presentation +packages/kbn-capture-oas-snapshot-cli @elastic/kibana-core +x-pack/test/cases_api_integration/common/plugins/cases @elastic/response-ops +packages/kbn-cases-components @elastic/response-ops +x-pack/plugins/cases @elastic/response-ops +packages/kbn-cbor @elastic/kibana-operations +packages/kbn-cell-actions @elastic/security-threat-hunting-explore +src/plugins/chart_expressions/common @elastic/kibana-visualizations +packages/kbn-chart-icons @elastic/kibana-visualizations +src/plugins/charts @elastic/kibana-visualizations +packages/kbn-check-mappings-update-cli @elastic/kibana-core +packages/kbn-check-prod-native-modules-cli @elastic/kibana-operations +packages/kbn-ci-stats-core @elastic/kibana-operations +packages/kbn-ci-stats-performance-metrics @elastic/kibana-operations +packages/kbn-ci-stats-reporter @elastic/kibana-operations +packages/kbn-ci-stats-shipper-cli @elastic/kibana-operations +packages/kbn-cli-dev-mode @elastic/kibana-operations +packages/cloud @elastic/kibana-core +x-pack/plugins/cloud_integrations/cloud_chat @elastic/kibana-core +x-pack/plugins/cloud_integrations/cloud_data_migration @elastic/kibana-management +x-pack/plugins/cloud_defend @elastic/kibana-cloud-security-posture +x-pack/plugins/cloud_integrations/cloud_experiments @elastic/kibana-core +x-pack/plugins/cloud_integrations/cloud_full_story @elastic/kibana-core +x-pack/test/cloud_integration/plugins/saml_provider @elastic/kibana-core +x-pack/plugins/cloud_integrations/cloud_links @elastic/kibana-core +x-pack/plugins/cloud @elastic/kibana-core +x-pack/packages/kbn-cloud-security-posture/public @elastic/kibana-cloud-security-posture +x-pack/packages/kbn-cloud-security-posture/common @elastic/kibana-cloud-security-posture +x-pack/packages/kbn-cloud-security-posture/graph @elastic/kibana-cloud-security-posture +x-pack/plugins/cloud_security_posture @elastic/kibana-cloud-security-posture +packages/shared-ux/code_editor/impl @elastic/appex-sharedux +packages/shared-ux/code_editor/mocks @elastic/appex-sharedux +packages/kbn-code-owners @elastic/appex-qa +packages/kbn-coloring @elastic/kibana-visualizations +packages/kbn-config @elastic/kibana-core +packages/kbn-config-mocks @elastic/kibana-core +packages/kbn-config-schema @elastic/kibana-core +src/plugins/console @elastic/kibana-management +packages/content-management/content_editor @elastic/appex-sharedux +packages/content-management/content_insights/content_insights_public @elastic/appex-sharedux +packages/content-management/content_insights/content_insights_server @elastic/appex-sharedux +examples/content_management_examples @elastic/appex-sharedux +packages/content-management/favorites/favorites_public @elastic/appex-sharedux +packages/content-management/favorites/favorites_server @elastic/appex-sharedux +src/plugins/content_management @elastic/appex-sharedux +packages/content-management/tabbed_table_list_view @elastic/appex-sharedux +packages/content-management/table_list_view @elastic/appex-sharedux +packages/content-management/table_list_view_common @elastic/appex-sharedux +packages/content-management/table_list_view_table @elastic/appex-sharedux +packages/content-management/user_profiles @elastic/appex-sharedux +packages/kbn-content-management-utils @elastic/kibana-data-discovery +examples/controls_example @elastic/kibana-presentation +src/plugins/controls @elastic/kibana-presentation +src/core @elastic/kibana-core +packages/core/analytics/core-analytics-browser @elastic/kibana-core +packages/core/analytics/core-analytics-browser-internal @elastic/kibana-core +packages/core/analytics/core-analytics-browser-mocks @elastic/kibana-core +packages/core/analytics/core-analytics-server @elastic/kibana-core +packages/core/analytics/core-analytics-server-internal @elastic/kibana-core +packages/core/analytics/core-analytics-server-mocks @elastic/kibana-core +test/plugin_functional/plugins/core_app_status @elastic/kibana-core +packages/core/application/core-application-browser @elastic/kibana-core +packages/core/application/core-application-browser-internal @elastic/kibana-core +packages/core/application/core-application-browser-mocks @elastic/kibana-core +packages/core/application/core-application-common @elastic/kibana-core +packages/core/apps/core-apps-browser-internal @elastic/kibana-core +packages/core/apps/core-apps-browser-mocks @elastic/kibana-core +packages/core/apps/core-apps-server-internal @elastic/kibana-core +packages/core/base/core-base-browser-internal @elastic/kibana-core +packages/core/base/core-base-browser-mocks @elastic/kibana-core +packages/core/base/core-base-common @elastic/kibana-core +packages/core/base/core-base-common-internal @elastic/kibana-core +packages/core/base/core-base-server-internal @elastic/kibana-core +packages/core/base/core-base-server-mocks @elastic/kibana-core +packages/core/capabilities/core-capabilities-browser-internal @elastic/kibana-core +packages/core/capabilities/core-capabilities-browser-mocks @elastic/kibana-core +packages/core/capabilities/core-capabilities-common @elastic/kibana-core +packages/core/capabilities/core-capabilities-server @elastic/kibana-core +packages/core/capabilities/core-capabilities-server-internal @elastic/kibana-core +packages/core/capabilities/core-capabilities-server-mocks @elastic/kibana-core +packages/core/chrome/core-chrome-browser @elastic/appex-sharedux +packages/core/chrome/core-chrome-browser-internal @elastic/appex-sharedux +packages/core/chrome/core-chrome-browser-mocks @elastic/appex-sharedux +packages/core/config/core-config-server-internal @elastic/kibana-core +packages/core/custom-branding/core-custom-branding-browser @elastic/appex-sharedux +packages/core/custom-branding/core-custom-branding-browser-internal @elastic/appex-sharedux +packages/core/custom-branding/core-custom-branding-browser-mocks @elastic/appex-sharedux +packages/core/custom-branding/core-custom-branding-common @elastic/appex-sharedux +packages/core/custom-branding/core-custom-branding-server @elastic/appex-sharedux +packages/core/custom-branding/core-custom-branding-server-internal @elastic/appex-sharedux +packages/core/custom-branding/core-custom-branding-server-mocks @elastic/appex-sharedux +packages/core/deprecations/core-deprecations-browser @elastic/kibana-core +packages/core/deprecations/core-deprecations-browser-internal @elastic/kibana-core +packages/core/deprecations/core-deprecations-browser-mocks @elastic/kibana-core +packages/core/deprecations/core-deprecations-common @elastic/kibana-core +packages/core/deprecations/core-deprecations-server @elastic/kibana-core +packages/core/deprecations/core-deprecations-server-internal @elastic/kibana-core +packages/core/deprecations/core-deprecations-server-mocks @elastic/kibana-core +packages/core/doc-links/core-doc-links-browser @elastic/kibana-core +packages/core/doc-links/core-doc-links-browser-internal @elastic/kibana-core +packages/core/doc-links/core-doc-links-browser-mocks @elastic/kibana-core +packages/core/doc-links/core-doc-links-server @elastic/kibana-core +packages/core/doc-links/core-doc-links-server-internal @elastic/kibana-core +packages/core/doc-links/core-doc-links-server-mocks @elastic/kibana-core +packages/core/elasticsearch/core-elasticsearch-client-server-internal @elastic/kibana-core +packages/core/elasticsearch/core-elasticsearch-client-server-mocks @elastic/kibana-core +packages/core/elasticsearch/core-elasticsearch-server @elastic/kibana-core +packages/core/elasticsearch/core-elasticsearch-server-internal @elastic/kibana-core +packages/core/elasticsearch/core-elasticsearch-server-mocks @elastic/kibana-core +packages/core/environment/core-environment-server-internal @elastic/kibana-core +packages/core/environment/core-environment-server-mocks @elastic/kibana-core +packages/core/execution-context/core-execution-context-browser @elastic/kibana-core +packages/core/execution-context/core-execution-context-browser-internal @elastic/kibana-core +packages/core/execution-context/core-execution-context-browser-mocks @elastic/kibana-core +packages/core/execution-context/core-execution-context-common @elastic/kibana-core +packages/core/execution-context/core-execution-context-server @elastic/kibana-core +packages/core/execution-context/core-execution-context-server-internal @elastic/kibana-core +packages/core/execution-context/core-execution-context-server-mocks @elastic/kibana-core +packages/core/fatal-errors/core-fatal-errors-browser @elastic/kibana-core +packages/core/fatal-errors/core-fatal-errors-browser-internal @elastic/kibana-core +packages/core/fatal-errors/core-fatal-errors-browser-mocks @elastic/kibana-core +packages/core/feature-flags/core-feature-flags-browser @elastic/kibana-core +packages/core/feature-flags/core-feature-flags-browser-internal @elastic/kibana-core +packages/core/feature-flags/core-feature-flags-browser-mocks @elastic/kibana-core +packages/core/feature-flags/core-feature-flags-server @elastic/kibana-core +packages/core/feature-flags/core-feature-flags-server-internal @elastic/kibana-core +packages/core/feature-flags/core-feature-flags-server-mocks @elastic/kibana-core +test/plugin_functional/plugins/core_history_block @elastic/kibana-core +packages/core/http/core-http-browser @elastic/kibana-core +packages/core/http/core-http-browser-internal @elastic/kibana-core +packages/core/http/core-http-browser-mocks @elastic/kibana-core +packages/core/http/core-http-common @elastic/kibana-core +packages/core/http/core-http-context-server-internal @elastic/kibana-core +packages/core/http/core-http-context-server-mocks @elastic/kibana-core +test/plugin_functional/plugins/core_http @elastic/kibana-core +packages/core/http/core-http-request-handler-context-server @elastic/kibana-core +packages/core/http/core-http-request-handler-context-server-internal @elastic/kibana-core +packages/core/http/core-http-resources-server @elastic/kibana-core +packages/core/http/core-http-resources-server-internal @elastic/kibana-core +packages/core/http/core-http-resources-server-mocks @elastic/kibana-core +packages/core/http/core-http-router-server-internal @elastic/kibana-core +packages/core/http/core-http-router-server-mocks @elastic/kibana-core +packages/core/http/core-http-server @elastic/kibana-core +packages/core/http/core-http-server-internal @elastic/kibana-core +packages/core/http/core-http-server-mocks @elastic/kibana-core +packages/core/i18n/core-i18n-browser @elastic/kibana-core +packages/core/i18n/core-i18n-browser-internal @elastic/kibana-core +packages/core/i18n/core-i18n-browser-mocks @elastic/kibana-core +packages/core/i18n/core-i18n-server @elastic/kibana-core +packages/core/i18n/core-i18n-server-internal @elastic/kibana-core +packages/core/i18n/core-i18n-server-mocks @elastic/kibana-core +packages/core/injected-metadata/core-injected-metadata-browser-internal @elastic/kibana-core +packages/core/injected-metadata/core-injected-metadata-browser-mocks @elastic/kibana-core +packages/core/injected-metadata/core-injected-metadata-common-internal @elastic/kibana-core +packages/core/integrations/core-integrations-browser-internal @elastic/kibana-core +packages/core/integrations/core-integrations-browser-mocks @elastic/kibana-core +packages/core/lifecycle/core-lifecycle-browser @elastic/kibana-core +packages/core/lifecycle/core-lifecycle-browser-internal @elastic/kibana-core +packages/core/lifecycle/core-lifecycle-browser-mocks @elastic/kibana-core +packages/core/lifecycle/core-lifecycle-server @elastic/kibana-core +packages/core/lifecycle/core-lifecycle-server-internal @elastic/kibana-core +packages/core/lifecycle/core-lifecycle-server-mocks @elastic/kibana-core +packages/core/logging/core-logging-browser-internal @elastic/kibana-core +packages/core/logging/core-logging-browser-mocks @elastic/kibana-core +packages/core/logging/core-logging-common-internal @elastic/kibana-core +packages/core/logging/core-logging-server @elastic/kibana-core +packages/core/logging/core-logging-server-internal @elastic/kibana-core +packages/core/logging/core-logging-server-mocks @elastic/kibana-core +packages/core/metrics/core-metrics-collectors-server-internal @elastic/kibana-core +packages/core/metrics/core-metrics-collectors-server-mocks @elastic/kibana-core +packages/core/metrics/core-metrics-server @elastic/kibana-core +packages/core/metrics/core-metrics-server-internal @elastic/kibana-core +packages/core/metrics/core-metrics-server-mocks @elastic/kibana-core +packages/core/mount-utils/core-mount-utils-browser @elastic/kibana-core +packages/core/mount-utils/core-mount-utils-browser-internal @elastic/kibana-core +packages/core/node/core-node-server @elastic/kibana-core +packages/core/node/core-node-server-internal @elastic/kibana-core +packages/core/node/core-node-server-mocks @elastic/kibana-core +packages/core/notifications/core-notifications-browser @elastic/kibana-core +packages/core/notifications/core-notifications-browser-internal @elastic/kibana-core +packages/core/notifications/core-notifications-browser-mocks @elastic/kibana-core +packages/core/overlays/core-overlays-browser @elastic/kibana-core +packages/core/overlays/core-overlays-browser-internal @elastic/kibana-core +packages/core/overlays/core-overlays-browser-mocks @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_a @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_appleave @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_b @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_chromeless @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_deep_links @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_deprecations @elastic/kibana-core +test/plugin_functional/plugins/core_dynamic_resolving_a @elastic/kibana-core +test/plugin_functional/plugins/core_dynamic_resolving_b @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_execution_context @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_helpmenu @elastic/kibana-core +test/node_roles_functional/plugins/core_plugin_initializer_context @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_route_timeouts @elastic/kibana-core +test/plugin_functional/plugins/core_plugin_static_assets @elastic/kibana-core +packages/core/plugins/core-plugins-base-server-internal @elastic/kibana-core +packages/core/plugins/core-plugins-browser @elastic/kibana-core +packages/core/plugins/core-plugins-browser-internal @elastic/kibana-core +packages/core/plugins/core-plugins-browser-mocks @elastic/kibana-core +packages/core/plugins/core-plugins-contracts-browser @elastic/kibana-core +packages/core/plugins/core-plugins-contracts-server @elastic/kibana-core +packages/core/plugins/core-plugins-server @elastic/kibana-core +packages/core/plugins/core-plugins-server-internal @elastic/kibana-core +packages/core/plugins/core-plugins-server-mocks @elastic/kibana-core +packages/core/preboot/core-preboot-server @elastic/kibana-core +packages/core/preboot/core-preboot-server-internal @elastic/kibana-core +packages/core/preboot/core-preboot-server-mocks @elastic/kibana-core +test/plugin_functional/plugins/core_provider_plugin @elastic/kibana-core +packages/core/rendering/core-rendering-browser-internal @elastic/kibana-core +packages/core/rendering/core-rendering-browser-mocks @elastic/kibana-core +packages/core/rendering/core-rendering-server-internal @elastic/kibana-core +packages/core/rendering/core-rendering-server-mocks @elastic/kibana-core +packages/core/root/core-root-browser-internal @elastic/kibana-core +packages/core/root/core-root-server-internal @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-api-browser @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-api-server @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-api-server-internal @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-api-server-mocks @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-base-server-internal @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-base-server-mocks @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-browser @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-browser-internal @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-browser-mocks @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-common @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-import-export-server-internal @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-import-export-server-mocks @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-migration-server-internal @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-migration-server-mocks @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-server @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-server-internal @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-server-mocks @elastic/kibana-core +packages/core/saved-objects/core-saved-objects-utils-server @elastic/kibana-core +packages/core/security/core-security-browser @elastic/kibana-core +packages/core/security/core-security-browser-internal @elastic/kibana-core +packages/core/security/core-security-browser-mocks @elastic/kibana-core +packages/core/security/core-security-common @elastic/kibana-core @elastic/kibana-security +packages/core/security/core-security-server @elastic/kibana-core +packages/core/security/core-security-server-internal @elastic/kibana-core +packages/core/security/core-security-server-mocks @elastic/kibana-core +packages/core/status/core-status-common @elastic/kibana-core +packages/core/status/core-status-common-internal @elastic/kibana-core +packages/core/status/core-status-server @elastic/kibana-core +packages/core/status/core-status-server-internal @elastic/kibana-core +packages/core/status/core-status-server-mocks @elastic/kibana-core +packages/core/test-helpers/core-test-helpers-deprecations-getters @elastic/kibana-core +packages/core/test-helpers/core-test-helpers-http-setup-browser @elastic/kibana-core +packages/core/test-helpers/core-test-helpers-kbn-server @elastic/kibana-core +packages/core/test-helpers/core-test-helpers-model-versions @elastic/kibana-core +packages/core/test-helpers/core-test-helpers-so-type-serializer @elastic/kibana-core +packages/core/test-helpers/core-test-helpers-test-utils @elastic/kibana-core +packages/core/theme/core-theme-browser @elastic/kibana-core +packages/core/theme/core-theme-browser-internal @elastic/kibana-core +packages/core/theme/core-theme-browser-mocks @elastic/kibana-core +packages/core/ui-settings/core-ui-settings-browser @elastic/appex-sharedux +packages/core/ui-settings/core-ui-settings-browser-internal @elastic/appex-sharedux +packages/core/ui-settings/core-ui-settings-browser-mocks @elastic/appex-sharedux +packages/core/ui-settings/core-ui-settings-common @elastic/appex-sharedux +packages/core/ui-settings/core-ui-settings-server @elastic/appex-sharedux +packages/core/ui-settings/core-ui-settings-server-internal @elastic/appex-sharedux +packages/core/ui-settings/core-ui-settings-server-mocks @elastic/appex-sharedux +packages/core/usage-data/core-usage-data-base-server-internal @elastic/kibana-core +packages/core/usage-data/core-usage-data-server @elastic/kibana-core +packages/core/usage-data/core-usage-data-server-internal @elastic/kibana-core +packages/core/usage-data/core-usage-data-server-mocks @elastic/kibana-core +packages/core/user-profile/core-user-profile-browser @elastic/kibana-core +packages/core/user-profile/core-user-profile-browser-internal @elastic/kibana-core +packages/core/user-profile/core-user-profile-browser-mocks @elastic/kibana-core +packages/core/user-profile/core-user-profile-common @elastic/kibana-core +packages/core/user-profile/core-user-profile-server @elastic/kibana-core +packages/core/user-profile/core-user-profile-server-internal @elastic/kibana-core +packages/core/user-profile/core-user-profile-server-mocks @elastic/kibana-core +packages/core/user-settings/core-user-settings-server @elastic/kibana-security +packages/core/user-settings/core-user-settings-server-internal @elastic/kibana-security +packages/core/user-settings/core-user-settings-server-mocks @elastic/kibana-security +x-pack/plugins/cross_cluster_replication @elastic/kibana-management +packages/kbn-crypto @elastic/kibana-security +packages/kbn-crypto-browser @elastic/kibana-core +x-pack/plugins/custom_branding @elastic/appex-sharedux +packages/kbn-custom-icons @elastic/obs-ux-logs-team +packages/kbn-custom-integrations @elastic/obs-ux-logs-team +src/plugins/custom_integrations @elastic/fleet +packages/kbn-cypress-config @elastic/kibana-operations +x-pack/plugins/dashboard_enhanced @elastic/kibana-presentation +src/plugins/dashboard @elastic/kibana-presentation +x-pack/packages/kbn-data-forge @elastic/obs-ux-management-team +src/plugins/data @elastic/kibana-visualizations @elastic/kibana-data-discovery +x-pack/plugins/data_quality @elastic/obs-ux-logs-team +test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery +packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery +packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore +x-pack/plugins/data_usage @elastic/obs-ai-assistant @elastic/security-solution +src/plugins/data_view_editor @elastic/kibana-data-discovery +examples/data_view_field_editor_example @elastic/kibana-data-discovery +src/plugins/data_view_field_editor @elastic/kibana-data-discovery +src/plugins/data_view_management @elastic/kibana-data-discovery +packages/kbn-data-view-utils @elastic/kibana-data-discovery +src/plugins/data_views @elastic/kibana-data-discovery +x-pack/plugins/data_visualizer @elastic/ml-ui +x-pack/plugins/observability_solution/dataset_quality @elastic/obs-ux-logs-team +packages/kbn-datemath @elastic/kibana-data-discovery +packages/deeplinks/analytics @elastic/kibana-data-discovery @elastic/kibana-presentation @elastic/kibana-visualizations +packages/deeplinks/devtools @elastic/kibana-management +packages/deeplinks/fleet @elastic/fleet +packages/deeplinks/management @elastic/kibana-management +packages/deeplinks/ml @elastic/ml-ui +packages/deeplinks/observability @elastic/obs-ux-management-team +packages/deeplinks/search @elastic/search-kibana +packages/deeplinks/security @elastic/security-solution +packages/deeplinks/shared @elastic/appex-sharedux +packages/default-nav/analytics @elastic/kibana-data-discovery @elastic/kibana-presentation @elastic/kibana-visualizations +packages/default-nav/devtools @elastic/kibana-management +packages/default-nav/management @elastic/kibana-management +packages/default-nav/ml @elastic/ml-ui +packages/kbn-dev-cli-errors @elastic/kibana-operations +packages/kbn-dev-cli-runner @elastic/kibana-operations +packages/kbn-dev-proc-runner @elastic/kibana-operations +src/plugins/dev_tools @elastic/kibana-management +packages/kbn-dev-utils @elastic/kibana-operations +examples/developer_examples @elastic/appex-sharedux +packages/kbn-discover-contextual-components @elastic/obs-ux-logs-team @elastic/kibana-data-discovery +examples/discover_customization_examples @elastic/kibana-data-discovery +x-pack/plugins/discover_enhanced @elastic/kibana-data-discovery +src/plugins/discover @elastic/kibana-data-discovery +src/plugins/discover_shared @elastic/kibana-data-discovery @elastic/obs-ux-logs-team +packages/kbn-discover-utils @elastic/kibana-data-discovery +packages/kbn-doc-links @elastic/docs +packages/kbn-docs-utils @elastic/kibana-operations +packages/kbn-dom-drag-drop @elastic/kibana-visualizations @elastic/kibana-data-discovery +packages/kbn-ebt-tools @elastic/kibana-core +x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore +x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore +packages/kbn-elastic-agent-utils @elastic/obs-ux-logs-team +x-pack/packages/kbn-elastic-assistant @elastic/security-generative-ai +x-pack/packages/kbn-elastic-assistant-common @elastic/security-generative-ai +x-pack/plugins/elastic_assistant @elastic/security-generative-ai +test/plugin_functional/plugins/elasticsearch_client_plugin @elastic/kibana-core +x-pack/test/plugin_api_integration/plugins/elasticsearch_client @elastic/kibana-core +x-pack/plugins/embeddable_enhanced @elastic/kibana-presentation +examples/embeddable_examples @elastic/kibana-presentation +src/plugins/embeddable @elastic/kibana-presentation +x-pack/examples/embedded_lens_example @elastic/kibana-visualizations +x-pack/plugins/encrypted_saved_objects @elastic/kibana-security +x-pack/plugins/enterprise_search @elastic/search-kibana +x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities +x-pack/packages/kbn-entities-schema @elastic/obs-entities +x-pack/test/api_integration/apis/entity_manager/fixture_plugin @elastic/obs-entities +x-pack/plugins/entity_manager @elastic/obs-entities +examples/error_boundary @elastic/appex-sharedux +packages/kbn-es @elastic/kibana-operations +packages/kbn-es-archiver @elastic/kibana-operations @elastic/appex-qa +packages/kbn-es-errors @elastic/kibana-core +packages/kbn-es-query @elastic/kibana-data-discovery +packages/kbn-es-types @elastic/kibana-core @elastic/obs-knowledge-team +src/plugins/es_ui_shared @elastic/kibana-management +packages/kbn-eslint-config @elastic/kibana-operations +packages/kbn-eslint-plugin-disable @elastic/kibana-operations +packages/kbn-eslint-plugin-eslint @elastic/kibana-operations +packages/kbn-eslint-plugin-i18n @elastic/obs-knowledge-team @elastic/kibana-operations +packages/kbn-eslint-plugin-imports @elastic/kibana-operations +packages/kbn-eslint-plugin-telemetry @elastic/obs-knowledge-team +examples/eso_model_version_example @elastic/kibana-security +x-pack/test/encrypted_saved_objects_api_integration/plugins/api_consumer_plugin @elastic/kibana-security +src/plugins/esql @elastic/kibana-esql +packages/kbn-esql-ast @elastic/kibana-esql +examples/esql_ast_inspector @elastic/kibana-esql +src/plugins/esql_datagrid @elastic/kibana-esql +packages/kbn-esql-editor @elastic/kibana-esql +packages/kbn-esql-utils @elastic/kibana-esql +packages/kbn-esql-validation-autocomplete @elastic/kibana-esql +examples/esql_validation_example @elastic/kibana-esql +test/plugin_functional/plugins/eui_provider_dev_warning @elastic/appex-sharedux +packages/kbn-event-annotation-common @elastic/kibana-visualizations +packages/kbn-event-annotation-components @elastic/kibana-visualizations +src/plugins/event_annotation_listing @elastic/kibana-visualizations +src/plugins/event_annotation @elastic/kibana-visualizations +x-pack/test/plugin_api_integration/plugins/event_log @elastic/response-ops +x-pack/plugins/event_log @elastic/response-ops +packages/kbn-expandable-flyout @elastic/security-threat-hunting-investigations +packages/kbn-expect @elastic/kibana-operations @elastic/appex-qa +x-pack/examples/exploratory_view_example @elastic/obs-ux-infra_services-team +x-pack/plugins/observability_solution/exploratory_view @elastic/obs-ux-management-team +src/plugins/expression_error @elastic/kibana-presentation +src/plugins/chart_expressions/expression_gauge @elastic/kibana-visualizations +src/plugins/chart_expressions/expression_heatmap @elastic/kibana-visualizations +src/plugins/expression_image @elastic/kibana-presentation +src/plugins/chart_expressions/expression_legacy_metric @elastic/kibana-visualizations +src/plugins/expression_metric @elastic/kibana-presentation +src/plugins/chart_expressions/expression_metric @elastic/kibana-visualizations +src/plugins/chart_expressions/expression_partition_vis @elastic/kibana-visualizations +src/plugins/expression_repeat_image @elastic/kibana-presentation +src/plugins/expression_reveal_image @elastic/kibana-presentation +src/plugins/expression_shape @elastic/kibana-presentation +src/plugins/chart_expressions/expression_tagcloud @elastic/kibana-visualizations +src/plugins/chart_expressions/expression_xy @elastic/kibana-visualizations +examples/expressions_explorer @elastic/kibana-visualizations +src/plugins/expressions @elastic/kibana-visualizations +packages/kbn-failed-test-reporter-cli @elastic/kibana-operations @elastic/appex-qa +examples/feature_control_examples @elastic/kibana-security +examples/feature_flags_example @elastic/kibana-core +x-pack/test/plugin_api_integration/plugins/feature_usage_test @elastic/kibana-security +x-pack/plugins/features @elastic/kibana-core +x-pack/test/security_api_integration/plugins/features_provider @elastic/kibana-security +x-pack/test/functional_execution_context/plugins/alerts @elastic/kibana-core +examples/field_formats_example @elastic/kibana-data-discovery +src/plugins/field_formats @elastic/kibana-data-discovery +packages/kbn-field-types @elastic/kibana-data-discovery +packages/kbn-field-utils @elastic/kibana-data-discovery +x-pack/plugins/fields_metadata @elastic/obs-ux-logs-team +x-pack/plugins/file_upload @elastic/kibana-gis @elastic/ml-ui +examples/files_example @elastic/appex-sharedux +src/plugins/files_management @elastic/appex-sharedux +src/plugins/files @elastic/appex-sharedux +packages/kbn-find-used-node-modules @elastic/kibana-operations +x-pack/plugins/fleet @elastic/fleet +packages/kbn-flot-charts @elastic/kibana-operations +x-pack/test/ui_capabilities/common/plugins/foo_plugin @elastic/kibana-security +packages/kbn-formatters @elastic/obs-ux-logs-team +src/plugins/ftr_apis @elastic/kibana-core +packages/kbn-ftr-common-functional-services @elastic/kibana-operations @elastic/appex-qa +packages/kbn-ftr-common-functional-ui-services @elastic/appex-qa +packages/kbn-ftr-screenshot-filename @elastic/kibana-operations @elastic/appex-qa +x-pack/test/functional_with_es_ssl/plugins/cases @elastic/response-ops +x-pack/examples/gen_ai_streaming_response_example @elastic/response-ops +packages/kbn-generate @elastic/kibana-operations +packages/kbn-generate-console-definitions @elastic/kibana-management +packages/kbn-generate-csv @elastic/appex-sharedux +packages/kbn-get-repo-files @elastic/kibana-operations +x-pack/plugins/global_search_bar @elastic/appex-sharedux +x-pack/plugins/global_search @elastic/appex-sharedux +x-pack/plugins/global_search_providers @elastic/appex-sharedux +x-pack/test/plugin_functional/plugins/global_search_test @elastic/kibana-core +x-pack/plugins/graph @elastic/kibana-visualizations +examples/grid_example @elastic/kibana-presentation +packages/kbn-grid-layout @elastic/kibana-presentation +x-pack/plugins/grokdebugger @elastic/kibana-management +packages/kbn-grouping @elastic/response-ops +packages/kbn-guided-onboarding @elastic/appex-sharedux +examples/guided_onboarding_example @elastic/appex-sharedux +src/plugins/guided_onboarding @elastic/appex-sharedux +packages/kbn-handlebars @elastic/kibana-security +packages/kbn-hapi-mocks @elastic/kibana-core +test/plugin_functional/plugins/hardening @elastic/kibana-security +packages/kbn-health-gateway-server @elastic/kibana-core +examples/hello_world @elastic/kibana-core +src/plugins/home @elastic/kibana-core +packages/home/sample_data_card @elastic/appex-sharedux +packages/home/sample_data_tab @elastic/appex-sharedux +packages/home/sample_data_types @elastic/appex-sharedux +packages/kbn-i18n @elastic/kibana-core +packages/kbn-i18n-react @elastic/kibana-core +x-pack/test/functional_embedded/plugins/iframe_embedded @elastic/kibana-core +src/plugins/image_embeddable @elastic/appex-sharedux +packages/kbn-import-locator @elastic/kibana-operations +packages/kbn-import-resolver @elastic/kibana-operations +x-pack/plugins/index_lifecycle_management @elastic/kibana-management +x-pack/plugins/index_management @elastic/kibana-management +x-pack/packages/index-management/index_management_shared_types @elastic/kibana-management +test/plugin_functional/plugins/index_patterns @elastic/kibana-data-discovery +x-pack/packages/ml/inference_integration_flyout @elastic/ml-ui +x-pack/packages/ai-infra/inference-common @elastic/appex-ai-infra +x-pack/plugins/inference @elastic/appex-ai-infra +x-pack/packages/kbn-infra-forge @elastic/obs-ux-management-team +x-pack/plugins/observability_solution/infra @elastic/obs-ux-logs-team @elastic/obs-ux-infra_services-team +x-pack/plugins/ingest_pipelines @elastic/kibana-management +src/plugins/input_control_vis @elastic/kibana-presentation +src/plugins/inspector @elastic/kibana-presentation +x-pack/plugins/integration_assistant @elastic/security-scalability +src/plugins/interactive_setup @elastic/kibana-security +test/interactive_setup_api_integration/plugins/test_endpoints @elastic/kibana-security +packages/kbn-interpreter @elastic/kibana-visualizations +x-pack/plugins/observability_solution/inventory/e2e @elastic/obs-ux-infra_services-team +x-pack/plugins/observability_solution/inventory @elastic/obs-ux-infra_services-team +x-pack/plugins/observability_solution/investigate_app @elastic/obs-ux-management-team +x-pack/plugins/observability_solution/investigate @elastic/obs-ux-management-team +packages/kbn-investigation-shared @elastic/obs-ux-management-team +packages/kbn-io-ts-utils @elastic/obs-knowledge-team +packages/kbn-ipynb @elastic/search-kibana +packages/kbn-jest-serializers @elastic/kibana-operations +packages/kbn-journeys @elastic/kibana-operations @elastic/appex-qa +packages/kbn-json-ast @elastic/kibana-operations +x-pack/packages/ml/json_schemas @elastic/ml-ui +test/health_gateway/plugins/status @elastic/kibana-core +test/plugin_functional/plugins/kbn_sample_panel_action @elastic/appex-sharedux +test/plugin_functional/plugins/kbn_top_nav @elastic/kibana-core +test/plugin_functional/plugins/kbn_tp_custom_visualizations @elastic/kibana-visualizations +test/interpreter_functional/plugins/kbn_tp_run_pipeline @elastic/kibana-core +x-pack/test/functional_cors/plugins/kibana_cors_test @elastic/kibana-security +packages/kbn-kibana-manifest-schema @elastic/kibana-operations +src/plugins/kibana_overview @elastic/appex-sharedux +src/plugins/kibana_react @elastic/appex-sharedux +src/plugins/kibana_usage_collection @elastic/kibana-core +src/plugins/kibana_utils @elastic/appex-sharedux +x-pack/plugins/kubernetes_security @elastic/kibana-cloud-security-posture +x-pack/packages/kbn-langchain @elastic/security-generative-ai +packages/kbn-language-documentation @elastic/kibana-esql +x-pack/examples/lens_config_builder_example @elastic/kibana-visualizations +packages/kbn-lens-embeddable-utils @elastic/obs-ux-infra_services-team @elastic/kibana-visualizations +packages/kbn-lens-formula-docs @elastic/kibana-visualizations +x-pack/examples/lens_embeddable_inline_editing_example @elastic/kibana-visualizations +x-pack/plugins/lens @elastic/kibana-visualizations +x-pack/plugins/license_api_guard @elastic/kibana-management +x-pack/plugins/license_management @elastic/kibana-management +x-pack/plugins/licensing @elastic/kibana-core +src/plugins/links @elastic/kibana-presentation +packages/kbn-lint-packages-cli @elastic/kibana-operations +packages/kbn-lint-ts-projects-cli @elastic/kibana-operations +x-pack/plugins/lists @elastic/security-detection-engine +examples/locator_examples @elastic/appex-sharedux +examples/locator_explorer @elastic/appex-sharedux +packages/kbn-logging @elastic/kibana-core +packages/kbn-logging-mocks @elastic/kibana-core +x-pack/plugins/observability_solution/logs_data_access @elastic/obs-knowledge-team @elastic/obs-ux-logs-team +x-pack/plugins/observability_solution/logs_explorer @elastic/obs-ux-logs-team +x-pack/plugins/observability_solution/logs_shared @elastic/obs-ux-logs-team +x-pack/plugins/logstash @elastic/logstash +packages/kbn-managed-content-badge @elastic/kibana-visualizations +packages/kbn-managed-vscode-config @elastic/kibana-operations +packages/kbn-managed-vscode-config-cli @elastic/kibana-operations +packages/kbn-management/cards_navigation @elastic/kibana-management +src/plugins/management @elastic/kibana-management +packages/kbn-management/settings/application @elastic/kibana-management +packages/kbn-management/settings/components/field_category @elastic/kibana-management +packages/kbn-management/settings/components/field_input @elastic/kibana-management +packages/kbn-management/settings/components/field_row @elastic/kibana-management +packages/kbn-management/settings/components/form @elastic/kibana-management +packages/kbn-management/settings/field_definition @elastic/kibana-management +packages/kbn-management/settings/setting_ids @elastic/appex-sharedux @elastic/kibana-management +packages/kbn-management/settings/section_registry @elastic/appex-sharedux @elastic/kibana-management +packages/kbn-management/settings/types @elastic/kibana-management +packages/kbn-management/settings/utilities @elastic/kibana-management +packages/kbn-management/storybook/config @elastic/kibana-management +test/plugin_functional/plugins/management_test_plugin @elastic/kibana-management +packages/kbn-manifest @elastic/kibana-core +packages/kbn-mapbox-gl @elastic/kibana-gis +x-pack/examples/third_party_maps_source_example @elastic/kibana-gis +src/plugins/maps_ems @elastic/kibana-gis +x-pack/plugins/maps @elastic/kibana-gis +x-pack/packages/maps/vector_tile_utils @elastic/kibana-gis +x-pack/plugins/observability_solution/metrics_data_access @elastic/obs-knowledge-team @elastic/obs-ux-infra_services-team +x-pack/packages/ml/agg_utils @elastic/ml-ui +x-pack/packages/ml/anomaly_utils @elastic/ml-ui +x-pack/packages/ml/cancellable_search @elastic/ml-ui +x-pack/packages/ml/category_validator @elastic/ml-ui +x-pack/packages/ml/chi2test @elastic/ml-ui +x-pack/packages/ml/creation_wizard_utils @elastic/ml-ui +x-pack/packages/ml/data_frame_analytics_utils @elastic/ml-ui +x-pack/packages/ml/data_grid @elastic/ml-ui +x-pack/packages/ml/data_view_utils @elastic/ml-ui +x-pack/packages/ml/date_picker @elastic/ml-ui +x-pack/packages/ml/date_utils @elastic/ml-ui +x-pack/packages/ml/error_utils @elastic/ml-ui +x-pack/packages/ml/field_stats_flyout @elastic/ml-ui +x-pack/packages/ml/in_memory_table @elastic/ml-ui +x-pack/packages/ml/is_defined @elastic/ml-ui +x-pack/packages/ml/is_populated_object @elastic/ml-ui +x-pack/packages/ml/kibana_theme @elastic/ml-ui +x-pack/packages/ml/local_storage @elastic/ml-ui +x-pack/packages/ml/nested_property @elastic/ml-ui +x-pack/packages/ml/number_utils @elastic/ml-ui +x-pack/packages/ml/parse_interval @elastic/ml-ui +x-pack/plugins/ml @elastic/ml-ui +x-pack/packages/ml/query_utils @elastic/ml-ui +x-pack/packages/ml/random_sampler_utils @elastic/ml-ui +x-pack/packages/ml/response_stream @elastic/ml-ui +x-pack/packages/ml/route_utils @elastic/ml-ui +x-pack/packages/ml/runtime_field_utils @elastic/ml-ui +x-pack/packages/ml/string_hash @elastic/ml-ui +x-pack/packages/ml/time_buckets @elastic/ml-ui +x-pack/packages/ml/trained_models_utils @elastic/ml-ui +x-pack/packages/ml/ui_actions @elastic/ml-ui +x-pack/packages/ml/url_state @elastic/ml-ui +x-pack/packages/ml/validators @elastic/ml-ui +packages/kbn-mock-idp-plugin @elastic/kibana-security +packages/kbn-mock-idp-utils @elastic/kibana-security +packages/kbn-monaco @elastic/appex-sharedux +x-pack/plugins/monitoring_collection @elastic/stack-monitoring +x-pack/plugins/monitoring @elastic/stack-monitoring +src/plugins/navigation @elastic/appex-sharedux +src/plugins/newsfeed @elastic/kibana-core +test/common/plugins/newsfeed @elastic/kibana-core +src/plugins/no_data_page @elastic/appex-sharedux +x-pack/plugins/notifications @elastic/appex-sharedux +packages/kbn-object-versioning @elastic/appex-sharedux +packages/kbn-object-versioning-utils @elastic/appex-sharedux +x-pack/plugins/observability_solution/observability_ai_assistant_app @elastic/obs-ai-assistant +x-pack/plugins/observability_solution/observability_ai_assistant_management @elastic/obs-ai-assistant +x-pack/plugins/observability_solution/observability_ai_assistant @elastic/obs-ai-assistant +x-pack/packages/observability/alert_details @elastic/obs-ux-management-team +x-pack/packages/observability/alerting_rule_utils @elastic/obs-ux-management-team +x-pack/packages/observability/alerting_test_data @elastic/obs-ux-management-team +x-pack/test/cases_api_integration/common/plugins/observability @elastic/response-ops +x-pack/packages/observability/get_padded_alert_time_range_util @elastic/obs-ux-management-team +x-pack/plugins/observability_solution/observability_logs_explorer @elastic/obs-ux-logs-team +x-pack/packages/observability/logs_overview @elastic/obs-ux-logs-team +x-pack/plugins/observability_solution/observability_onboarding/e2e @elastic/obs-ux-logs-team @elastic/obs-ux-onboarding-team +x-pack/plugins/observability_solution/observability_onboarding @elastic/obs-ux-logs-team @elastic/obs-ux-onboarding-team +x-pack/plugins/observability_solution/observability @elastic/obs-ux-management-team +x-pack/plugins/observability_solution/observability_shared @elastic/observability-ui +x-pack/packages/observability/synthetics_test_data @elastic/obs-ux-management-team +x-pack/packages/observability/observability_utils @elastic/observability-ui +x-pack/test/security_api_integration/plugins/oidc_provider @elastic/kibana-security +test/common/plugins/otel_metrics @elastic/obs-ux-infra_services-team +packages/kbn-openapi-bundler @elastic/security-detection-rule-management +packages/kbn-openapi-common @elastic/security-detection-rule-management +packages/kbn-openapi-generator @elastic/security-detection-rule-management +packages/kbn-optimizer @elastic/kibana-operations +packages/kbn-optimizer-webpack-helpers @elastic/kibana-operations +packages/kbn-osquery-io-ts-types @elastic/security-asset-management +x-pack/plugins/osquery @elastic/security-defend-workflows +examples/partial_results_example @elastic/kibana-data-discovery +x-pack/plugins/painless_lab @elastic/kibana-management +packages/kbn-panel-loader @elastic/kibana-presentation +packages/kbn-peggy @elastic/kibana-operations +packages/kbn-peggy-loader @elastic/kibana-operations +packages/kbn-performance-testing-dataset-extractor @elastic/kibana-performance-testing +packages/kbn-picomatcher @elastic/kibana-operations +packages/kbn-plugin-check @elastic/appex-sharedux +packages/kbn-plugin-generator @elastic/kibana-operations +packages/kbn-plugin-helpers @elastic/kibana-operations +examples/portable_dashboards_example @elastic/kibana-presentation +examples/preboot_example @elastic/kibana-security @elastic/kibana-core +packages/presentation/presentation_containers @elastic/kibana-presentation +src/plugins/presentation_panel @elastic/kibana-presentation +packages/presentation/presentation_publishing @elastic/kibana-presentation +src/plugins/presentation_util @elastic/kibana-presentation +x-pack/packages/ai-infra/product-doc-artifact-builder @elastic/appex-ai-infra +x-pack/plugins/observability_solution/profiling_data_access @elastic/obs-ux-infra_services-team +x-pack/plugins/observability_solution/profiling @elastic/obs-ux-infra_services-team +packages/kbn-profiling-utils @elastic/obs-ux-infra_services-team +x-pack/packages/kbn-random-sampling @elastic/kibana-visualizations +packages/kbn-react-field @elastic/kibana-data-discovery +packages/kbn-react-hooks @elastic/obs-ux-logs-team +packages/react/kibana_context/common @elastic/appex-sharedux +packages/react/kibana_context/render @elastic/appex-sharedux +packages/react/kibana_context/root @elastic/appex-sharedux +packages/react/kibana_context/styled @elastic/appex-sharedux +packages/react/kibana_context/theme @elastic/appex-sharedux +packages/react/kibana_mount @elastic/appex-sharedux +packages/kbn-recently-accessed @elastic/appex-sharedux +x-pack/plugins/remote_clusters @elastic/kibana-management +test/plugin_functional/plugins/rendering_plugin @elastic/kibana-core +packages/kbn-repo-file-maps @elastic/kibana-operations +packages/kbn-repo-info @elastic/kibana-operations +packages/kbn-repo-linter @elastic/kibana-operations +packages/kbn-repo-packages @elastic/kibana-operations +packages/kbn-repo-path @elastic/kibana-operations +packages/kbn-repo-source-classifier @elastic/kibana-operations +packages/kbn-repo-source-classifier-cli @elastic/kibana-operations +packages/kbn-reporting/common @elastic/appex-sharedux +packages/kbn-reporting/get_csv_panel_actions @elastic/appex-sharedux +packages/kbn-reporting/export_types/csv @elastic/appex-sharedux +packages/kbn-reporting/export_types/csv_common @elastic/appex-sharedux +packages/kbn-reporting/export_types/pdf @elastic/appex-sharedux +packages/kbn-reporting/export_types/pdf_common @elastic/appex-sharedux +packages/kbn-reporting/export_types/png @elastic/appex-sharedux +packages/kbn-reporting/export_types/png_common @elastic/appex-sharedux +packages/kbn-reporting/mocks_server @elastic/appex-sharedux +x-pack/plugins/reporting @elastic/appex-sharedux +packages/kbn-reporting/public @elastic/appex-sharedux +packages/kbn-reporting/server @elastic/appex-sharedux +packages/kbn-resizable-layout @elastic/kibana-data-discovery +examples/resizable_layout_examples @elastic/kibana-data-discovery +x-pack/test/plugin_functional/plugins/resolver_test @elastic/security-solution +packages/response-ops/feature_flag_service @elastic/response-ops +packages/response-ops/rule_params @elastic/response-ops +examples/response_stream @elastic/ml-ui +packages/kbn-rison @elastic/kibana-operations +x-pack/packages/rollup @elastic/kibana-management +x-pack/plugins/rollup @elastic/kibana-management +packages/kbn-router-to-openapispec @elastic/kibana-core +packages/kbn-router-utils @elastic/obs-ux-logs-team +examples/routing_example @elastic/kibana-core +packages/kbn-rrule @elastic/response-ops +packages/kbn-rule-data-utils @elastic/security-detections-response @elastic/response-ops @elastic/obs-ux-management-team +x-pack/plugins/rule_registry @elastic/response-ops @elastic/obs-ux-management-team +x-pack/plugins/runtime_fields @elastic/kibana-management +packages/kbn-safer-lodash-set @elastic/kibana-security +x-pack/test/security_api_integration/plugins/saml_provider @elastic/kibana-security +x-pack/test/plugin_api_integration/plugins/sample_task_plugin @elastic/response-ops +x-pack/test/task_manager_claimer_update_by_query/plugins/sample_task_plugin_mget @elastic/response-ops +test/plugin_functional/plugins/saved_object_export_transforms @elastic/kibana-core +test/plugin_functional/plugins/saved_object_import_warnings @elastic/kibana-core +x-pack/test/saved_object_api_integration/common/plugins/saved_object_test_plugin @elastic/kibana-security +src/plugins/saved_objects_finder @elastic/kibana-data-discovery +test/plugin_functional/plugins/saved_objects_hidden_from_http_apis_type @elastic/kibana-core +test/plugin_functional/plugins/saved_objects_hidden_type @elastic/kibana-core +src/plugins/saved_objects_management @elastic/kibana-core +src/plugins/saved_objects @elastic/kibana-core +packages/kbn-saved-objects-settings @elastic/appex-sharedux +src/plugins/saved_objects_tagging_oss @elastic/appex-sharedux +x-pack/plugins/saved_objects_tagging @elastic/appex-sharedux +src/plugins/saved_search @elastic/kibana-data-discovery +examples/screenshot_mode_example @elastic/appex-sharedux +src/plugins/screenshot_mode @elastic/appex-sharedux +x-pack/examples/screenshotting_example @elastic/appex-sharedux +x-pack/plugins/screenshotting @elastic/kibana-reporting-services +packages/kbn-screenshotting-server @elastic/appex-sharedux +packages/kbn-search-api-keys-components @elastic/search-kibana +packages/kbn-search-api-keys-server @elastic/search-kibana +packages/kbn-search-api-panels @elastic/search-kibana +x-pack/plugins/search_assistant @elastic/search-kibana +packages/kbn-search-connectors @elastic/search-kibana +x-pack/plugins/search_connectors @elastic/search-kibana +packages/kbn-search-errors @elastic/kibana-data-discovery +examples/search_examples @elastic/kibana-data-discovery +x-pack/plugins/search_homepage @elastic/search-kibana +packages/kbn-search-index-documents @elastic/search-kibana +x-pack/plugins/search_indices @elastic/search-kibana +x-pack/plugins/search_inference_endpoints @elastic/search-kibana +x-pack/plugins/search_notebooks @elastic/search-kibana +x-pack/plugins/search_playground @elastic/search-kibana +packages/kbn-search-response-warnings @elastic/kibana-data-discovery +x-pack/packages/search/shared_ui @elastic/search-kibana +packages/kbn-search-types @elastic/kibana-data-discovery +x-pack/plugins/searchprofiler @elastic/kibana-management +x-pack/test/security_api_integration/packages/helpers @elastic/kibana-security +x-pack/packages/security/api_key_management @elastic/kibana-security +x-pack/packages/security/authorization_core @elastic/kibana-security +x-pack/packages/security/authorization_core_common @elastic/kibana-security +x-pack/packages/security/form_components @elastic/kibana-security +packages/kbn-security-hardening @elastic/kibana-security +x-pack/plugins/security @elastic/kibana-security +x-pack/packages/security/plugin_types_common @elastic/kibana-security +x-pack/packages/security/plugin_types_public @elastic/kibana-security +x-pack/packages/security/plugin_types_server @elastic/kibana-security +x-pack/packages/security/role_management_model @elastic/kibana-security +x-pack/packages/security-solution/distribution_bar @elastic/kibana-cloud-security-posture +x-pack/plugins/security_solution_ess @elastic/security-solution +x-pack/packages/security-solution/features @elastic/security-threat-hunting-explore +x-pack/test/cases_api_integration/common/plugins/security_solution @elastic/response-ops +x-pack/packages/security-solution/navigation @elastic/security-threat-hunting-explore +x-pack/plugins/security_solution @elastic/security-solution +x-pack/plugins/security_solution_serverless @elastic/security-solution +x-pack/packages/security-solution/side_nav @elastic/security-threat-hunting-explore +x-pack/packages/security-solution/storybook/config @elastic/security-threat-hunting-explore +x-pack/packages/security-solution/upselling @elastic/security-threat-hunting-explore +x-pack/test/security_functional/plugins/test_endpoints @elastic/kibana-security +x-pack/packages/security/ui_components @elastic/kibana-security +packages/kbn-securitysolution-autocomplete @elastic/security-detection-engine +x-pack/packages/security-solution/data_table @elastic/security-threat-hunting-investigations +packages/kbn-securitysolution-ecs @elastic/security-threat-hunting-explore +packages/kbn-securitysolution-endpoint-exceptions-common @elastic/security-detection-engine +packages/kbn-securitysolution-es-utils @elastic/security-detection-engine +packages/kbn-securitysolution-exception-list-components @elastic/security-detection-engine +packages/kbn-securitysolution-exceptions-common @elastic/security-detection-engine +packages/kbn-securitysolution-hook-utils @elastic/security-detection-engine +packages/kbn-securitysolution-io-ts-alerting-types @elastic/security-detection-engine +packages/kbn-securitysolution-io-ts-list-types @elastic/security-detection-engine +packages/kbn-securitysolution-io-ts-types @elastic/security-detection-engine +packages/kbn-securitysolution-io-ts-utils @elastic/security-detection-engine +packages/kbn-securitysolution-list-api @elastic/security-detection-engine +packages/kbn-securitysolution-list-constants @elastic/security-detection-engine +packages/kbn-securitysolution-list-hooks @elastic/security-detection-engine +packages/kbn-securitysolution-list-utils @elastic/security-detection-engine +packages/kbn-securitysolution-lists-common @elastic/security-detection-engine +packages/kbn-securitysolution-rules @elastic/security-detection-engine +packages/kbn-securitysolution-t-grid @elastic/security-detection-engine +packages/kbn-securitysolution-utils @elastic/security-detection-engine +packages/kbn-server-http-tools @elastic/kibana-core +packages/kbn-server-route-repository @elastic/obs-knowledge-team +packages/kbn-server-route-repository-client @elastic/obs-knowledge-team +packages/kbn-server-route-repository-utils @elastic/obs-knowledge-team +x-pack/plugins/serverless @elastic/appex-sharedux +packages/serverless/settings/common @elastic/appex-sharedux @elastic/kibana-management +x-pack/plugins/serverless_observability @elastic/obs-ux-management-team +packages/serverless/settings/observability_project @elastic/appex-sharedux @elastic/kibana-management @elastic/obs-ux-management-team +packages/serverless/project_switcher @elastic/appex-sharedux +x-pack/plugins/serverless_search @elastic/search-kibana +packages/serverless/settings/search_project @elastic/search-kibana @elastic/kibana-management +packages/serverless/settings/security_project @elastic/security-solution @elastic/kibana-management +packages/serverless/storybook/config @elastic/appex-sharedux +packages/serverless/types @elastic/appex-sharedux +test/plugin_functional/plugins/session_notifications @elastic/kibana-core +x-pack/plugins/session_view @elastic/kibana-cloud-security-posture +packages/kbn-set-map @elastic/kibana-operations +examples/share_examples @elastic/appex-sharedux +src/plugins/share @elastic/appex-sharedux +packages/kbn-shared-svg @elastic/obs-ux-infra_services-team +packages/shared-ux/avatar/solution @elastic/appex-sharedux +packages/shared-ux/button/exit_full_screen @elastic/appex-sharedux +packages/shared-ux/button_toolbar @elastic/appex-sharedux +packages/shared-ux/card/no_data/impl @elastic/appex-sharedux +packages/shared-ux/card/no_data/mocks @elastic/appex-sharedux +packages/shared-ux/card/no_data/types @elastic/appex-sharedux +packages/shared-ux/chrome/navigation @elastic/appex-sharedux +packages/shared-ux/error_boundary @elastic/appex-sharedux +packages/shared-ux/file/context @elastic/appex-sharedux +packages/shared-ux/file/image/impl @elastic/appex-sharedux +packages/shared-ux/file/image/mocks @elastic/appex-sharedux +packages/shared-ux/file/mocks @elastic/appex-sharedux +packages/shared-ux/file/file_picker/impl @elastic/appex-sharedux +packages/shared-ux/file/types @elastic/appex-sharedux +packages/shared-ux/file/file_upload/impl @elastic/appex-sharedux +packages/shared-ux/file/util @elastic/appex-sharedux +packages/shared-ux/link/redirect_app/impl @elastic/appex-sharedux +packages/shared-ux/link/redirect_app/mocks @elastic/appex-sharedux +packages/shared-ux/link/redirect_app/types @elastic/appex-sharedux +packages/shared-ux/markdown/impl @elastic/appex-sharedux +packages/shared-ux/markdown/mocks @elastic/appex-sharedux +packages/shared-ux/markdown/types @elastic/appex-sharedux +packages/shared-ux/page/analytics_no_data/impl @elastic/appex-sharedux +packages/shared-ux/page/analytics_no_data/mocks @elastic/appex-sharedux +packages/shared-ux/page/analytics_no_data/types @elastic/appex-sharedux +packages/shared-ux/page/kibana_no_data/impl @elastic/appex-sharedux +packages/shared-ux/page/kibana_no_data/mocks @elastic/appex-sharedux +packages/shared-ux/page/kibana_no_data/types @elastic/appex-sharedux +packages/shared-ux/page/kibana_template/impl @elastic/appex-sharedux +packages/shared-ux/page/kibana_template/mocks @elastic/appex-sharedux +packages/shared-ux/page/kibana_template/types @elastic/appex-sharedux +packages/shared-ux/page/no_data/impl @elastic/appex-sharedux +packages/shared-ux/page/no_data_config/impl @elastic/appex-sharedux +packages/shared-ux/page/no_data_config/mocks @elastic/appex-sharedux +packages/shared-ux/page/no_data_config/types @elastic/appex-sharedux +packages/shared-ux/page/no_data/mocks @elastic/appex-sharedux +packages/shared-ux/page/no_data/types @elastic/appex-sharedux +packages/shared-ux/page/solution_nav @elastic/appex-sharedux +packages/shared-ux/prompt/no_data_views/impl @elastic/appex-sharedux +packages/shared-ux/prompt/no_data_views/mocks @elastic/appex-sharedux +packages/shared-ux/prompt/no_data_views/types @elastic/appex-sharedux +packages/shared-ux/prompt/not_found @elastic/appex-sharedux +packages/shared-ux/router/impl @elastic/appex-sharedux +packages/shared-ux/router/mocks @elastic/appex-sharedux +packages/shared-ux/router/types @elastic/appex-sharedux +packages/shared-ux/storybook/config @elastic/appex-sharedux +packages/shared-ux/storybook/mock @elastic/appex-sharedux +packages/shared-ux/modal/tabbed @elastic/appex-sharedux +packages/shared-ux/table_persist @elastic/appex-sharedux +packages/kbn-shared-ux-utility @elastic/appex-sharedux +x-pack/plugins/observability_solution/slo @elastic/obs-ux-management-team +x-pack/packages/kbn-slo-schema @elastic/obs-ux-management-team +x-pack/plugins/snapshot_restore @elastic/kibana-management +packages/kbn-some-dev-log @elastic/kibana-operations +packages/kbn-sort-package-json @elastic/kibana-operations +packages/kbn-sort-predicates @elastic/kibana-visualizations +x-pack/plugins/spaces @elastic/kibana-security +x-pack/test/spaces_api_integration/common/plugins/spaces_test_plugin @elastic/kibana-security +packages/kbn-spec-to-console @elastic/kibana-management +packages/kbn-sse-utils @elastic/obs-knowledge-team +packages/kbn-sse-utils-client @elastic/obs-knowledge-team +packages/kbn-sse-utils-server @elastic/obs-knowledge-team +x-pack/plugins/stack_alerts @elastic/response-ops +x-pack/plugins/stack_connectors @elastic/response-ops +x-pack/test/usage_collection/plugins/stack_management_usage_test @elastic/kibana-management +examples/state_containers_examples @elastic/appex-sharedux +test/server_integration/plugins/status_plugin_a @elastic/kibana-core +test/server_integration/plugins/status_plugin_b @elastic/kibana-core +packages/kbn-std @elastic/kibana-core +packages/kbn-stdio-dev-helpers @elastic/kibana-operations +packages/kbn-storybook @elastic/kibana-operations +x-pack/plugins/observability_solution/synthetics/e2e @elastic/obs-ux-management-team +x-pack/plugins/observability_solution/synthetics @elastic/obs-ux-management-team +x-pack/packages/kbn-synthetics-private-location @elastic/obs-ux-management-team +x-pack/test/alerting_api_integration/common/plugins/task_manager_fixture @elastic/response-ops +x-pack/test/plugin_api_perf/plugins/task_manager_performance @elastic/response-ops +x-pack/plugins/task_manager @elastic/response-ops +src/plugins/telemetry_collection_manager @elastic/kibana-core +x-pack/plugins/telemetry_collection_xpack @elastic/kibana-core +src/plugins/telemetry_management_section @elastic/kibana-core +src/plugins/telemetry @elastic/kibana-core +test/plugin_functional/plugins/telemetry @elastic/kibana-core +packages/kbn-telemetry-tools @elastic/kibana-core +packages/kbn-test @elastic/kibana-operations @elastic/appex-qa +packages/kbn-test-eui-helpers @elastic/kibana-visualizations +x-pack/test/licensing_plugin/plugins/test_feature_usage @elastic/kibana-security +packages/kbn-test-jest-helpers @elastic/kibana-operations @elastic/appex-qa +packages/kbn-test-subj-selector @elastic/kibana-operations @elastic/appex-qa +x-pack/test_serverless +test +x-pack/test +x-pack/performance @elastic/appex-qa +x-pack/examples/testing_embedded_lens @elastic/kibana-visualizations +x-pack/examples/third_party_lens_navigation_prompt @elastic/kibana-visualizations +x-pack/examples/third_party_vis_lens_example @elastic/kibana-visualizations +x-pack/plugins/threat_intelligence @elastic/security-threat-hunting-investigations +x-pack/plugins/timelines @elastic/security-threat-hunting-investigations +packages/kbn-timelion-grammar @elastic/kibana-visualizations +packages/kbn-timerange @elastic/obs-ux-logs-team +packages/kbn-tinymath @elastic/kibana-visualizations +packages/kbn-tooling-log @elastic/kibana-operations +x-pack/plugins/transform @elastic/ml-ui +x-pack/plugins/translations @elastic/kibana-localization +packages/kbn-transpose-utils @elastic/kibana-visualizations +x-pack/examples/triggers_actions_ui_example @elastic/response-ops +x-pack/plugins/triggers_actions_ui @elastic/response-ops +packages/kbn-triggers-actions-ui-types @elastic/response-ops +packages/kbn-try-in-console @elastic/search-kibana +packages/kbn-ts-projects @elastic/kibana-operations +packages/kbn-ts-type-check-cli @elastic/kibana-operations +packages/kbn-typed-react-router-config @elastic/obs-knowledge-team @elastic/obs-ux-management-team +packages/kbn-ui-actions-browser @elastic/appex-sharedux +x-pack/examples/ui_actions_enhanced_examples @elastic/appex-sharedux +src/plugins/ui_actions_enhanced @elastic/appex-sharedux +examples/ui_action_examples @elastic/appex-sharedux +examples/ui_actions_explorer @elastic/appex-sharedux +src/plugins/ui_actions @elastic/appex-sharedux +test/plugin_functional/plugins/ui_settings_plugin @elastic/kibana-core +packages/kbn-ui-shared-deps-npm @elastic/kibana-operations +packages/kbn-ui-shared-deps-src @elastic/kibana-operations +packages/kbn-ui-theme @elastic/kibana-operations +packages/kbn-unified-data-table @elastic/kibana-data-discovery @elastic/security-threat-hunting-investigations +packages/kbn-unified-doc-viewer @elastic/kibana-data-discovery +examples/unified_doc_viewer @elastic/kibana-core +src/plugins/unified_doc_viewer @elastic/kibana-data-discovery +packages/kbn-unified-field-list @elastic/kibana-data-discovery +examples/unified_field_list_examples @elastic/kibana-data-discovery +src/plugins/unified_histogram @elastic/kibana-data-discovery +src/plugins/unified_search @elastic/kibana-visualizations +packages/kbn-unsaved-changes-badge @elastic/kibana-data-discovery +packages/kbn-unsaved-changes-prompt @elastic/kibana-management +x-pack/plugins/upgrade_assistant @elastic/kibana-management +x-pack/plugins/observability_solution/uptime @elastic/obs-ux-management-team +x-pack/plugins/drilldowns/url_drilldown @elastic/appex-sharedux +src/plugins/url_forwarding @elastic/kibana-visualizations +src/plugins/usage_collection @elastic/kibana-core +test/plugin_functional/plugins/usage_collection @elastic/kibana-core +packages/kbn-use-tracked-promise @elastic/obs-ux-logs-team +packages/kbn-user-profile-components @elastic/kibana-security +examples/user_profile_examples @elastic/kibana-security +x-pack/test/security_api_integration/plugins/user_profiles_consumer @elastic/kibana-security +packages/kbn-utility-types @elastic/kibana-core +packages/kbn-utility-types-jest @elastic/kibana-operations +packages/kbn-utils @elastic/kibana-operations +x-pack/plugins/observability_solution/ux @elastic/obs-ux-infra_services-team +examples/v8_profiler_examples @elastic/response-ops +packages/kbn-validate-next-docs-cli @elastic/kibana-operations +src/plugins/vis_default_editor @elastic/kibana-visualizations +src/plugins/vis_types/gauge @elastic/kibana-visualizations +src/plugins/vis_types/heatmap @elastic/kibana-visualizations +src/plugins/vis_type_markdown @elastic/kibana-presentation +src/plugins/vis_types/metric @elastic/kibana-visualizations +src/plugins/vis_types/pie @elastic/kibana-visualizations +src/plugins/vis_types/table @elastic/kibana-visualizations +src/plugins/vis_types/tagcloud @elastic/kibana-visualizations +src/plugins/vis_types/timelion @elastic/kibana-visualizations +src/plugins/vis_types/timeseries @elastic/kibana-visualizations +src/plugins/vis_types/vega @elastic/kibana-visualizations +src/plugins/vis_types/vislib @elastic/kibana-visualizations +src/plugins/vis_types/xy @elastic/kibana-visualizations +packages/kbn-visualization-ui-components @elastic/kibana-visualizations +packages/kbn-visualization-utils @elastic/kibana-visualizations +src/plugins/visualizations @elastic/kibana-visualizations +x-pack/plugins/watcher @elastic/kibana-management +packages/kbn-web-worker-stub @elastic/kibana-operations +packages/kbn-whereis-pkg-cli @elastic/kibana-operations +packages/kbn-xstate-utils @elastic/obs-ux-logs-team +packages/kbn-yarn-lock-validator @elastic/kibana-operations +packages/kbn-zod @elastic/kibana-core +packages/kbn-zod-helpers @elastic/security-detection-rule-management +#### +## Everything below this line overrides the default assignments for each package. +## Items lower in the file have higher precedence: +## https://help.github.com/articles/about-codeowners/ +#### + +# The #CC# prefix delineates Code Coverage, +# used for the 'team' designator within Kibana Stats + +/x-pack/test/api_integration/apis/metrics_ui @elastic/obs-ux-logs-team @elastic/obs-ux-infra_services-team +x-pack/test_serverless/api_integration/test_suites/common/platform_security @elastic/kibana-security + +# Observability Entities Team (@elastic/obs-entities) +/x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities +/x-pack/packages/kbn-entities-schema @elastic/obs-entities +/x-pack/test/api_integration/apis/entity_manager/fixture_plugin @elastic/obs-entities +/x-pack/plugins/entity_manager @elastic/obs-entities +/x-pack/test/api_integration/apis/entity_manager @elastic/obs-entities + +# Data Discovery +/x-pack/test/api_integration/apis/kibana/kql_telemetry @elastic/kibana-data-discovery @elastic/kibana-visualizations +/x-pack/test_serverless/functional/es_archives/pre_calculated_histogram @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/es_archives/kibana_sample_data_flights_index_pattern @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/security/config.examples.ts @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/security/config.context_awareness.ts @elastic/kibana-data-discovery +/test/accessibility/apps/discover.ts @elastic/kibana-data-discovery +/test/api_integration/apis/data_views @elastic/kibana-data-discovery +/test/api_integration/apis/data_view_field_editor @elastic/kibana-data-discovery +/test/api_integration/apis/kql_telemetry @elastic/kibana-data-discovery +/test/api_integration/apis/scripts @elastic/kibana-data-discovery +/test/api_integration/apis/search @elastic/kibana-data-discovery +/test/examples/data_view_field_editor_example @elastic/kibana-data-discovery +/test/examples/discover_customization_examples @elastic/kibana-data-discovery +/test/examples/field_formats @elastic/kibana-data-discovery +/test/examples/partial_results @elastic/kibana-data-discovery +/test/examples/search @elastic/kibana-data-discovery +/test/examples/unified_field_list_examples @elastic/kibana-data-discovery +/test/functional/apps/context @elastic/kibana-data-discovery +/test/functional/apps/discover @elastic/kibana-data-discovery +/test/functional/apps/management/ccs_compatibility/_data_views_ccs.ts @elastic/kibana-data-discovery +/test/functional/apps/management/data_views @elastic/kibana-data-discovery +/test/plugin_functional/test_suites/data_plugin @elastic/kibana-data-discovery +/x-pack/test/accessibility/apps/group3/search_sessions.ts @elastic/kibana-data-discovery +/x-pack/test/api_integration/apis/management/rollup/index_patterns_extensions.js @elastic/kibana-data-discovery +/x-pack/test/api_integration/apis/search @elastic/kibana-data-discovery +/x-pack/test/examples/search_examples @elastic/kibana-data-discovery +/x-pack/test/functional/apps/data_views @elastic/kibana-data-discovery +/x-pack/test/functional/apps/discover @elastic/kibana-data-discovery +/x-pack/test/functional/apps/saved_query_management @elastic/kibana-data-discovery +/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/discover @elastic/kibana-data-discovery +/x-pack/test/search_sessions_integration @elastic/kibana-data-discovery +/x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js @elastic/kibana-data-discovery +/x-pack/test/stack_functional_integration/apps/management/_index_pattern_create.js @elastic/kibana-data-discovery +/x-pack/test/upgrade/apps/discover @elastic/kibana-data-discovery +/x-pack/test_serverless/api_integration/test_suites/common/data_views @elastic/kibana-data-discovery +/x-pack/test_serverless/api_integration/test_suites/common/data_view_field_editor @elastic/kibana-data-discovery +/x-pack/test_serverless/api_integration/test_suites/common/kql_telemetry @elastic/kibana-data-discovery +/x-pack/test_serverless/api_integration/test_suites/common/scripts_tests @elastic/kibana-data-discovery +/x-pack/test_serverless/api_integration/test_suites/common/search_oss @elastic/kibana-data-discovery +/x-pack/test_serverless/api_integration/test_suites/common/search_xpack @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/context @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/discover @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/discover_ml_uptime/discover @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/examples/data_view_field_editor_example @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/examples/discover_customization_examples @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/examples/field_formats @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/examples/partial_results @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/examples/search @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/examples/search_examples @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples @elastic/kibana-data-discovery +/x-pack/test_serverless/functional/test_suites/common/management/data_views @elastic/kibana-data-discovery +src/plugins/discover/public/context_awareness/profile_providers/security @elastic/kibana-data-discovery @elastic/security-threat-hunting-investigations + +# Platform Docs +/x-pack/test_serverless/functional/test_suites/security/screenshot_creation/index.ts @elastic/platform-docs +/x-pack/test_serverless/functional/test_suites/security/config.screenshots.ts @elastic/platform-docs + +# Visualizations +/x-pack/test/accessibility/apps/group3/graph.ts @elastic/kibana-visualizations +/x-pack/test/accessibility/apps/group2/lens.ts @elastic/kibana-visualizations +/src/plugins/visualize/ @elastic/kibana-visualizations +/x-pack/test/functional/apps/lens @elastic/kibana-visualizations +/x-pack/test/api_integration/apis/lens/ @elastic/kibana-visualizations +/test/functional/apps/visualize/ @elastic/kibana-visualizations +/x-pack/test/functional/apps/graph @elastic/kibana-visualizations +/test/api_integration/apis/event_annotations @elastic/kibana-visualizations +/x-pack/test_serverless/functional/test_suites/common/visualizations/ @elastic/kibana-visualizations +/x-pack/test_serverless/functional/fixtures/kbn_archiver/lens/ @elastic/kibana-visualizations +packages/kbn-monaco/src/esql @elastic/kibana-esql + +# Global Experience + +### Global Experience Reporting +/x-pack/test/functional/apps/dashboard/reporting/ @elastic/appex-sharedux +/x-pack/test/functional/apps/reporting/ @elastic/appex-sharedux +/x-pack/test/functional/apps/reporting_management/ @elastic/appex-sharedux +/x-pack/test/examples/screenshotting/ @elastic/appex-sharedux +/x-pack/test/functional/es_archives/lens/reporting/ @elastic/appex-sharedux +/x-pack/test/functional/es_archives/reporting/ @elastic/appex-sharedux +/x-pack/test/functional/fixtures/kbn_archiver/reporting/ @elastic/appex-sharedux +/x-pack/test/reporting_api_integration/ @elastic/appex-sharedux +/x-pack/test/reporting_functional/ @elastic/appex-sharedux +/x-pack/test/stack_functional_integration/apps/reporting/ @elastic/appex-sharedux +/docs/user/reporting @elastic/appex-sharedux +/docs/settings/reporting-settings.asciidoc @elastic/appex-sharedux +/docs/setup/configuring-reporting.asciidoc @elastic/appex-sharedux +/x-pack/test_serverless/**/test_suites/common/reporting/ @elastic/appex-sharedux +/x-pack/test/accessibility/apps/group3/reporting.ts @elastic/appex-sharedux + +### Global Experience Tagging +/x-pack/test/saved_object_tagging/ @elastic/appex-sharedux + +### Kibana React (to be deprecated) +/src/plugins/kibana_react/public/@elastic/appex-sharedux @elastic/kibana-presentation + +### Home Plugin and Packages +/src/plugins/home/public @elastic/appex-sharedux +/src/plugins/home/server/*.ts @elastic/appex-sharedux +/src/plugins/home/server/services/ @elastic/appex-sharedux + +### Code Coverage +#CC# /src/plugins/home/public @elastic/appex-sharedux +#CC# /src/plugins/home/server/services/ @elastic/appex-sharedux +#CC# /src/plugins/home/ @elastic/appex-sharedux +#CC# /x-pack/plugins/reporting/ @elastic/appex-sharedux +#CC# /x-pack/plugins/security_solution_serverless/ @elastic/appex-sharedux + +### Observability Plugins + + +# Observability AI Assistant +x-pack/test/observability_ai_assistant_api_integration @elastic/obs-ai-assistant +x-pack/test/observability_ai_assistant_functional @elastic/obs-ai-assistant +x-pack/test_serverless/**/test_suites/observability/ai_assistant @elastic/obs-ai-assistant + +# Infra Monitoring +## This plugin mostly contains the codebase for the infra services, but also includes some code for the Logs UI app. +## To keep @elastic/obs-ux-logs-team as codeowner of the plugin manifest without requiring a review for all the other code changes +## the priority on codeownership will be as follow: +## - infra -> both teams (automatically generated by script) +## - infra/{common,docs,public,server}/{sub-folders}/ -> @elastic/obs-ux-infra_services-team +## - Logs UI code exceptions -> @elastic/obs-ux-logs-team +## This should allow the infra team to work without dependencies on the @elastic/obs-ux-logs-team, which will maintain ownership of the Logs UI code only. + +## infra/{common,docs,public,server}/{sub-folders}/ -> @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/common @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/docs @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/alerting @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/apps @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/common @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/components @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/containers @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/hooks @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/images @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/lib @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/pages @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/services @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/test_utils @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/public/utils @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/server/lib @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/server/routes @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/server/saved_objects @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/server/services @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/server/usage @elastic/obs-ux-infra_services-team +/x-pack/plugins/observability_solution/infra/server/utils @elastic/obs-ux-infra_services-team + +## Logs UI code exceptions -> @elastic/obs-ux-logs-team +/x-pack/test_serverless/functional/page_objects/svl_oblt_onboarding_stream_log_file.ts @elastic/obs-ux-logs-team +/x-pack/test_serverless/functional/page_objects/svl_oblt_onboarding_page.ts @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/common/http_api/log_alerts @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/common/http_api/log_analysis @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/common/log_analysis @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/common/log_search_result @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/common/log_search_summary @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/common/log_text_scale @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/common/performance_tracing.ts @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/common/search_strategies/log_entries @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/docs/state_machines @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/public/apps/logs_app.tsx @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/public/components/log_stream @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/public/components/logging @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/public/containers/logs @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/public/observability_logs @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/public/pages/logs @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/server/lib/log_analysis @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/server/routes/log_alerts @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/server/routes/log_analysis @elastic/obs-ux-logs-team +/x-pack/plugins/observability_solution/infra/server/services/rules @elastic/obs-ux-infra_services-team @elastic/obs-ux-logs-team +# Infra Monitoring tests +/x-pack/test/api_integration/apis/infra @elastic/obs-ux-infra_services-team +/x-pack/test/functional/apps/infra @elastic/obs-ux-infra_services-team +/x-pack/test/functional/apps/infra/logs @elastic/obs-ux-logs-team + +# Observability UX management team +/x-pack/test/api_integration/apis/slos @elastic/obs-ux-management-team +/x-pack/test/accessibility/apps/group1/uptime.ts @elastic/obs-ux-management-team +/x-pack/test/accessibility/apps/group3/observability.ts @elastic/obs-ux-management-team +/x-pack/packages/observability/alert_details @elastic/obs-ux-management-team +/x-pack/test/observability_functional @elastic/obs-ux-management-team +/x-pack/plugins/observability_solution/infra/public/alerting @elastic/obs-ux-management-team +/x-pack/plugins/observability_solution/infra/server/lib/alerting @elastic/obs-ux-management-team +/x-pack/test_serverless/**/test_suites/observability/custom_threshold_rule/ @elastic/obs-ux-management-team +/x-pack/test_serverless/**/test_suites/observability/slos/ @elastic/obs-ux-management-team +/x-pack/test_serverless/api_integration/test_suites/observability/es_query_rule @elastic/obs-ux-management-team +/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/burn_rate_rule @elastic/obs-ux-management-team +/x-pack/test/api_integration/deployment_agnostic/services/alerting_api @elastic/obs-ux-management-team +/x-pack/test/api_integration/deployment_agnostic/services/slo_api @elastic/obs-ux-management-team +/x-pack/test_serverless/**/test_suites/observability/infra/ @elastic/obs-ux-infra_services-team + +# Elastic Stack Monitoring +/x-pack/test/functional/apps/monitoring @elastic/stack-monitoring +/x-pack/test/api_integration/apis/monitoring @elastic/stack-monitoring +/x-pack/test/api_integration/apis/monitoring_collection @elastic/stack-monitoring +/x-pack/test/accessibility/apps/group1/kibana_overview.ts @elastic/stack-monitoring +/x-pack/test/accessibility/apps/group3/stack_monitoring.ts @elastic/stack-monitoring + +# Fleet +/x-pack/test/fleet_api_integration @elastic/fleet +/x-pack/test/fleet_cypress @elastic/fleet +/x-pack/test/fleet_functional @elastic/fleet +/src/dev/build/tasks/bundle_fleet_packages.ts @elastic/fleet @elastic/kibana-operations +/x-pack/plugins/fleet/server/services/elastic_agent_manifest.ts @elastic/fleet @elastic/obs-cloudnative-monitoring +/x-pack/test_serverless/**/test_suites/**/fleet/ @elastic/fleet + +# APM +/x-pack/test/functional/apps/apm/ @elastic/obs-ux-infra_services-team +/x-pack/test/apm_api_integration/ @elastic/obs-ux-infra_services-team +/src/apm.js @elastic/kibana-core @vigneshshanmugam +/packages/kbn-utility-types/src/dot.ts @dgieselaar +/packages/kbn-utility-types/src/dot_test.ts @dgieselaar +/x-pack/test_serverless/api_integration/test_suites/observability/apm_api_integration/ @elastic/obs-ux-infra_services-team +#CC# /src/plugins/apm_oss/ @elastic/apm-ui +#CC# /x-pack/plugins/observability_solution/observability/ @elastic/apm-ui + +# Uptime +/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/uptime/ @elastic/obs-ux-management-team +/x-pack/test/functional/apps/uptime @elastic/obs-ux-management-team +/x-pack/test/functional/es_archives/uptime @elastic/obs-ux-management-team +/x-pack/test/functional/services/uptime @elastic/obs-ux-management-team +/x-pack/test/api_integration/apis/uptime @elastic/obs-ux-management-team +/x-pack/test/api_integration/apis/synthetics @elastic/obs-ux-management-team +/x-pack/test/alerting_api_integration/observability/synthetics_rule.ts @elastic/obs-ux-management-team +/x-pack/test/alerting_api_integration/observability/index.ts @elastic/obs-ux-management-team +/x-pack/test_serverless/api_integration/test_suites/observability/synthetics @elastic/obs-ux-management-team + +# obs-ux-logs-team +/x-pack/test_serverless/api_integration/test_suites/observability/index.feature_flags.ts @elastic/obs-ux-logs-team +/x-pack/test/api_integration/apis/logs_ui @elastic/obs-ux-logs-team +/x-pack/test/dataset_quality_api_integration @elastic/obs-ux-logs-team +/x-pack/test_serverless/api_integration/test_suites/observability/dataset_quality_api_integration @elastic/obs-ux-logs-team +/x-pack/test/functional/apps/observability_logs_explorer @elastic/obs-ux-logs-team +/x-pack/test_serverless/functional/test_suites/observability/observability_logs_explorer @elastic/obs-ux-logs-team +/x-pack/test/functional/apps/dataset_quality @elastic/obs-ux-logs-team +/x-pack/test_serverless/functional/test_suites/observability/dataset_quality @elastic/obs-ux-logs-team +/x-pack/test_serverless/functional/test_suites/observability/ @elastic/obs-ux-logs-team +/src/plugins/unified_doc_viewer/public/components/doc_viewer_logs_overview @elastic/obs-ux-logs-team +/x-pack/test/api_integration/apis/logs_shared @elastic/obs-ux-logs-team + +# Observability onboarding tour +/x-pack/plugins/observability_solution/observability_shared/public/components/tour @elastic/appex-sharedux +/x-pack/test/functional/apps/infra/tour.ts @elastic/appex-sharedux + +# Observability settings +/x-pack/plugins/observability_solution/observability/server/ui_settings.ts @elastic/obs-docs + +### END Observability Plugins + +# Presentation +/x-pack/test/functional/apps/dashboard @elastic/kibana-presentation +/x-pack/test/accessibility/apps/group3/maps.ts @elastic/kibana-presentation +/x-pack/test/accessibility/apps/group1/dashboard_panel_options.ts @elastic/kibana-presentation +/x-pack/test/accessibility/apps/group1/dashboard_links.ts @elastic/kibana-presentation +/x-pack/test/accessibility/apps/group1/dashboard_controls.ts @elastic/kibana-presentation +/test/functional/apps/dashboard/ @elastic/kibana-presentation +/test/functional/apps/dashboard_elements/ @elastic/kibana-presentation +/test/functional/services/dashboard/ @elastic/kibana-presentation +/x-pack/test/functional/apps/canvas/ @elastic/kibana-presentation +/x-pack/test_serverless/functional/test_suites/search/dashboards/ @elastic/kibana-presentation +/test/plugin_functional/test_suites/panel_actions @elastic/kibana-presentation +/x-pack/test/functional/es_archives/canvas/logstash_lens @elastic/kibana-presentation +#CC# /src/plugins/kibana_react/public/code_editor/ @elastic/kibana-presentation + +# Machine Learning +/x-pack/test/api_integration/apis/file_upload @elastic/ml-ui +/x-pack/test/accessibility/apps/group2/ml.ts @elastic/ml-ui +/x-pack/test/accessibility/apps/group3/ml_embeddables_in_dashboard.ts @elastic/ml-ui +/x-pack/test/api_integration/apis/ml/ @elastic/ml-ui +/x-pack/test/api_integration_basic/apis/ml/ @elastic/ml-ui +/x-pack/test/functional/apps/ml/ @elastic/ml-ui +/x-pack/test/functional/es_archives/ml/ @elastic/ml-ui +/x-pack/test/functional/services/ml/ @elastic/ml-ui +/x-pack/test/functional_basic/apps/ml/ @elastic/ml-ui +/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/ml/ @elastic/ml-ui +/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/ml_rule_types/ @elastic/ml-ui +/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/transform_rule_types/ @elastic/ml-ui +/x-pack/test/screenshot_creation/apps/ml_docs @elastic/ml-ui +/x-pack/test/screenshot_creation/services/ml_screenshots.ts @elastic/ml-ui +/x-pack/test_serverless/**/test_suites/**/ml/ @elastic/ml-ui +/x-pack/test_serverless/**/test_suites/common/management/transforms/ @elastic/ml-ui + +# Additional plugins and packages maintained by the ML team. +/x-pack/test/accessibility/apps/group2/transform.ts @elastic/ml-ui +/x-pack/test/api_integration/apis/aiops/ @elastic/ml-ui +/x-pack/test/api_integration/apis/transform/ @elastic/ml-ui +/x-pack/test/api_integration_basic/apis/transform/ @elastic/ml-ui +/x-pack/test/functional/apps/transform/ @elastic/ml-ui +/x-pack/test/functional/services/transform/ @elastic/ml-ui +/x-pack/test/functional_basic/apps/transform/ @elastic/ml-ui + +# Maps +#CC# /x-pack/plugins/maps/ @elastic/kibana-gis +/x-pack/test/api_integration/apis/maps/ @elastic/kibana-gis +/x-pack/test/functional/apps/maps/ @elastic/kibana-gis +/x-pack/test/functional/es_archives/maps/ @elastic/kibana-gis +/x-pack/plugins/stack_alerts/server/rule_types/geo_containment @elastic/kibana-gis +/x-pack/plugins/stack_alerts/public/rule_types/geo_containment @elastic/kibana-gis +#CC# /x-pack/plugins/file_upload @elastic/kibana-gis + +# Operations +/src/dev/license_checker/config.ts @elastic/kibana-operations +/src/dev/ @elastic/kibana-operations +/src/setup_node_env/ @elastic/kibana-operations +/src/cli/keystore/ @elastic/kibana-operations +/src/cli/serve/ @elastic/kibana-operations +/src/cli_keystore/ @elastic/kibana-operations +/.github/workflows/ @elastic/kibana-operations +/vars/ @elastic/kibana-operations +/.bazelignore @elastic/kibana-operations +/.bazeliskversion @elastic/kibana-operations +/.bazelrc @elastic/kibana-operations +/.bazelrc.common @elastic/kibana-operations +/.bazelversion @elastic/kibana-operations +/WORKSPACE.bazel @elastic/kibana-operations +/.buildkite/ @elastic/kibana-operations +/.buildkite/scripts/steps/esql_grammar_sync.sh @elastic/kibana-esql +/.buildkite/scripts/steps/esql_generate_function_metadata.sh @elastic/kibana-esql +/.buildkite/pipelines/esql_grammar_sync.yml @elastic/kibana-esql +/.buildkite/scripts/steps/code_generation/security_solution_codegen.sh @elastic/security-detection-rule-management +/kbn_pm/ @elastic/kibana-operations +/x-pack/dev-tools @elastic/kibana-operations +/catalog-info.yaml @elastic/kibana-operations @elastic/kibana-tech-leads +/.devcontainer/ @elastic/kibana-operations +/.eslintrc.js @elastic/kibana-operations +/.eslintignore @elastic/kibana-operations + +# Appex QA +/x-pack/test/functional/config.*.* @elastic/appex-qa +/x-pack/test/api_integration/ftr_provider_context.d.ts @elastic/appex-qa # Maybe this should be a glob? +/x-pack/test/accessibility/services.ts @elastic/appex-qa +/x-pack/test/accessibility/page_objects.ts @elastic/appex-qa +/x-pack/test/accessibility/ftr_provider_context.d.ts @elastic/appex-qa +/x-pack/test_serverless/tsconfig.json @elastic/appex-qa +/x-pack/test_serverless/kibana.jsonc @elastic/appex-qa +/x-pack/test_serverless/functional/test_suites/common/README.md @elastic/appex-qa +/x-pack/test_serverless/functional/page_objects/index.ts @elastic/appex-qa +/x-pack/test_serverless/functional/ftr_provider_context.d.ts @elastic/appex-qa +/x-pack/test_serverless/functional/test_suites/common/management/index.ts @elastic/appex-qa +/x-pack/test_serverless/functional/test_suites/common/examples/index.ts @elastic/appex-qa +/x-pack/test_serverless/functional/page_objects/svl_common_page.ts @elastic/appex-qa +/x-pack/test_serverless/README.md @elastic/appex-qa +/x-pack/test_serverless/api_integration/ftr_provider_context.d.ts @elastic/appex-qa +/x-pack/test_serverless/api_integration/test_suites/common/README.md @elastic/appex-qa +/src/dev/code_coverage @elastic/appex-qa +/test/functional/services/common @elastic/appex-qa +/test/functional/services/lib @elastic/appex-qa +/test/functional/services/remote @elastic/appex-qa +/test/visual_regression @elastic/appex-qa +/x-pack/test/visual_regression @elastic/appex-qa +/packages/kbn-test/src/functional_test_runner @elastic/appex-qa +/packages/kbn-performance-testing-dataset-extractor @elastic/appex-qa +/x-pack/test_serverless/**/*config.base.ts @elastic/appex-qa +/x-pack/test_serverless/**/deployment_agnostic_services.ts @elastic/appex-qa +/x-pack/test_serverless/shared/ @elastic/appex-qa +/x-pack/test_serverless/**/test_suites/**/common_configs/ @elastic/appex-qa +/x-pack/test_serverless/api_integration/test_suites/common/elasticsearch_api @elastic/appex-qa +/x-pack/test_serverless/functional/test_suites/security/ftr/ @elastic/appex-qa +/x-pack/test_serverless/functional/test_suites/common/home_page/ @elastic/appex-qa +/x-pack/test_serverless/**/services/ @elastic/appex-qa +/packages/kbn-es/src/stateful_resources/roles.yml @elastic/appex-qa +x-pack/test/api_integration/deployment_agnostic/default_configs/ @elastic/appex-qa +x-pack/test/api_integration/deployment_agnostic/services/ @elastic/appex-qa +x-pack/test/**/deployment_agnostic/ @elastic/appex-qa #temporarily to monitor tests migration + +# Core +/x-pack/test/api_integration/apis/telemetry @elastic/kibana-core +/x-pack/test/api_integration/apis/status @elastic/kibana-core +/x-pack/test/api_integration/apis/stats @elastic/kibana-core +/x-pack/test/api_integration/apis/kibana/stats @elastic/kibana-core +/x-pack/test_serverless/functional/test_suites/security/config.saved_objects_management.ts @elastic/kibana-core +/config/ @elastic/kibana-core +/config/serverless.yml @elastic/kibana-core @elastic/kibana-security +/config/serverless.es.yml @elastic/kibana-core @elastic/kibana-security +/config/serverless.oblt.yml @elastic/kibana-core @elastic/kibana-security +/config/serverless.security.yml @elastic/kibana-core @elastic/kibana-security +/typings/ @elastic/kibana-core +/test/analytics @elastic/kibana-core +/packages/kbn-test/src/jest/setup/mocks.kbn_i18n_react.js @elastic/kibana-core +/x-pack/test/saved_objects_field_count/ @elastic/kibana-core +/x-pack/test_serverless/**/test_suites/common/saved_objects_management/ @elastic/kibana-core +/x-pack/test_serverless/api_integration/test_suites/common/core/ @elastic/kibana-core +/x-pack/test_serverless/api_integration/test_suites/**/telemetry/ @elastic/kibana-core +/x-pack/test/functional/es_archives/cases/migrations/8.8.0 @elastic/response-ops + +#CC# /src/core/server/csp/ @elastic/kibana-core +#CC# /src/plugins/saved_objects/ @elastic/kibana-core +#CC# /x-pack/plugins/cloud/ @elastic/kibana-core +#CC# /x-pack/plugins/features/ @elastic/kibana-core +#CC# /x-pack/plugins/global_search/ @elastic/kibana-core +#CC# /src/plugins/newsfeed @elastic/kibana-core +#CC# /x-pack/plugins/global_search_providers/ @elastic/kibana-core + +# AppEx AI Infra +/x-pack/plugins/inference @elastic/appex-ai-infra @elastic/obs-ai-assistant @elastic/security-generative-ai + +# AppEx Platform Services Security +//x-pack/test_serverless/api_integration/test_suites/common/security_response_headers.ts @elastic/kibana-security +/x-pack/test/api_integration/apis/es @elastic/kibana-security + +/x-pack/test/api_integration/apis/features @elastic/kibana-security + +# Kibana Telemetry +/.telemetryrc.json @elastic/kibana-core +/x-pack/.telemetryrc.json @elastic/kibana-core +/src/plugins/telemetry/schema/ @elastic/kibana-core +/x-pack/plugins/telemetry_collection_xpack/schema/ @elastic/kibana-core +x-pack/plugins/cloud_integrations/cloud_full_story/server/config.ts @elastic/kibana-core @shahinakmal + +# Kibana Localization +/src/dev/i18n_tools/ @elastic/kibana-localization @elastic/kibana-core +/src/core/public/i18n/ @elastic/kibana-localization @elastic/kibana-core +#CC# /x-pack/plugins/translations/ @elastic/kibana-localization @elastic/kibana-core + +# Kibana Platform Security +/.github/codeql @elastic/kibana-security +/.github/workflows/codeql.yml @elastic/kibana-security +/.github/workflows/codeql-stats.yml @elastic/kibana-security +/src/dev/eslint/security_eslint_rule_tests.ts @elastic/kibana-security +/src/core/server/integration_tests/config/check_dynamic_config.test.ts @elastic/kibana-security +/src/plugins/telemetry/server/config/telemetry_labels.ts @elastic/kibana-security +/packages/kbn-std/src/is_internal_url.test.ts @elastic/kibana-core @elastic/kibana-security +/packages/kbn-std/src/is_internal_url.ts @elastic/kibana-core @elastic/kibana-security +/packages/kbn-std/src/parse_next_url.test.ts @elastic/kibana-core @elastic/kibana-security +/packages/kbn-std/src/parse_next_url.ts @elastic/kibana-core @elastic/kibana-security +/test/interactive_setup_api_integration/ @elastic/kibana-security +/test/interactive_setup_functional/ @elastic/kibana-security +/test/plugin_functional/plugins/hardening @elastic/kibana-security +/test/plugin_functional/test_suites/core_plugins/rendering.ts @elastic/kibana-security +/test/plugin_functional/test_suites/hardening @elastic/kibana-security +/x-pack/test/accessibility/apps/group1/login_page.ts @elastic/kibana-security +/x-pack/test/accessibility/apps/group1/roles.ts @elastic/kibana-security +/x-pack/test/accessibility/apps/group1/spaces.ts @elastic/kibana-security +/x-pack/test/accessibility/apps/group1/users.ts @elastic/kibana-security +/x-pack/test/api_integration/apis/security/ @elastic/kibana-security +/x-pack/test/api_integration/apis/spaces/ @elastic/kibana-security +/x-pack/test/ui_capabilities/ @elastic/kibana-security +/x-pack/test/encrypted_saved_objects_api_integration/ @elastic/kibana-security +/x-pack/test/functional/apps/security/ @elastic/kibana-security +/x-pack/test/functional/apps/spaces/ @elastic/kibana-security +/x-pack/test/security_api_integration/ @elastic/kibana-security +/x-pack/test/security_functional/ @elastic/kibana-security +/x-pack/test/spaces_api_integration/ @elastic/kibana-security +/x-pack/test/saved_object_api_integration/ @elastic/kibana-security +/x-pack/test_serverless/**/test_suites/common/platform_security/ @elastic/kibana-security +/x-pack/test_serverless/**/test_suites/search/platform_security/ @elastic/kibana-security +/x-pack/test_serverless/**/test_suites/security/platform_security/ @elastic/kibana-security +/x-pack/test_serverless/**/test_suites/observability/platform_security/ @elastic/kibana-security +/packages/core/http/core-http-server-internal/src/cdn_config/ @elastic/kibana-security @elastic/kibana-core +#CC# /x-pack/plugins/security/ @elastic/kibana-security + +# Response Ops team +/x-pack/test/accessibility/apps/group3/rules_connectors.ts @elastic/response-ops +/x-pack/test/functional/es_archives/cases/default @elastic/response-ops +/x-pack/test_serverless/api_integration/test_suites/observability/config.ts @elastic/response-ops +/x-pack/test_serverless/api_integration/test_suites/observability/index.ts @elastic/response-ops +/x-pack/test_serverless/functional/page_objects/svl_triggers_actions_ui_page.ts @elastic/response-ops +/x-pack/test_serverless/functional/page_objects/svl_rule_details_ui_page.ts @elastic/response-ops +/x-pack/test_serverless/functional/page_objects/svl_oblt_overview_page.ts @elastic/response-ops +/x-pack/test/alerting_api_integration/ @elastic/response-ops +/x-pack/test/alerting_api_integration/observability @elastic/obs-ux-management-team +/x-pack/test/plugin_api_integration/test_suites/task_manager/ @elastic/response-ops +/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/ @elastic/response-ops +/x-pack/test/task_manager_claimer_mget/ @elastic/response-ops +/docs/user/alerting/ @elastic/response-ops +/docs/management/connectors/ @elastic/response-ops +/x-pack/test/cases_api_integration/ @elastic/response-ops +/x-pack/test/functional/services/cases/ @elastic/response-ops +/x-pack/test/functional_with_es_ssl/apps/cases/ @elastic/response-ops +/x-pack/test/api_integration/apis/cases/ @elastic/response-ops +/x-pack/test_serverless/functional/test_suites/observability/cases @elastic/response-ops +/x-pack/test_serverless/functional/test_suites/search/cases/ @elastic/response-ops +/x-pack/test_serverless/functional/test_suites/security/ftr/cases/ @elastic/response-ops +/x-pack/test_serverless/api_integration/test_suites/search/cases/ @elastic/response-ops +/x-pack/test_serverless/api_integration/test_suites/observability/cases/ @elastic/response-ops +/x-pack/test_serverless/api_integration/test_suites/security/cases/ @elastic/response-ops +/x-pack/test_serverless/functional/test_suites/search/screenshot_creation/response_ops_docs @elastic/response-ops +/x-pack/test_serverless/functional/test_suites/security/screenshot_creation/response_ops_docs @elastic/response-ops +/x-pack/test_serverless/functional/test_suites/observability/screenshot_creation/response_ops_docs @elastic/response-ops +/x-pack/test_serverless/api_integration/test_suites/common/alerting/ @elastic/response-ops +/x-pack/test/functional/es_archives/action_task_params @elastic/response-ops +/x-pack/test/functional/es_archives/actions @elastic/response-ops +/x-pack/test/functional/es_archives/alerting @elastic/response-ops +/x-pack/test/functional/es_archives/alerts @elastic/response-ops +/x-pack/test/functional/es_archives/alerts_legacy @elastic/response-ops +/x-pack/test/functional/es_archives/observability/alerts @elastic/response-ops +/x-pack/test/functional/es_archives/actions @elastic/response-ops +/x-pack/test/functional/es_archives/rules_scheduled_task_id @elastic/response-ops +/x-pack/test/functional/es_archives/alerting/8_2_0 @elastic/response-ops +/x-pack/test/functional/es_archives/cases/signals/default @elastic/response-ops +/x-pack/test/functional/es_archives/cases/signals/hosts_users @elastic/response-ops + +# Enterprise Search +/x-pack/test_serverless/functional/page_objects/svl_ingest_pipelines.ts @elastic/search-kibana +/x-pack/test/functional/apps/dev_tools/embedded_console.ts @elastic/search-kibana +/x-pack/test/functional/apps/ingest_pipelines/feature_controls/ingest_pipelines_security.ts @elastic/search-kibana +/x-pack/test/functional/page_objects/embedded_console.ts @elastic/search-kibana +/x-pack/test/functional_enterprise_search/ @elastic/search-kibana +/x-pack/plugins/enterprise_search/public/applications/shared/doc_links @elastic/platform-docs +/x-pack/test_serverless/api_integration/test_suites/search/serverless_search @elastic/search-kibana +/x-pack/test_serverless/functional/test_suites/search/ @elastic/search-kibana +/x-pack/test_serverless/functional/test_suites/search/config.ts @elastic/search-kibana @elastic/appex-qa +x-pack/test/api_integration/apis/management/index_management/inference_endpoints.ts @elastic/search-kibana +/x-pack/test_serverless/api_integration/test_suites/search @elastic/search-kibana +/x-pack/test_serverless/functional/page_objects/svl_api_keys.ts @elastic/search-kibana +/x-pack/test_serverless/functional/page_objects/svl_search_* @elastic/search-kibana +/x-pack/test/functional_search/ @elastic/search-kibana + +# Management Experience - Deployment Management +/x-pack/test/api_integration/services/index_management.ts @elastic/kibana-management +/x-pack/test/functional/services/grok_debugger.js @elastic/kibana-management +/x-pack/test/functional/apps/grok_debugger @elastic/kibana-management +/x-pack/test/functional/apps/index_lifecycle_management @elastic/kibana-management +/x-pack/test/functional/apps/index_management @elastic/kibana-management +/x-pack/test/api_integration/services/ingest_pipelines @elastic/kibana-management +/x-pack/test/functional/apps/watcher @elastic/kibana-management +/x-pack/test/api_integration/apis/watcher @elastic/kibana-management +/x-pack/test/api_integration/apis/upgrade_assistant @elastic/kibana-management +/x-pack/test/api_integration/apis/searchprofiler @elastic/kibana-management +/x-pack/test/api_integration/apis/console @elastic/kibana-management +/x-pack/test_serverless/**/test_suites/common/index_management/ @elastic/kibana-management +/x-pack/test_serverless/**/test_suites/common/management/index_management/ @elastic/kibana-management +/x-pack/test_serverless/**/test_suites/common/painless_lab/ @elastic/kibana-management +/x-pack/test_serverless/**/test_suites/common/console/ @elastic/kibana-management +/x-pack/test_serverless/api_integration/test_suites/common/management/ @elastic/kibana-management +/x-pack/test_serverless/api_integration/test_suites/common/search_profiler/ @elastic/kibana-management +/x-pack/test_serverless/functional/test_suites/**/advanced_settings.ts @elastic/kibana-management +/x-pack/test_serverless/functional/test_suites/common/management/disabled_uis.ts @elastic/kibana-management +/x-pack/test_serverless/functional/test_suites/common/management/ingest_pipelines.ts @elastic/kibana-management +/x-pack/test_serverless/functional/test_suites/common/management/landing_page.ts @elastic/kibana-management +/x-pack/test_serverless/functional/test_suites/common/dev_tools/ @elastic/kibana-management +/x-pack/test_serverless/**/test_suites/common/grok_debugger/ @elastic/kibana-management +/x-pack/test/api_integration/apis/management/ @elastic/kibana-management +/x-pack/test/functional/apps/rollup_job/ @elastic/kibana-management +/x-pack/test/api_integration/apis/grok_debugger @elastic/kibana-management +/x-pack/test/accessibility/apps/group1/advanced_settings.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/**/grok_debugger.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group1/helpers.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group1/home.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group1/index_lifecycle_management.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group1/ingest_node_pipelines.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group1/management.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group1/painless_lab.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group1/search_profiler.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group3/cross_cluster_replication.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group3/license_management.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group3/remote_clusters.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group3/rollup_jobs.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group3/upgrade_assistant.ts @elastic/kibana-management +/x-pack/test/accessibility/apps/group3/watcher.ts @elastic/kibana-management + +#CC# /x-pack/plugins/cross_cluster_replication/ @elastic/kibana-management + +# Security Solution +/x-pack/test/common/services/security_solution @elastic/security-solution +/x-pack/test/api_integration/services/security_solution_*.gen.ts @elastic/security-solution +/x-pack/test/accessibility/apps/group3/security_solution.ts @elastic/security-solution +/x-pack/test_serverless/functional/test_suites/security/config.ts @elastic/security-solution @elastic/appex-qa +/x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts @elastic/security-solution +/x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts @elastic/security-solution +/x-pack/test_serverless/functional/test_suites/common/spaces/multiple_spaces_enabled.ts @elastic/security-solution +/x-pack/test/functional/es_archives/endpoint/ @elastic/security-solution +/x-pack/test/plugin_functional/test_suites/resolver/ @elastic/security-solution +/x-pack/test/security_solution_api_integration @elastic/security-solution +/x-pack/test/api_integration/apis/security_solution @elastic/security-solution +/x-pack/test/functional/es_archives/auditbeat/default @elastic/security-solution +/x-pack/test/functional/es_archives/auditbeat/hosts @elastic/security-solution +/x-pack/test_serverless/functional/page_objects/svl_management_page.ts @elastic/security-solution +/x-pack/test_serverless/api_integration/test_suites/security @elastic/security-solution + +/x-pack/test_serverless/functional/test_suites/security/index.feature_flags.ts @elastic/security-solution +/x-pack/test_serverless/functional/test_suites/security/index.ts @elastic/security-solution +#CC# /x-pack/plugins/security_solution/ @elastic/security-solution +/x-pack/test/functional/es_archives/cases/signals/duplicate_ids @elastic/response-ops + +# Security Solution OpenAPI bundles +/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_detections_api_* @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_endpoint_management_api_* @elastic/security-defend-workflows +/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_* @elastic/security-entity-analytics +/x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_timeline_api_* @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_detections_api_* @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_endpoint_management_api_* @elastic/security-defend-workflows +/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_* @elastic/security-entity-analytics +/x-pack/plugins/security_solution/docs/openapi/ess/security_solution_timeline_api_* @elastic/security-threat-hunting-investigations + +# Security Solution Offering plugins +# TODO: assign sub directories to sub teams +/x-pack/plugins/security_solution_ess/ @elastic/security-solution +/x-pack/plugins/security_solution_serverless/ @elastic/security-solution + +# GenAI in Security Solution +/x-pack/plugins/security_solution/public/assistant @elastic/security-generative-ai +/x-pack/plugins/security_solution/public/attack_discovery @elastic/security-generative-ai +/x-pack/test/security_solution_cypress/cypress/e2e/ai_assistant @elastic/security-generative-ai + +# Security Solution cross teams ownership +/x-pack/test/security_solution_cypress/cypress/fixtures @elastic/security-detections-response @elastic/security-threat-hunting +/x-pack/test/security_solution_cypress/cypress/helpers @elastic/security-detections-response @elastic/security-threat-hunting +/x-pack/test/security_solution_cypress/cypress/objects @elastic/security-detections-response @elastic/security-threat-hunting +/x-pack/test/security_solution_cypress/cypress/plugins @elastic/security-detections-response @elastic/security-threat-hunting +/x-pack/test/security_solution_cypress/cypress/screens/common @elastic/security-detections-response @elastic/security-threat-hunting +/x-pack/test/security_solution_cypress/cypress/support @elastic/security-detections-response @elastic/security-threat-hunting +/x-pack/test/security_solution_cypress/cypress/urls @elastic/security-threat-hunting-investigations @elastic/security-detection-engine + +/x-pack/plugins/security_solution/common/ecs @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/common/test @elastic/security-detections-response @elastic/security-threat-hunting + +/x-pack/plugins/security_solution/public/common/components/callouts @elastic/security-detections-response +/x-pack/plugins/security_solution/public/common/components/hover_actions @elastic/security-threat-hunting-explore @elastic/security-threat-hunting-investigations + +/x-pack/plugins/security_solution/server/routes @elastic/security-detections-response @elastic/security-threat-hunting +/x-pack/plugins/security_solution/server/utils @elastic/security-detections-response @elastic/security-threat-hunting +x-pack/test/security_solution_api_integration/test_suites/detections_response/utils @elastic/security-detections-response +x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry @elastic/security-detections-response +x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles @elastic/security-detections-response +x-pack/test/security_solution_api_integration/test_suites/explore @elastic/security-threat-hunting-explore +x-pack/test/security_solution_api_integration/test_suites/investigations @elastic/security-threat-hunting-investigations +x-pack/test/security_solution_api_integration/test_suites/sources @elastic/security-detections-response +/x-pack/test/common/utils/security_solution/detections_response @elastic/security-detections-response + +# Security Solution sub teams + +## Security Solution sub teams - security-engineering-productivity +## NOTE: It's important to keep this above other teams' sections because test automation doesn't process +## the CODEOWNERS file correctly. See https://github.com/elastic/kibana/issues/173307#issuecomment-1855858929 +/x-pack/test/security_solution_cypress/* @elastic/security-engineering-productivity +/x-pack/test/security_solution_cypress/cypress/* @elastic/security-engineering-productivity +/x-pack/test/security_solution_cypress/cypress/tasks/login.ts @elastic/security-engineering-productivity +/x-pack/test/security_solution_cypress/es_archives @elastic/security-engineering-productivity +/x-pack/test/security_solution_playwright @elastic/security-engineering-productivity +/x-pack/plugins/security_solution/scripts/run_cypress @MadameSheema @patrykkopycinski @maximpn @banderror + +## Security Solution sub teams - Threat Hunting + +/x-pack/plugins/security_solution/server/lib/siem_migrations @elastic/security-threat-hunting +/x-pack/plugins/security_solution/common/siem_migrations @elastic/security-threat-hunting + +## Security Solution Threat Hunting areas - Threat Hunting Investigations + +/x-pack/plugins/security_solution/common/api/timeline @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/common/search_strategy/timeline @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/common/types/timeline @elastic/security-threat-hunting-investigations + +/x-pack/test/security_solution_cypress/cypress/e2e/investigations @elastic/security-threat-hunting-investigations +/x-pack/test/security_solution_cypress/cypress/e2e/sourcerer/sourcerer_timeline.cy.ts @elastic/security-threat-hunting-investigations + +x-pack/test/security_solution_cypress/cypress/screens/expandable_flyout @elastic/security-threat-hunting-investigations +x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout @elastic/security-threat-hunting-investigations + +/x-pack/plugins/security_solution/common/timelines @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/common/components/alerts_viewer @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_action @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/common/components/event_details @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/common/components/events_viewer @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/common/components/markdown_editor @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/detections/components/alerts_kpis @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/detections/components/alerts_table @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/detections/components/alerts_info @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/flyout/document_details @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/flyout/shared @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/notes @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/resolver @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/threat_intelligence @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/timelines @elastic/security-threat-hunting-investigations + +/x-pack/plugins/security_solution/server/lib/timeline @elastic/security-threat-hunting-investigations + +## Security Solution Threat Hunting areas - Threat Hunting Explore +/x-pack/plugins/security_solution/common/api/tags @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/common/search_strategy/security_solution/network @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/common/search_strategy/security_solution/user @elastic/security-threat-hunting-explore + +/x-pack/test/security_solution_cypress/cypress/e2e/explore @elastic/security-threat-hunting-explore +/x-pack/test/security_solution_cypress/cypress/screens/hosts @elastic/security-threat-hunting-explore +/x-pack/test/security_solution_cypress/cypress/screens/network @elastic/security-threat-hunting-explore +/x-pack/test/security_solution_cypress/cypress/tasks/hosts @elastic/security-threat-hunting-explore +/x-pack/test/security_solution_cypress/cypress/tasks/network @elastic/security-threat-hunting-explore + +/x-pack/plugins/security_solution/public/app/actions @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/guided_onboarding_tour @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/charts @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/detections/components/alerts_table/grouping_settings @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/header_page @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/header_section @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/inspect @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/last_event_time @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/links @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/matrix_histogram @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/navigation @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/news_feed @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/overview_description_list @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/page @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/sidebar_header @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/tables @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/top_n @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/components/with_hover_actions @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/containers/matrix_histogram @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/common/lib/cell_actions @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/cases @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/explore @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/overview @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/public/dashboards @elastic/security-threat-hunting-explore + +/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/matrix_histogram @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network @elastic/security-threat-hunting-explore +/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/users @elastic/security-threat-hunting-explore + +/x-pack/test/functional/es_archives/auditbeat/overview @elastic/security-threat-hunting-explore +/x-pack/test/functional/es_archives/auditbeat/users @elastic/security-threat-hunting-explore + +/x-pack/test/functional/es_archives/auditbeat/uncommon_processes @elastic/security-threat-hunting-explore + +## Generative AI owner connectors +# OpenAI +/x-pack/plugins/stack_connectors/public/connector_types/openai @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra +/x-pack/plugins/stack_connectors/server/connector_types/openai @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra +/x-pack/plugins/stack_connectors/common/openai @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra +# Bedrock +/x-pack/plugins/stack_connectors/public/connector_types/bedrock @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra +/x-pack/plugins/stack_connectors/server/connector_types/bedrock @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra +/x-pack/plugins/stack_connectors/common/bedrock @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra + +# Gemini +/x-pack/plugins/stack_connectors/public/connector_types/gemini @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra +/x-pack/plugins/stack_connectors/server/connector_types/gemini @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra +/x-pack/plugins/stack_connectors/common/gemini @elastic/security-generative-ai @elastic/obs-ai-assistant @elastic/appex-ai-infra + +# Inference API +/x-pack/plugins/stack_connectors/public/connector_types/inference @elastic/appex-ai-infra @elastic/security-generative-ai @elastic/obs-ai-assistant +/x-pack/plugins/stack_connectors/server/connector_types/inference @elastic/appex-ai-infra @elastic/security-generative-ai @elastic/obs-ai-assistant +/x-pack/plugins/stack_connectors/common/inference @elastic/appex-ai-infra @elastic/security-generative-ai @elastic/obs-ai-assistant + +## Defend Workflows owner connectors +/x-pack/plugins/stack_connectors/public/connector_types/sentinelone @elastic/security-defend-workflows +/x-pack/plugins/stack_connectors/server/connector_types/sentinelone @elastic/security-defend-workflows +/x-pack/plugins/stack_connectors/common/sentinelone @elastic/security-defend-workflows +/x-pack/plugins/stack_connectors/server/connector_types/crowdstrike @elastic/security-defend-workflows +/x-pack/plugins/stack_connectors/common/crowdstrike @elastic/security-defend-workflows + +## Security Solution shared OAS schemas +/x-pack/plugins/security_solution/common/api/model @elastic/security-detection-rule-management @elastic/security-detection-engine + +## Security Solution sub teams - Detection Rule Management +/x-pack/plugins/security_solution/common/api/detection_engine/fleet_integrations @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema @elastic/security-detection-rule-management @elastic/security-detection-engine +/x-pack/plugins/security_solution/common/api/detection_engine/prebuilt_rules @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/common/api/detection_engine/rule_management @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/common/api/detection_engine/rule_monitoring @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/common/detection_engine/rule_management @elastic/security-detection-rule-management + +/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/docs/rfcs/detection_response @elastic/security-detection-rule-management @elastic/security-detection-engine +/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/rule_management @elastic/security-detection-rule-management +/x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management @elastic/security-detection-rule-management + +/x-pack/plugins/security_solution/public/common/components/health_truncate_text @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/common/components/links_to_docs @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/common/components/ml_popover @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/common/components/popover_items @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detection_engine/fleet_integrations @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detection_engine/endpoint_exceptions @elastic/security-defend-workflows +/x-pack/plugins/security_solution/public/detection_engine/rule_details_ui @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detection_engine/rule_management @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detection_engine/rule_management_ui @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detection_engine/rule_monitoring @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detections/components/callouts @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detections/components/modals/ml_job_upgrade_modal @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detections/components/rules @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detections/components/rules/rule_preview @elastic/security-detection-engine +/x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detections/mitre @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/public/rules @elastic/security-detection-rule-management + +/x-pack/plugins/security_solution/server/lib/detection_engine/fleet_integrations @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/server/lib/detection_engine/prebuilt_rules @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring @elastic/security-detection-rule-management +/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema @elastic/security-detection-rule-management @elastic/security-detection-engine + +/x-pack/plugins/security_solution/scripts/openapi @elastic/security-detection-rule-management + +## Security Solution sub teams - Detection Engine +/x-pack/plugins/security_solution/common/api/detection_engine/alert_tags @elastic/security-detection-engine +/x-pack/plugins/security_solution/common/api/detection_engine/index_management @elastic/security-detection-engine +/x-pack/plugins/security_solution/common/api/detection_engine/model/alerts @elastic/security-detection-engine +/x-pack/plugins/security_solution/common/api/detection_engine/rule_exceptions @elastic/security-detection-engine +/x-pack/plugins/security_solution/common/api/detection_engine/rule_preview @elastic/security-detection-engine +/x-pack/plugins/security_solution/common/api/detection_engine/signals @elastic/security-detection-engine +/x-pack/plugins/security_solution/common/api/detection_engine/signals_migration @elastic/security-detection-engine +/x-pack/plugins/security_solution/common/cti @elastic/security-detection-engine +/x-pack/plugins/security_solution/common/field_maps @elastic/security-detection-engine +/x-pack/test/functional/es_archives/entity/risks @elastic/security-detection-engine +/x-pack/test/functional/es_archives/entity/host_risk @elastic/security-detection-engine +/x-pack/test/api_integration/apis/lists @elastic/security-detection-engine + +/x-pack/plugins/security_solution/public/sourcerer @elastic/security-threat-hunting-investigations +/x-pack/plugins/security_solution/public/detection_engine/rule_creation @elastic/security-detection-engine +/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui @elastic/security-detection-engine +/x-pack/plugins/security_solution/public/detection_engine/rule_exceptions @elastic/security-detection-engine +/x-pack/plugins/security_solution/public/detection_engine/rule_gaps @elastic/security-detection-engine +/x-pack/plugins/security_solution/public/detections/containers/detection_engine/lists @elastic/security-detection-engine +/x-pack/plugins/security_solution/public/detections/pages/alerts @elastic/security-detection-engine +/x-pack/plugins/security_solution/public/exceptions @elastic/security-detection-engine + +/x-pack/plugins/security_solution/server/lib/detection_engine/migrations @elastic/security-detection-engine +/x-pack/plugins/security_solution/server/lib/detection_engine/rule_actions_legacy @elastic/security-detection-engine +/x-pack/plugins/security_solution/server/lib/detection_engine/rule_exceptions @elastic/security-detection-engine +/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview @elastic/security-detection-engine +/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types @elastic/security-detection-engine +/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index @elastic/security-detection-engine +/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals @elastic/security-detection-engine + +/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine @elastic/security-detection-engine + +/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine @elastic/security-detection-engine +/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/rule_gaps.ts @elastic/security-detection-engine +/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists @elastic/security-detection-engine +/x-pack/test/functional/es_archives/asset_criticality @elastic/security-detection-engine + +## Security Threat Intelligence - Under Security Platform +/x-pack/plugins/security_solution/public/common/components/threat_match @elastic/security-detection-engine + +## Security Solution sub teams - security-defend-workflows +/x-pack/test/api_integration/apis/osquery @elastic/security-defend-workflows +/x-pack/plugins/security_solution/public/management/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/public/common/lib/endpoint/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/public/common/components/endpoint/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/public/common/hooks/endpoint/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/public/common/mock/endpoint @elastic/security-defend-workflows +/x-pack/plugins/security_solution/public/flyout/document_details/isolate_host/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/common/endpoint/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/common/api/endpoint/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/server/endpoint/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/server/lists_integration/endpoint/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/server/lib/license/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/server/fleet_integration/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution/scripts/endpoint/ @elastic/security-defend-workflows +/x-pack/test/security_solution_endpoint/ @elastic/security-defend-workflows +/x-pack/test/security_solution_api_integration/test_suites/edr_workflows/ @elastic/security-defend-workflows +/x-pack/test_serverless/shared/lib/security/kibana_roles/ @elastic/security-defend-workflows +/x-pack/plugins/security_solution_serverless/public/upselling/sections/endpoint_management @elastic/security-defend-workflows +/x-pack/plugins/security_solution_serverless/public/upselling/pages/endpoint_management @elastic/security-defend-workflows +/x-pack/plugins/security_solution_serverless/server/endpoint @elastic/security-defend-workflows + +## Security Solution sub teams - security-telemetry (Data Engineering) +x-pack/plugins/security_solution/server/usage/ @elastic/security-data-analytics +x-pack/plugins/security_solution/server/lib/telemetry/ @elastic/security-data-analytics + +## Security Solution sub teams - adaptive-workload-protection +x-pack/plugins/security_solution/public/common/components/sessions_viewer @elastic/kibana-cloud-security-posture +x-pack/plugins/security_solution/public/kubernetes @elastic/kibana-cloud-security-posture + +## Security Solution sub teams - Entity Analytics +x-pack/plugins/security_solution/common/entity_analytics @elastic/security-entity-analytics +x-pack/plugins/security_solution/common/search_strategy/security_solution/risk_score @elastic/security-entity-analytics +x-pack/plugins/security_solution/public/entity_analytics @elastic/security-entity-analytics +x-pack/plugins/security_solution/server/lib/entity_analytics @elastic/security-entity-analytics +x-pack/plugins/security_solution/server/lib/risk_score @elastic/security-entity-analytics +x-pack/test/security_solution_api_integration/test_suites/entity_analytics @elastic/security-entity-analytics +x-pack/test/security_solution_cypress/cypress/e2e/entity_analytics @elastic/security-entity-analytics +x-pack/plugins/security_solution/public/flyout/entity_details @elastic/security-entity-analytics +x-pack/plugins/security_solution/common/api/entity_analytics @elastic/security-entity-analytics + +## Security Solution sub teams - GenAI +x-pack/test/security_solution_api_integration/test_suites/genai @elastic/security-generative-ai + +# Security Defend Workflows - OSQuery Ownership +x-pack/plugins/osquery @elastic/security-defend-workflows +/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_response_actions @elastic/security-defend-workflows +/x-pack/plugins/security_solution/public/detection_engine/rule_response_actions @elastic/security-defend-workflows +/x-pack/plugins/security_solution/server/lib/detection_engine/rule_response_actions @elastic/security-defend-workflows +/x-pack/plugins/security_solution/public/detections/components/osquery @elastic/security-defend-workflows + +# Cloud Defend +/x-pack/plugins/cloud_defend/ @elastic/kibana-cloud-security-posture +/x-pack/plugins/security_solution/public/cloud_defend @elastic/kibana-cloud-security-posture + +# Cloud Security Posture +/x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.* @elastic/kibana-cloud-security-posture +/x-pack/plugins/security_solution/public/cloud_security_posture @elastic/kibana-cloud-security-posture +/x-pack/test/api_integration/apis/cloud_security_posture/ @elastic/kibana-cloud-security-posture +/x-pack/test/cloud_security_posture_functional/ @elastic/kibana-cloud-security-posture +/x-pack/test/cloud_security_posture_api/ @elastic/kibana-cloud-security-posture +/x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/ @elastic/kibana-cloud-security-posture +/x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.basic.ts @elastic/kibana-cloud-security-posture +/x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.essentials.ts @elastic/kibana-cloud-security-posture +/x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/ @elastic/kibana-cloud-security-posture +/x-pack/plugins/fleet/public/components/cloud_security_posture @elastic/fleet @elastic/kibana-cloud-security-posture +/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/components/cloud_security_posture @elastic/fleet @elastic/kibana-cloud-security-posture +/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/hooks/setup_technology.* @elastic/fleet @elastic/kibana-cloud-security-posture +/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/cloud_posture_third_party_support_callout.* @elastic/fleet @elastic/kibana-cloud-security-posture +/x-pack/plugins/security_solution/public/cloud_security_posture @elastic/kibana-cloud-security-posture +/x-pack/test/security_solution_cypress/cypress/e2e/cloud_security_posture/misconfiguration_contextual_flyout.cy.ts @elastic/kibana-cloud-security-posture +/x-pack/test/security_solution_cypress/cypress/e2e/cloud_security_posture/vulnerabilities_contextual_flyout.cy.ts @elastic/kibana-cloud-security-posture + +# Security Solution onboarding tour +/x-pack/plugins/security_solution/public/common/components/guided_onboarding @elastic/security-threat-hunting-explore + +# Security Service Integrations +x-pack/plugins/security_solution/common/security_integrations @elastic/security-service-integrations +x-pack/plugins/security_solution/public/security_integrations @elastic/security-service-integrations +x-pack/plugins/security_solution/server/security_integrations @elastic/security-service-integrations +x-pack/plugins/security_solution/server/lib/security_integrations @elastic/security-service-integrations + +# Kibana design +# scss overrides should be below this line for specificity +**/*.scss @elastic/kibana-design + +# Observability design +/x-pack/plugins/fleet/**/*.scss @elastic/observability-design +/x-pack/plugins/monitoring/**/*.scss @elastic/observability-design + +# Ent. Search design +/x-pack/plugins/enterprise_search/**/*.scss @elastic/search-design +/x-pack/test/accessibility/apps/group3/enterprise_search.ts @elastic/search-kibana + +# Security design +/x-pack/plugins/endpoint/**/*.scss @elastic/security-design +/x-pack/plugins/security_solution/**/*.scss @elastic/security-design +/x-pack/plugins/security_solution_ess/**/*.scss @elastic/security-design +/x-pack/plugins/security_solution_serverless/**/*.scss @elastic/security-design + +# Logstash +/x-pack/test/api_integration/apis/logstash @elastic/logstash +#CC# /x-pack/plugins/logstash/ @elastic/logstash + +# EUI team +/src/plugins/kibana_react/public/page_template/ @elastic/eui-team @elastic/appex-sharedux + +# Landing page for guided onboarding in Home plugin +/src/plugins/home/public/application/components/guided_onboarding @elastic/appex-sharedux + +# Changes to translation files should not ping code reviewers +x-pack/plugins/translations/translations + +# Profiling api integration testing +x-pack/test/profiling_api_integration @elastic/obs-ux-infra_services-team + +# Observability shared profiling +x-pack/plugins/observability_solution/observability_shared/public/components/profiling @elastic/obs-ux-infra_services-team + +# Shared UX +/x-pack/test/api_integration/apis/content_management @elastic/appex-sharedux +/x-pack/test/accessibility/apps/group3/tags.ts @elastic/appex-sharedux +/x-pack/test/accessibility/apps/group3/snapshot_and_restore.ts @elastic/appex-sharedux +/x-pack/test_serverless/functional/test_suites/common/spaces/spaces_selection.ts @elastic/appex-sharedux +/x-pack/test_serverless/functional/test_suites/common/spaces/index.ts @elastic/appex-sharedux +packages/react @elastic/appex-sharedux +test/functional/page_objects/solution_navigation.ts @elastic/appex-sharedux +/x-pack/test_serverless/functional/page_objects/svl_common_navigation.ts @elastic/appex-sharedux +/x-pack/test_serverless/functional/fixtures/kbn_archiver/reporting @elastic/appex-sharedux +/x-pack/test_serverless/functional/page_objects/svl_sec_landing_page.ts @elastic/appex-sharedux +/x-pack/test_serverless/functional/test_suites/security/ftr/navigation.ts @elastic/appex-sharedux + +# OpenAPI spec files +oas_docs/.spectral.yaml @elastic/platform-docs +oas_docs/kibana.info.serverless.yaml @elastic/platform-docs +oas_docs/kibana.info.yaml @elastic/platform-docs + +# Plugin manifests +/src/plugins/**/kibana.jsonc @elastic/kibana-core +/x-pack/plugins/**/kibana.jsonc @elastic/kibana-core + +# Temporary Encrypted Saved Objects (ESO) guarding +# This additional code-ownership is meant to be a temporary precaution to notify the Kibana platform security team +# when an encrypted saved object is changed. Very careful review is necessary to ensure any changes are compatible +# with serverless zero downtime upgrades (ZDT). This section should be removed only when proper guidance for +# maintaining ESOs has been documented and consuming teams have acclimated to ZDT changes. +x-pack/plugins/actions/server/saved_objects/index.ts @elastic/response-ops @elastic/kibana-security +x-pack/plugins/alerting/server/saved_objects/index.ts @elastic/response-ops @elastic/kibana-security +x-pack/plugins/fleet/server/saved_objects/index.ts @elastic/fleet @elastic/kibana-security +x-pack/plugins/observability_solution/synthetics/server/saved_objects/saved_objects.ts @elastic/obs-ux-management-team @elastic/kibana-security +x-pack/plugins/observability_solution/synthetics/server/saved_objects/synthetics_monitor.ts @elastic/obs-ux-management-team @elastic/kibana-security +x-pack/plugins/observability_solution/synthetics/server/saved_objects/synthetics_param.ts @elastic/obs-ux-management-team @elastic/kibana-security + +# Specialised GitHub workflows for the Observability robots +/.github/workflows/deploy-my-kibana.yml @elastic/observablt-robots @elastic/kibana-operations +/.github/workflows/oblt-github-commands @elastic/observablt-robots @elastic/kibana-operations +/.github/workflows/undeploy-my-kibana.yml @elastic/observablt-robots @elastic/kibana-operations + +#### +## These rules are always last so they take ultimate priority over everything else +#### diff --git a/x-pack/plugins/osquery/cypress/cypress_base.config.ts b/x-pack/plugins/osquery/cypress/cypress_base.config.ts index 820df131c700f..d37ebf246576e 100644 --- a/x-pack/plugins/osquery/cypress/cypress_base.config.ts +++ b/x-pack/plugins/osquery/cypress/cypress_base.config.ts @@ -10,8 +10,8 @@ import path from 'path'; import { safeLoad as loadYaml } from 'js-yaml'; import { readFileSync } from 'fs'; import type { YamlRoleDefinitions } from '@kbn/test-suites-serverless/shared/lib'; -import { setupUserDataLoader } from '@kbn/test-suites-serverless/functional/test_suites/security/cypress/support/setup_data_loader_tasks'; import { samlAuthentication } from '@kbn/security-solution-plugin/public/management/cypress/support/saml_authentication'; +import { setupUserDataLoader } from './support/setup_data_loader_tasks'; import { getFailedSpecVideos } from './support/filter_videos'; const ROLES_YAML_FILE_PATH = path.join( diff --git a/x-pack/plugins/osquery/cypress/support/e2e.ts b/x-pack/plugins/osquery/cypress/support/e2e.ts index 3a989aa235575..7426498cd2832 100644 --- a/x-pack/plugins/osquery/cypress/support/e2e.ts +++ b/x-pack/plugins/osquery/cypress/support/e2e.ts @@ -34,11 +34,16 @@ registerCypressGrep(); import type { SecuritySolutionDescribeBlockFtrConfig } from '@kbn/security-solution-plugin/scripts/run_cypress/utils'; import { login } from '@kbn/security-solution-plugin/public/management/cypress/tasks/login'; +import type { LoadedRoleAndUser } from '@kbn/test-suites-serverless/shared/lib'; import type { ServerlessRoleName } from './roles'; import { waitUntil } from '../tasks/wait_until'; import { isCloudServerless, isServerless } from '../tasks/serverless'; +export interface LoadUserAndRoleCyTaskOptions { + name: ServerlessRoleName; +} + declare global { // eslint-disable-next-line @typescript-eslint/no-namespace namespace Cypress { @@ -49,6 +54,12 @@ declare global { } interface Chainable { + task( + name: 'loadUserAndRole', + arg: LoadUserAndRoleCyTaskOptions, + options?: Partial + ): Chainable; + getBySel(...args: Parameters): Chainable>; getBySelContains( diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/support/setup_data_loader_tasks.ts b/x-pack/plugins/osquery/cypress/support/setup_data_loader_tasks.ts similarity index 77% rename from x-pack/test_serverless/functional/test_suites/security/cypress/support/setup_data_loader_tasks.ts rename to x-pack/plugins/osquery/cypress/support/setup_data_loader_tasks.ts index 65cbcf5aac212..938fa67585f88 100644 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/support/setup_data_loader_tasks.ts +++ b/x-pack/plugins/osquery/cypress/support/setup_data_loader_tasks.ts @@ -6,12 +6,12 @@ */ import { createRuntimeServices } from '@kbn/security-solution-plugin/scripts/endpoint/common/stack_services'; -import { LoadUserAndRoleCyTaskOptions } from '../cypress'; -import { +import { SecurityRoleAndUserLoader } from '@kbn/test-suites-serverless/shared/lib'; +import type { LoadedRoleAndUser, - SecurityRoleAndUserLoader, YamlRoleDefinitions, -} from '../../../../../shared/lib'; +} from '@kbn/test-suites-serverless/shared/lib'; +import type { LoadUserAndRoleCyTaskOptions } from './e2e'; interface AdditionalDefinitions { roleDefinitions?: YamlRoleDefinitions; @@ -33,9 +33,7 @@ export const setupUserDataLoader = ( }); const roleAndUserLoaderPromise: Promise = stackServicesPromise.then( - ({ kbnClient, log }) => { - return new SecurityRoleAndUserLoader(kbnClient, log, roleDefinitions); - } + ({ kbnClient, log }) => new SecurityRoleAndUserLoader(kbnClient, log, roleDefinitions) ); on('task', { @@ -43,8 +41,7 @@ export const setupUserDataLoader = ( * Loads a user/role into Kibana. Used from `login()` task. * @param name */ - loadUserAndRole: async ({ name }: LoadUserAndRoleCyTaskOptions): Promise => { - return (await roleAndUserLoaderPromise).load(name, additionalRoleName); - }, + loadUserAndRole: async ({ name }: LoadUserAndRoleCyTaskOptions): Promise => + (await roleAndUserLoaderPromise).load(name, additionalRoleName), }); }; diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/security_config.base.ts b/x-pack/test/defend_workflows_cypress/serverless_config.base.ts similarity index 93% rename from x-pack/test_serverless/functional/test_suites/security/cypress/security_config.base.ts rename to x-pack/test/defend_workflows_cypress/serverless_config.base.ts index 515ea0c52efee..07d514687e954 100644 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/security_config.base.ts +++ b/x-pack/test/defend_workflows_cypress/serverless_config.base.ts @@ -9,7 +9,7 @@ import { FtrConfigProviderContext } from '@kbn/test'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const svlSharedConfig = await readConfigFile( - require.resolve('../../../../shared/config.base.ts') + require.resolve('@kbn/test-suites-serverless/shared/config.base') ); return { diff --git a/x-pack/test/defend_workflows_cypress/serverless_config.ts b/x-pack/test/defend_workflows_cypress/serverless_config.ts index 38c9c5040e8d3..c8dde0ebcff5d 100644 --- a/x-pack/test/defend_workflows_cypress/serverless_config.ts +++ b/x-pack/test/defend_workflows_cypress/serverless_config.ts @@ -14,9 +14,7 @@ import { DefendWorkflowsCypressCliTestRunner } from './runner'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const defendWorkflowsCypressConfig = await readConfigFile( - require.resolve( - '@kbn/test-suites-serverless/functional/test_suites/security/cypress/security_config.base' - ) + require.resolve('./serverless_config.base.ts') ); const config = defendWorkflowsCypressConfig.getAll(); const hostIp = getLocalhostRealIp(); diff --git a/x-pack/test/osquery_cypress/serverless_cli_config.ts b/x-pack/test/osquery_cypress/serverless_cli_config.ts index 0ed1be5e332d3..67df33aa34a68 100644 --- a/x-pack/test/osquery_cypress/serverless_cli_config.ts +++ b/x-pack/test/osquery_cypress/serverless_cli_config.ts @@ -12,9 +12,7 @@ import { startOsqueryCypress } from './runner'; export default async function ({ readConfigFile }: FtrConfigProviderContext) { const securitySolutionCypressConfig = await readConfigFile( - require.resolve( - '@kbn/test-suites-serverless/functional/test_suites/security/cypress/security_config.base' - ) + require.resolve('./serverless_config.base.ts') ); return { diff --git a/x-pack/test/osquery_cypress/serverless_config.base.ts b/x-pack/test/osquery_cypress/serverless_config.base.ts new file mode 100644 index 0000000000000..07d514687e954 --- /dev/null +++ b/x-pack/test/osquery_cypress/serverless_config.base.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const svlSharedConfig = await readConfigFile( + require.resolve('@kbn/test-suites-serverless/shared/config.base') + ); + + return { + ...svlSharedConfig.getAll(), + esTestCluster: { + ...svlSharedConfig.get('esTestCluster'), + serverArgs: [ + ...svlSharedConfig.get('esTestCluster.serverArgs'), + // define custom es server here + // API Keys is enabled at the top level + ], + }, + kbnTestServer: { + ...svlSharedConfig.get('kbnTestServer'), + serverArgs: [ + ...svlSharedConfig.get('kbnTestServer.serverArgs'), + '--csp.strict=false', + '--csp.warnLegacyBrowsers=false', + '--serverless=security', + ], + }, + }; +} diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/.eslintrc.json b/x-pack/test_serverless/functional/test_suites/security/cypress/.eslintrc.json deleted file mode 100644 index 22a4d052afdc5..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/.eslintrc.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "plugins": ["cypress"], - "extends": [ - "plugin:cypress/recommended" - ], - "env": { - "cypress/globals": true - }, - "rules": { - "cypress/no-force": "warn", - "import/no-extraneous-dependencies": "off" - } -} diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/.gitignore b/x-pack/test_serverless/functional/test_suites/security/cypress/.gitignore deleted file mode 100644 index c23080c54def2..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -videos -screenshots -downloads \ No newline at end of file diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/README.md b/x-pack/test_serverless/functional/test_suites/security/cypress/README.md deleted file mode 100644 index da13d4e99ce85..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Security Serverless Tests - -Before considering adding a new Cypress tests, please make sure you have added unit and API tests first and the behaviour can only be exercised with Cypress. - -Note that, the aim of Cypress is to test that the user interface operates as expected, hence, you should not be using this tool to test REST API or data contracts. - -## Folder Structure - -Below you can find the folder structure used on our Cypress tests. - -### e2e/ - -Cypress convention starting version 10 (previously known as integration). Contains the specs that are going to be executed. - -### fixtures/ - -Cypress convention. Fixtures are used as external pieces of static data when we stub responses. - -### screens/ - -Contains the elements we want to interact with in our tests. - -Each file inside the screens folder represents a screen in our application. When the screens are complex, e.g. Hosts with its multiple tabs, the page is represented by a folder and the different important parts are represented by files. - -Example: - -- screens -- hosts -- all_hosts.ts -- authentications.ts -- events.ts -- main.ts -- uncommon_processes.ts - -### tasks/ - -_Tasks_ are functions that may be reused across tests. - -Each file inside the tasks folder represents a screen of our application. When the screens are complex, e.g. Hosts with its multiple tabs, the page is represented by a folder and the different important parts are represented by files. - -Example: -- tasks -- hosts -- all_hosts.ts -- authentications.ts -- events.ts -- main.ts -- uncommon_processes.ts - -## Run tests - -Currently serverless tests are not included in any pipeline, so the execution for now should be done in our local machines. - -### Visual mode - -- Navigate to `x-pack/test_serverless/functional/test_suites/security/cypress` -- Execute `yarn cypress:serverless:open` -- Select `E2E testing` -- Click on `Start E2E testing in chrome` -- Click on the test - -### Headless mode - -- Navigate to `x-pack/test_serverless/functional/test_suites/security/cypress` -- Execute `yarn cypress:serverless:run` diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts deleted file mode 100644 index 1db2cc6e0119f..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { defineCypressConfig } from '@kbn/cypress-config'; -import { dataLoaders as setupEndpointDataLoaders } from '@kbn/security-solution-plugin/public/management/cypress/support/data_loaders'; -import { setupUserDataLoader } from './support/setup_data_loader_tasks'; - -export default defineCypressConfig({ - defaultCommandTimeout: 60000, - execTimeout: 60000, - pageLoadTimeout: 60000, - responseTimeout: 60000, - screenshotsFolder: '../../../../../../target/kibana-security-solution/cypress/screenshots', - trashAssetsBeforeRuns: false, - video: false, - viewportHeight: 946, - viewportWidth: 1680, - numTestsKeptInMemory: 10, - env: { - KIBANA_USERNAME: 'system_indices_superuser', - KIBANA_PASSWORD: 'changeme', - ELASTICSEARCH_USERNAME: 'system_indices_superuser', - ELASTICSEARCH_PASSWORD: 'changeme', - }, - e2e: { - experimentalRunAllSpecs: true, - experimentalMemoryManagement: true, - supportFile: './support/e2e.js', - specPattern: './e2e/**/*.cy.ts', - setupNodeEvents: (on, config) => { - // Reuse data loaders from endpoint management cypress setup - setupEndpointDataLoaders(on, config); - setupUserDataLoader(on, config, {}); - }, - }, -}); diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/cypress.d.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/cypress.d.ts deleted file mode 100644 index a3e6066621aa1..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/cypress.d.ts +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// / - -import { SecuritySolutionDescribeBlockFtrConfig } from '@kbn/security-solution-plugin/scripts/run_cypress/utils'; -import { - DeleteIndexedFleetEndpointPoliciesResponse, - IndexedFleetEndpointPolicyResponse, -} from '@kbn/security-solution-plugin/common/endpoint/data_loaders/index_fleet_endpoint_policy'; -import { CasePostRequest } from '@kbn/cases-plugin/common/api'; -import { - DeletedIndexedCase, - IndexedCase, -} from '@kbn/security-solution-plugin/common/endpoint/data_loaders/index_case'; -import { - HostActionResponse, - IndexEndpointHostsCyTaskOptions, -} from '@kbn/security-solution-plugin/public/management/cypress/types'; -import { IndexedHostsAndAlertsResponse } from '@kbn/security-solution-plugin/common/endpoint/index_data'; -import { DeleteIndexedEndpointHostsResponse } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/index_endpoint_hosts'; -import { - DeletedIndexedEndpointRuleAlerts, - IndexedEndpointRuleAlerts, -} from '@kbn/security-solution-plugin/common/endpoint/data_loaders/index_endpoint_rule_alerts'; -import { - HostPolicyResponse, - LogsEndpointActionResponse, -} from '@kbn/security-solution-plugin/common/endpoint/types'; -import { IndexedEndpointPolicyResponse } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/index_endpoint_policy_response'; -import { DeleteAllEndpointDataResponse } from '@kbn/security-solution-plugin/scripts/endpoint/common/delete_all_endpoint_data'; -import { LoadedRoleAndUser, ServerlessRoleName } from '../../../../shared/lib'; - -export interface LoadUserAndRoleCyTaskOptions { - name: ServerlessRoleName; -} - -declare global { - namespace Cypress { - interface SuiteConfigOverrides { - env?: { - ftrConfig: SecuritySolutionDescribeBlockFtrConfig; - }; - } - - interface Chainable { - /** - * Get Elements by `data-test-subj`. Note that his is a parent query and can only be used - * from `cy` - * - * @param args - * - * @example - * // Correct: - * cy.getByTestSubj('some-subject); - * - * // Incorrect: - * cy.get('someElement').getByTestSubj('some-subject); - */ - getByTestSubj( - ...args: Parameters['get']> - ): Chainable>; - - /** - * Finds elements by `data-test-subj` from within another. Can not be used directly from `cy`. - * - * @example - * // Correct: - * cy.get('someElement').findByTestSubj('some-subject); - * - * // Incorrect: - * cy.findByTestSubj('some-subject); - */ - findByTestSubj( - ...args: Parameters['find']> - ): Chainable>; - - /** - * Continuously call provided callback function until it either return `true` - * or fail if `timeout` is reached. - * @param fn - * @param options - */ - waitUntil( - fn: (subject?: any) => boolean | Promise | Chainable, - options?: Partial<{ - interval: number; - timeout: number; - }> - ): Chainable; - - // ---------------------------------------------------- - // - // TASKS - // - // ---------------------------------------------------- - task( - name: 'loadUserAndRole', - arg: LoadUserAndRoleCyTaskOptions, - options?: Partial - ): Chainable; - - task( - name: 'indexFleetEndpointPolicy', - arg: { - policyName: string; - endpointPackageVersion: string; - }, - options?: Partial - ): Chainable; - - task( - name: 'deleteIndexedFleetEndpointPolicies', - arg: IndexedFleetEndpointPolicyResponse, - options?: Partial - ): Chainable; - - task( - name: 'indexCase', - arg?: Partial, - options?: Partial - ): Chainable; - - task( - name: 'deleteIndexedCase', - arg: IndexedCase['data'], - options?: Partial - ): Chainable; - - task( - name: 'indexEndpointHosts', - arg?: IndexEndpointHostsCyTaskOptions, - options?: Partial - ): Chainable; - - task( - name: 'deleteIndexedEndpointHosts', - arg: IndexedHostsAndAlertsResponse, - options?: Partial - ): Chainable; - - task( - name: 'indexEndpointRuleAlerts', - arg?: { endpointAgentId: string; count?: number }, - options?: Partial - ): Chainable; - - task( - name: 'deleteIndexedEndpointRuleAlerts', - arg: IndexedEndpointRuleAlerts['alerts'], - options?: Partial - ): Chainable; - - task( - name: 'indexEndpointPolicyResponse', - arg: HostPolicyResponse, - options?: Partial - ): Chainable; - - task( - name: 'deleteIndexedEndpointPolicyResponse', - arg: IndexedEndpointPolicyResponse, - options?: Partial - ): Chainable; - - task( - name: 'sendHostActionResponse', - arg: HostActionResponse, - options?: Partial - ): Chainable; - - task( - name: 'deleteAllEndpointData', - arg: { endpointAgentIds: string[] }, - options?: Partial - ): Chainable; - - task( - name: 'createFileOnEndpoint', - arg: { hostname: string; path: string; content: string }, - options?: Partial - ): Chainable; - - task( - name: 'uploadFileToEndpoint', - arg: { hostname: string; srcPath: string; destPath: string }, - options?: Partial - ): Chainable; - - task( - name: 'installPackagesOnEndpoint', - arg: { hostname: string; packages: string[] }, - options?: Partial - ): Chainable; - - task( - name: 'readZippedFileContentOnEndpoint', - arg: { hostname: string; path: string; password?: string }, - options?: Partial - ): Chainable; - } - } -} diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/e2e/serverless.cy.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/e2e/serverless.cy.ts deleted file mode 100644 index 7000fe8ecca16..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/e2e/serverless.cy.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { LEFT_NAVIGATION } from '../screens/landing_page'; -import { navigatesToLandingPage } from '../tasks/navigation'; - -describe('Serverless', () => { - it('Should navigate to the landing page', () => { - cy.visit('/', { - auth: { - username: 'elastic_serverless', - password: 'changeme', - }, - }); - navigatesToLandingPage(); - cy.get(LEFT_NAVIGATION).should('exist'); - }); -}); diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/package.json b/x-pack/test_serverless/functional/test_suites/security/cypress/package.json deleted file mode 100644 index ef8534585d4d0..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "author": "Elastic", - "name": "@kbn/security-solution-serverless", - "version": "1.0.0", - "private": true, - "license": "Elastic License 2.0", - "scripts": { - "cypress": "NODE_OPTIONS=--openssl-legacy-provider node ../../../../../../node_modules/.bin/cypress", - "cypress:open": "NODE_OPTIONS=--openssl-legacy-provider node ../../../../../plugins/security_solution/scripts/start_cypress_parallel open --config-file ../../../x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts --ftr-config-file ../../../../../../x-pack/test_serverless/functional/test_suites/security/cypress/security_config", - "cypress:run": "NODE_OPTIONS=--openssl-legacy-provider node ../../../../../plugins/security_solution/scripts/start_cypress_parallel run --browser chrome --config-file ../../../x-pack/test_serverless/functional/test_suites/security/cypress/cypress.config.ts --ftr-config-file ../../../../../../x-pack/test_serverless/functional/test_suites/security/cypress/security_config --reporter ../../../../../../node_modules/cypress-multi-reporters --reporter-options configFile=./reporter_config.json; status=$?; yarn junit:merge && exit $status", - "junit:merge": "../../../../../../node_modules/.bin/mochawesome-merge ../../../../../../target/kibana-security-serverless/cypress/results/mochawesome*.json > ../../../../../../target/kibana-security-serverless/cypress/results/output.json && ../../../../../../node_modules/.bin/marge ../../../../../../target/kibana-security-serverless/cypress/results/output.json --reportDir ../../../../../../target/kibana-security-serverless/cypress/results && mkdir -p ../../../../../../target/junit && cp ../../../../../../target/kibana-security-serverless/cypress/results/*.xml ../../../../../../target/junit/" - } -} \ No newline at end of file diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/reporter_config.json b/x-pack/test_serverless/functional/test_suites/security/cypress/reporter_config.json deleted file mode 100644 index 616e2382a8516..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/reporter_config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "reporterEnabled": "mochawesome, mocha-junit-reporter", - "reporterOptions": { - "html": false, - "json": true, - "mochaFile": "../../../../../../target/kibana-security-serverless/cypress/results/TEST-security-solution-cypress-[hash].xml", - "overwrite": false, - "reportDir": "../../../../../../target/kibana-security-serverless/cypress/results" - } -} diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/runner.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/runner.ts deleted file mode 100644 index a83d8afbaefdc..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/runner.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../../../ftr_provider_context'; - -export type { FtrProviderContext } from '../../../ftr_provider_context'; - -export async function SecuritySolutionCypressTestRunner( - { getService }: FtrProviderContext, - envVars?: Record -) { - const config = getService('config'); - - return { - FORCE_COLOR: '1', - ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'), - ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'), - ...envVars, - }; -} diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/screens/index.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/screens/index.ts deleted file mode 100644 index 194bf6301191a..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/screens/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './landing_page'; diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/screens/landing_page.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/screens/landing_page.ts deleted file mode 100644 index 5b7450bd0492d..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/screens/landing_page.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const LEFT_NAVIGATION = '[data-test-subj="securitySolutionNavHeading"]'; diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/security_config.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/security_config.ts deleted file mode 100644 index e9b8a16c0b9c7..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/security_config.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrConfigProviderContext } from '@kbn/test'; - -import { ES_RESOURCES } from '@kbn/security-solution-plugin/scripts/endpoint/common/roles_users/serverless'; -import type { FtrProviderContext } from './runner'; -import { SecuritySolutionCypressTestRunner } from './runner'; - -export default async function ({ readConfigFile }: FtrConfigProviderContext) { - const securitySolutionCypressConfig = await readConfigFile( - require.resolve('./security_config.base.ts') - ); - - return { - ...securitySolutionCypressConfig.getAll(), - - esServerlessOptions: { - ...(securitySolutionCypressConfig.has('esServerlessOptions') - ? securitySolutionCypressConfig.get('esServerlessOptions') ?? {} - : {}), - resources: Object.values(ES_RESOURCES), - }, - - testRunner: (context: FtrProviderContext) => SecuritySolutionCypressTestRunner(context), - }; -} diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/support/commands.js b/x-pack/test_serverless/functional/test_suites/security/cypress/support/commands.js deleted file mode 100644 index 73895fbbec589..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/support/commands.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// *********************************************** -// This example commands.js shows you how to -// create various custom commands and overwrite -// existing commands. -// -// For more comprehensive examples of custom -// commands please read more here: -// https://on.cypress.io/custom-commands -// *********************************************** -// -// -// -- This is a parent command -- -// Cypress.Commands.add("login", (email, password) => { ... }) -// -// -// -- This is a child command -- -// Cypress.Commands.add("drag", { prevSubject: 'element'}, (subject, options) => { ... }) -// -// -// -- This is a dual command -- -// Cypress.Commands.add("dismiss", { prevSubject: 'optional'}, (subject, options) => { ... }) -// -// -// -- This is will overwrite an existing command -- -// Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... }) diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/support/e2e.js b/x-pack/test_serverless/functional/test_suites/security/cypress/support/e2e.js deleted file mode 100644 index 6095b2ada6c81..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/support/e2e.js +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// *********************************************************** -// This example support/index.js is processed and -// loaded automatically before your test files. -// -// This is a great place to put global configuration and -// behavior that modifies Cypress. -// -// You can change the location of this file or turn off -// automatically serving support files with the -// 'supportFile' configuration option. -// -// You can read more here: -// https://on.cypress.io/configuration -// *********************************************************** - -import './commands'; -import 'cypress-real-events/support'; -import '@kbn/security-solution-plugin/public/management/cypress/support/e2e'; - -Cypress.on('uncaught:exception', () => { - return false; -}); diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/support/index.d.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/support/index.d.ts deleted file mode 100644 index 6928ba89a56f0..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/support/index.d.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -declare namespace Cypress { - interface Chainable { - promisify(): Promise; - attachFile(fileName: string, fileType?: string): Chainable; - waitUntil( - fn: (subject: Subject) => boolean | Chainable, - options?: { - interval: number; - timeout: number; - } - ): Chainable; - } -} - -declare namespace Mocha { - interface SuiteFunction { - (title: string, ftrConfig: Record, fn: (this: Suite) => void): Suite; - ( - title: string, - ftrConfig?: Record, - config: Cypress.TestConfigOverrides, - fn: (this: Suite) => void - ): Suite; - } - - interface ExclusiveSuiteFunction { - (title: string, ftrConfig: Record, fn: (this: Suite) => void): Suite; - ( - title: string, - ftrConfig?: Record, - config: Cypress.TestConfigOverrides, - fn: (this: Suite) => void - ): Suite; - } - - interface PendingSuiteFunction { - (title: string, ftrConfig: Record, fn: (this: Suite) => void): Suite; - ( - title: string, - ftrConfig?: Record, - config: Cypress.TestConfigOverrides, - fn: (this: Suite) => void - ): Suite | void; - } -} diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/endpoint_management/index_endpoint_hosts.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/endpoint_management/index_endpoint_hosts.ts deleted file mode 100644 index 81ea6d009814d..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/endpoint_management/index_endpoint_hosts.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - DeleteIndexedHostsAndAlertsResponse, - IndexedHostsAndAlertsResponse, -} from '@kbn/security-solution-plugin/common/endpoint/index_data'; -import { IndexEndpointHostsCyTaskOptions } from '@kbn/security-solution-plugin/public/management/cypress/types'; - -export interface CyIndexEndpointHosts { - data: IndexedHostsAndAlertsResponse; - cleanup: () => Cypress.Chainable; -} - -export const indexEndpointHosts = ( - options: IndexEndpointHostsCyTaskOptions = {} -): Cypress.Chainable => { - return cy.task('indexEndpointHosts', options, { timeout: 240000 }).then((indexHosts) => { - return { - data: indexHosts, - cleanup: () => { - cy.log( - 'Deleting Endpoint Host data', - indexHosts.hosts.map((host) => `${host.host.name} (${host.host.id})`) - ); - - return cy.task('deleteIndexedEndpointHosts', indexHosts); - }, - }; - }); -}; diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts deleted file mode 100644 index 7ff366ea2cd14..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/login.ts +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { request } from '@kbn/security-solution-plugin/public/management/cypress/tasks/common'; -import { LoginState } from '@kbn/security-plugin/common/login_state'; -import type { ServerlessRoleName } from '../../../../../shared/lib'; -import { ServerlessRoleName as RoleName } from '../../../../../shared/lib/security/types'; -import { STANDARD_HTTP_HEADERS } from '../../../../../shared/lib/security/default_http_headers'; - -/** - * Send login via API - * @param username - * @param password - * - * @private - */ -const sendApiLoginRequest = ( - username: string, - password: string -): Cypress.Chainable<{ username: string; password: string }> => { - const baseUrl = Cypress.config().baseUrl; - - cy.log(`Authenticating [${username}] via ${baseUrl}`); - - const headers = { ...STANDARD_HTTP_HEADERS }; - return request({ headers, url: `${baseUrl}/internal/security/login_state` }) - .then((loginState) => { - const basicProvider = loginState.body.selector.providers.find( - (provider) => provider.type === 'basic' - ); - return request({ - url: `${baseUrl}/internal/security/login`, - method: 'POST', - headers, - body: { - providerType: basicProvider?.type, - providerName: basicProvider?.name, - currentURL: '/', - params: { username, password }, - }, - }); - }) - .then(() => ({ username, password })); -}; - -interface CyLoginTask { - (user?: ServerlessRoleName | 'elastic'): ReturnType; - - /** - * Login using any username/password - * @param username - * @param password - */ - with(username: string, password: string): ReturnType; -} - -/** - * Login to Kibana using API (not login page). By default, user will be logged in using - * the username and password defined via `KIBANA_USERNAME` and `KIBANA_PASSWORD` cypress env - * variables. - * @param user Defaults to `soc_manager` - */ -export const login: CyLoginTask = ( - user: ServerlessRoleName | 'elastic' = RoleName.SOC_MANAGER -): ReturnType => { - let username = Cypress.env('KIBANA_USERNAME'); - let password = Cypress.env('KIBANA_PASSWORD'); - - if (user && user !== 'elastic') { - return cy.task('loadUserAndRole', { name: user }).then((loadedUser) => { - username = loadedUser.username; - password = loadedUser.password; - - return sendApiLoginRequest(username, password); - }); - } else { - return sendApiLoginRequest(username, password); - } -}; - -login.with = (username: string, password: string): ReturnType => { - return sendApiLoginRequest(username, password); -}; diff --git a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/navigation.ts b/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/navigation.ts deleted file mode 100644 index af00f418747c5..0000000000000 --- a/x-pack/test_serverless/functional/test_suites/security/cypress/tasks/navigation.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const navigatesToLandingPage = () => { - cy.visit('/app/security/get_started'); -}; diff --git a/x-pack/test_serverless/tsconfig.json b/x-pack/test_serverless/tsconfig.json index fc52752b513b2..4f34639270fe9 100644 --- a/x-pack/test_serverless/tsconfig.json +++ b/x-pack/test_serverless/tsconfig.json @@ -37,7 +37,6 @@ "@kbn/server-route-repository", "@kbn/core-chrome-browser", "@kbn/security-plugin", - "@kbn/security-solution-plugin", "@kbn/security-solution-plugin/public/management/cypress", "@kbn/tooling-log", "@kbn/cases-plugin", From de6da8aa280609b722df7eddff57cbfab48b7cfa Mon Sep 17 00:00:00 2001 From: Sergi Massaneda Date: Thu, 7 Nov 2024 11:41:13 +0100 Subject: [PATCH 03/12] [8.x] [SecuritySolution][SIEM migrations] Implement background task API (#197997) (#199209) # Backport This will backport the following commits from `main` to `8.x`: - [[SecuritySolution][SIEM migrations] Implement background task API (#197997)](https://github.com/elastic/kibana/pull/197997) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --- .../common/api/quickstart_client.gen.ts | 89 +++++- .../common/siem_migrations/constants.ts | 21 +- .../model/api/rules/rules_migration.gen.ts | 63 +++- .../api/rules/rules_migration.schema.yaml | 139 ++++++++- .../model/rule_migration.gen.ts | 68 ++++- .../model/rule_migration.schema.yaml | 80 ++++- x-pack/plugins/security_solution/kibana.jsonc | 3 +- .../routes/__mocks__/request_context.ts | 6 +- .../lib/siem_migrations/__mocks__/mocks.ts | 8 +- .../__mocks__/siem_migrations_service.ts | 9 + .../siem_migrations/rules/__mocks__/mocks.ts | 51 +++- .../__mocks__/siem_rule_migrations_client.ts | 9 + .../lib/siem_migrations/rules/api/create.ts | 30 +- .../lib/siem_migrations/rules/api/get.ts | 47 +++ .../lib/siem_migrations/rules/api/index.ts | 10 + .../lib/siem_migrations/rules/api/start.ts | 91 ++++++ .../lib/siem_migrations/rules/api/stats.ts | 47 +++ .../siem_migrations/rules/api/stats_all.ts | 39 +++ .../lib/siem_migrations/rules/api/stop.ts | 50 +++ .../rules/data_stream/__mocks__/mocks.ts | 4 +- .../rule_migrations_data_client.ts | 275 +++++++++++++++++ .../rule_migrations_data_stream.test.ts | 57 ++-- .../rule_migrations_data_stream.ts | 44 ++- .../data_stream/rule_migrations_field_map.ts | 3 +- .../siem_rule_migrations_service.test.ts | 72 ++--- .../rules/siem_rule_migrations_service.ts | 59 ++-- .../siem_migrations/rules/task/agent/graph.ts | 43 +++ .../siem_migrations/rules/task/agent/index.ts | 8 + .../agent/nodes/match_prebuilt_rule/index.ts | 8 + .../match_prebuilt_rule.ts | 59 ++++ .../nodes/match_prebuilt_rule/prompts.ts | 35 +++ .../esql_knowledge_base_caller.ts | 36 +++ .../task/agent/nodes/translate_query/index.ts | 7 + .../agent/nodes/translate_query/prompt.ts | 39 +++ .../nodes/translate_query/translate_query.ts | 56 ++++ .../siem_migrations/rules/task/agent/state.ts | 32 ++ .../siem_migrations/rules/task/agent/types.ts | 23 ++ .../rules/task/rule_migrations_task_runner.ts | 285 ++++++++++++++++++ .../lib/siem_migrations/rules/task/types.ts | 70 +++++ .../rules/task/util/actions_client_chat.ts | 93 ++++++ .../rules/task/util/prebuilt_rules.test.ts | 105 +++++++ .../rules/task/util/prebuilt_rules.ts | 77 +++++ .../server/lib/siem_migrations/rules/types.ts | 48 ++- .../siem_migrations_service.test.ts | 22 +- .../siem_migrations_service.ts | 14 +- .../server/lib/siem_migrations/types.ts | 8 +- .../server/plugin_contract.ts | 2 + .../server/request_context_factory.ts | 10 +- .../plugins/security_solution/server/types.ts | 6 +- .../plugins/security_solution/tsconfig.json | 1 + .../services/security_solution_api.gen.ts | 87 +++++- 51 files changed, 2346 insertions(+), 202 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts diff --git a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts index 065601f1cf4ab..ca28715a1524e 100644 --- a/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts +++ b/x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts @@ -364,7 +364,16 @@ import type { import type { CreateRuleMigrationRequestBodyInput, CreateRuleMigrationResponse, + GetAllStatsRuleMigrationResponse, + GetRuleMigrationRequestParamsInput, GetRuleMigrationResponse, + GetRuleMigrationStatsRequestParamsInput, + GetRuleMigrationStatsResponse, + StartRuleMigrationRequestParamsInput, + StartRuleMigrationRequestBodyInput, + StartRuleMigrationResponse, + StopRuleMigrationRequestParamsInput, + StopRuleMigrationResponse, } from '../siem_migrations/model/api/rules/rules_migration.gen'; export interface ClientOptions { @@ -1205,6 +1214,21 @@ finalize it. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Retrieves the rule migrations stats for all migrations stored in the system + */ + async getAllStatsRuleMigration() { + this.log.info(`${new Date().toISOString()} Calling API GetAllStatsRuleMigration`); + return this.kbnClient + .request({ + path: '/internal/siem_migrations/rules/stats', + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'GET', + }) + .catch(catchAxiosErrorFormatAndThrow); + } /** * Get the criticality record for a specific asset. */ @@ -1395,13 +1419,28 @@ finalize it. .catch(catchAxiosErrorFormatAndThrow); } /** - * Retrieves the rule migrations stored in the system + * Retrieves the rule documents stored in the system given the rule migration id */ - async getRuleMigration() { + async getRuleMigration(props: GetRuleMigrationProps) { this.log.info(`${new Date().toISOString()} Calling API GetRuleMigration`); return this.kbnClient .request({ - path: '/internal/siem_migrations/rules', + path: replaceParams('/internal/siem_migrations/rules/{migration_id}', props.params), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'GET', + }) + .catch(catchAxiosErrorFormatAndThrow); + } + /** + * Retrieves the stats of a SIEM rules migration using the migration id provided + */ + async getRuleMigrationStats(props: GetRuleMigrationStatsProps) { + this.log.info(`${new Date().toISOString()} Calling API GetRuleMigrationStats`); + return this.kbnClient + .request({ + path: replaceParams('/internal/siem_migrations/rules/{migration_id}/stats', props.params), headers: { [ELASTIC_HTTP_VERSION_HEADER]: '1', }, @@ -1913,6 +1952,22 @@ detection engine rules. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Starts a SIEM rules migration using the migration id provided + */ + async startRuleMigration(props: StartRuleMigrationProps) { + this.log.info(`${new Date().toISOString()} Calling API StartRuleMigration`); + return this.kbnClient + .request({ + path: replaceParams('/internal/siem_migrations/rules/{migration_id}/start', props.params), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'PUT', + body: props.body, + }) + .catch(catchAxiosErrorFormatAndThrow); + } async stopEntityEngine(props: StopEntityEngineProps) { this.log.info(`${new Date().toISOString()} Calling API StopEntityEngine`); return this.kbnClient @@ -1925,6 +1980,21 @@ detection engine rules. }) .catch(catchAxiosErrorFormatAndThrow); } + /** + * Stops a running SIEM rules migration using the migration id provided + */ + async stopRuleMigration(props: StopRuleMigrationProps) { + this.log.info(`${new Date().toISOString()} Calling API StopRuleMigration`); + return this.kbnClient + .request({ + path: replaceParams('/internal/siem_migrations/rules/{migration_id}/stop', props.params), + headers: { + [ELASTIC_HTTP_VERSION_HEADER]: '1', + }, + method: 'PUT', + }) + .catch(catchAxiosErrorFormatAndThrow); + } /** * Suggests user profiles. */ @@ -2161,6 +2231,12 @@ export interface GetRuleExecutionResultsProps { query: GetRuleExecutionResultsRequestQueryInput; params: GetRuleExecutionResultsRequestParamsInput; } +export interface GetRuleMigrationProps { + params: GetRuleMigrationRequestParamsInput; +} +export interface GetRuleMigrationStatsProps { + params: GetRuleMigrationStatsRequestParamsInput; +} export interface GetTimelineProps { query: GetTimelineRequestQueryInput; } @@ -2237,9 +2313,16 @@ export interface SetAlertTagsProps { export interface StartEntityEngineProps { params: StartEntityEngineRequestParamsInput; } +export interface StartRuleMigrationProps { + params: StartRuleMigrationRequestParamsInput; + body: StartRuleMigrationRequestBodyInput; +} export interface StopEntityEngineProps { params: StopEntityEngineRequestParamsInput; } +export interface StopRuleMigrationProps { + params: StopRuleMigrationRequestParamsInput; +} export interface SuggestUserProfilesProps { query: SuggestUserProfilesRequestQueryInput; } diff --git a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts index 96ca75679f112..f2efc646a8101 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/constants.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/constants.ts @@ -8,9 +8,24 @@ export const SIEM_MIGRATIONS_PATH = '/internal/siem_migrations' as const; export const SIEM_RULE_MIGRATIONS_PATH = `${SIEM_MIGRATIONS_PATH}/rules` as const; -export enum SiemMigrationsStatus { +export const SIEM_RULE_MIGRATIONS_ALL_STATS_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/stats` as const; +export const SIEM_RULE_MIGRATIONS_GET_PATH = `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}` as const; +export const SIEM_RULE_MIGRATIONS_START_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/start` as const; +export const SIEM_RULE_MIGRATIONS_STATS_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stats` as const; +export const SIEM_RULE_MIGRATIONS_STOP_PATH = + `${SIEM_RULE_MIGRATIONS_PATH}/{migration_id}/stop` as const; + +export enum SiemMigrationStatus { PENDING = 'pending', PROCESSING = 'processing', - FINISHED = 'finished', - ERROR = 'error', + COMPLETED = 'completed', + FAILED = 'failed', +} + +export enum SiemMigrationRuleTranslationResult { + FULL = 'full', + PARTIAL = 'partial', + UNTRANSLATABLE = 'untranslatable', } diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts index fa8a1cc8a6778..120505ec43cb7 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.gen.ts @@ -16,7 +16,13 @@ import { z } from '@kbn/zod'; -import { OriginalRule, RuleMigration } from '../../rule_migration.gen'; +import { + OriginalRule, + RuleMigrationAllTaskStats, + RuleMigration, + RuleMigrationTaskStats, +} from '../../rule_migration.gen'; +import { ConnectorId, LangSmithOptions } from '../common.gen'; export type CreateRuleMigrationRequestBody = z.infer; export const CreateRuleMigrationRequestBody = z.array(OriginalRule); @@ -30,5 +36,60 @@ export const CreateRuleMigrationResponse = z.object({ migration_id: z.string(), }); +export type GetAllStatsRuleMigrationResponse = z.infer; +export const GetAllStatsRuleMigrationResponse = RuleMigrationAllTaskStats; + +export type GetRuleMigrationRequestParams = z.infer; +export const GetRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type GetRuleMigrationRequestParamsInput = z.input; + export type GetRuleMigrationResponse = z.infer; export const GetRuleMigrationResponse = z.array(RuleMigration); + +export type GetRuleMigrationStatsRequestParams = z.infer; +export const GetRuleMigrationStatsRequestParams = z.object({ + migration_id: z.string(), +}); +export type GetRuleMigrationStatsRequestParamsInput = z.input< + typeof GetRuleMigrationStatsRequestParams +>; + +export type GetRuleMigrationStatsResponse = z.infer; +export const GetRuleMigrationStatsResponse = RuleMigrationTaskStats; + +export type StartRuleMigrationRequestParams = z.infer; +export const StartRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type StartRuleMigrationRequestParamsInput = z.input; + +export type StartRuleMigrationRequestBody = z.infer; +export const StartRuleMigrationRequestBody = z.object({ + connector_id: ConnectorId, + langsmith_options: LangSmithOptions.optional(), +}); +export type StartRuleMigrationRequestBodyInput = z.input; + +export type StartRuleMigrationResponse = z.infer; +export const StartRuleMigrationResponse = z.object({ + /** + * Indicates the migration has been started. `false` means the migration does not need to be started. + */ + started: z.boolean(), +}); + +export type StopRuleMigrationRequestParams = z.infer; +export const StopRuleMigrationRequestParams = z.object({ + migration_id: z.string(), +}); +export type StopRuleMigrationRequestParamsInput = z.input; + +export type StopRuleMigrationResponse = z.infer; +export const StopRuleMigrationResponse = z.object({ + /** + * Indicates the migration has been stopped. + */ + stopped: z.boolean(), +}); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml index 40596ba7e712d..7b06c3d6a22ac 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/api/rules/rules_migration.schema.yaml @@ -10,8 +10,7 @@ paths: x-codegen-enabled: true description: Creates a new SIEM rules migration using the original vendor rules provided tags: - - SIEM Migrations - - Rule Migrations + - SIEM Rule Migrations requestBody: required: true content: @@ -33,20 +32,146 @@ paths: migration_id: type: string description: The migration id created. + + /internal/siem_migrations/rules/stats: get: - summary: Retrieves rule migrations - operationId: GetRuleMigration + summary: Retrieves the stats for all rule migrations + operationId: GetAllStatsRuleMigration x-codegen-enabled: true - description: Retrieves the rule migrations stored in the system + description: Retrieves the rule migrations stats for all migrations stored in the system tags: - - SIEM Migrations - - Rule Migrations + - SIEM Rule Migrations responses: 200: description: Indicates rule migrations have been retrieved correctly. + content: + application/json: + schema: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationAllTaskStats' + + /internal/siem_migrations/rules/{migration_id}: + get: + summary: Retrieves all the rules of a migration + operationId: GetRuleMigration + x-codegen-enabled: true + description: Retrieves the rule documents stored in the system given the rule migration id + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to start + responses: + 200: + description: Indicates rule migration have been retrieved correctly. content: application/json: schema: type: array items: $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigration' + 204: + description: Indicates the migration id was not found. + + /internal/siem_migrations/rules/{migration_id}/start: + put: + summary: Starts a rule migration + operationId: StartRuleMigration + x-codegen-enabled: true + description: Starts a SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to start + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - connector_id + properties: + connector_id: + $ref: '../common.schema.yaml#/components/schemas/ConnectorId' + langsmith_options: + $ref: '../common.schema.yaml#/components/schemas/LangSmithOptions' + responses: + 200: + description: Indicates the migration start request has been processed successfully. + content: + application/json: + schema: + type: object + required: + - started + properties: + started: + type: boolean + description: Indicates the migration has been started. `false` means the migration does not need to be started. + 204: + description: Indicates the migration id was not found. + + /internal/siem_migrations/rules/{migration_id}/stats: + get: + summary: Gets a rule migration task stats + operationId: GetRuleMigrationStats + x-codegen-enabled: true + description: Retrieves the stats of a SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to start + responses: + 200: + description: Indicates the migration stats has been retrieved correctly. + content: + application/json: + schema: + $ref: '../../rule_migration.schema.yaml#/components/schemas/RuleMigrationTaskStats' + 204: + description: Indicates the migration id was not found. + + /internal/siem_migrations/rules/{migration_id}/stop: + put: + summary: Stops an existing rule migration + operationId: StopRuleMigration + x-codegen-enabled: true + description: Stops a running SIEM rules migration using the migration id provided + tags: + - SIEM Rule Migrations + parameters: + - name: migration_id + in: path + required: true + schema: + type: string + description: The migration id to stop + responses: + 200: + description: Indicates migration task stop has been processed successfully. + content: + application/json: + schema: + type: object + required: + - stopped + properties: + stopped: + type: boolean + description: Indicates the migration has been stopped. + 204: + description: Indicates the migration id was not found running. diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts index 0e07ef2f208da..fe00c4b4df1c6 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.gen.ts @@ -71,11 +71,11 @@ export const ElasticRule = z.object({ /** * The translated elastic query. */ - query: z.string(), + query: z.string().optional(), /** * The translated elastic query language. */ - query_language: z.literal('esql').default('esql'), + query_language: z.literal('esql').optional(), /** * The Elastic prebuilt rule id matched. */ @@ -99,16 +99,20 @@ export const RuleMigration = z.object({ * The migration id. */ migration_id: z.string(), + /** + * The username of the user who created the migration. + */ + created_by: z.string(), original_rule: OriginalRule, elastic_rule: ElasticRule.optional(), /** - * The translation state. + * The rule translation result. */ - translation_state: z.enum(['complete', 'partial', 'untranslatable']).optional(), + translation_result: z.enum(['full', 'partial', 'untranslatable']).optional(), /** - * The status of the rule migration. + * The status of the rule migration process. */ - status: z.enum(['pending', 'processing', 'finished', 'error']).default('pending'), + status: z.enum(['pending', 'processing', 'completed', 'failed']).default('pending'), /** * The comments for the migration including a summary from the LLM in markdown. */ @@ -122,3 +126,55 @@ export const RuleMigration = z.object({ */ updated_by: z.string().optional(), }); + +/** + * The rule migration task stats object. + */ +export type RuleMigrationTaskStats = z.infer; +export const RuleMigrationTaskStats = z.object({ + /** + * Indicates if the migration task status. + */ + status: z.enum(['ready', 'running', 'stopped', 'finished']), + /** + * The rules migration stats. + */ + rules: z.object({ + /** + * The total number of rules to migrate. + */ + total: z.number().int(), + /** + * The number of rules that are pending migration. + */ + pending: z.number().int(), + /** + * The number of rules that are being migrated. + */ + processing: z.number().int(), + /** + * The number of rules that have been migrated successfully. + */ + completed: z.number().int(), + /** + * The number of rules that have failed migration. + */ + failed: z.number().int(), + }), + /** + * The moment of the last update. + */ + last_updated_at: z.string().optional(), +}); + +export type RuleMigrationAllTaskStats = z.infer; +export const RuleMigrationAllTaskStats = z.array( + RuleMigrationTaskStats.merge( + z.object({ + /** + * The migration id + */ + migration_id: z.string(), + }) + ) +); diff --git a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml index 9ec825389a52b..c9841856a6914 100644 --- a/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml +++ b/x-pack/plugins/security_solution/common/siem_migrations/model/rule_migration.schema.yaml @@ -48,8 +48,6 @@ components: description: The migrated elastic rule. required: - title - - query - - query_language properties: title: type: string @@ -68,7 +66,6 @@ components: description: The translated elastic query language. enum: - esql - default: esql prebuilt_rule_id: type: string description: The Elastic prebuilt rule id matched. @@ -84,32 +81,36 @@ components: - migration_id - original_rule - status + - created_by properties: - "@timestamp": + '@timestamp': type: string description: The moment of creation migration_id: type: string description: The migration id. + created_by: + type: string + description: The username of the user who created the migration. original_rule: $ref: '#/components/schemas/OriginalRule' elastic_rule: $ref: '#/components/schemas/ElasticRule' - translation_state: + translation_result: type: string - description: The translation state. - enum: - - complete + description: The rule translation result. + enum: # should match SiemMigrationRuleTranslationResult enum at ../constants.ts + - full - partial - untranslatable status: type: string - description: The status of the rule migration. + description: The status of the rule migration process. enum: # should match SiemMigrationsStatus enum at ../constants.ts - pending - processing - - finished - - error + - completed + - failed default: pending comments: type: array @@ -122,3 +123,60 @@ components: updated_by: type: string description: The user who last updated the migration + + RuleMigrationTaskStats: + type: object + description: The rule migration task stats object. + required: + - status + - rules + properties: + status: + type: string + description: Indicates if the migration task status. + enum: + - ready + - running + - stopped + - finished + rules: + type: object + description: The rules migration stats. + required: + - total + - pending + - processing + - completed + - failed + properties: + total: + type: integer + description: The total number of rules to migrate. + pending: + type: integer + description: The number of rules that are pending migration. + processing: + type: integer + description: The number of rules that are being migrated. + completed: + type: integer + description: The number of rules that have been migrated successfully. + failed: + type: integer + description: The number of rules that have failed migration. + last_updated_at: + type: string + description: The moment of the last update. + + RuleMigrationAllTaskStats: + type: array + items: + allOf: + - $ref: '#/components/schemas/RuleMigrationTaskStats' + - type: object + required: + - migration_id + properties: + migration_id: + type: string + description: The migration id diff --git a/x-pack/plugins/security_solution/kibana.jsonc b/x-pack/plugins/security_solution/kibana.jsonc index e48a9794b7e5c..8c8b77d48bc9f 100644 --- a/x-pack/plugins/security_solution/kibana.jsonc +++ b/x-pack/plugins/security_solution/kibana.jsonc @@ -54,7 +54,8 @@ "savedSearch", "unifiedDocViewer", "charts", - "entityManager" + "entityManager", + "inference" ], "optionalPlugins": [ "encryptedSavedObjects", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts index ebc1706b309f8..d2aacbdeaeeaf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts @@ -79,7 +79,8 @@ export const createMockClients = () => { internalFleetServices: { packages: packageServiceMock.createClient(), }, - siemMigrationsClient: siemMigrationsServiceMock.createClient(), + siemRuleMigrationsClient: siemMigrationsServiceMock.createRulesClient(), + getInferenceClient: jest.fn(), }; }; @@ -165,7 +166,8 @@ const createSecuritySolutionRequestContextMock = ( getAssetCriticalityDataClient: jest.fn(() => clients.assetCriticalityDataClient), getAuditLogger: jest.fn(() => mockAuditLogger), getEntityStoreDataClient: jest.fn(() => clients.entityStoreDataClient), - getSiemMigrationsClient: jest.fn(() => clients.siemMigrationsClient), + getSiemRuleMigrationsClient: jest.fn(() => clients.siemRuleMigrationsClient), + getInferenceClient: jest.fn(() => clients.getInferenceClient()), }; }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts index fcf119e19ece5..af961d48db5b1 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/mocks.ts @@ -7,18 +7,16 @@ import { createRuleMigrationClient } from '../rules/__mocks__/mocks'; -const createClient = () => ({ rules: createRuleMigrationClient() }); - export const mockSetup = jest.fn().mockResolvedValue(undefined); -export const mockCreateClient = jest.fn().mockReturnValue(createClient()); +export const mockCreateClient = jest.fn().mockReturnValue(createRuleMigrationClient()); export const mockStop = jest.fn(); export const siemMigrationsServiceMock = { create: () => jest.fn().mockImplementation(() => ({ setup: mockSetup, - createClient: mockCreateClient, + createRulesClient: mockCreateClient, stop: mockStop, })), - createClient: () => createClient(), + createRulesClient: () => createRuleMigrationClient(), }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts new file mode 100644 index 0000000000000..659929d47570f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/__mocks__/siem_migrations_service.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { siemMigrationsServiceMock } from './mocks'; +export const SiemMigrationsService = siemMigrationsServiceMock.create(); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts index 8233151f513e4..8811a54195e2b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/mocks.ts @@ -5,17 +5,56 @@ * 2.0. */ -import type { SiemRuleMigrationsClient } from '../types'; - -export const createRuleMigrationClient = (): SiemRuleMigrationsClient => ({ +export const createRuleMigrationDataClient = jest.fn().mockImplementation(() => ({ create: jest.fn().mockResolvedValue({ success: true }), - search: jest.fn().mockResolvedValue([]), + getRules: jest.fn().mockResolvedValue([]), + takePending: jest.fn().mockResolvedValue([]), + saveFinished: jest.fn().mockResolvedValue({ success: true }), + saveError: jest.fn().mockResolvedValue({ success: true }), + releaseProcessing: jest.fn().mockResolvedValue({ success: true }), + releaseProcessable: jest.fn().mockResolvedValue({ success: true }), + getStats: jest.fn().mockResolvedValue({ + status: 'done', + rules: { + total: 1, + finished: 1, + processing: 0, + pending: 0, + failed: 0, + }, + }), + getAllStats: jest.fn().mockResolvedValue([]), +})); + +export const createRuleMigrationTaskClient = () => ({ + start: jest.fn().mockResolvedValue({ started: true }), + stop: jest.fn().mockResolvedValue({ stopped: true }), + getStats: jest.fn().mockResolvedValue({ + status: 'done', + rules: { + total: 1, + finished: 1, + processing: 0, + pending: 0, + failed: 0, + }, + }), + getAllStats: jest.fn().mockResolvedValue([]), }); +export const createRuleMigrationClient = () => ({ + data: createRuleMigrationDataClient(), + task: createRuleMigrationTaskClient(), +}); + +export const MockSiemRuleMigrationsClient = jest.fn().mockImplementation(createRuleMigrationClient); + export const mockSetup = jest.fn(); -export const mockGetClient = jest.fn().mockReturnValue(createRuleMigrationClient()); +export const mockCreateClient = jest.fn().mockReturnValue(createRuleMigrationClient()); +export const mockStop = jest.fn(); export const MockSiemRuleMigrationsService = jest.fn().mockImplementation(() => ({ setup: mockSetup, - getClient: mockGetClient, + createClient: mockCreateClient, + stop: mockStop, })); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts new file mode 100644 index 0000000000000..98032605ed233 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/__mocks__/siem_rule_migrations_client.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockSiemRuleMigrationsClient } from './mocks'; +export const SiemRuleMigrationsClient = MockSiemRuleMigrationsClient; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts index f4c52e9b444b8..e2505ca83beed 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/create.ts @@ -8,14 +8,11 @@ import type { IKibanaResponse, Logger } from '@kbn/core/server'; import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; import { v4 as uuidV4 } from 'uuid'; -import type { RuleMigration } from '../../../../../common/siem_migrations/model/rule_migration.gen'; import type { CreateRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; import { CreateRuleMigrationRequestBody } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; -import { - SIEM_RULE_MIGRATIONS_PATH, - SiemMigrationsStatus, -} from '../../../../../common/siem_migrations/constants'; +import { SIEM_RULE_MIGRATIONS_PATH } from '../../../../../common/siem_migrations/constants'; import type { SecuritySolutionPluginRouter } from '../../../../types'; +import type { CreateRuleMigrationInput } from '../data_stream/rule_migrations_data_client'; export const registerSiemRuleMigrationsCreateRoute = ( router: SecuritySolutionPluginRouter, @@ -25,11 +22,7 @@ export const registerSiemRuleMigrationsCreateRoute = ( .post({ path: SIEM_RULE_MIGRATIONS_PATH, access: 'internal', - security: { - authz: { - requiredPrivileges: ['securitySolution'], - }, - }, + security: { authz: { requiredPrivileges: ['securitySolution'] } }, }) .addVersion( { @@ -41,27 +34,22 @@ export const registerSiemRuleMigrationsCreateRoute = ( async (context, req, res): Promise> => { const originalRules = req.body; try { - const ctx = await context.resolve(['core', 'actions', 'securitySolution']); - - const siemMigrationClient = ctx.securitySolution.getSiemMigrationsClient(); + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); const migrationId = uuidV4(); - const timestamp = new Date().toISOString(); - const ruleMigrations = originalRules.map((originalRule) => ({ - '@timestamp': timestamp, + const ruleMigrations = originalRules.map((originalRule) => ({ migration_id: migrationId, original_rule: originalRule, - status: SiemMigrationsStatus.PENDING, })); - await siemMigrationClient.rules.create(ruleMigrations); + + await ruleMigrationsClient.data.create(ruleMigrations); return res.ok({ body: { migration_id: migrationId } }); } catch (err) { logger.error(err); - return res.badRequest({ - body: err.message, - }); + return res.badRequest({ body: err.message }); } } ); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts new file mode 100644 index 0000000000000..0efb6706918f5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/get.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { GetRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_GET_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsGetRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATIONS_GET_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(GetRuleMigrationRequestParams) }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const migrationRules = await ruleMigrationsClient.data.getRules(migrationId); + + return res.ok({ body: migrationRules }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts index 0de49eb7df92b..f37eb2108a8a4 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/index.ts @@ -8,10 +8,20 @@ import type { Logger } from '@kbn/core/server'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { registerSiemRuleMigrationsCreateRoute } from './create'; +import { registerSiemRuleMigrationsGetRoute } from './get'; +import { registerSiemRuleMigrationsStartRoute } from './start'; +import { registerSiemRuleMigrationsStatsRoute } from './stats'; +import { registerSiemRuleMigrationsStopRoute } from './stop'; +import { registerSiemRuleMigrationsStatsAllRoute } from './stats_all'; export const registerSiemRuleMigrationsRoutes = ( router: SecuritySolutionPluginRouter, logger: Logger ) => { registerSiemRuleMigrationsCreateRoute(router, logger); + registerSiemRuleMigrationsStatsAllRoute(router, logger); + registerSiemRuleMigrationsGetRoute(router, logger); + registerSiemRuleMigrationsStartRoute(router, logger); + registerSiemRuleMigrationsStatsRoute(router, logger); + registerSiemRuleMigrationsStopRoute(router, logger); }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts new file mode 100644 index 0000000000000..f97a4f2ce2398 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/start.ts @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import { APMTracer } from '@kbn/langchain/server/tracers/apm'; +import { getLangSmithTracer } from '@kbn/langchain/server/tracers/langsmith'; +import type { StartRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { + StartRuleMigrationRequestBody, + StartRuleMigrationRequestParams, +} from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_START_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStartRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .put({ + path: SIEM_RULE_MIGRATIONS_START_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { + params: buildRouteValidationWithZod(StartRuleMigrationRequestParams), + body: buildRouteValidationWithZod(StartRuleMigrationRequestBody), + }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + const { langsmith_options: langsmithOptions, connector_id: connectorId } = req.body; + + try { + const ctx = await context.resolve([ + 'core', + 'actions', + 'alerting', + 'securitySolution', + 'licensing', + ]); + if (!ctx.licensing.license.hasAtLeast('enterprise')) { + return res.forbidden({ + body: 'You must have a trial or enterprise license to use this feature', + }); + } + + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + const inferenceClient = ctx.securitySolution.getInferenceClient(); + const actionsClient = ctx.actions.getActionsClient(); + const soClient = ctx.core.savedObjects.client; + const rulesClient = ctx.alerting.getRulesClient(); + + const invocationConfig = { + callbacks: [ + new APMTracer({ projectName: langsmithOptions?.project_name ?? 'default' }, logger), + ...getLangSmithTracer({ ...langsmithOptions, logger }), + ], + }; + + const { exists, started } = await ruleMigrationsClient.task.start({ + migrationId, + connectorId, + invocationConfig, + inferenceClient, + actionsClient, + soClient, + rulesClient, + }); + + if (!exists) { + return res.noContent(); + } + return res.ok({ body: { started } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts new file mode 100644 index 0000000000000..8316e01fc6a9b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { GetRuleMigrationStatsResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationStatsRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_STATS_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStatsRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATIONS_STATS_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(GetRuleMigrationStatsRequestParams) }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const stats = await ruleMigrationsClient.task.getStats(migrationId); + + return res.ok({ body: stats }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts new file mode 100644 index 0000000000000..dd2f2f503e19d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stats_all.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import type { GetAllStatsRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_ALL_STATS_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStatsAllRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .get({ + path: SIEM_RULE_MIGRATIONS_ALL_STATS_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { version: '1', validate: {} }, + async (context, req, res): Promise> => { + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const allStats = await ruleMigrationsClient.task.getAllStats(); + + return res.ok({ body: allStats }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts new file mode 100644 index 0000000000000..4767106910186 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/api/stop.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IKibanaResponse, Logger } from '@kbn/core/server'; +import { buildRouteValidationWithZod } from '@kbn/zod-helpers'; +import type { StopRuleMigrationResponse } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { StopRuleMigrationRequestParams } from '../../../../../common/siem_migrations/model/api/rules/rules_migration.gen'; +import { SIEM_RULE_MIGRATIONS_STOP_PATH } from '../../../../../common/siem_migrations/constants'; +import type { SecuritySolutionPluginRouter } from '../../../../types'; + +export const registerSiemRuleMigrationsStopRoute = ( + router: SecuritySolutionPluginRouter, + logger: Logger +) => { + router.versioned + .put({ + path: SIEM_RULE_MIGRATIONS_STOP_PATH, + access: 'internal', + security: { authz: { requiredPrivileges: ['securitySolution'] } }, + }) + .addVersion( + { + version: '1', + validate: { + request: { params: buildRouteValidationWithZod(StopRuleMigrationRequestParams) }, + }, + }, + async (context, req, res): Promise> => { + const migrationId = req.params.migration_id; + try { + const ctx = await context.resolve(['securitySolution']); + const ruleMigrationsClient = ctx.securitySolution.getSiemRuleMigrationsClient(); + + const { exists, stopped } = await ruleMigrationsClient.task.stop(migrationId); + + if (!exists) { + return res.noContent(); + } + return res.ok({ body: { stopped } }); + } catch (err) { + logger.error(err); + return res.badRequest({ body: err.message }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts index 103c0f9b0c952..1d9a181d2de5b 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/__mocks__/mocks.ts @@ -7,9 +7,9 @@ export const mockIndexName = 'mocked_data_stream_name'; export const mockInstall = jest.fn().mockResolvedValue(undefined); -export const mockInstallSpace = jest.fn().mockResolvedValue(mockIndexName); +export const mockCreateClient = jest.fn().mockReturnValue({}); export const MockRuleMigrationsDataStream = jest.fn().mockImplementation(() => ({ install: mockInstall, - installSpace: mockInstallSpace, + createClient: mockCreateClient, })); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts new file mode 100644 index 0000000000000..83808901a0bd1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_client.ts @@ -0,0 +1,275 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; +import assert from 'assert'; +import type { + AggregationsFilterAggregate, + AggregationsMaxAggregate, + AggregationsStringTermsAggregate, + AggregationsStringTermsBucket, + QueryDslQueryContainer, + SearchHit, + SearchResponse, +} from '@elastic/elasticsearch/lib/api/types'; +import type { StoredRuleMigration } from '../types'; +import { SiemMigrationStatus } from '../../../../../common/siem_migrations/constants'; +import type { + RuleMigration, + RuleMigrationTaskStats, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; + +export type CreateRuleMigrationInput = Omit; +export type RuleMigrationDataStats = Omit; +export type RuleMigrationAllDataStats = Array; + +export class RuleMigrationsDataClient { + constructor( + private dataStreamNamePromise: Promise, + private currentUser: AuthenticatedUser, + private esClient: ElasticsearchClient, + private logger: Logger + ) {} + + /** Indexes an array of rule migrations to be processed */ + async create(ruleMigrations: CreateRuleMigrationInput[]): Promise { + const index = await this.dataStreamNamePromise; + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: ruleMigrations.flatMap((ruleMigration) => [ + { create: { _index: index } }, + { + ...ruleMigration, + '@timestamp': new Date().toISOString(), + status: SiemMigrationStatus.PENDING, + created_by: this.currentUser.username, + }, + ]), + }) + .catch((error) => { + this.logger.error(`Error creating rule migrations: ${error.message}`); + throw error; + }); + } + + /** Retrieves an array of rule documents of a specific migrations */ + async getRules(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId); + + const storedRuleMigrations = await this.esClient + .search({ index, query, sort: '_doc' }) + .catch((error) => { + this.logger.error(`Error searching getting rule migrations: ${error.message}`); + throw error; + }) + .then((response) => this.processHits(response.hits.hits)); + return storedRuleMigrations; + } + + /** + * Retrieves `pending` rule migrations with the provided id and updates their status to `processing`. + * This operation is not atomic at migration level: + * - Multiple tasks can process different migrations simultaneously. + * - Multiple tasks should not process the same migration simultaneously. + */ + async takePending(migrationId: string, size: number): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PENDING); + + const storedRuleMigrations = await this.esClient + .search({ index, query, sort: '_doc', size }) + .catch((error) => { + this.logger.error(`Error searching for rule migrations: ${error.message}`); + throw error; + }) + .then((response) => + this.processHits(response.hits.hits, { status: SiemMigrationStatus.PROCESSING }) + ); + + await this.esClient + .bulk({ + refresh: 'wait_for', + operations: storedRuleMigrations.flatMap(({ _id, _index, status }) => [ + { update: { _id, _index } }, + { + doc: { + status, + updated_by: this.currentUser.username, + updated_at: new Date().toISOString(), + }, + }, + ]), + }) + .catch((error) => { + this.logger.error( + `Error updating for rule migrations status to processing: ${error.message}` + ); + throw error; + }); + + return storedRuleMigrations; + } + + /** Updates one rule migration with the provided data and sets the status to `completed` */ + async saveFinished({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { + const doc = { + ...ruleMigration, + status: SiemMigrationStatus.COMPLETED, + updated_by: this.currentUser.username, + updated_at: new Date().toISOString(), + }; + await this.esClient + .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) + .catch((error) => { + this.logger.error(`Error updating rule migration status to completed: ${error.message}`); + throw error; + }); + } + + /** Updates one rule migration with the provided data and sets the status to `failed` */ + async saveError({ _id, _index, ...ruleMigration }: StoredRuleMigration): Promise { + const doc = { + ...ruleMigration, + status: SiemMigrationStatus.FAILED, + updated_by: this.currentUser.username, + updated_at: new Date().toISOString(), + }; + await this.esClient + .update({ index: _index, id: _id, doc, refresh: 'wait_for' }) + .catch((error) => { + this.logger.error(`Error updating rule migration status to completed: ${error.message}`); + throw error; + }); + } + + /** Updates all the rule migration with the provided id with status `processing` back to `pending` */ + async releaseProcessing(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId, SiemMigrationStatus.PROCESSING); + const script = { source: `ctx._source['status'] = '${SiemMigrationStatus.PENDING}'` }; + await this.esClient.updateByQuery({ index, query, script, refresh: false }).catch((error) => { + this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); + throw error; + }); + } + + /** Updates all the rule migration with the provided id with status `processing` or `failed` back to `pending` */ + async releaseProcessable(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId, [ + SiemMigrationStatus.PROCESSING, + SiemMigrationStatus.FAILED, + ]); + const script = { source: `ctx._source['status'] = '${SiemMigrationStatus.PENDING}'` }; + await this.esClient.updateByQuery({ index, query, script, refresh: true }).catch((error) => { + this.logger.error(`Error releasing rule migrations status to pending: ${error.message}`); + throw error; + }); + } + + /** Retrieves the stats for the rule migrations with the provided id */ + async getStats(migrationId: string): Promise { + const index = await this.dataStreamNamePromise; + const query = this.getFilterQuery(migrationId); + const aggregations = { + pending: { filter: { term: { status: SiemMigrationStatus.PENDING } } }, + processing: { filter: { term: { status: SiemMigrationStatus.PROCESSING } } }, + completed: { filter: { term: { status: SiemMigrationStatus.COMPLETED } } }, + failed: { filter: { term: { status: SiemMigrationStatus.FAILED } } }, + lastUpdatedAt: { max: { field: 'updated_at' } }, + }; + const result = await this.esClient + .search({ index, query, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting rule migrations stats: ${error.message}`); + throw error; + }); + + const { pending, processing, completed, lastUpdatedAt, failed } = result.aggregations ?? {}; + return { + rules: { + total: this.getTotalHits(result), + pending: (pending as AggregationsFilterAggregate)?.doc_count ?? 0, + processing: (processing as AggregationsFilterAggregate)?.doc_count ?? 0, + completed: (completed as AggregationsFilterAggregate)?.doc_count ?? 0, + failed: (failed as AggregationsFilterAggregate)?.doc_count ?? 0, + }, + last_updated_at: (lastUpdatedAt as AggregationsMaxAggregate)?.value_as_string, + }; + } + + /** Retrieves the stats for all the rule migrations aggregated by migration id */ + async getAllStats(): Promise { + const index = await this.dataStreamNamePromise; + const aggregations = { + migrationIds: { + terms: { field: 'migration_id' }, + aggregations: { + pending: { filter: { term: { status: SiemMigrationStatus.PENDING } } }, + processing: { filter: { term: { status: SiemMigrationStatus.PROCESSING } } }, + completed: { filter: { term: { status: SiemMigrationStatus.COMPLETED } } }, + failed: { filter: { term: { status: SiemMigrationStatus.FAILED } } }, + lastUpdatedAt: { max: { field: 'updated_at' } }, + }, + }, + }; + const result = await this.esClient + .search({ index, aggregations, _source: false }) + .catch((error) => { + this.logger.error(`Error getting all rule migrations stats: ${error.message}`); + throw error; + }); + + const migrationsAgg = result.aggregations?.migrationIds as AggregationsStringTermsAggregate; + const buckets = (migrationsAgg?.buckets as AggregationsStringTermsBucket[]) ?? []; + return buckets.map((bucket) => ({ + migration_id: bucket.key, + rules: { + total: bucket.doc_count, + pending: bucket.pending?.doc_count ?? 0, + processing: bucket.processing?.doc_count ?? 0, + completed: bucket.completed?.doc_count ?? 0, + failed: bucket.failed?.doc_count ?? 0, + }, + last_updated_at: bucket.lastUpdatedAt?.value_as_string, + })); + } + + private getFilterQuery( + migrationId: string, + status?: SiemMigrationStatus | SiemMigrationStatus[] + ): QueryDslQueryContainer { + const filter: QueryDslQueryContainer[] = [{ term: { migration_id: migrationId } }]; + if (status) { + if (Array.isArray(status)) { + filter.push({ terms: { status } }); + } else { + filter.push({ term: { status } }); + } + } + return { bool: { filter } }; + } + + private processHits( + hits: Array>, + override: Partial = {} + ): StoredRuleMigration[] { + return hits.map(({ _id, _index, _source }) => { + assert(_id, 'RuleMigration document should have _id'); + assert(_source, 'RuleMigration document should have _source'); + return { ..._source, ...override, _id, _index }; + }); + } + + private getTotalHits(response: SearchResponse) { + return typeof response.hits.total === 'number' + ? response.hits.total + : response.hits.total?.value ?? 0; + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts index 56510da48f1bb..467d26a380945 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.test.ts @@ -11,9 +11,19 @@ import type { InstallParams } from '@kbn/data-stream-adapter'; import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { loggerMock } from '@kbn/logging-mocks'; +import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; +import { securityServiceMock } from '@kbn/core-security-server-mocks'; jest.mock('@kbn/data-stream-adapter'); +// This mock is required to have a way to await the data stream name promise +const mockDataStreamNamePromise = jest.fn(); +jest.mock('./rule_migrations_data_client', () => ({ + RuleMigrationsDataClient: jest.fn((dataStreamNamePromise: Promise) => { + mockDataStreamNamePromise.mockReturnValue(dataStreamNamePromise); + }), +})); + const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest.MockedClass< typeof DataStreamSpacesAdapter >; @@ -21,18 +31,21 @@ const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest const esClient = elasticsearchServiceMock.createStart().client.asInternalUser; describe('SiemRuleMigrationsDataStream', () => { + const kibanaVersion = '8.16.0'; + const logger = loggingSystemMock.createLogger(); + beforeEach(() => { jest.clearAllMocks(); }); describe('constructor', () => { it('should create DataStreamSpacesAdapter', () => { - new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + new RuleMigrationsDataStream(logger, kibanaVersion); expect(MockedDataStreamSpacesAdapter).toHaveBeenCalledTimes(1); }); it('should create component templates', () => { - new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + new RuleMigrationsDataStream(logger, kibanaVersion); const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: '.kibana.siem-rule-migrations' }) @@ -40,7 +53,7 @@ describe('SiemRuleMigrationsDataStream', () => { }); it('should create index templates', () => { - new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + new RuleMigrationsDataStream(logger, kibanaVersion); const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; expect(dataStreamSpacesAdapter.setIndexTemplate).toHaveBeenCalledWith( expect.objectContaining({ name: '.kibana.siem-rule-migrations' }) @@ -50,22 +63,20 @@ describe('SiemRuleMigrationsDataStream', () => { describe('install', () => { it('should install data stream', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); - const params: InstallParams = { + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); + const params: Omit = { esClient, - logger: loggerMock.create(), pluginStop$: new Subject(), }; await dataStream.install(params); const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; - expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(params); + expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(expect.objectContaining(params)); }); it('should log error', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); - const params: InstallParams = { + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); + const params: Omit = { esClient, - logger: loggerMock.create(), pluginStop$: new Subject(), }; const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; @@ -73,13 +84,16 @@ describe('SiemRuleMigrationsDataStream', () => { (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); await dataStream.install(params); - expect(params.logger.error).toHaveBeenCalledWith(expect.any(String), error); + expect(logger.error).toHaveBeenCalledWith(expect.any(String), error); }); }); - describe('installSpace', () => { + describe('createClient', () => { + const currentUser = securityServiceMock.createMockAuthenticatedUser(); + const createClientParams = { spaceId: 'space1', currentUser, esClient }; + it('should install space data stream', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); const params: InstallParams = { esClient, logger: loggerMock.create(), @@ -89,19 +103,23 @@ describe('SiemRuleMigrationsDataStream', () => { (dataStreamSpacesAdapter.install as jest.Mock).mockResolvedValueOnce(undefined); await dataStream.install(params); - await dataStream.installSpace('space1'); + dataStream.createClient(createClientParams); + await mockDataStreamNamePromise(); expect(dataStreamSpacesAdapter.getInstalledSpaceName).toHaveBeenCalledWith('space1'); expect(dataStreamSpacesAdapter.installSpace).toHaveBeenCalledWith('space1'); }); it('should not install space data stream if install not executed', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); - await expect(dataStream.installSpace('space1')).rejects.toThrowError(); + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); + await expect(async () => { + dataStream.createClient(createClientParams); + await mockDataStreamNamePromise(); + }).rejects.toThrowError(); }); it('should throw error if main install had error', async () => { - const dataStream = new RuleMigrationsDataStream({ kibanaVersion: '8.13.0' }); + const dataStream = new RuleMigrationsDataStream(logger, kibanaVersion); const params: InstallParams = { esClient, logger: loggerMock.create(), @@ -112,7 +130,10 @@ describe('SiemRuleMigrationsDataStream', () => { (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); await dataStream.install(params); - await expect(dataStream.installSpace('space1')).rejects.toThrowError(error); + await expect(async () => { + dataStream.createClient(createClientParams); + await mockDataStreamNamePromise(); + }).rejects.toThrowError(error); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts index 83eb471e0cee3..a5855cefb1324 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_data_stream.ts @@ -6,51 +6,69 @@ */ import { DataStreamSpacesAdapter, type InstallParams } from '@kbn/data-stream-adapter'; +import type { AuthenticatedUser, ElasticsearchClient, Logger } from '@kbn/core/server'; import { ruleMigrationsFieldMap } from './rule_migrations_field_map'; +import { RuleMigrationsDataClient } from './rule_migrations_data_client'; const TOTAL_FIELDS_LIMIT = 2500; const DATA_STREAM_NAME = '.kibana.siem-rule-migrations'; -const ECS_COMPONENT_TEMPLATE_NAME = 'ecs'; + +interface RuleMigrationsDataStreamCreateClientParams { + spaceId: string; + currentUser: AuthenticatedUser; + esClient: ElasticsearchClient; +} export class RuleMigrationsDataStream { - private readonly dataStream: DataStreamSpacesAdapter; + private readonly dataStreamAdapter: DataStreamSpacesAdapter; private installPromise?: Promise; - constructor({ kibanaVersion }: { kibanaVersion: string }) { - this.dataStream = new DataStreamSpacesAdapter(DATA_STREAM_NAME, { + constructor(private logger: Logger, kibanaVersion: string) { + this.dataStreamAdapter = new DataStreamSpacesAdapter(DATA_STREAM_NAME, { kibanaVersion, totalFieldsLimit: TOTAL_FIELDS_LIMIT, }); - this.dataStream.setComponentTemplate({ + this.dataStreamAdapter.setComponentTemplate({ name: DATA_STREAM_NAME, fieldMap: ruleMigrationsFieldMap, }); - this.dataStream.setIndexTemplate({ + this.dataStreamAdapter.setIndexTemplate({ name: DATA_STREAM_NAME, - componentTemplateRefs: [DATA_STREAM_NAME, ECS_COMPONENT_TEMPLATE_NAME], + componentTemplateRefs: [DATA_STREAM_NAME], }); } - async install(params: InstallParams) { + async install(params: Omit) { try { - this.installPromise = this.dataStream.install(params); + this.installPromise = this.dataStreamAdapter.install({ ...params, logger: this.logger }); await this.installPromise; } catch (err) { - params.logger.error(`Error installing siem rule migrations data stream. ${err.message}`, err); + this.logger.error(`Error installing siem rule migrations data stream. ${err.message}`, err); } } - async installSpace(spaceId: string): Promise { + createClient({ + spaceId, + currentUser, + esClient, + }: RuleMigrationsDataStreamCreateClientParams): RuleMigrationsDataClient { + const dataStreamNamePromise = this.installSpace(spaceId); + return new RuleMigrationsDataClient(dataStreamNamePromise, currentUser, esClient, this.logger); + } + + // Installs the data stream for the specific space. it will only install if it hasn't been installed yet. + // The adapter stores the data stream name promise, it will return it directly when the data stream is known to be installed. + private async installSpace(spaceId: string): Promise { if (!this.installPromise) { throw new Error('Siem rule migrations data stream not installed'); } // wait for install to complete, may reject if install failed, routes should handle this await this.installPromise; - let dataStreamName = await this.dataStream.getInstalledSpaceName(spaceId); + let dataStreamName = await this.dataStreamAdapter.getInstalledSpaceName(spaceId); if (!dataStreamName) { - dataStreamName = await this.dataStream.installSpace(spaceId); + dataStreamName = await this.dataStreamAdapter.installSpace(spaceId); } return dataStreamName; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts index ba9a706957bcb..a65cd45b832e9 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/data_stream/rule_migrations_field_map.ts @@ -11,6 +11,7 @@ import type { RuleMigration } from '../../../../../common/siem_migrations/model/ export const ruleMigrationsFieldMap: FieldMap> = { '@timestamp': { type: 'date', required: false }, migration_id: { type: 'keyword', required: true }, + created_by: { type: 'keyword', required: true }, status: { type: 'keyword', required: true }, original_rule: { type: 'nested', required: true }, 'original_rule.vendor': { type: 'keyword', required: true }, @@ -28,7 +29,7 @@ export const ruleMigrationsFieldMap: FieldMap> 'elastic_rule.severity': { type: 'keyword', required: false }, 'elastic_rule.prebuilt_rule_id': { type: 'keyword', required: false }, 'elastic_rule.id': { type: 'keyword', required: false }, - translation_state: { type: 'keyword', required: false }, + translation_result: { type: 'keyword', required: false }, comments: { type: 'text', array: true, required: false }, updated_at: { type: 'date', required: false }, updated_by: { type: 'keyword', required: false }, diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts index 390d302264cea..5c611d85e0464 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.test.ts @@ -8,25 +8,28 @@ import { loggingSystemMock, elasticsearchServiceMock, httpServerMock, + securityServiceMock, } from '@kbn/core/server/mocks'; import { SiemRuleMigrationsService } from './siem_rule_migrations_service'; import { Subject } from 'rxjs'; -import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen'; import { MockRuleMigrationsDataStream, mockInstall, - mockInstallSpace, - mockIndexName, + mockCreateClient, } from './data_stream/__mocks__/mocks'; -import type { KibanaRequest } from '@kbn/core/server'; +import type { SiemRuleMigrationsCreateClientParams } from './types'; jest.mock('./data_stream/rule_migrations_data_stream'); +jest.mock('./task/rule_migrations_task_runner', () => ({ + RuleMigrationsTaskRunner: jest.fn(), +})); describe('SiemRuleMigrationsService', () => { let ruleMigrationsService: SiemRuleMigrationsService; const kibanaVersion = '8.16.0'; const esClusterClient = elasticsearchServiceMock.createClusterClient(); + const currentUser = securityServiceMock.createMockAuthenticatedUser(); const logger = loggingSystemMock.createLogger(); const pluginStop$ = new Subject(); @@ -36,7 +39,7 @@ describe('SiemRuleMigrationsService', () => { }); it('should instantiate the rule migrations data stream adapter', () => { - expect(MockRuleMigrationsDataStream).toHaveBeenCalledWith({ kibanaVersion }); + expect(MockRuleMigrationsDataStream).toHaveBeenCalledWith(logger, kibanaVersion); }); describe('when setup is called', () => { @@ -45,22 +48,26 @@ describe('SiemRuleMigrationsService', () => { expect(mockInstall).toHaveBeenCalledWith({ esClient: esClusterClient.asInternalUser, - logger, pluginStop$, }); }); }); - describe('when getClient is called', () => { - let request: KibanaRequest; + describe('when createClient is called', () => { + let createClientParams: SiemRuleMigrationsCreateClientParams; + beforeEach(() => { - request = httpServerMock.createKibanaRequest(); + createClientParams = { + spaceId: 'default', + currentUser, + request: httpServerMock.createKibanaRequest(), + }; }); describe('without setup', () => { it('should throw an error', () => { expect(() => { - ruleMigrationsService.getClient({ spaceId: 'default', request }); + ruleMigrationsService.createClient(createClientParams); }).toThrowError('ES client not available, please call setup first'); }); }); @@ -71,44 +78,19 @@ describe('SiemRuleMigrationsService', () => { }); it('should call installSpace', () => { - ruleMigrationsService.getClient({ spaceId: 'default', request }); - - expect(mockInstallSpace).toHaveBeenCalledWith('default'); - }); - - it('should return a client with create and search methods after setup', () => { - const client = ruleMigrationsService.getClient({ spaceId: 'default', request }); - - expect(client).toHaveProperty('create'); - expect(client).toHaveProperty('search'); + ruleMigrationsService.createClient(createClientParams); + expect(mockCreateClient).toHaveBeenCalledWith({ + spaceId: createClientParams.spaceId, + currentUser: createClientParams.currentUser, + esClient: esClusterClient.asScoped().asCurrentUser, + }); }); - it('should call ES bulk create API with the correct parameters with create is called', async () => { - const client = ruleMigrationsService.getClient({ spaceId: 'default', request }); - - const ruleMigrations = [{ migration_id: 'dummy_migration_id' } as RuleMigration]; - await client.create(ruleMigrations); - - expect(esClusterClient.asScoped().asCurrentUser.bulk).toHaveBeenCalledWith( - expect.objectContaining({ - body: [{ create: { _index: mockIndexName } }, { migration_id: 'dummy_migration_id' }], - refresh: 'wait_for', - }) - ); - }); - - it('should call ES search API with the correct parameters with search is called', async () => { - const client = ruleMigrationsService.getClient({ spaceId: 'default', request }); - - const term = { migration_id: 'dummy_migration_id' }; - await client.search(term); + it('should return data and task clients', () => { + const client = ruleMigrationsService.createClient(createClientParams); - expect(esClusterClient.asScoped().asCurrentUser.search).toHaveBeenCalledWith( - expect.objectContaining({ - index: mockIndexName, - body: { query: { term } }, - }) - ); + expect(client).toHaveProperty('data'); + expect(client).toHaveProperty('task'); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts index 5b20f957cb6fa..1bf9dcf11fd95 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/siem_rule_migrations_service.ts @@ -5,52 +5,67 @@ * 2.0. */ +import assert from 'assert'; import type { IClusterClient, Logger } from '@kbn/core/server'; import { RuleMigrationsDataStream } from './data_stream/rule_migrations_data_stream'; import type { - SiemRuleMigrationsClient, SiemRulesMigrationsSetupParams, - SiemRuleMigrationsGetClientParams, + SiemRuleMigrationsCreateClientParams, + SiemRuleMigrationsClient, } from './types'; +import { RuleMigrationsTaskRunner } from './task/rule_migrations_task_runner'; export class SiemRuleMigrationsService { - private dataStreamAdapter: RuleMigrationsDataStream; + private rulesDataStream: RuleMigrationsDataStream; private esClusterClient?: IClusterClient; + private taskRunner: RuleMigrationsTaskRunner; constructor(private logger: Logger, kibanaVersion: string) { - this.dataStreamAdapter = new RuleMigrationsDataStream({ kibanaVersion }); + this.rulesDataStream = new RuleMigrationsDataStream(this.logger, kibanaVersion); + this.taskRunner = new RuleMigrationsTaskRunner(this.logger); } setup({ esClusterClient, ...params }: SiemRulesMigrationsSetupParams) { this.esClusterClient = esClusterClient; const esClient = esClusterClient.asInternalUser; - this.dataStreamAdapter.install({ ...params, esClient, logger: this.logger }).catch((err) => { + + this.rulesDataStream.install({ ...params, esClient }).catch((err) => { this.logger.error(`Error installing data stream for rule migrations: ${err.message}`); throw err; }); } - getClient({ spaceId, request }: SiemRuleMigrationsGetClientParams): SiemRuleMigrationsClient { - if (!this.esClusterClient) { - throw new Error('ES client not available, please call setup first'); - } - // Installs the data stream for the specific space. it will only install if it hasn't been installed yet. - // The adapter stores the data stream name promise, it will return it directly when the data stream is known to be installed. - const dataStreamNamePromise = this.dataStreamAdapter.installSpace(spaceId); + createClient({ + spaceId, + currentUser, + request, + }: SiemRuleMigrationsCreateClientParams): SiemRuleMigrationsClient { + assert(currentUser, 'Current user must be authenticated'); + assert(this.esClusterClient, 'ES client not available, please call setup first'); const esClient = this.esClusterClient.asScoped(request).asCurrentUser; + const dataClient = this.rulesDataStream.createClient({ spaceId, currentUser, esClient }); + return { - create: async (ruleMigrations) => { - const _index = await dataStreamNamePromise; - return esClient.bulk({ - refresh: 'wait_for', - body: ruleMigrations.flatMap((ruleMigration) => [{ create: { _index } }, ruleMigration]), - }); - }, - search: async (term) => { - const index = await dataStreamNamePromise; - return esClient.search({ index, body: { query: { term } } }); + data: dataClient, + task: { + start: (params) => { + return this.taskRunner.start({ ...params, currentUser, dataClient }); + }, + stop: (migrationId) => { + return this.taskRunner.stop({ migrationId, dataClient }); + }, + getStats: async (migrationId) => { + return this.taskRunner.getStats({ migrationId, dataClient }); + }, + getAllStats: async () => { + return this.taskRunner.getAllStats({ dataClient }); + }, }, }; } + + stop() { + this.taskRunner.stopAll(); + } } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts new file mode 100644 index 0000000000000..a44197d82850f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { END, START, StateGraph } from '@langchain/langgraph'; +import { migrateRuleState } from './state'; +import type { MigrateRuleGraphParams, MigrateRuleState } from './types'; +import { getTranslateQueryNode } from './nodes/translate_query'; +import { getMatchPrebuiltRuleNode } from './nodes/match_prebuilt_rule'; + +export function getRuleMigrationAgent({ + model, + inferenceClient, + prebuiltRulesMap, + connectorId, + logger, +}: MigrateRuleGraphParams) { + const matchPrebuiltRuleNode = getMatchPrebuiltRuleNode({ model, prebuiltRulesMap, logger }); + const translationNode = getTranslateQueryNode({ inferenceClient, connectorId, logger }); + + const translateRuleGraph = new StateGraph(migrateRuleState) + // Nodes + .addNode('matchPrebuiltRule', matchPrebuiltRuleNode) + .addNode('translation', translationNode) + // Edges + .addEdge(START, 'matchPrebuiltRule') + .addConditionalEdges('matchPrebuiltRule', matchedPrebuiltRuleConditional) + .addEdge('translation', END); + + const graph = translateRuleGraph.compile(); + graph.name = 'Rule Migration Graph'; // Customizes the name displayed in LangSmith + return graph; +} + +const matchedPrebuiltRuleConditional = (state: MigrateRuleState) => { + if (state.elastic_rule?.prebuilt_rule_id) { + return END; + } + return 'translation'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts new file mode 100644 index 0000000000000..febf5fc85f5a0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getRuleMigrationAgent } from './graph'; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts new file mode 100644 index 0000000000000..2d8b81d00eafb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { getMatchPrebuiltRuleNode } from './match_prebuilt_rule'; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts new file mode 100644 index 0000000000000..4a0404acf653d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import type { ChatModel } from '../../../util/actions_client_chat'; +import type { GraphNode } from '../../types'; +import { filterPrebuiltRules, type PrebuiltRulesMapByName } from '../../../util/prebuilt_rules'; +import { MATCH_PREBUILT_RULE_PROMPT } from './prompts'; + +interface GetMatchPrebuiltRuleNodeParams { + model: ChatModel; + prebuiltRulesMap: PrebuiltRulesMapByName; + logger: Logger; +} + +export const getMatchPrebuiltRuleNode = + ({ model, prebuiltRulesMap }: GetMatchPrebuiltRuleNodeParams): GraphNode => + async (state) => { + const mitreAttackIds = state.original_rule.mitre_attack_ids; + if (!mitreAttackIds?.length) { + return {}; + } + const filteredPrebuiltRulesMap = filterPrebuiltRules(prebuiltRulesMap, mitreAttackIds); + if (filteredPrebuiltRulesMap.size === 0) { + return {}; + } + + const outputParser = new StringOutputParser(); + const matchPrebuiltRule = MATCH_PREBUILT_RULE_PROMPT.pipe(model).pipe(outputParser); + + const elasticSecurityRules = Array(filteredPrebuiltRulesMap.keys()).join('\n'); + const response = await matchPrebuiltRule.invoke({ + elasticSecurityRules, + ruleTitle: state.original_rule.title, + }); + const cleanResponse = response.trim(); + if (cleanResponse === 'no_match') { + return {}; + } + + const result = filteredPrebuiltRulesMap.get(cleanResponse); + if (result != null) { + return { + elastic_rule: { + title: result.rule.name, + description: result.rule.description, + prebuilt_rule_id: result.rule.rule_id, + id: result.installedRuleId, + }, + }; + } + + return {}; + }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts new file mode 100644 index 0000000000000..434636d0519b1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/prompts.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ChatPromptTemplate } from '@langchain/core/prompts'; +export const MATCH_PREBUILT_RULE_PROMPT = ChatPromptTemplate.fromMessages([ + [ + 'system', + `You are an expert assistant in Cybersecurity, your task is to help migrating a SIEM detection rule, from Splunk Security to Elastic Security. +You will be provided with a Splunk Detection Rule name by the user, your goal is to try find an Elastic Detection Rule that covers the same threat, if any. +The list of Elastic Detection Rules suggested is provided in the context below. + +Guidelines: +If there is no Elastic rule in the list that covers the same threat, answer only with the string: no_match +If there is one Elastic rule in the list that covers the same threat, answer only with its name without any further explanation. +If there are multiple rules in the list that cover the same threat, answer with the most specific of them, for example: "Linux User Account Creation" is more specific than "User Account Creation". + + +{elasticSecurityRules} + +`, + ], + [ + 'human', + `The Splunk Detection Rule is: +<> +{ruleTitle} +<> +`, + ], + ['ai', 'Please find the answer below:'], +]); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts new file mode 100644 index 0000000000000..2277f2fae41a9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/esql_knowledge_base_caller.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { naturalLanguageToEsql, type InferenceClient } from '@kbn/inference-plugin/server'; +import { lastValueFrom } from 'rxjs'; + +export type EsqlKnowledgeBaseCaller = (input: string) => Promise; + +type GetEsqlTranslatorToolParams = (params: { + inferenceClient: InferenceClient; + connectorId: string; + logger: Logger; +}) => EsqlKnowledgeBaseCaller; + +export const getEsqlKnowledgeBase: GetEsqlTranslatorToolParams = + ({ inferenceClient: client, connectorId, logger }) => + async (input: string) => { + const { content } = await lastValueFrom( + naturalLanguageToEsql({ + client, + connectorId, + input, + logger: { + debug: (source) => { + logger.debug(typeof source === 'function' ? source() : source); + }, + }, + }) + ); + return content; + }; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts new file mode 100644 index 0000000000000..7d247f755e9da --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export { getTranslateQueryNode } from './translate_query'; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts new file mode 100644 index 0000000000000..0b97faf7dc96f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/prompt.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { MigrateRuleState } from '../../types'; + +export const getEsqlTranslationPrompt = ( + state: MigrateRuleState +): string => `You are a helpful cybersecurity (SIEM) expert agent. Your task is to migrate "detection rules" from Splunk to Elastic Security. +Below you will find Splunk rule information: the title, description and the SPL (Search Processing Language) query. +Your goal is to translate the SPL query into an equivalent Elastic Security Query Language (ES|QL) query. + +Guidelines: +- Start the translation process by analyzing the SPL query and identifying the key components. +- Always use logs* index pattern for the ES|QL translated query. +- If, in the SPL query, you find a lookup list or macro that, based only on its name, you can not translate with confidence to ES|QL, mention it in the summary and +add a placeholder in the query with the format [macro:(parameters)] or [lookup:] including the [] keys, example: [macro:my_macro(first_param,second_param)] or [lookup:my_lookup]. + +The output will be parsed and should contain: +- First, the ES|QL query inside an \`\`\`esql code block. +- At the end, the summary of the translation process followed in markdown, starting with "## Migration Summary". + +This is the Splunk rule information: + +<> +${state.original_rule.title} +<> + +<> +${state.original_rule.description} +<> + +<> +${state.original_rule.query} +<> +`; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts new file mode 100644 index 0000000000000..00e1e60c7b5f3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/translate_query/translate_query.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { GraphNode } from '../../types'; +import { getEsqlKnowledgeBase } from './esql_knowledge_base_caller'; +import { getEsqlTranslationPrompt } from './prompt'; +import { SiemMigrationRuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants'; + +interface GetTranslateQueryNodeParams { + inferenceClient: InferenceClient; + connectorId: string; + logger: Logger; +} + +export const getTranslateQueryNode = ({ + inferenceClient, + connectorId, + logger, +}: GetTranslateQueryNodeParams): GraphNode => { + const esqlKnowledgeBaseCaller = getEsqlKnowledgeBase({ inferenceClient, connectorId, logger }); + return async (state) => { + const input = getEsqlTranslationPrompt(state); + const response = await esqlKnowledgeBaseCaller(input); + + const esqlQuery = response.match(/```esql\n([\s\S]*?)\n```/)?.[1] ?? ''; + const summary = response.match(/## Migration Summary[\s\S]*$/)?.[0] ?? ''; + + const translationResult = getTranslationResult(esqlQuery); + + return { + response, + comments: [summary], + translation_result: translationResult, + elastic_rule: { + title: state.original_rule.title, + description: state.original_rule.description, + severity: 'low', + query: esqlQuery, + query_language: 'esql', + }, + }; + }; +}; + +const getTranslationResult = (esqlQuery: string): SiemMigrationRuleTranslationResult => { + if (esqlQuery.match(/\[(macro|lookup):[\s\S]*\]/)) { + return SiemMigrationRuleTranslationResult.PARTIAL; + } + return SiemMigrationRuleTranslationResult.FULL; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts new file mode 100644 index 0000000000000..c1e510bdc052d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/state.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BaseMessage } from '@langchain/core/messages'; +import { Annotation, messagesStateReducer } from '@langchain/langgraph'; +import type { + ElasticRule, + OriginalRule, + RuleMigration, +} from '../../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { SiemMigrationRuleTranslationResult } from '../../../../../../common/siem_migrations/constants'; + +export const migrateRuleState = Annotation.Root({ + messages: Annotation({ + reducer: messagesStateReducer, + default: () => [], + }), + original_rule: Annotation(), + elastic_rule: Annotation({ + reducer: (state, action) => ({ ...state, ...action }), + }), + translation_result: Annotation(), + comments: Annotation({ + reducer: (current, value) => (value ? (current ?? []).concat(value) : current), + default: () => [], + }), + response: Annotation(), +}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts new file mode 100644 index 0000000000000..643d200e4b0bf --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { migrateRuleState } from './state'; +import type { ChatModel } from '../util/actions_client_chat'; +import type { PrebuiltRulesMapByName } from '../util/prebuilt_rules'; + +export type MigrateRuleState = typeof migrateRuleState.State; +export type GraphNode = (state: MigrateRuleState) => Promise>; + +export interface MigrateRuleGraphParams { + inferenceClient: InferenceClient; + model: ChatModel; + connectorId: string; + prebuiltRulesMap: PrebuiltRulesMapByName; + logger: Logger; +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts new file mode 100644 index 0000000000000..6ae7294fb5257 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/rule_migrations_task_runner.ts @@ -0,0 +1,285 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { AbortError, abortSignalToPromise } from '@kbn/kibana-utils-plugin/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { + RuleMigrationAllTaskStats, + RuleMigrationTaskStats, +} from '../../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationDataStats } from '../data_stream/rule_migrations_data_client'; +import type { + RuleMigrationTaskStartParams, + RuleMigrationTaskStartResult, + RuleMigrationTaskStatsParams, + RuleMigrationTaskStopParams, + RuleMigrationTaskStopResult, + RuleMigrationTaskPrepareParams, + RuleMigrationTaskRunParams, + MigrationAgent, + RuleMigrationAllTaskStatsParams, +} from './types'; +import { getRuleMigrationAgent } from './agent'; +import type { MigrateRuleState } from './agent/types'; +import { retrievePrebuiltRulesMap } from './util/prebuilt_rules'; +import { ActionsClientChat } from './util/actions_client_chat'; + +interface TaskLogger { + info: (msg: string) => void; + debug: (msg: string) => void; + error: (msg: string, error: Error) => void; +} +const getTaskLogger = (logger: Logger): TaskLogger => { + const prefix = '[ruleMigrationsTask]: '; + return { + info: (msg) => logger.info(`${prefix}${msg}`), + debug: (msg) => logger.debug(`${prefix}${msg}`), + error: (msg, error) => logger.error(`${prefix}${msg}: ${error.message}`), + }; +}; + +const ITERATION_BATCH_SIZE = 50 as const; +const ITERATION_SLEEP_SECONDS = 10 as const; + +export class RuleMigrationsTaskRunner { + private migrationsRunning: Map; + private taskLogger: TaskLogger; + + constructor(private logger: Logger) { + this.migrationsRunning = new Map(); + this.taskLogger = getTaskLogger(logger); + } + + /** Starts a rule migration task */ + async start(params: RuleMigrationTaskStartParams): Promise { + const { migrationId, dataClient } = params; + if (this.migrationsRunning.has(migrationId)) { + return { exists: true, started: false }; + } + // Just in case some previous execution was interrupted without releasing + await dataClient.releaseProcessable(migrationId); + + const { rules } = await dataClient.getStats(migrationId); + if (rules.total === 0) { + return { exists: false, started: false }; + } + if (rules.pending === 0) { + return { exists: true, started: false }; + } + + const abortController = new AbortController(); + + // Await the preparation to make sure the agent is created properly so the task can run + const agent = await this.prepare({ ...params, abortController }); + + // not awaiting the `run` promise to execute the task in the background + this.run({ ...params, agent, abortController }).catch((err) => { + // All errors in the `run` method are already catch, this should never happen, but just in case + this.taskLogger.error(`Unexpected error running the migration ID:${migrationId}`, err); + }); + + return { exists: true, started: true }; + } + + private async prepare({ + connectorId, + inferenceClient, + actionsClient, + rulesClient, + soClient, + abortController, + }: RuleMigrationTaskPrepareParams): Promise { + const prebuiltRulesMap = await retrievePrebuiltRulesMap({ soClient, rulesClient }); + + const actionsClientChat = new ActionsClientChat(connectorId, actionsClient, this.logger); + const model = await actionsClientChat.createModel({ + signal: abortController.signal, + temperature: 0.05, + }); + + const agent = getRuleMigrationAgent({ + connectorId, + model, + inferenceClient, + prebuiltRulesMap, + logger: this.logger, + }); + return agent; + } + + private async run({ + migrationId, + agent, + dataClient, + currentUser, + invocationConfig, + abortController, + }: RuleMigrationTaskRunParams): Promise { + if (this.migrationsRunning.has(migrationId)) { + // This should never happen, but just in case + throw new Error(`Task already running for migration ID:${migrationId} `); + } + this.taskLogger.info(`Starting migration ID:${migrationId}`); + + this.migrationsRunning.set(migrationId, { user: currentUser.username, abortController }); + const config: RunnableConfig = { + ...invocationConfig, + // signal: abortController.signal, // not working properly https://github.com/langchain-ai/langgraphjs/issues/319 + }; + + const abortPromise = abortSignalToPromise(abortController.signal); + + try { + const sleep = async (seconds: number) => { + this.taskLogger.debug(`Sleeping ${seconds}s for migration ID:${migrationId}`); + await Promise.race([ + new Promise((resolve) => setTimeout(resolve, seconds * 1000)), + abortPromise.promise, + ]); + }; + + let isDone: boolean = false; + do { + const ruleMigrations = await dataClient.takePending(migrationId, ITERATION_BATCH_SIZE); + this.taskLogger.debug( + `Processing ${ruleMigrations.length} rules for migration ID:${migrationId}` + ); + + await Promise.all( + ruleMigrations.map(async (ruleMigration) => { + this.taskLogger.debug( + `Starting migration of rule "${ruleMigration.original_rule.title}"` + ); + try { + const start = Date.now(); + + const ruleMigrationResult: MigrateRuleState = await Promise.race([ + agent.invoke({ original_rule: ruleMigration.original_rule }, config), + abortPromise.promise, // workaround for the issue with the langGraph signal + ]); + + const duration = (Date.now() - start) / 1000; + this.taskLogger.debug( + `Migration of rule "${ruleMigration.original_rule.title}" finished in ${duration}s` + ); + + await dataClient.saveFinished({ + ...ruleMigration, + elastic_rule: ruleMigrationResult.elastic_rule, + translation_result: ruleMigrationResult.translation_result, + comments: ruleMigrationResult.comments, + }); + } catch (error) { + if (error instanceof AbortError) { + throw error; + } + this.taskLogger.error( + `Error migrating rule "${ruleMigration.original_rule.title}"`, + error + ); + await dataClient.saveError({ + ...ruleMigration, + comments: [`Error migrating rule: ${error.message}`], + }); + } + }) + ); + + this.taskLogger.debug(`Batch processed successfully for migration ID:${migrationId}`); + + const { rules } = await dataClient.getStats(migrationId); + isDone = rules.pending === 0; + if (!isDone) { + await sleep(ITERATION_SLEEP_SECONDS); + } + } while (!isDone); + + this.taskLogger.info(`Finished migration ID:${migrationId}`); + } catch (error) { + await dataClient.releaseProcessing(migrationId); + + if (error instanceof AbortError) { + this.taskLogger.info(`Abort signal received, stopping migration ID:${migrationId}`); + return; + } else { + this.taskLogger.error(`Error processing migration ID:${migrationId}`, error); + } + } finally { + this.migrationsRunning.delete(migrationId); + abortPromise.cleanup(); + } + } + + /** Returns the stats of a migration */ + async getStats({ + migrationId, + dataClient, + }: RuleMigrationTaskStatsParams): Promise { + const dataStats = await dataClient.getStats(migrationId); + const status = this.getTaskStatus(migrationId, dataStats.rules); + return { status, ...dataStats }; + } + + /** Returns the stats of all migrations */ + async getAllStats({ + dataClient, + }: RuleMigrationAllTaskStatsParams): Promise { + const allDataStats = await dataClient.getAllStats(); + return allDataStats.map((dataStats) => { + const status = this.getTaskStatus(dataStats.migration_id, dataStats.rules); + return { status, ...dataStats }; + }); + } + + private getTaskStatus( + migrationId: string, + dataStats: RuleMigrationDataStats['rules'] + ): RuleMigrationTaskStats['status'] { + if (this.migrationsRunning.has(migrationId)) { + return 'running'; + } + if (dataStats.pending === dataStats.total) { + return 'ready'; + } + if (dataStats.completed + dataStats.failed === dataStats.total) { + return 'finished'; + } + return 'stopped'; + } + + /** Stops one running migration */ + async stop({ + migrationId, + dataClient, + }: RuleMigrationTaskStopParams): Promise { + try { + const migrationRunning = this.migrationsRunning.get(migrationId); + if (migrationRunning) { + migrationRunning.abortController.abort(); + return { exists: true, stopped: true }; + } + + const { rules } = await dataClient.getStats(migrationId); + if (rules.total > 0) { + return { exists: true, stopped: false }; + } + return { exists: false, stopped: false }; + } catch (err) { + this.taskLogger.error(`Error stopping migration ID:${migrationId}`, err); + return { exists: true, stopped: false }; + } + } + + /** Stops all running migrations */ + stopAll() { + this.migrationsRunning.forEach((migrationRunning) => { + migrationRunning.abortController.abort(); + }); + this.migrationsRunning.clear(); + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts new file mode 100644 index 0000000000000..e26a5b7216f48 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/types.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuthenticatedUser, SavedObjectsClientContract } from '@kbn/core/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { RuleMigrationsDataClient } from '../data_stream/rule_migrations_data_client'; +import type { getRuleMigrationAgent } from './agent'; + +export type MigrationAgent = ReturnType; + +export interface RuleMigrationTaskStartParams { + migrationId: string; + currentUser: AuthenticatedUser; + connectorId: string; + invocationConfig: RunnableConfig; + inferenceClient: InferenceClient; + actionsClient: ActionsClient; + rulesClient: RulesClient; + soClient: SavedObjectsClientContract; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskPrepareParams { + connectorId: string; + inferenceClient: InferenceClient; + actionsClient: ActionsClient; + rulesClient: RulesClient; + soClient: SavedObjectsClientContract; + abortController: AbortController; +} + +export interface RuleMigrationTaskRunParams { + migrationId: string; + currentUser: AuthenticatedUser; + invocationConfig: RunnableConfig; + agent: MigrationAgent; + dataClient: RuleMigrationsDataClient; + abortController: AbortController; +} + +export interface RuleMigrationTaskStopParams { + migrationId: string; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskStatsParams { + migrationId: string; + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationAllTaskStatsParams { + dataClient: RuleMigrationsDataClient; +} + +export interface RuleMigrationTaskStartResult { + started: boolean; + exists: boolean; +} + +export interface RuleMigrationTaskStopResult { + stopped: boolean; + exists: boolean; +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts new file mode 100644 index 0000000000000..204978c901df6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/actions_client_chat.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ActionsClientSimpleChatModel } from '@kbn/langchain/server'; +import { + ActionsClientBedrockChatModel, + ActionsClientChatOpenAI, + ActionsClientChatVertexAI, +} from '@kbn/langchain/server'; +import type { Logger } from '@kbn/core/server'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { ActionsClientChatOpenAIParams } from '@kbn/langchain/server/language_models/chat_openai'; +import type { CustomChatModelInput as ActionsClientBedrockChatModelParams } from '@kbn/langchain/server/language_models/bedrock_chat'; +import type { CustomChatModelInput as ActionsClientChatVertexAIParams } from '@kbn/langchain/server/language_models/gemini_chat'; +import type { CustomChatModelInput as ActionsClientSimpleChatModelParams } from '@kbn/langchain/server/language_models/simple_chat_model'; + +export type ChatModel = + | ActionsClientSimpleChatModel + | ActionsClientChatOpenAI + | ActionsClientBedrockChatModel + | ActionsClientChatVertexAI; + +export type ActionsClientChatModelClass = + | typeof ActionsClientSimpleChatModel + | typeof ActionsClientChatOpenAI + | typeof ActionsClientBedrockChatModel + | typeof ActionsClientChatVertexAI; + +export type ChatModelParams = Partial & + Partial & + Partial & + Partial & { + /** Enables the streaming mode of the response, disabled by default */ + streaming?: boolean; + }; + +const llmTypeDictionary: Record = { + [`.gen-ai`]: `openai`, + [`.bedrock`]: `bedrock`, + [`.gemini`]: `gemini`, +}; + +export class ActionsClientChat { + constructor( + private readonly connectorId: string, + private readonly actionsClient: ActionsClient, + private readonly logger: Logger + ) {} + + public async createModel(params?: ChatModelParams): Promise { + const connector = await this.actionsClient.get({ id: this.connectorId }); + if (!connector) { + throw new Error(`Connector not found: ${this.connectorId}`); + } + + const llmType = this.getLLMType(connector.actionTypeId); + const ChatModelClass = this.getLLMClass(llmType); + + const model = new ChatModelClass({ + actionsClient: this.actionsClient, + connectorId: this.connectorId, + logger: this.logger, + llmType, + model: connector.config?.defaultModel, + ...params, + streaming: params?.streaming ?? false, // disabling streaming by default, for some reason is enabled when omitted + }); + return model; + } + + private getLLMType(actionTypeId: string): string | undefined { + if (llmTypeDictionary[actionTypeId]) { + return llmTypeDictionary[actionTypeId]; + } + throw new Error(`Unknown LLM type for action type ID: ${actionTypeId}`); + } + + private getLLMClass(llmType?: string): ActionsClientChatModelClass { + switch (llmType) { + case 'bedrock': + return ActionsClientBedrockChatModel; + case 'gemini': + return ActionsClientChatVertexAI; + case 'openai': + default: + return ActionsClientChatOpenAI; + } + } +} diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts new file mode 100644 index 0000000000000..55256d0ad0fdd --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; +import type { PrebuiltRulesMapByName } from './prebuilt_rules'; +import { filterPrebuiltRules, retrievePrebuiltRulesMap } from './prebuilt_rules'; +import { rulesClientMock } from '@kbn/alerting-plugin/server/mocks'; + +jest.mock( + '../../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client', + () => ({ createPrebuiltRuleObjectsClient: jest.fn() }) +); +jest.mock( + '../../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client', + () => ({ createPrebuiltRuleAssetsClient: jest.fn() }) +); + +const mitreAttackIds = 'T1234'; +const rule1 = { + name: 'rule one', + id: 'rule1', + threat: [ + { + framework: 'MITRE ATT&CK', + technique: [{ id: mitreAttackIds, name: 'tactic one' }], + }, + ], +}; +const rule2 = { + name: 'rule two', + id: 'rule2', +}; + +const defaultRuleVersionsTriad = new Map([ + ['rule1', { target: rule1 }], + ['rule2', { target: rule2, current: rule2 }], +]); +const mockFetchRuleVersionsTriad = jest.fn().mockResolvedValue(defaultRuleVersionsTriad); +jest.mock( + '../../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad', + () => ({ + fetchRuleVersionsTriad: () => mockFetchRuleVersionsTriad(), + }) +); + +const defaultParams = { + soClient: savedObjectsClientMock.create(), + rulesClient: rulesClientMock.create(), +}; + +describe('retrievePrebuiltRulesMap', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('when prebuilt rule is installed', () => { + it('should return isInstalled flag', async () => { + const prebuiltRulesMap = await retrievePrebuiltRulesMap(defaultParams); + expect(prebuiltRulesMap.size).toBe(2); + expect(prebuiltRulesMap.get('rule one')).toEqual( + expect.objectContaining({ installedRuleId: undefined }) + ); + expect(prebuiltRulesMap.get('rule two')).toEqual( + expect.objectContaining({ installedRuleId: rule2.id }) + ); + }); + }); +}); + +describe('filterPrebuiltRules', () => { + let prebuiltRulesMap: PrebuiltRulesMapByName; + + beforeEach(async () => { + prebuiltRulesMap = await retrievePrebuiltRulesMap(defaultParams); + jest.clearAllMocks(); + }); + + describe('when splunk rule contains empty mitreAttackIds', () => { + it('should return empty rules map', async () => { + const filteredPrebuiltRules = filterPrebuiltRules(prebuiltRulesMap, []); + expect(filteredPrebuiltRules.size).toBe(0); + }); + }); + + describe('when splunk rule does not match mitreAttackIds', () => { + it('should return empty rules map', async () => { + const filteredPrebuiltRules = filterPrebuiltRules(prebuiltRulesMap, [`${mitreAttackIds}_2`]); + expect(filteredPrebuiltRules.size).toBe(0); + }); + }); + + describe('when splunk rule contains matching mitreAttackIds', () => { + it('should return the filtered rules map', async () => { + const filteredPrebuiltRules = filterPrebuiltRules(prebuiltRulesMap, [mitreAttackIds]); + expect(filteredPrebuiltRules.size).toBe(1); + expect(filteredPrebuiltRules.get('rule one')).toEqual( + expect.objectContaining({ rule: rule1 }) + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts new file mode 100644 index 0000000000000..ade6632aaa5b5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/util/prebuilt_rules.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import type { PrebuiltRuleAsset } from '../../../../detection_engine/prebuilt_rules'; +import { fetchRuleVersionsTriad } from '../../../../detection_engine/prebuilt_rules/logic/rule_versions/fetch_rule_versions_triad'; +import { createPrebuiltRuleObjectsClient } from '../../../../detection_engine/prebuilt_rules/logic/rule_objects/prebuilt_rule_objects_client'; +import { createPrebuiltRuleAssetsClient } from '../../../../detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client'; + +export interface PrebuiltRuleMapped { + rule: PrebuiltRuleAsset; + installedRuleId?: string; +} + +export type PrebuiltRulesMapByName = Map; + +interface RetrievePrebuiltRulesParams { + soClient: SavedObjectsClientContract; + rulesClient: RulesClient; +} + +export const retrievePrebuiltRulesMap = async ({ + soClient, + rulesClient, +}: RetrievePrebuiltRulesParams): Promise => { + const ruleAssetsClient = createPrebuiltRuleAssetsClient(soClient); + const ruleObjectsClient = createPrebuiltRuleObjectsClient(rulesClient); + + const prebuiltRulesMap = await fetchRuleVersionsTriad({ + ruleAssetsClient, + ruleObjectsClient, + }); + const prebuiltRulesByName: PrebuiltRulesMapByName = new Map(); + prebuiltRulesMap.forEach((ruleVersions) => { + const rule = ruleVersions.target || ruleVersions.current; + if (rule) { + prebuiltRulesByName.set(rule.name, { + rule, + installedRuleId: ruleVersions.current?.id, + }); + } + }); + return prebuiltRulesByName; +}; + +export const filterPrebuiltRules = ( + prebuiltRulesByName: PrebuiltRulesMapByName, + mitreAttackIds: string[] +) => { + const filteredPrebuiltRulesByName = new Map(); + if (mitreAttackIds?.length) { + // If this rule has MITRE ATT&CK IDs, remove unrelated prebuilt rules + prebuiltRulesByName.forEach(({ rule }, ruleName) => { + const mitreAttackThreat = rule.threat?.filter( + ({ framework }) => framework === 'MITRE ATT&CK' + ); + if (!mitreAttackThreat) { + // If this rule has no MITRE ATT&CK reference we skip it + return; + } + + const sameTechnique = mitreAttackThreat.find((threat) => + threat.technique?.some(({ id }) => mitreAttackIds?.includes(id)) + ); + + if (sameTechnique) { + filteredPrebuiltRulesByName.set(ruleName, prebuiltRulesByName.get(ruleName)); + } + }); + } + return filteredPrebuiltRulesByName; +}; diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts index 1892032a21723..78ec2ef89c7a3 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/rules/types.ts @@ -5,10 +5,29 @@ * 2.0. */ -import type { BulkResponse, SearchResponse } from '@elastic/elasticsearch/lib/api/types'; -import type { IClusterClient, KibanaRequest } from '@kbn/core/server'; +import type { + AuthenticatedUser, + IClusterClient, + KibanaRequest, + SavedObjectsClientContract, +} from '@kbn/core/server'; import type { Subject } from 'rxjs'; -import type { RuleMigration } from '../../../../common/siem_migrations/model/rule_migration.gen'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import type { ActionsClient } from '@kbn/actions-plugin/server'; +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { + RuleMigration, + RuleMigrationAllTaskStats, + RuleMigrationTaskStats, +} from '../../../../common/siem_migrations/model/rule_migration.gen'; +import type { RuleMigrationsDataClient } from './data_stream/rule_migrations_data_client'; +import type { RuleMigrationTaskStopResult, RuleMigrationTaskStartResult } from './task/types'; + +export interface StoredRuleMigration extends RuleMigration { + _id: string; + _index: string; +} export interface SiemRulesMigrationsSetupParams { esClusterClient: IClusterClient; @@ -16,15 +35,28 @@ export interface SiemRulesMigrationsSetupParams { tasksTimeoutMs?: number; } -export interface SiemRuleMigrationsGetClientParams { +export interface SiemRuleMigrationsCreateClientParams { request: KibanaRequest; + currentUser: AuthenticatedUser | null; spaceId: string; } -export interface RuleMigrationSearchParams { - migration_id?: string; +export interface SiemRuleMigrationsStartTaskParams { + migrationId: string; + connectorId: string; + invocationConfig: RunnableConfig; + inferenceClient: InferenceClient; + actionsClient: ActionsClient; + rulesClient: RulesClient; + soClient: SavedObjectsClientContract; } + export interface SiemRuleMigrationsClient { - create: (body: RuleMigration[]) => Promise; - search: (params: RuleMigrationSearchParams) => Promise; + data: RuleMigrationsDataClient; + task: { + start: (params: SiemRuleMigrationsStartTaskParams) => Promise; + stop: (migrationId: string) => Promise; + getStats: (migrationId: string) => Promise; + getAllStats: () => Promise; + }; } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts index 3d9e5b9fe179b..adf77756cce34 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.test.ts @@ -8,9 +8,15 @@ import { loggingSystemMock, elasticsearchServiceMock, httpServerMock, + securityServiceMock, } from '@kbn/core/server/mocks'; import { SiemMigrationsService } from './siem_migrations_service'; -import { MockSiemRuleMigrationsService, mockSetup, mockGetClient } from './rules/__mocks__/mocks'; +import { + MockSiemRuleMigrationsService, + mockSetup, + mockCreateClient, + mockStop, +} from './rules/__mocks__/mocks'; import type { ConfigType } from '../../config'; jest.mock('./rules/siem_rule_migrations_service'); @@ -25,6 +31,7 @@ describe('SiemMigrationsService', () => { let siemMigrationsService: SiemMigrationsService; const kibanaVersion = '8.16.0'; + const currentUser = securityServiceMock.createMockAuthenticatedUser(); const esClusterClient = elasticsearchServiceMock.createClusterClient(); const logger = loggingSystemMock.createLogger(); @@ -57,17 +64,22 @@ describe('SiemMigrationsService', () => { }); }); - describe('when createClient is called', () => { + describe('when createRulesClient is called', () => { it('should create rules client', async () => { - const request = httpServerMock.createKibanaRequest(); - siemMigrationsService.createClient({ spaceId: 'default', request }); - expect(mockGetClient).toHaveBeenCalledWith({ spaceId: 'default', request }); + const createRulesClientParams = { + spaceId: 'default', + request: httpServerMock.createKibanaRequest(), + currentUser, + }; + siemMigrationsService.createRulesClient(createRulesClientParams); + expect(mockCreateClient).toHaveBeenCalledWith(createRulesClientParams); }); }); describe('when stop is called', () => { it('should trigger the pluginStop subject', async () => { siemMigrationsService.stop(); + expect(mockStop).toHaveBeenCalled(); expect(mockReplaySubject$.next).toHaveBeenCalled(); expect(mockReplaySubject$.complete).toHaveBeenCalled(); }); diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts index b84281eb13d9b..7a85dd625feec 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/siem_migrations_service.ts @@ -9,11 +9,8 @@ import type { Logger } from '@kbn/core/server'; import { ReplaySubject, type Subject } from 'rxjs'; import type { ConfigType } from '../../config'; import { SiemRuleMigrationsService } from './rules/siem_rule_migrations_service'; -import type { - SiemMigrationsClient, - SiemMigrationsSetupParams, - SiemMigrationsGetClientParams, -} from './types'; +import type { SiemMigrationsSetupParams, SiemMigrationsCreateClientParams } from './types'; +import type { SiemRuleMigrationsClient } from './rules/types'; export class SiemMigrationsService { private pluginStop$: Subject; @@ -30,13 +27,12 @@ export class SiemMigrationsService { } } - createClient(params: SiemMigrationsGetClientParams): SiemMigrationsClient { - return { - rules: this.rules.getClient(params), - }; + createRulesClient(params: SiemMigrationsCreateClientParams): SiemRuleMigrationsClient { + return this.rules.createClient(params); } stop() { + this.rules.stop(); this.pluginStop$.next(); this.pluginStop$.complete(); } diff --git a/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts b/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts index b5647ff65e214..d2af1e2518722 100644 --- a/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts +++ b/x-pack/plugins/security_solution/server/lib/siem_migrations/types.ts @@ -6,15 +6,11 @@ */ import type { IClusterClient } from '@kbn/core/server'; -import type { SiemRuleMigrationsClient, SiemRuleMigrationsGetClientParams } from './rules/types'; +import type { SiemRuleMigrationsCreateClientParams } from './rules/types'; export interface SiemMigrationsSetupParams { esClusterClient: IClusterClient; tasksTimeoutMs?: number; } -export type SiemMigrationsGetClientParams = SiemRuleMigrationsGetClientParams; - -export interface SiemMigrationsClient { - rules: SiemRuleMigrationsClient; -} +export type SiemMigrationsCreateClientParams = SiemRuleMigrationsCreateClientParams; diff --git a/x-pack/plugins/security_solution/server/plugin_contract.ts b/x-pack/plugins/security_solution/server/plugin_contract.ts index c7ec67c1b07fc..c178f0654d9bd 100644 --- a/x-pack/plugins/security_solution/server/plugin_contract.ts +++ b/x-pack/plugins/security_solution/server/plugin_contract.ts @@ -45,6 +45,7 @@ import type { SharePluginStart } from '@kbn/share-plugin/server'; import type { GuidedOnboardingPluginSetup } from '@kbn/guided-onboarding-plugin/server'; import type { PluginSetup as UnifiedSearchServerPluginSetup } from '@kbn/unified-search-plugin/server'; import type { ElasticAssistantPluginStart } from '@kbn/elastic-assistant-plugin/server'; +import type { InferenceServerStart } from '@kbn/inference-plugin/server'; import type { ProductFeaturesService } from './lib/product_features_service/product_features_service'; import type { ExperimentalFeatures } from '../common'; @@ -88,6 +89,7 @@ export interface SecuritySolutionPluginStartDependencies { telemetry?: TelemetryPluginStart; share: SharePluginStart; actions: ActionsPluginStartContract; + inference: InferenceServerStart; } export interface SecuritySolutionPluginSetup { diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index f91f3c055a25b..fbe7be692e523 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -166,10 +166,16 @@ export class RequestContextFactory implements IRequestContextFactory { }) ), - getSiemMigrationsClient: memoize(() => - siemMigrationsService.createClient({ request, spaceId: getSpaceId() }) + getSiemRuleMigrationsClient: memoize(() => + siemMigrationsService.createRulesClient({ + request, + currentUser: coreContext.security.authc.getCurrentUser(), + spaceId: getSpaceId(), + }) ), + getInferenceClient: memoize(() => startPlugins.inference.getClient({ request })), + getExceptionListClient: () => { if (!lists) { return null; diff --git a/x-pack/plugins/security_solution/server/types.ts b/x-pack/plugins/security_solution/server/types.ts index 1355904dbe7f7..7afbf5dcff6d2 100644 --- a/x-pack/plugins/security_solution/server/types.ts +++ b/x-pack/plugins/security_solution/server/types.ts @@ -20,6 +20,7 @@ import type { AlertsClient, IRuleDataService } from '@kbn/rule-registry-plugin/s import type { Readable } from 'stream'; import type { AuditLogger } from '@kbn/security-plugin-types-server'; +import type { InferenceClient } from '@kbn/inference-plugin/server'; import type { Immutable } from '../common/endpoint/types'; import { AppClient } from './client'; import type { ConfigType } from './config'; @@ -35,7 +36,7 @@ import type { RiskScoreDataClient } from './lib/entity_analytics/risk_score/risk import type { AssetCriticalityDataClient } from './lib/entity_analytics/asset_criticality'; import type { IDetectionRulesClient } from './lib/detection_engine/rule_management/logic/detection_rules_client/detection_rules_client_interface'; import type { EntityStoreDataClient } from './lib/entity_analytics/entity_store/entity_store_data_client'; -import type { SiemMigrationsClient } from './lib/siem_migrations/types'; +import type { SiemRuleMigrationsClient } from './lib/siem_migrations/rules/types'; export { AppClient }; export interface SecuritySolutionApiRequestHandlerContext { @@ -58,7 +59,8 @@ export interface SecuritySolutionApiRequestHandlerContext { getRiskScoreDataClient: () => RiskScoreDataClient; getAssetCriticalityDataClient: () => AssetCriticalityDataClient; getEntityStoreDataClient: () => EntityStoreDataClient; - getSiemMigrationsClient: () => SiemMigrationsClient; + getSiemRuleMigrationsClient: () => SiemRuleMigrationsClient; + getInferenceClient: () => InferenceClient; } export type SecuritySolutionRequestHandlerContext = CustomRequestHandlerContext<{ diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index cbdd2aed3496f..df743a666108e 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -229,5 +229,6 @@ "@kbn/data-stream-adapter", "@kbn/core-lifecycle-server", "@kbn/core-user-profile-common", + "@kbn/langchain", ] } diff --git a/x-pack/test/api_integration/services/security_solution_api.gen.ts b/x-pack/test/api_integration/services/security_solution_api.gen.ts index 1ddbbf2ed7365..0b1338fee46e2 100644 --- a/x-pack/test/api_integration/services/security_solution_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_api.gen.ts @@ -95,6 +95,8 @@ import { GetRuleExecutionResultsRequestQueryInput, GetRuleExecutionResultsRequestParamsInput, } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.gen'; +import { GetRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; +import { GetRuleMigrationStatsRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; import { GetTimelineRequestQueryInput } from '@kbn/security-solution-plugin/common/api/timeline/get_timeline/get_timeline_route.gen'; import { GetTimelinesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/timeline/get_timelines/get_timelines_route.gen'; import { ImportRulesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/import_rules/import_rules_route.gen'; @@ -127,7 +129,12 @@ import { SetAlertAssigneesRequestBodyInput } from '@kbn/security-solution-plugin import { SetAlertsStatusRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/signals/set_signal_status/set_signals_status_route.gen'; import { SetAlertTagsRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/alert_tags/set_alert_tags/set_alert_tags.gen'; import { StartEntityEngineRequestParamsInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/engine/start.gen'; +import { + StartRuleMigrationRequestParamsInput, + StartRuleMigrationRequestBodyInput, +} from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; import { StopEntityEngineRequestParamsInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/entity_store/engine/stop.gen'; +import { StopRuleMigrationRequestParamsInput } from '@kbn/security-solution-plugin/common/siem_migrations/model/api/rules/rules_migration.gen'; import { SuggestUserProfilesRequestQueryInput } from '@kbn/security-solution-plugin/common/api/detection_engine/users/suggest_user_profiles_route.gen'; import { TriggerRiskScoreCalculationRequestBodyInput } from '@kbn/security-solution-plugin/common/api/entity_analytics/risk_engine/entity_calculation_route.gen'; import { UpdateRuleRequestBodyInput } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_management/crud/update_rule/update_rule_route.gen'; @@ -755,6 +762,16 @@ finalize it. .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Retrieves the rule migrations stats for all migrations stored in the system + */ + getAllStatsRuleMigration(kibanaSpace: string = 'default') { + return supertest + .get(routeWithNamespace('/internal/siem_migrations/rules/stats', kibanaSpace)) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, /** * Get the criticality record for a specific asset. */ @@ -909,11 +926,31 @@ finalize it. .query(props.query); }, /** - * Retrieves the rule migrations stored in the system + * Retrieves the rule documents stored in the system given the rule migration id */ - getRuleMigration(kibanaSpace: string = 'default') { + getRuleMigration(props: GetRuleMigrationProps, kibanaSpace: string = 'default') { return supertest - .get(routeWithNamespace('/internal/siem_migrations/rules', kibanaSpace)) + .get( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, + /** + * Retrieves the stats of a SIEM rules migration using the migration id provided + */ + getRuleMigrationStats(props: GetRuleMigrationStatsProps, kibanaSpace: string = 'default') { + return supertest + .get( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/stats', props.params), + kibanaSpace + ) + ) .set('kbn-xsrf', 'true') .set(ELASTIC_HTTP_VERSION_HEADER, '1') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); @@ -1260,6 +1297,22 @@ detection engine rules. .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Starts a SIEM rules migration using the migration id provided + */ + startRuleMigration(props: StartRuleMigrationProps, kibanaSpace: string = 'default') { + return supertest + .put( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/start', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') + .send(props.body as object); + }, stopEntityEngine(props: StopEntityEngineProps, kibanaSpace: string = 'default') { return supertest .post( @@ -1272,6 +1325,21 @@ detection engine rules. .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Stops a running SIEM rules migration using the migration id provided + */ + stopRuleMigration(props: StopRuleMigrationProps, kibanaSpace: string = 'default') { + return supertest + .put( + routeWithNamespace( + replaceParams('/internal/siem_migrations/rules/{migration_id}/stop', props.params), + kibanaSpace + ) + ) + .set('kbn-xsrf', 'true') + .set(ELASTIC_HTTP_VERSION_HEADER, '1') + .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); + }, /** * Suggests user profiles. */ @@ -1490,6 +1558,12 @@ export interface GetRuleExecutionResultsProps { query: GetRuleExecutionResultsRequestQueryInput; params: GetRuleExecutionResultsRequestParamsInput; } +export interface GetRuleMigrationProps { + params: GetRuleMigrationRequestParamsInput; +} +export interface GetRuleMigrationStatsProps { + params: GetRuleMigrationStatsRequestParamsInput; +} export interface GetTimelineProps { query: GetTimelineRequestQueryInput; } @@ -1562,9 +1636,16 @@ export interface SetAlertTagsProps { export interface StartEntityEngineProps { params: StartEntityEngineRequestParamsInput; } +export interface StartRuleMigrationProps { + params: StartRuleMigrationRequestParamsInput; + body: StartRuleMigrationRequestBodyInput; +} export interface StopEntityEngineProps { params: StopEntityEngineRequestParamsInput; } +export interface StopRuleMigrationProps { + params: StopRuleMigrationRequestParamsInput; +} export interface SuggestUserProfilesProps { query: SuggestUserProfilesRequestQueryInput; } From 3b2a5729316fa1bbd33b0149c7f54d01da496b61 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 21:41:39 +1100 Subject: [PATCH 04/12] [8.x] [Obs AI Assistant] Add uuid to knowledge base entries to avoid overwriting accidentally (#191043) (#199263) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Backport This will backport the following commits from `main` to `8.x`: - [[Obs AI Assistant] Add uuid to knowledge base entries to avoid overwriting accidentally (#191043)](https://github.com/elastic/kibana/pull/191043) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Søren Louv-Jansen --- .../common/types.ts | 6 +- .../common/utils/short_id_table.test.ts | 10 + .../server/analytics/recall_ranking.ts | 4 +- .../server/functions/summarize.ts | 25 +- .../server/routes/chat/route.ts | 2 +- .../server/routes/functions/route.ts | 15 +- .../server/routes/knowledge_base/route.ts | 129 ++-- .../server/service/client/index.ts | 64 +- .../server/service/index.ts | 102 +-- .../server/service/kb_component_template.ts | 11 +- .../service/knowledge_base_service/index.ts | 292 +++------ .../knowledge_base_service/kb_docs/lens.ts | 596 ------------------ .../server/service/types.ts | 5 +- ...t_system_message_from_instructions.test.ts | 6 +- .../get_system_message_from_instructions.ts | 9 +- .../server/service/util/split_kb_text.ts | 36 -- .../server/utils/recall/recall_and_score.ts | 22 +- .../utils/recall/retrieve_suggestions.ts | 24 - .../server/utils/recall/score_suggestions.ts | 60 +- .../server/utils/recall/types.ts | 10 - .../public/helpers/categorize_entries.ts | 29 +- .../hooks/use_create_knowledge_base_entry.ts | 12 +- ..._create_knowledge_base_user_instruction.ts | 4 +- .../use_import_knowledge_base_entries.ts | 2 +- .../knowledge_base_bulk_import_flyout.tsx | 6 +- .../knowledge_base_category_flyout.tsx | 4 +- ...nowledge_base_edit_manual_entry_flyout.tsx | 61 +- ...edge_base_edit_user_instruction_flyout.tsx | 6 +- .../components/knowledge_base_tab.test.tsx | 19 +- .../routes/components/knowledge_base_tab.tsx | 23 +- .../translations/translations/fr-FR.json | 1 - .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - .../common/config.ts | 20 +- .../common/users/users.ts | 14 +- .../tests/complete/complete.spec.ts | 12 +- .../tests/complete/functions/helpers.ts | 2 +- .../complete/functions/summarize.spec.ts | 14 +- .../tests/connectors/connectors.spec.ts | 6 +- .../tests/conversations/conversations.spec.ts | 24 +- .../knowledge_base/knowledge_base.spec.ts | 189 +++--- .../knowledge_base_setup.spec.ts | 4 +- .../knowledge_base_status.spec.ts | 6 +- .../knowledge_base_user_instructions.spec.ts | 73 +-- .../public_complete/public_complete.spec.ts | 2 +- .../common/config.ts | 12 +- .../common/ui/index.ts | 5 + .../tests/conversations/index.spec.ts | 16 +- .../knowledge_base_management/index.spec.ts | 124 ++++ .../check_registered_task_types.ts | 1 - .../knowledge_base/knowledge_base.spec.ts | 4 + 51 files changed, 687 insertions(+), 1438 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/kb_docs/lens.ts delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/retrieve_suggestions.ts delete mode 100644 x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/types.ts create mode 100644 x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts index 51ae37b39d90f..210eb08b31e1a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/types.ts @@ -82,8 +82,8 @@ export type ConversationUpdateRequest = ConversationRequestBase & { export interface KnowledgeBaseEntry { '@timestamp': string; id: string; + title?: string; text: string; - doc_id: string; confidence: 'low' | 'medium' | 'high'; is_correction: boolean; type?: 'user_instruction' | 'contextual'; @@ -96,12 +96,12 @@ export interface KnowledgeBaseEntry { } export interface Instruction { - doc_id: string; + id: string; text: string; } export interface AdHocInstruction { - doc_id?: string; + id?: string; text: string; instruction_type: 'user_instruction' | 'application_instruction'; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/short_id_table.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/short_id_table.test.ts index 784cf67530652..03d1cb177826e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/short_id_table.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/common/utils/short_id_table.test.ts @@ -7,6 +7,16 @@ import { ShortIdTable } from './short_id_table'; describe('shortIdTable', () => { + it('generates a short id from a uuid', () => { + const table = new ShortIdTable(); + + const uuid = 'd877f65c-4036-42c4-b105-19e2f1a1c045'; + const shortId = table.take(uuid); + + expect(shortId.length).toBe(4); + expect(table.lookup(shortId)).toBe(uuid); + }); + it('generates at least 10k unique ids consistently', () => { const ids = new Set(); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/analytics/recall_ranking.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/analytics/recall_ranking.ts index 4c82f79fcba8d..4371310811edf 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/analytics/recall_ranking.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/analytics/recall_ranking.ts @@ -52,9 +52,9 @@ const schema: RootSchema = { }, }; -export const RecallRankingEventType = 'observability_ai_assistant_recall_ranking'; +export const recallRankingEventType = 'observability_ai_assistant_recall_ranking'; export const recallRankingEvent: EventTypeOpts = { - eventType: RecallRankingEventType, + eventType: recallRankingEventType, schema, }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts index 8865861d81f45..1f4afdbdd56bb 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/functions/summarize.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { KnowledgeBaseType } from '../../common/types'; +import { v4 } from 'uuid'; import type { FunctionRegistrationParameters } from '.'; import { KnowledgeBaseEntryRole } from '../../common'; @@ -14,6 +14,7 @@ export const SUMMARIZE_FUNCTION_NAME = 'summarize'; export function registerSummarizationFunction({ client, functions, + resources, }: FunctionRegistrationParameters) { functions.registerFunction( { @@ -28,10 +29,10 @@ export function registerSummarizationFunction({ parameters: { type: 'object', properties: { - id: { + title: { type: 'string', description: - 'An id for the document. This should be a short human-readable keyword field with only alphabetic characters and underscores, that allow you to update it later.', + 'A human readable title that can be used to identify the document later. This should be no longer than 255 characters', }, text: { type: 'string', @@ -54,7 +55,7 @@ export function registerSummarizationFunction({ }, }, required: [ - 'id' as const, + 'title' as const, 'text' as const, 'is_correction' as const, 'confidence' as const, @@ -62,21 +63,23 @@ export function registerSummarizationFunction({ ], }, }, - ( - { arguments: { id, text, is_correction: isCorrection, confidence, public: isPublic } }, + async ( + { arguments: { title, text, is_correction: isCorrection, confidence, public: isPublic } }, signal ) => { + const id = v4(); + resources.logger.debug(`Creating new knowledge base entry with id: ${id}`); + return client .addKnowledgeBaseEntry({ entry: { - doc_id: id, - role: KnowledgeBaseEntryRole.AssistantSummarization, id, + title, text, - is_correction: isCorrection, - type: KnowledgeBaseType.Contextual, - confidence, public: isPublic, + role: KnowledgeBaseEntryRole.AssistantSummarization, + confidence, + is_correction: isCorrection, labels: {}, }, // signal, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts index 8bc88cca10b01..a6fe57cb58adc 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/chat/route.ts @@ -42,7 +42,7 @@ const chatCompleteBaseRt = t.type({ ]), instructions: t.array( t.intersection([ - t.partial({ doc_id: t.string }), + t.partial({ id: t.string }), t.type({ text: t.string, instruction_type: t.union([ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts index c402a0506736f..1571487765c09 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/functions/route.ts @@ -7,6 +7,7 @@ import { notImplemented } from '@hapi/boom'; import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; +import { v4 } from 'uuid'; import { FunctionDefinition } from '../../../common/functions/types'; import { KnowledgeBaseEntryRole } from '../../../common/types'; import type { RecalledEntry } from '../../service/knowledge_base_service'; @@ -114,7 +115,8 @@ const functionRecallRoute = createObservabilityAIAssistantServerRoute({ throw notImplemented(); } - return client.recall({ queries, categories }); + const entries = await client.recall({ queries, categories }); + return { entries }; }, }); @@ -122,11 +124,10 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', params: t.type({ body: t.type({ - id: t.string, + title: t.string, text: nonEmptyStringRt, confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), is_correction: toBooleanRt, - type: t.union([t.literal('user_instruction'), t.literal('contextual')]), public: toBooleanRt, labels: t.record(t.string, t.string), }), @@ -142,10 +143,9 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ } const { + title, confidence, - id, is_correction: isCorrection, - type, text, public: isPublic, labels, @@ -153,11 +153,10 @@ const functionSummariseRoute = createObservabilityAIAssistantServerRoute({ return client.addKnowledgeBaseEntry({ entry: { + title, confidence, - id, - doc_id: id, + id: v4(), is_correction: isCorrection, - type, text, public: isPublic, labels, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts index 1eb1650545781..0f1852c0e396c 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/routes/knowledge_base/route.ts @@ -9,16 +9,12 @@ import type { MlDeploymentAllocationState, MlDeploymentState, } from '@elastic/elasticsearch/lib/api/types'; +import pLimit from 'p-limit'; import { notImplemented } from '@hapi/boom'; import { nonEmptyStringRt, toBooleanRt } from '@kbn/io-ts-utils'; import * as t from 'io-ts'; import { createObservabilityAIAssistantServerRoute } from '../create_observability_ai_assistant_server_route'; -import { - Instruction, - KnowledgeBaseEntry, - KnowledgeBaseEntryRole, - KnowledgeBaseType, -} from '../../../common/types'; +import { Instruction, KnowledgeBaseEntry, KnowledgeBaseEntryRole } from '../../../common/types'; const getKnowledgeBaseStatus = createObservabilityAIAssistantServerRoute({ endpoint: 'GET /internal/observability_ai_assistant/kb/status', @@ -108,18 +104,8 @@ const saveKnowledgeBaseUserInstruction = createObservabilityAIAssistantServerRou } const { id, text, public: isPublic } = resources.params.body; - return client.addKnowledgeBaseEntry({ - entry: { - id, - doc_id: id, - text, - public: isPublic, - confidence: 'high', - is_correction: false, - type: KnowledgeBaseType.UserInstruction, - labels: {}, - role: KnowledgeBaseEntryRole.UserEntry, - }, + return client.addUserInstruction({ + entry: { id, text, public: isPublic }, }); }, }); @@ -153,26 +139,29 @@ const getKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ }, }); +const knowledgeBaseEntryRt = t.intersection([ + t.type({ + id: t.string, + title: t.string, + text: nonEmptyStringRt, + }), + t.partial({ + confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), + is_correction: toBooleanRt, + public: toBooleanRt, + labels: t.record(t.string, t.string), + role: t.union([ + t.literal(KnowledgeBaseEntryRole.AssistantSummarization), + t.literal(KnowledgeBaseEntryRole.UserEntry), + t.literal(KnowledgeBaseEntryRole.Elastic), + ]), + }), +]); + const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/save', params: t.type({ - body: t.intersection([ - t.type({ - id: t.string, - text: nonEmptyStringRt, - }), - t.partial({ - confidence: t.union([t.literal('low'), t.literal('medium'), t.literal('high')]), - is_correction: toBooleanRt, - public: toBooleanRt, - labels: t.record(t.string, t.string), - role: t.union([ - t.literal('assistant_summarization'), - t.literal('user_entry'), - t.literal('elastic'), - ]), - }), - ]), + body: knowledgeBaseEntryRt, }), options: { tags: ['access:ai_assistant'], @@ -184,27 +173,15 @@ const saveKnowledgeBaseEntry = createObservabilityAIAssistantServerRoute({ throw notImplemented(); } - const { - id, - text, - public: isPublic, - confidence, - is_correction: isCorrection, - labels, - role, - } = resources.params.body; - + const entry = resources.params.body; return client.addKnowledgeBaseEntry({ entry: { - id, - text, - doc_id: id, - confidence: confidence ?? 'high', - is_correction: isCorrection ?? false, - type: 'contextual', - public: isPublic ?? true, - labels: labels ?? {}, - role: (role as KnowledgeBaseEntryRole) ?? KnowledgeBaseEntryRole.UserEntry, + confidence: 'high', + is_correction: false, + public: true, + labels: {}, + role: KnowledgeBaseEntryRole.UserEntry, + ...entry, }, }); }, @@ -235,12 +212,7 @@ const importKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', params: t.type({ body: t.type({ - entries: t.array( - t.type({ - id: t.string, - text: nonEmptyStringRt, - }) - ), + entries: t.array(knowledgeBaseEntryRt), }), }), options: { @@ -253,18 +225,29 @@ const importKnowledgeBaseEntries = createObservabilityAIAssistantServerRoute({ throw notImplemented(); } - const entries = resources.params.body.entries.map((entry) => ({ - doc_id: entry.id, - confidence: 'high' as KnowledgeBaseEntry['confidence'], - is_correction: false, - type: 'contextual' as const, - public: true, - labels: {}, - role: KnowledgeBaseEntryRole.UserEntry, - ...entry, - })); - - return await client.importKnowledgeBaseEntries({ entries }); + const status = await client.getKnowledgeBaseStatus(); + if (!status.ready) { + throw new Error('Knowledge base is not ready'); + } + + const limiter = pLimit(5); + + const promises = resources.params.body.entries.map(async (entry) => { + return limiter(async () => { + return client.addKnowledgeBaseEntry({ + entry: { + confidence: 'high', + is_correction: false, + public: true, + labels: {}, + role: KnowledgeBaseEntryRole.UserEntry, + ...entry, + }, + }); + }); + }); + + await Promise.all(promises); }, }); @@ -273,8 +256,8 @@ export const knowledgeBaseRoutes = { ...getKnowledgeBaseStatus, ...getKnowledgeBaseEntries, ...saveKnowledgeBaseUserInstruction, - ...getKnowledgeBaseUserInstructions, ...importKnowledgeBaseEntries, + ...getKnowledgeBaseUserInstructions, ...saveKnowledgeBaseEntry, ...deleteKnowledgeBaseEntry, }; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts index 162220ec7a7f1..048bbd2d362c2 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/client/index.ts @@ -47,21 +47,19 @@ import { } from '../../../common/conversation_complete'; import { CompatibleJSONSchema } from '../../../common/functions/types'; import { - AdHocInstruction, + type AdHocInstruction, type Conversation, type ConversationCreateRequest, type ConversationUpdateRequest, type KnowledgeBaseEntry, type Message, + KnowledgeBaseType, + KnowledgeBaseEntryRole, } from '../../../common/types'; import { withoutTokenCountEvents } from '../../../common/utils/without_token_count_events'; import { CONTEXT_FUNCTION_NAME } from '../../functions/context'; import type { ChatFunctionClient } from '../chat_function_client'; -import { - KnowledgeBaseEntryOperationType, - KnowledgeBaseService, - RecalledEntry, -} from '../knowledge_base_service'; +import { KnowledgeBaseService, RecalledEntry } from '../knowledge_base_service'; import { getAccessQuery } from '../util/get_access_query'; import { getSystemMessageFromInstructions } from '../util/get_system_message_from_instructions'; import { replaceSystemMessage } from '../util/replace_system_message'; @@ -709,7 +707,7 @@ export class ObservabilityAIAssistantClient { }: { queries: Array<{ text: string; boost?: number }>; categories?: string[]; - }): Promise<{ entries: RecalledEntry[] }> => { + }): Promise => { return ( this.dependencies.knowledgeBaseService?.recall({ namespace: this.dependencies.namespace, @@ -718,7 +716,7 @@ export class ObservabilityAIAssistantClient { categories, esClient: this.dependencies.esClient, uiSettingsClient: this.dependencies.uiSettingsClient, - }) || { entries: [] } + }) || [] ); }; @@ -730,29 +728,55 @@ export class ObservabilityAIAssistantClient { return this.dependencies.knowledgeBaseService.setup(); }; - addKnowledgeBaseEntry = async ({ + addUserInstruction = async ({ entry, }: { - entry: Omit; + entry: Omit< + KnowledgeBaseEntry, + '@timestamp' | 'confidence' | 'is_correction' | 'type' | 'role' + >; }): Promise => { + // for now we want to limit the number of user instructions to 1 per user + // if a user instruction already exists for the user, we get the id and update it + this.dependencies.logger.debug('Adding user instruction entry'); + const existingId = await this.dependencies.knowledgeBaseService.getPersonalUserInstructionId({ + isPublic: entry.public, + namespace: this.dependencies.namespace, + user: this.dependencies.user, + }); + + if (existingId) { + entry.id = existingId; + this.dependencies.logger.debug(`Updating user instruction with id "${existingId}"`); + } + return this.dependencies.knowledgeBaseService.addEntry({ namespace: this.dependencies.namespace, user: this.dependencies.user, - entry, + entry: { + ...entry, + confidence: 'high', + is_correction: false, + type: KnowledgeBaseType.UserInstruction, + labels: {}, + role: KnowledgeBaseEntryRole.UserEntry, + }, }); }; - importKnowledgeBaseEntries = async ({ - entries, + addKnowledgeBaseEntry = async ({ + entry, }: { - entries: Array>; + entry: Omit; }): Promise => { - const operations = entries.map((entry) => ({ - type: KnowledgeBaseEntryOperationType.Index, - document: { ...entry, '@timestamp': new Date().toISOString() }, - })); - - await this.dependencies.knowledgeBaseService.addEntries({ operations }); + return this.dependencies.knowledgeBaseService.addEntry({ + namespace: this.dependencies.namespace, + user: this.dependencies.user, + entry: { + ...entry, + type: KnowledgeBaseType.Contextual, + }, + }); }; getKnowledgeBaseEntries = async ({ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts index d1aba4f232b0d..eb7eab19340ce 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/index.ts @@ -13,18 +13,14 @@ import { getSpaceIdFromPath } from '@kbn/spaces-plugin/common'; import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; import { once } from 'lodash'; import type { AssistantScope } from '@kbn/ai-assistant-common'; -import { - KnowledgeBaseEntryRole, - ObservabilityAIAssistantScreenContextRequest, -} from '../../common/types'; +import { ObservabilityAIAssistantScreenContextRequest } from '../../common/types'; import type { ObservabilityAIAssistantPluginStartDependencies } from '../types'; import { ChatFunctionClient } from './chat_function_client'; import { ObservabilityAIAssistantClient } from './client'; import { conversationComponentTemplate } from './conversation_component_template'; import { kbComponentTemplate } from './kb_component_template'; -import { KnowledgeBaseEntryOperationType, KnowledgeBaseService } from './knowledge_base_service'; +import { KnowledgeBaseService } from './knowledge_base_service'; import type { RegistrationCallback, RespondFunctionResources } from './types'; -import { splitKbText } from './util/split_kb_text'; function getResourceName(resource: string) { return `.kibana-observability-ai-assistant-${resource}`; @@ -52,24 +48,11 @@ export const resourceNames = { }, }; -export const INDEX_QUEUED_DOCUMENTS_TASK_ID = 'observabilityAIAssistant:indexQueuedDocumentsTask'; - -export const INDEX_QUEUED_DOCUMENTS_TASK_TYPE = INDEX_QUEUED_DOCUMENTS_TASK_ID + 'Type'; - -type KnowledgeBaseEntryRequest = { id: string; labels?: Record } & ( - | { - text: string; - } - | { - texts: string[]; - } -); - export class ObservabilityAIAssistantService { private readonly core: CoreSetup; private readonly logger: Logger; private readonly getModelId: () => Promise; - private kbService?: KnowledgeBaseService; + public kbService?: KnowledgeBaseService; private enableKnowledgeBase: boolean; private readonly registrations: RegistrationCallback[] = []; @@ -93,26 +76,6 @@ export class ObservabilityAIAssistantService { this.enableKnowledgeBase = enableKnowledgeBase; this.allowInit(); - if (enableKnowledgeBase) { - taskManager.registerTaskDefinitions({ - [INDEX_QUEUED_DOCUMENTS_TASK_TYPE]: { - title: 'Index queued KB articles', - description: - 'Indexes previously registered entries into the knowledge base when it is ready', - timeout: '30m', - maxAttempts: 2, - createTaskRunner: (context) => { - return { - run: async () => { - if (this.kbService) { - await this.kbService.processQueue(); - } - }, - }; - }, - }, - }); - } } getKnowledgeBaseStatus() { @@ -336,65 +299,6 @@ export class ObservabilityAIAssistantService { return fnClient; } - addToKnowledgeBaseQueue(entries: KnowledgeBaseEntryRequest[]): void { - if (this.enableKnowledgeBase) { - this.init() - .then(() => { - this.kbService!.queue( - entries.flatMap((entry) => { - const entryWithSystemProperties = { - ...entry, - '@timestamp': new Date().toISOString(), - doc_id: entry.id, - public: true, - confidence: 'high' as const, - type: 'contextual' as const, - is_correction: false, - labels: { - ...entry.labels, - }, - role: KnowledgeBaseEntryRole.Elastic, - }; - - const operations = - 'texts' in entryWithSystemProperties - ? splitKbText(entryWithSystemProperties) - : [ - { - type: KnowledgeBaseEntryOperationType.Index, - document: entryWithSystemProperties, - }, - ]; - - return operations; - }) - ); - }) - .catch((error) => { - this.logger.error( - `Could not index ${entries.length} entries because of an initialisation error` - ); - this.logger.error(error); - }); - } - } - - addCategoryToKnowledgeBase(categoryId: string, entries: KnowledgeBaseEntryRequest[]) { - if (this.enableKnowledgeBase) { - this.addToKnowledgeBaseQueue( - entries.map((entry) => { - return { - ...entry, - labels: { - ...entry.labels, - category: categoryId, - }, - }; - }) - ); - } - } - register(cb: RegistrationCallback) { this.registrations.push(cb); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts index a4c6dc25d2e57..b1b2d3293a234 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/kb_component_template.ts @@ -31,7 +31,16 @@ export const kbComponentTemplate: ClusterComponentTemplate['component_template'] properties: { '@timestamp': date, id: keyword, - doc_id: { type: 'text', fielddata: true }, + doc_id: { type: 'text', fielddata: true }, // deprecated but kept for backwards compatibility + title: { + type: 'text', + fields: { + keyword: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, user: { properties: { id: keyword, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts index 7306a0df7c572..92ce3a4a7e03b 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/index.ts @@ -9,16 +9,11 @@ import { serverUnavailable, gatewayTimeout, badRequest } from '@hapi/boom'; import type { ElasticsearchClient, IUiSettingsClient } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; -import pLimit from 'p-limit'; import pRetry from 'p-retry'; -import { map, orderBy } from 'lodash'; +import { orderBy } from 'lodash'; import { encode } from 'gpt-tokenizer'; import { MlTrainedModelDeploymentNodesStats } from '@elastic/elasticsearch/lib/api/types'; -import { - INDEX_QUEUED_DOCUMENTS_TASK_ID, - INDEX_QUEUED_DOCUMENTS_TASK_TYPE, - resourceNames, -} from '..'; +import { resourceNames } from '..'; import { Instruction, KnowledgeBaseEntry, @@ -63,36 +58,11 @@ function throwKnowledgeBaseNotReady(body: any) { throw serverUnavailable(`Knowledge base is not ready yet`, body); } -export enum KnowledgeBaseEntryOperationType { - Index = 'index', - Delete = 'delete', -} - -interface KnowledgeBaseDeleteOperation { - type: KnowledgeBaseEntryOperationType.Delete; - doc_id?: string; - labels?: Record; -} - -interface KnowledgeBaseIndexOperation { - type: KnowledgeBaseEntryOperationType.Index; - document: KnowledgeBaseEntry; -} - -export type KnowledgeBaseEntryOperation = - | KnowledgeBaseDeleteOperation - | KnowledgeBaseIndexOperation; - export class KnowledgeBaseService { - private hasSetup: boolean = false; - - private _queue: KnowledgeBaseEntryOperation[] = []; - - constructor(private readonly dependencies: Dependencies) { - this.ensureTaskScheduled(); - } + constructor(private readonly dependencies: Dependencies) {} setup = async () => { + this.dependencies.logger.debug('Setting up knowledge base'); if (!this.dependencies.enabled) { return; } @@ -192,7 +162,7 @@ export class KnowledgeBaseService { ); if (isReady) { - return Promise.resolve(); + return; } this.dependencies.logger.debug(`${elserModelId} model is not allocated yet`); @@ -202,116 +172,10 @@ export class KnowledgeBaseService { }, retryOptions); this.dependencies.logger.info(`${elserModelId} model is ready`); - this.ensureTaskScheduled(); }; - private ensureTaskScheduled() { - if (!this.dependencies.enabled) { - return; - } - this.dependencies.taskManagerStart - .ensureScheduled({ - taskType: INDEX_QUEUED_DOCUMENTS_TASK_TYPE, - id: INDEX_QUEUED_DOCUMENTS_TASK_ID, - state: {}, - params: {}, - schedule: { - interval: '1h', - }, - }) - .then(() => { - this.dependencies.logger.debug('Scheduled queue task'); - return this.dependencies.taskManagerStart.runSoon(INDEX_QUEUED_DOCUMENTS_TASK_ID); - }) - .then(() => { - this.dependencies.logger.debug('Queue task ran'); - }) - .catch((err) => { - this.dependencies.logger.error(`Failed to schedule queue task`); - this.dependencies.logger.error(err); - }); - } - - private async processOperation(operation: KnowledgeBaseEntryOperation) { - if (operation.type === KnowledgeBaseEntryOperationType.Delete) { - await this.dependencies.esClient.asInternalUser.deleteByQuery({ - index: resourceNames.aliases.kb, - query: { - bool: { - filter: [ - ...(operation.doc_id ? [{ term: { _id: operation.doc_id } }] : []), - ...(operation.labels - ? map(operation.labels, (value, key) => { - return { term: { [key]: value } }; - }) - : []), - ], - }, - }, - }); - return; - } - - await this.addEntry({ - entry: operation.document, - }); - } - - async processQueue() { - if (!this._queue.length || !this.dependencies.enabled) { - return; - } - - if (!(await this.status()).ready) { - this.dependencies.logger.debug(`Bailing on queue task: KB is not ready yet`); - return; - } - - this.dependencies.logger.debug(`Processing queue`); - - this.hasSetup = true; - - this.dependencies.logger.info(`Processing ${this._queue.length} queue operations`); - - const limiter = pLimit(5); - - const operations = this._queue.concat(); - - await Promise.all( - operations.map((operation) => - limiter(async () => { - this._queue.splice(operations.indexOf(operation), 1); - await this.processOperation(operation); - }) - ) - ); - - this.dependencies.logger.info('Processed all queued operations'); - } - - queue(operations: KnowledgeBaseEntryOperation[]): void { - if (!operations.length) { - return; - } - - if (!this.hasSetup) { - this._queue.push(...operations); - return; - } - - const limiter = pLimit(5); - - const limitedFunctions = this._queue.map((operation) => - limiter(() => this.processOperation(operation)) - ); - - Promise.all(limitedFunctions).catch((err) => { - this.dependencies.logger.error(`Failed to process all queued operations`); - this.dependencies.logger.error(err); - }); - } - status = async () => { + this.dependencies.logger.debug('Checking model status'); if (!this.dependencies.enabled) { return { ready: false, enabled: false }; } @@ -324,15 +188,24 @@ export class KnowledgeBaseService { const elserModelStats = modelStats.trained_model_stats[0]; const deploymentState = elserModelStats.deployment_stats?.state; const allocationState = elserModelStats.deployment_stats?.allocation_status.state; + const ready = deploymentState === 'started' && allocationState === 'fully_allocated'; + + this.dependencies.logger.debug( + `Model deployment state: ${deploymentState}, allocation state: ${allocationState}, ready: ${ready}` + ); return { - ready: deploymentState === 'started' && allocationState === 'fully_allocated', + ready, deployment_state: deploymentState, allocation_state: allocationState, model_name: elserModelId, enabled: true, }; } catch (error) { + this.dependencies.logger.debug( + `Failed to get status for model "${elserModelId}" due to ${error.message}` + ); + return { error: error instanceof errors.ResponseError ? error.body.error : String(error), ready: false, @@ -380,18 +253,21 @@ export class KnowledgeBaseService { }; const response = await this.dependencies.esClient.asInternalUser.search< - Pick + Pick & { doc_id?: string } >({ index: [resourceNames.aliases.kb], query: esQuery, size: 20, _source: { - includes: ['text', 'is_correction', 'labels'], + includes: ['text', 'is_correction', 'labels', 'doc_id', 'title'], }, }); return response.hits.hits.map((hit) => ({ - ...hit._source!, + text: hit._source?.text!, + is_correction: hit._source?.is_correction, + labels: hit._source?.labels, + title: hit._source?.title ?? hit._source?.doc_id, // use `doc_id` as fallback title for backwards compatibility score: hit._score!, id: hit._id!, })); @@ -411,12 +287,11 @@ export class KnowledgeBaseService { namespace: string; esClient: { asCurrentUser: ElasticsearchClient; asInternalUser: ElasticsearchClient }; uiSettingsClient: IUiSettingsClient; - }): Promise<{ - entries: RecalledEntry[]; - }> => { + }): Promise => { if (!this.dependencies.enabled) { - return { entries: [] }; + return []; } + this.dependencies.logger.debug( () => `Recalling entries from KB for queries: "${JSON.stringify(queries)}"` ); @@ -480,9 +355,7 @@ export class KnowledgeBaseService { this.dependencies.logger.info(`Dropped ${droppedEntries} entries because of token limit`); } - return { - entries: returnedEntries, - }; + return returnedEntries; }; getUserInstructions = async ( @@ -508,11 +381,11 @@ export class KnowledgeBaseService { }, }, size: 500, - _source: ['doc_id', 'text', 'public'], + _source: ['id', 'text', 'public'], }); return response.hits.hits.map((hit) => ({ - doc_id: hit._source?.doc_id ?? '', + id: hit._id!, text: hit._source?.text ?? '', public: hit._source?.public, })); @@ -536,13 +409,17 @@ export class KnowledgeBaseService { return { entries: [] }; } try { - const response = await this.dependencies.esClient.asInternalUser.search({ + const response = await this.dependencies.esClient.asInternalUser.search< + KnowledgeBaseEntry & { doc_id?: string } + >({ index: resourceNames.aliases.kb, query: { bool: { filter: [ - // filter title by query - ...(query ? [{ wildcard: { doc_id: { value: `${query}*` } } }] : []), + // filter by search query + ...(query + ? [{ query_string: { query: `${query}*`, fields: ['doc_id', 'title'] } }] + : []), { // exclude user instructions bool: { must_not: { term: { type: KnowledgeBaseType.UserInstruction } } }, @@ -550,16 +427,17 @@ export class KnowledgeBaseService { ], }, }, - sort: [ - { - [String(sortBy)]: { - order: sortDirection, - }, - }, - ], + sort: + sortBy === 'title' + ? [ + { ['title.keyword']: { order: sortDirection } }, + { doc_id: { order: sortDirection } }, // sort by doc_id for backwards compatibility + ] + : [{ [String(sortBy)]: { order: sortDirection } }], size: 500, _source: { includes: [ + 'title', 'doc_id', 'text', 'is_correction', @@ -577,6 +455,7 @@ export class KnowledgeBaseService { return { entries: response.hits.hits.map((hit) => ({ ...hit._source!, + title: hit._source!.title ?? hit._source!.doc_id, // use `doc_id` as fallback title for backwards compatibility role: hit._source!.role ?? KnowledgeBaseEntryRole.UserEntry, score: hit._score, id: hit._id!, @@ -590,7 +469,7 @@ export class KnowledgeBaseService { } }; - getExistingUserInstructionId = async ({ + getPersonalUserInstructionId = async ({ isPublic, user, namespace, @@ -602,9 +481,7 @@ export class KnowledgeBaseService { if (!this.dependencies.enabled) { return null; } - const res = await this.dependencies.esClient.asInternalUser.search< - Pick - >({ + const res = await this.dependencies.esClient.asInternalUser.search({ index: resourceNames.aliases.kb, query: { bool: { @@ -616,14 +493,47 @@ export class KnowledgeBaseService { }, }, size: 1, - _source: ['doc_id'], + _source: false, }); - return res.hits.hits[0]?._source?.doc_id; + return res.hits.hits[0]?._id; + }; + + getUuidFromDocId = async ({ + docId, + user, + namespace, + }: { + docId: string; + user?: { name: string; id?: string }; + namespace?: string; + }) => { + const query = { + bool: { + filter: [ + { term: { doc_id: docId } }, + + // exclude user instructions + { bool: { must_not: { term: { type: KnowledgeBaseType.UserInstruction } } } }, + + // restrict access to user's own entries + ...getAccessQuery({ user, namespace }), + ], + }, + }; + + const response = await this.dependencies.esClient.asInternalUser.search({ + size: 1, + index: resourceNames.aliases.kb, + query, + _source: false, + }); + + return response.hits.hits[0]?._id; }; addEntry = async ({ - entry: { id, ...document }, + entry: { id, ...doc }, user, namespace, }: { @@ -634,19 +544,6 @@ export class KnowledgeBaseService { if (!this.dependencies.enabled) { return; } - // for now we want to limit the number of user instructions to 1 per user - if (document.type === KnowledgeBaseType.UserInstruction) { - const existingId = await this.getExistingUserInstructionId({ - isPublic: document.public, - user, - namespace, - }); - - if (existingId) { - id = existingId; - document.doc_id = existingId; - } - } try { await this.dependencies.esClient.asInternalUser.index({ @@ -654,7 +551,7 @@ export class KnowledgeBaseService { id, document: { '@timestamp': new Date().toISOString(), - ...document, + ...doc, user, namespace, }, @@ -669,29 +566,6 @@ export class KnowledgeBaseService { } }; - addEntries = async ({ - operations, - }: { - operations: KnowledgeBaseEntryOperation[]; - }): Promise => { - if (!this.dependencies.enabled) { - return; - } - this.dependencies.logger.info(`Starting import of ${operations.length} entries`); - - const limiter = pLimit(5); - - await Promise.all( - operations.map((operation) => - limiter(async () => { - await this.processOperation(operation); - }) - ) - ); - - this.dependencies.logger.info(`Completed import of ${operations.length} entries`); - }; - deleteEntry = async ({ id }: { id: string }): Promise => { try { await this.dependencies.esClient.asInternalUser.delete({ diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/kb_docs/lens.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/kb_docs/lens.ts deleted file mode 100644 index 9baf75f6ff552..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/knowledge_base_service/kb_docs/lens.ts +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import dedent from 'dedent'; -import type { Logger } from '@kbn/logging'; -import type { ObservabilityAIAssistantService } from '../..'; - -export function addLensDocsToKb({ - service, -}: { - service: ObservabilityAIAssistantService; - logger: Logger; -}) { - service.addCategoryToKnowledgeBase('lens', [ - { - id: 'lens_formulas_how_it_works', - texts: [ - `Lens formulas let you do math using a combination of Elasticsearch aggregations and - math functions. There are three main types of functions: - - * Elasticsearch metrics, like \`sum(bytes)\` - * Time series functions use Elasticsearch metrics as input, like \`cumulative_sum()\` - * Math functions like \`round()\` - - An example formula that uses all of these: - - \`\`\` - round(100 * moving_average( - average(cpu.load.pct), - window=10, - kql='datacenter.name: east*' - )) - \`\`\` - `, - `Elasticsearch functions take a field name, which can be in quotes. \`sum(bytes)\` is the same - as \`sum('bytes')\`. - - Some functions take named arguments, like \`moving_average(count(), window=5)\`. - - Elasticsearch metrics can be filtered using KQL or Lucene syntax. To add a filter, use the named - parameter \`kql='field: value'\` or \`lucene=''\`. Always use single quotes when writing KQL or Lucene - queries. If your search has a single quote in it, use a backslash to escape, like: \`kql='Women's'\' - - Math functions can take positional arguments, like pow(count(), 3) is the same as count() * count() * count() - - Use the symbols +, -, /, and * to perform basic math.`, - ], - }, - { - id: 'lens_common_formulas', - texts: [ - `The most common formulas are dividing two values to produce a percent. To display accurately, set - "value format" to "percent"`, - `### Filter ratio: - - Use \`kql=''\` to filter one set of documents and compare it to other documents within the same grouping. - For example, to see how the error rate changes over time: - - \`\`\` - count(kql='response.status_code > 400') / count() - \`\`\``, - `### Week over week: - - Use \`shift='1w'\` to get the value of each grouping from - the previous week. Time shift should not be used with the *Top values* function. - - \`\`\` - percentile(system.network.in.bytes, percentile=99) / - percentile(system.network.in.bytes, percentile=99, shift='1w') - \`\`\``, - - `### Percent of total - - Formulas can calculate \`overall_sum\` for all the groupings, - which lets you convert each grouping into a percent of total: - - \`\`\` - sum(products.base_price) / overall_sum(sum(products.base_price)) - \`\`\``, - - `### Recent change - - Use \`reducedTimeRange='30m'\` to add an additional filter on the - time range of a metric aligned with the end of the global time range. - This can be used to calculate how much a value changed recently. - - \`\`\` - max(system.network.in.bytes, reducedTimeRange="30m") - - min(system.network.in.bytes, reducedTimeRange="30m") - \`\`\` - `, - ], - }, - { - id: 'lens_formulas_elasticsearch_functions', - texts: [ - `## Elasticsearch functions - - These functions will be executed on the raw documents for each row of the - resulting table, aggregating all documents matching the break down - dimensions into a single value.`, - - `#### average(field: string) - Returns the average of a field. This function only works for number fields. - - Example: Get the average of price: \`average(price)\` - - Example: Get the average of price for orders from the UK: \`average(price, - kql='location:UK')\``, - - `#### count([field: string]) - The total number of documents. When you provide a field, the total number of - field values is counted. When you use the Count function for fields that have - multiple values in a single document, all values are counted. - - To calculate the total number of documents, use \`count().\` - - To calculate the number of products in all orders, use \`count(products.id)\`. - - To calculate the number of documents that match a specific filter, use - \`count(kql='price > 500')\`.`, - - `#### last_value(field: string) - Returns the value of a field from the last document, ordered by the default - time field of the data view. - - This function is usefull the retrieve the latest state of an entity. - - Example: Get the current status of server A: \`last_value(server.status, - kql='server.name="A"')\``, - - `#### max(field: string) - Returns the max of a field. This function only works for number fields. - - Example: Get the max of price: \`max(price)\` - - Example: Get the max of price for orders from the UK: \`max(price, - kql='location:UK')\``, - - `#### median(field: string) - Returns the median of a field. This function only works for number fields. - - Example: Get the median of price: \`median(price)\` - - Example: Get the median of price for orders from the UK: \`median(price, - kql='location:UK')\``, - - `#### min(field: string) - Returns the min of a field. This function only works for number fields. - - Example: Get the min of price: \`min(price)\` - - Example: Get the min of price for orders from the UK: \`min(price, - kql='location:UK')\``, - - `#### percentile(field: string, [percentile]: number) - Returns the specified percentile of the values of a field. This is the value n - percent of the values occuring in documents are smaller. - - Example: Get the number of bytes larger than 95 % of values: - \`percentile(bytes, percentile=95)\``, - - `#### percentile_rank(field: string, [value]: number) - Returns the percentage of values which are below a certain value. For example, - if a value is greater than or equal to 95% of the observed values it is said to - be at the 95th percentile rank - - Example: Get the percentage of values which are below of 100: - \`percentile_rank(bytes, value=100)\``, - - `#### standard_deviation(field: string) - Returns the amount of variation or dispersion of the field. The function works - only for number fields. - - Example: To get the standard deviation of price, use - \`standard_deviation(price).\` - - Example: To get the variance of price for orders from the UK, use - \`square(standard_deviation(price, kql='location:UK'))\`.`, - - `#### sum(field: string) - Returns the sum of a field. This function only works for number fields. - - Example: Get the sum of price: sum(price) - - Example: Get the sum of price for orders from the UK: \`sum(price, - kql='location:UK')\``, - - `#### unique_count(field: string) - Calculates the number of unique values of a specified field. Works for number, - string, date and boolean values. - - Example: Calculate the number of different products: - \`unique_count(product.name)\` - - Example: Calculate the number of different products from the "clothes" group: - \`unique_count(product.name, kql='product.group=clothes')\` - - `, - ], - }, - { - id: 'lens_formulas_column_functions', - texts: [ - `## Column calculations - These functions are executed for each row, but are provided with the whole - column as context. This is also known as a window function.`, - - `#### counter_rate(metric: number) - Calculates the rate of an ever increasing counter. This function will only - yield helpful results on counter metric fields which contain a measurement of - some kind monotonically growing over time. If the value does get smaller, it - will interpret this as a counter reset. To get most precise results, - counter_rate should be calculated on the max of a field. - - This calculation will be done separately for separate series defined by filters - or top values dimensions. It uses the current interval when used in Formula. - - Example: Visualize the rate of bytes received over time by a memcached server: - counter_rate(max(memcached.stats.read.bytes))`, - - `cumulative_sum(metric: number) - Calculates the cumulative sum of a metric over time, adding all previous values - of a series to each value. To use this function, you need to configure a date - histogram dimension as well. - - This calculation will be done separately for separate series defined by filters - or top values dimensions. - - Example: Visualize the received bytes accumulated over time: - cumulative_sum(sum(bytes))`, - - `differences(metric: number) - Calculates the difference to the last value of a metric over time. To use this - function, you need to configure a date histogram dimension as well. Differences - requires the data to be sequential. If your data is empty when using - differences, try increasing the date histogram interval. - - This calculation will be done separately for separate series defined by filters - or top values dimensions. - - Example: Visualize the change in bytes received over time: - differences(sum(bytes))`, - - `moving_average(metric: number, [window]: number) - Calculates the moving average of a metric over time, averaging the last n-th - values to calculate the current value. To use this function, you need to - configure a date histogram dimension as well. The default window value is 5. - - This calculation will be done separately for separate series defined by filters - or top values dimensions. - - Takes a named parameter window which specifies how many last values to include - in the average calculation for the current value. - - Example: Smooth a line of measurements: moving_average(sum(bytes), window=5)`, - - `normalize_by_unit(metric: number, unit: s|m|h|d|w|M|y) - This advanced function is useful for normalizing counts and sums to a specific - time interval. It allows for integration with metrics that are stored already - normalized to a specific time interval. - - This function can only be used if there's a date histogram function used in the - current chart. - - Example: A ratio comparing an already normalized metric to another metric that - needs to be normalized. - normalize_by_unit(counter_rate(max(system.diskio.write.bytes)), unit='s') / - last_value(apache.status.bytes_per_second)`, - - `overall_average(metric: number) - Calculates the average of a metric for all data points of a series in the - current chart. A series is defined by a dimension using a date histogram or - interval function. Other dimensions breaking down the data like top values or - filter are treated as separate series. - - If no date histograms or interval functions are used in the current chart, - overall_average is calculating the average over all dimensions no matter the - used function - - Example: Divergence from the mean: sum(bytes) - overall_average(sum(bytes))`, - - `overall_max(metric: number) - Calculates the maximum of a metric for all data points of a series in the - current chart. A series is defined by a dimension using a date histogram or - interval function. Other dimensions breaking down the data like top values or - filter are treated as separate series. - - If no date histograms or interval functions are used in the current chart, - overall_max is calculating the maximum over all dimensions no matter the used - function - - Example: Percentage of range (sum(bytes) - overall_min(sum(bytes))) / - (overall_max(sum(bytes)) - overall_min(sum(bytes)))`, - - `overall_min(metric: number) - Calculates the minimum of a metric for all data points of a series in the - current chart. A series is defined by a dimension using a date histogram or - interval function. Other dimensions breaking down the data like top values or - filter are treated as separate series. - - If no date histograms or interval functions are used in the current chart, - overall_min is calculating the minimum over all dimensions no matter the used - function - - Example: Percentage of range (sum(bytes) - overall_min(sum(bytes)) / - (overall_max(sum(bytes)) - overall_min(sum(bytes)))`, - - `overall_sum(metric: number) - Calculates the sum of a metric of all data points of a series in the current - chart. A series is defined by a dimension using a date histogram or interval - function. Other dimensions breaking down the data like top values or filter are - treated as separate series. - - If no date histograms or interval functions are used in the current chart, - overall_sum is calculating the sum over all dimensions no matter the used - function. - - Example: Percentage of total sum(bytes) / overall_sum(sum(bytes))`, - ], - }, - { - id: 'lens_formulas_math_functions', - texts: [ - `Math - These functions will be executed for reach row of the resulting table using single values from the same row calculated using other functions.`, - - `abs([value]: number) - Calculates absolute value. A negative value is multiplied by -1, a positive value stays the same. - - Example: Calculate average distance to sea level abs(average(altitude))`, - - `add([left]: number, [right]: number) - Adds up two numbers. - - Also works with + symbol. - - Example: Calculate the sum of two fields - - sum(price) + sum(tax) - - Example: Offset count by a static value - - add(count(), 5)`, - - `cbrt([value]: number) - Cube root of value. - - Example: Calculate side length from volume - - cbrt(last_value(volume)) - - ceil([value]: number) - Ceiling of value, rounds up. - - Example: Round up price to the next dollar - - ceil(sum(price))`, - - `clamp([value]: number, [min]: number, [max]: number) - Limits the value from a minimum to maximum. - - Example: Make sure to catch outliers - - clamp( - average(bytes), - percentile(bytes, percentile=5), - percentile(bytes, percentile=95) - )`, - `cube([value]: number) - Calculates the cube of a number. - - Example: Calculate volume from side length - - cube(last_value(length))`, - - `defaults([value]: number, [default]: number) - Returns a default numeric value when value is null. - - Example: Return -1 when a field has no data - - defaults(average(bytes), -1)`, - - `divide([left]: number, [right]: number) - Divides the first number by the second number. - - Also works with / symbol - - Example: Calculate profit margin - - sum(profit) / sum(revenue) - - Example: divide(sum(bytes), 2)`, - - `exp([value]: number) - Raises e to the nth power. - - Example: Calculate the natural exponential function - - exp(last_value(duration))`, - - `fix([value]: number) - For positive values, takes the floor. For negative values, takes the ceiling. - - Example: Rounding towards zero - - fix(sum(profit))`, - - `floor([value]: number) - Round down to nearest integer value - - Example: Round down a price - - floor(sum(price))`, - - `log([value]: number, [base]?: number) - Logarithm with optional base. The natural base e is used as default. - - Example: Calculate number of bits required to store values - - log(sum(bytes)) - log(sum(bytes), 2)`, - `mod([value]: number, [base]: number) - Remainder after dividing the function by a number - - Example: Calculate last three digits of a value - - mod(sum(price), 1000)`, - - `multiply([left]: number, [right]: number) - Multiplies two numbers. - - Also works with * symbol. - - Example: Calculate price after current tax rate - - sum(bytes) * last_value(tax_rate) - - Example: Calculate price after constant tax rate - - multiply(sum(price), 1.2)`, - - `pick_max([left]: number, [right]: number) - Finds the maximum value between two numbers. - - Example: Find the maximum between two fields averages - - pick_max(average(bytes), average(memory))`, - - `pick_min([left]: number, [right]: number) - Finds the minimum value between two numbers. - - Example: Find the minimum between two fields averages - - pick_min(average(bytes), average(memory))`, - - `pow([value]: number, [base]: number) - Raises the value to a certain power. The second argument is required - - Example: Calculate volume based on side length - - pow(last_value(length), 3)`, - - `round([value]: number, [decimals]?: number) - Rounds to a specific number of decimal places, default of 0 - - Examples: Round to the cent - - round(sum(bytes)) - round(sum(bytes), 2)`, - `sqrt([value]: number) - Square root of a positive value only - - Example: Calculate side length based on area - - sqrt(last_value(area))`, - - `square([value]: number) - Raise the value to the 2nd power - - Example: Calculate area based on side length - - square(last_value(length))`, - - `subtract([left]: number, [right]: number) - Subtracts the first number from the second number. - - Also works with - symbol. - - Example: Calculate the range of a field - - subtract(max(bytes), min(bytes))`, - ], - }, - { - id: 'lens_formulas_comparison_functions', - texts: [ - `Comparison - These functions are used to perform value comparison.`, - - `eq([left]: number, [right]: number) - Performs an equality comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with == symbol. - - Example: Returns true if the average of bytes is exactly the same amount of average memory - - average(bytes) == average(memory) - - Example: eq(sum(bytes), 1000000)`, - - `gt([left]: number, [right]: number) - Performs a greater than comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with > symbol. - - Example: Returns true if the average of bytes is greater than the average amount of memory - - average(bytes) > average(memory) - - Example: gt(average(bytes), 1000)`, - - `gte([left]: number, [right]: number) - Performs a greater than comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with >= symbol. - - Example: Returns true if the average of bytes is greater than or equal to the average amount of memory - - average(bytes) >= average(memory) - - Example: gte(average(bytes), 1000)`, - - `ifelse([condition]: boolean, [left]: number, [right]: number) - Returns a value depending on whether the element of condition is true or false. - - Example: Average revenue per customer but in some cases customer id is not provided which counts as additional customer - - sum(total)/(unique_count(customer_id) + ifelse( count() > count(kql='customer_id:*'), 1, 0))`, - - `lt([left]: number, [right]: number) - Performs a lower than comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with < symbol. - - Example: Returns true if the average of bytes is lower than the average amount of memory - - average(bytes) <= average(memory) - - Example: lt(average(bytes), 1000)`, - - `lte([left]: number, [right]: number) - Performs a lower than or equal comparison between two values. - - To be used as condition for ifelse comparison function. - - Also works with <= symbol. - - Example: Returns true if the average of bytes is lower than or equal to the average amount of memory - - average(bytes) <= average(memory) - - Example: lte(average(bytes), 1000)`, - ], - }, - { - id: 'lens_formulas_kibana_context', - text: dedent(`Kibana context - - These functions are used to retrieve Kibana context variables, which are the - date histogram \`interval\`, the current \`now\` and the selected \`time_range\` - and help you to compute date math operations. - - interval() - The specified minimum interval for the date histogram, in milliseconds (ms). - - now() - The current now moment used in Kibana expressed in milliseconds (ms). - - time_range() - The specified time range, in milliseconds (ms).`), - }, - ]); -} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts index 2df3f36163972..2e24cf25902e0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/types.ts @@ -17,7 +17,6 @@ import type { import type { Message, ObservabilityAIAssistantScreenContextRequest, - InstructionOrPlainText, AdHocInstruction, } from '../../common/types'; import type { ObservabilityAIAssistantRouteHandlerResources } from '../routes/types'; @@ -67,13 +66,13 @@ export interface FunctionHandler { respond: RespondFunction; } -export type InstructionOrCallback = InstructionOrPlainText | RegisterInstructionCallback; +export type InstructionOrCallback = string | RegisterInstructionCallback; export type RegisterInstructionCallback = ({ availableFunctionNames, }: { availableFunctionNames: string[]; -}) => InstructionOrPlainText | InstructionOrPlainText[] | undefined; +}) => string | string[] | undefined; export type RegisterInstruction = (...instruction: InstructionOrCallback[]) => void; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts index 8e4075bed7b9d..82f22e08d1fc7 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.test.ts @@ -41,10 +41,10 @@ describe('getSystemMessageFromInstructions', () => { expect( getSystemMessageFromInstructions({ applicationInstructions: ['first'], - userInstructions: [{ doc_id: 'second', text: 'second from kb' }], + userInstructions: [{ id: 'second', text: 'second from kb' }], adHocInstructions: [ { - doc_id: 'second', + id: 'second', text: 'second from adhoc instruction', instruction_type: 'user_instruction', }, @@ -58,7 +58,7 @@ describe('getSystemMessageFromInstructions', () => { expect( getSystemMessageFromInstructions({ applicationInstructions: ['first'], - userInstructions: [{ doc_id: 'second', text: 'second_kb' }], + userInstructions: [{ id: 'second', text: 'second_kb' }], adHocInstructions: [], availableFunctionNames: [], }) diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts index 570449673084b..7b59b4ce59219 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/get_system_message_from_instructions.ts @@ -45,7 +45,7 @@ export function getSystemMessageFromInstructions({ const adHocInstructionsWithId = adHocInstructions.map((adHocInstruction) => ({ ...adHocInstruction, - doc_id: adHocInstruction?.doc_id ?? v4(), + id: adHocInstruction?.id ?? v4(), })); // split ad hoc instructions into user instructions and application instructions @@ -55,15 +55,16 @@ export function getSystemMessageFromInstructions({ ); // all adhoc instructions and KB instructions. - // adhoc instructions will be prioritized over Knowledge Base instructions if the doc_id is the same + // adhoc instructions will be prioritized over Knowledge Base instructions if the id is the same const allUserInstructions = withTokenBudget( - uniqBy([...adHocUserInstructions, ...kbUserInstructions], (i) => i.doc_id), + uniqBy([...adHocUserInstructions, ...kbUserInstructions], (i) => i.id), 1000 ); return [ // application instructions - ...allApplicationInstructions.concat(adHocApplicationInstructions), + ...allApplicationInstructions, + ...adHocApplicationInstructions, // user instructions ...(allUserInstructions.length ? [USER_INSTRUCTIONS_HEADER, ...allUserInstructions] : []), diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts deleted file mode 100644 index 9a2f047b60f9b..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/service/util/split_kb_text.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { merge } from 'lodash'; -import type { KnowledgeBaseEntry } from '../../../common/types'; -import { - type KnowledgeBaseEntryOperation, - KnowledgeBaseEntryOperationType, -} from '../knowledge_base_service'; - -export function splitKbText({ - id, - texts, - ...rest -}: Omit & { texts: string[] }): KnowledgeBaseEntryOperation[] { - return [ - { - type: KnowledgeBaseEntryOperationType.Delete, - doc_id: id, - labels: {}, - }, - ...texts.map((text, index) => ({ - type: KnowledgeBaseEntryOperationType.Index, - document: merge({}, rest, { - id: [id, index].join('_'), - doc_id: id, - labels: {}, - text, - }), - })), - ]; -} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts index 8885ff7e1d7a2..fefe324805e59 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/recall_and_score.ts @@ -7,13 +7,14 @@ import type { Logger } from '@kbn/logging'; import { AnalyticsServiceStart } from '@kbn/core/server'; +import { scoreSuggestions } from './score_suggestions'; import type { Message } from '../../../common'; import type { ObservabilityAIAssistantClient } from '../../service/client'; import type { FunctionCallChatFunction } from '../../service/types'; -import { retrieveSuggestions } from './retrieve_suggestions'; -import { scoreSuggestions } from './score_suggestions'; -import type { RetrievedSuggestion } from './types'; -import { RecallRanking, RecallRankingEventType } from '../../analytics/recall_ranking'; +import { RecallRanking, recallRankingEventType } from '../../analytics/recall_ranking'; +import { RecalledEntry } from '../../service/knowledge_base_service'; + +export type RecalledSuggestion = Pick; export async function recallAndScore({ recall, @@ -34,19 +35,18 @@ export async function recallAndScore({ logger: Logger; signal: AbortSignal; }): Promise<{ - relevantDocuments?: RetrievedSuggestion[]; + relevantDocuments?: RecalledSuggestion[]; scores?: Array<{ id: string; score: number }>; - suggestions: RetrievedSuggestion[]; + suggestions: RecalledSuggestion[]; }> { const queries = [ { text: userPrompt, boost: 3 }, { text: context, boost: 1 }, ].filter((query) => query.text.trim()); - const suggestions = await retrieveSuggestions({ - recall, - queries, - }); + const suggestions: RecalledSuggestion[] = (await recall({ queries })).map( + ({ id, text, score }) => ({ id, text, score }) + ); if (!suggestions.length) { return { @@ -67,7 +67,7 @@ export async function recallAndScore({ chat, }); - analytics.reportEvent(RecallRankingEventType, { + analytics.reportEvent(recallRankingEventType, { prompt: queries.map((query) => query.text).join('\n\n'), scoredDocuments: suggestions.map((suggestion) => { const llmScore = scores.find((score) => score.id === suggestion.id); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/retrieve_suggestions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/retrieve_suggestions.ts deleted file mode 100644 index 3c680229cd5d2..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/retrieve_suggestions.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { omit } from 'lodash'; -import { ObservabilityAIAssistantClient } from '../../service/client'; -import { RetrievedSuggestion } from './types'; - -export async function retrieveSuggestions({ - queries, - recall, -}: { - queries: Array<{ text: string; boost?: number }>; - recall: ObservabilityAIAssistantClient['recall']; -}): Promise { - const recallResponse = await recall({ - queries, - }); - - return recallResponse.entries.map((entry) => omit(entry, 'labels', 'is_correction')); -} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts index 009b91a7a8c2c..7d1a19463cceb 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/score_suggestions.ts @@ -5,15 +5,15 @@ * 2.0. */ import * as t from 'io-ts'; -import { omit } from 'lodash'; import { Logger } from '@kbn/logging'; import dedent from 'dedent'; import { lastValueFrom } from 'rxjs'; import { decodeOrThrow, jsonRt } from '@kbn/io-ts-utils'; +import { omit } from 'lodash'; import { concatenateChatCompletionChunks, Message, MessageRole } from '../../../common'; import type { FunctionCallChatFunction } from '../../service/types'; -import type { RetrievedSuggestion } from './types'; import { parseSuggestionScores } from './parse_suggestion_scores'; +import { RecalledSuggestion } from './recall_and_score'; import { ShortIdTable } from '../../../common/utils/short_id_table'; const scoreFunctionRequestRt = t.type({ @@ -38,7 +38,7 @@ export async function scoreSuggestions({ signal, logger, }: { - suggestions: RetrievedSuggestion[]; + suggestions: RecalledSuggestion[]; messages: Message[]; userPrompt: string; context: string; @@ -46,28 +46,21 @@ export async function scoreSuggestions({ signal: AbortSignal; logger: Logger; }): Promise<{ - relevantDocuments: RetrievedSuggestion[]; + relevantDocuments: RecalledSuggestion[]; scores: Array<{ id: string; score: number }>; }> { const shortIdTable = new ShortIdTable(); - const suggestionsWithShortId = suggestions.map((suggestion) => ({ - ...omit(suggestion, 'score', 'id'), // To not bias the LLM - originalId: suggestion.id, - shortId: shortIdTable.take(suggestion.id), - })); - const newUserMessageContent = - dedent(`Given the following question, score the documents that are relevant to the question. on a scale from 0 to 7, - 0 being completely irrelevant, and 7 being extremely relevant. Information is relevant to the question if it helps in - answering the question. Judge it according to the following criteria: - - - The document is relevant to the question, and the rest of the conversation - - The document has information relevant to the question that is not mentioned, - or more detailed than what is available in the conversation - - The document has a high amount of information relevant to the question compared to other documents - - The document contains new information not mentioned before in the conversation - + dedent(`Given the following prompt, score the documents that are relevant to the prompt on a scale from 0 to 7, + 0 being completely irrelevant, and 7 being extremely relevant. Information is relevant to the prompt if it helps in + answering the prompt. Judge the document according to the following criteria: + + - The document is relevant to the prompt, and the rest of the conversation + - The document has information relevant to the prompt that is not mentioned, or more detailed than what is available in the conversation + - The document has a high amount of information relevant to the prompt compared to other documents + - The document contains new information not mentioned before in the conversation or provides a correction to previously stated information. + User prompt: ${userPrompt} @@ -76,9 +69,9 @@ export async function scoreSuggestions({ Documents: ${JSON.stringify( - suggestionsWithShortId.map((suggestion) => ({ - id: suggestion.shortId, - content: suggestion.text, + suggestions.map((suggestion) => ({ + ...omit(suggestion, 'score'), // Omit score to not bias the LLM + id: shortIdTable.take(suggestion.id), // Shorten id to save tokens })), null, 2 @@ -127,15 +120,9 @@ export async function scoreSuggestions({ scoreFunctionRequest.message.function_call.arguments ); - const scores = parseSuggestionScores(scoresAsString).map(({ id, score }) => { - const originalSuggestion = suggestionsWithShortId.find( - (suggestion) => suggestion.shortId === id - ); - return { - originalId: originalSuggestion?.originalId, - score, - }; - }); + const scores = parseSuggestionScores(scoresAsString) + // Restore original IDs + .map(({ id, score }) => ({ id: shortIdTable.lookup(id)!, score })); if (scores.length === 0) { // seemingly invalid or no scores, return all @@ -144,12 +131,13 @@ export async function scoreSuggestions({ const suggestionIds = suggestions.map((document) => document.id); + // get top 5 documents ids with scores > 4 const relevantDocumentIds = scores - .filter((document) => suggestionIds.includes(document.originalId ?? '')) // Remove hallucinated documents - .filter((document) => document.score > 4) + .filter(({ score }) => score > 4) .sort((a, b) => b.score - a.score) .slice(0, 5) - .map((document) => document.originalId); + .filter(({ id }) => suggestionIds.includes(id ?? '')) // Remove hallucinated documents + .map(({ id }) => id); const relevantDocuments = suggestions.filter((suggestion) => relevantDocumentIds.includes(suggestion.id) @@ -159,6 +147,6 @@ export async function scoreSuggestions({ return { relevantDocuments, - scores: scores.map((score) => ({ id: score.originalId!, score: score.score })), + scores: scores.map((score) => ({ id: score.id, score: score.score })), }; } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/types.ts b/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/types.ts deleted file mode 100644 index 3774df64c1ee1..0000000000000 --- a/x-pack/plugins/observability_solution/observability_ai_assistant/server/utils/recall/types.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { RecalledEntry } from '../../service/knowledge_base_service'; - -export type RetrievedSuggestion = Omit; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts index 8d32ea4664f9f..7ea25eab8c661 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/helpers/categorize_entries.ts @@ -9,21 +9,30 @@ import type { KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/ export interface KnowledgeBaseEntryCategory { '@timestamp': string; - categoryName: string; + categoryKey: string; + title: string; entries: KnowledgeBaseEntry[]; } -export function categorizeEntries({ entries }: { entries: KnowledgeBaseEntry[] }) { +export function categorizeEntries({ + entries, +}: { + entries: KnowledgeBaseEntry[]; +}): KnowledgeBaseEntryCategory[] { return entries.reduce((acc, entry) => { - const categoryName = entry.labels?.category ?? entry.id; - - const index = acc.findIndex((item) => item.categoryName === categoryName); + const categoryKey = entry.labels?.category ?? entry.id; - if (index > -1) { - acc[index].entries.push(entry); + const existingEntry = acc.find((item) => item.categoryKey === categoryKey); + if (existingEntry) { + existingEntry.entries.push(entry); return acc; - } else { - return acc.concat({ categoryName, entries: [entry], '@timestamp': entry['@timestamp'] }); } - }, [] as Array<{ categoryName: string; entries: KnowledgeBaseEntry[]; '@timestamp': string }>); + + return acc.concat({ + categoryKey, + title: entry.labels?.category ?? entry.title ?? 'No title', + entries: [entry], + '@timestamp': entry['@timestamp'], + }); + }, [] as Array<{ categoryKey: string; title: string; entries: KnowledgeBaseEntry[]; '@timestamp': string }>); } diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts index 459de7be2d528..4cc6c8e2b9bf1 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_entry.ts @@ -28,10 +28,9 @@ export function useCreateKnowledgeBaseEntry() { void, ServerError, { - entry: Omit< - KnowledgeBaseEntry, - '@timestamp' | 'confidence' | 'is_correction' | 'role' | 'doc_id' - >; + entry: Omit & { + title: string; + }; } >( [REACT_QUERY_KEYS.CREATE_KB_ENTRIES], @@ -41,10 +40,7 @@ export function useCreateKnowledgeBaseEntry() { { signal: null, params: { - body: { - ...entry, - role: 'user_entry', - }, + body: entry, }, } ); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_user_instruction.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_user_instruction.ts index b51e45a3bdd6b..8adf5a7d8cfd0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_user_instruction.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_create_knowledge_base_user_instruction.ts @@ -32,7 +32,7 @@ export function useCreateKnowledgeBaseUserInstruction() { signal: null, params: { body: { - id: entry.doc_id, + id: entry.id, text: entry.text, public: entry.public, }, @@ -62,7 +62,7 @@ export function useCreateKnowledgeBaseUserInstruction() { 'xpack.observabilityAiAssistantManagement.kb.addUserInstruction.errorNotification', { defaultMessage: 'Something went wrong while creating {name}', - values: { name: entry.doc_id }, + values: { name: entry.id }, } ), }); diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_import_knowledge_base_entries.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_import_knowledge_base_entries.ts index 9ff0748793bc8..239e72d99109e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_import_knowledge_base_entries.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/hooks/use_import_knowledge_base_entries.ts @@ -30,7 +30,7 @@ export function useImportKnowledgeBaseEntries() { Omit< KnowledgeBaseEntry, '@timestamp' | 'confidence' | 'is_correction' | 'public' | 'labels' - > + > & { title: string } >; } >( diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_bulk_import_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_bulk_import_flyout.tsx index ac7632243a2a0..23d5ad00af12a 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_bulk_import_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_bulk_import_flyout.tsx @@ -47,15 +47,13 @@ export function KnowledgeBaseBulkImportFlyout({ onClose }: { onClose: () => void }; const handleSubmitNewEntryClick = async () => { - let entries: Array> = []; + let entries: Array & { title: string }> = []; const text = await files[0].text(); const elements = text.split('\n').filter(Boolean); try { - entries = elements.map((el) => JSON.parse(el)) as Array< - Omit - >; + entries = elements.map((el) => JSON.parse(el)); } catch (_) { toasts.addError( new Error( diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_category_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_category_flyout.tsx index fab3b34809c19..8dcf76e4bc56e 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_category_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_category_flyout.tsx @@ -104,13 +104,13 @@ export function KnowledgeBaseCategoryFlyout({ ]; const hasDescription = - CATEGORY_MAP[category.categoryName as unknown as keyof typeof CATEGORY_MAP]?.description; + CATEGORY_MAP[category.categoryKey as unknown as keyof typeof CATEGORY_MAP]?.description; return ( -

{capitalize(category.categoryName)}

+

{capitalize(category.categoryKey)}

 diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx index d809b6cd96d6d..20c7a75a401a8 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_manual_entry_flyout.tsx @@ -26,7 +26,9 @@ import { EuiTitle, } from '@elastic/eui'; import moment from 'moment'; -import type { KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common/types'; +import { KnowledgeBaseEntryRole } from '@kbn/observability-ai-assistant-plugin/public'; +import { type KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; +import { v4 } from 'uuid'; import { useCreateKnowledgeBaseEntry } from '../../hooks/use_create_knowledge_base_entry'; import { useDeleteKnowledgeBaseEntry } from '../../hooks/use_delete_knowledge_base_entry'; import { useKibana } from '../../hooks/use_kibana'; @@ -45,20 +47,24 @@ export function KnowledgeBaseEditManualEntryFlyout({ const { mutateAsync: deleteEntry, isLoading: isDeleting } = useDeleteKnowledgeBaseEntry(); const [isPublic, setIsPublic] = useState(entry?.public ?? false); - - const [newEntryId, setNewEntryId] = useState(entry?.id ?? ''); + const [newEntryTitle, setNewEntryTitle] = useState(entry?.title ?? ''); const [newEntryText, setNewEntryText] = useState(entry?.text ?? ''); - const isEntryIdInvalid = newEntryId.trim() === ''; + const isEntryTitleInvalid = newEntryTitle.trim() === ''; const isEntryTextInvalid = newEntryText.trim() === ''; - const isFormInvalid = isEntryIdInvalid || isEntryTextInvalid; + const isFormInvalid = isEntryTitleInvalid || isEntryTextInvalid; const handleSubmit = async () => { await createEntry({ entry: { - id: newEntryId, + id: entry?.id ?? v4(), + title: newEntryTitle, text: newEntryText, public: isPublic, + role: KnowledgeBaseEntryRole.UserEntry, + confidence: 'high', + is_correction: false, + labels: {}, }, }); @@ -85,8 +91,8 @@ export function KnowledgeBaseEditManualEntryFlyout({ : i18n.translate( 'xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.editEntryLabel', { - defaultMessage: 'Edit {id}', - values: { id: entry.id }, + defaultMessage: 'Edit {title}', + values: { title: entry?.title }, } )} @@ -94,23 +100,7 @@ export function KnowledgeBaseEditManualEntryFlyout({ - {!entry ? ( - - setNewEntryId(e.target.value)} - isInvalid={isEntryIdInvalid} - /> - - ) : ( + {entry ? ( @@ -136,7 +126,26 @@ export function KnowledgeBaseEditManualEntryFlyout({ - )} + ) : null} + + + + + setNewEntryTitle(e.target.value)} + isInvalid={isEntryTitleInvalid} + /> + + diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx index e8152738e3807..1c05ca6d52b3f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_edit_user_instruction_flyout.tsx @@ -30,19 +30,19 @@ export function KnowledgeBaseEditUserInstructionFlyout({ onClose }: { onClose: ( const { userInstructions, isLoading: isFetching } = useGetUserInstructions(); const { mutateAsync: createEntry, isLoading: isSaving } = useCreateKnowledgeBaseUserInstruction(); const [newEntryText, setNewEntryText] = useState(''); - const [newEntryDocId, setNewEntryDocId] = useState(); + const [newEntryId, setNewEntryId] = useState(); const isSubmitDisabled = newEntryText.trim() === ''; useEffect(() => { const userInstruction = userInstructions?.find((entry) => !entry.public); - setNewEntryDocId(userInstruction?.doc_id); setNewEntryText(userInstruction?.text ?? ''); + setNewEntryId(userInstruction?.id); }, [userInstructions]); const handleSubmit = async () => { await createEntry({ entry: { - doc_id: newEntryDocId ?? uuidv4(), + id: newEntryId ?? uuidv4(), text: newEntryText, public: false, // limit user instructions to private (for now) }, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx index d8e2897c6878c..50d0cb8ba47c8 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.test.tsx @@ -81,7 +81,18 @@ describe('KnowledgeBaseTab', () => { getByTestId('knowledgeBaseEditManualEntryFlyoutSaveButton').click(); - expect(createMock).toHaveBeenCalledWith({ entry: { id: 'foo', public: false, text: 'bar' } }); + expect(createMock).toHaveBeenCalledWith({ + entry: { + id: expect.any(String), + title: 'foo', + public: false, + text: 'bar', + role: 'user_entry', + confidence: 'high', + is_correction: false, + labels: expect.any(Object), + }, + }); }); it('should require an id', () => { @@ -126,7 +137,7 @@ describe('KnowledgeBaseTab', () => { entries: [ { id: 'test', - doc_id: 'test', + title: 'test', text: 'test', '@timestamp': 1638340456, labels: {}, @@ -134,7 +145,7 @@ describe('KnowledgeBaseTab', () => { }, { id: 'test2', - doc_id: 'test2', + title: 'test2', text: 'test', '@timestamp': 1638340456, labels: { @@ -144,7 +155,7 @@ describe('KnowledgeBaseTab', () => { }, { id: 'test3', - doc_id: 'test3', + title: 'test3', text: 'test', '@timestamp': 1638340456, labels: { diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx index 6ba09101b6227..23fbf796290c8 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_management/public/routes/components/knowledge_base_tab.tsx @@ -57,10 +57,10 @@ export function KnowledgeBaseTab() { data-test-subj="pluginsColumnsButton" onClick={() => setSelectedCategory(category)} aria-label={ - category.categoryName === selectedCategory?.categoryName ? 'Collapse' : 'Expand' + category.categoryKey === selectedCategory?.categoryKey ? 'Collapse' : 'Expand' } iconType={ - category.categoryName === selectedCategory?.categoryName ? 'minimize' : 'expand' + category.categoryKey === selectedCategory?.categoryKey ? 'minimize' : 'expand' } /> ); @@ -85,7 +85,8 @@ export function KnowledgeBaseTab() { width: '40px', }, { - field: 'categoryName', + 'data-test-subj': 'knowledgeBaseTableTitleCell', + field: 'title', name: i18n.translate('xpack.observabilityAiAssistantManagement.kbTab.columns.name', { defaultMessage: 'Name', }), @@ -107,6 +108,7 @@ export function KnowledgeBaseTab() { }, }, { + 'data-test-subj': 'knowledgeBaseTableAuthorCell', name: i18n.translate('xpack.observabilityAiAssistantManagement.kbTab.columns.author', { defaultMessage: 'Author', }), @@ -183,7 +185,7 @@ export function KnowledgeBaseTab() { const [isNewEntryPopoverOpen, setIsNewEntryPopoverOpen] = useState(false); const [isEditUserInstructionFlyoutOpen, setIsEditUserInstructionFlyoutOpen] = useState(false); const [query, setQuery] = useState(''); - const [sortBy, setSortBy] = useState<'doc_id' | '@timestamp'>('doc_id'); + const [sortBy, setSortBy] = useState('title'); const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>('asc'); const { @@ -193,17 +195,10 @@ export function KnowledgeBaseTab() { } = useGetKnowledgeBaseEntries({ query, sortBy, sortDirection }); const categorizedEntries = categorizeEntries({ entries }); - const handleChangeSort = ({ - sort, - }: Criteria) => { + const handleChangeSort = ({ sort }: Criteria) => { if (sort) { const { field, direction } = sort; - if (field === '@timestamp') { - setSortBy(field); - } - if (field === 'categoryName') { - setSortBy('doc_id'); - } + setSortBy(field); setSortDirection(direction); } }; @@ -329,7 +324,7 @@ export function KnowledgeBaseTab() { loading={isLoading} sorting={{ sort: { - field: sortBy === 'doc_id' ? 'categoryName' : sortBy, + field: sortBy, direction: sortDirection, }, }} diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 31982f6fddd72..7a41f7703eb2a 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -32593,7 +32593,6 @@ "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.contentsLabel": "Contenu", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.idLabel": "Nom", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiMarkdownEditor.enterContentsLabel": "Entrer du contenu", - "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.editEntryLabel": "Modifier {id}", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.newEntryLabel": "Nouvelle entrée", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.cancelButtonEmptyLabel": "Annuler", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.euiMarkdownEditor.observabilityAiAssistantKnowledgeBaseViewMarkdownEditorLabel": "observabilityAiAssistantKnowledgeBaseViewMarkdownEditor", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 277fc3eb639bf..ccdc4069314e8 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -32340,7 +32340,6 @@ "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.contentsLabel": "目次", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.idLabel": "名前", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiMarkdownEditor.enterContentsLabel": "コンテンツを入力", - "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.editEntryLabel": "{id}を編集", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.newEntryLabel": "新しいエントリー", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.cancelButtonEmptyLabel": "キャンセル", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.euiMarkdownEditor.observabilityAiAssistantKnowledgeBaseViewMarkdownEditorLabel": "observabilityAiAssistantKnowledgeBaseViewMarkdownEditor", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index ec974a4b38349..341eec24125a2 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -32383,7 +32383,6 @@ "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.contentsLabel": "内容", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiFormRow.idLabel": "名称", "xpack.observabilityAiAssistantManagement.knowledgeBaseEditManualEntryFlyout.euiMarkdownEditor.enterContentsLabel": "输入内容", - "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.editEntryLabel": "编辑 {id}", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewEntryFlyout.h2.newEntryLabel": "新条目", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.cancelButtonEmptyLabel": "取消", "xpack.observabilityAiAssistantManagement.knowledgeBaseNewManualEntryFlyout.euiMarkdownEditor.observabilityAiAssistantKnowledgeBaseViewMarkdownEditorLabel": "observabilityAiAssistantKnowledgeBaseViewMarkdownEditor", diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts index 198fcefdc2bc8..427258a6e2910 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/config.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/config.ts @@ -11,7 +11,7 @@ import { ObservabilityAIAssistantFtrConfigName } from '../configs'; import { getApmSynthtraceEsClient } from './create_synthtrace_client'; import { InheritedFtrProviderContext, InheritedServices } from './ftr_provider_context'; import { getScopedApiClient } from './observability_ai_assistant_api_client'; -import { editorUser, viewerUser } from './users/users'; +import { editor, secondaryEditor, viewer } from './users/users'; export interface ObservabilityAIAssistantFtrConfig { name: ObservabilityAIAssistantFtrConfigName; @@ -23,6 +23,10 @@ export type CreateTestConfig = ReturnType; export type CreateTest = ReturnType; +export type ObservabilityAIAssistantApiClients = Awaited< + ReturnType +>; + export type ObservabilityAIAssistantAPIClient = Awaited< ReturnType >; @@ -46,21 +50,23 @@ export function createObservabilityAIAssistantAPIConfig({ const apmSynthtraceKibanaClient = services.apmSynthtraceKibanaClient(); const allConfigs = config.getAll() as Record; + const getScopedApiClientForUsername = (username: string) => + getScopedApiClient(kibanaServer, username); + return { ...allConfigs, servers, services: { ...services, - getScopedApiClientForUsername: () => { - return (username: string) => getScopedApiClient(kibanaServer, username); - }, + getScopedApiClientForUsername: () => getScopedApiClientForUsername, apmSynthtraceEsClient: (context: InheritedFtrProviderContext) => getApmSynthtraceEsClient(context, apmSynthtraceKibanaClient), observabilityAIAssistantAPIClient: async () => { return { - adminUser: await getScopedApiClient(kibanaServer, 'elastic'), - viewerUser: await getScopedApiClient(kibanaServer, viewerUser.username), - editorUser: await getScopedApiClient(kibanaServer, editorUser.username), + admin: getScopedApiClientForUsername('elastic'), + viewer: getScopedApiClientForUsername(viewer.username), + editor: getScopedApiClientForUsername(editor.username), + secondaryEditor: getScopedApiClientForUsername(secondaryEditor.username), }; }, }, diff --git a/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts b/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts index b6fa38e52e60b..898954a9bfb97 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/common/users/users.ts @@ -9,21 +9,27 @@ import { kbnTestConfig } from '@kbn/test'; const password = kbnTestConfig.getUrlParts().password!; export interface User { - username: 'elastic' | 'editor' | 'viewer'; + username: 'elastic' | 'editor' | 'viewer' | 'secondary_editor'; password: string; roles: string[]; } -export const editorUser: User = { +export const editor: User = { username: 'editor', password, roles: ['editor'], }; -export const viewerUser: User = { +export const secondaryEditor: User = { + username: 'secondary_editor', + password, + roles: ['editor'], +}; + +export const viewer: User = { username: 'viewer', password, roles: ['viewer'], }; -export const allUsers = [editorUser, viewerUser]; +export const allUsers = [editor, secondaryEditor, viewer]; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts index a7606d21408c5..2eb7c6f986cfd 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/complete.spec.ts @@ -290,7 +290,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { )[0]?.conversation.id; await observabilityAIAssistantAPIClient - .adminUser({ + .admin({ endpoint: 'DELETE /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -366,7 +366,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { ).to.eql(0); const conversations = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }) .expect(200); @@ -396,7 +396,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .completeAfterIntercept(); const createResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', params: { body: { @@ -415,7 +415,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { conversationCreatedEvent = getConversationCreatedEvent(createResponse.body); const conversationId = conversationCreatedEvent.conversation.id; - const fullConversation = await observabilityAIAssistantAPIClient.editorUser({ + const fullConversation = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -429,7 +429,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .completeAfterIntercept(); const updatedResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', params: { body: { @@ -460,7 +460,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { after(async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'DELETE /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts index b83221869baec..6ce506c502b5e 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/helpers.ts @@ -43,7 +43,7 @@ export async function invokeChatCompleteWithFunctionRequest({ scopes?: AssistantScope[]; }) { const { body } = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/chat/complete', params: { body: { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts index 923e8b0206070..34da4270f7721 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/complete/functions/summarize.spec.ts @@ -32,10 +32,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { let connectorId: string; before(async () => { - await clearKnowledgeBase(es); await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(200); @@ -55,7 +54,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { name: 'summarize', trigger: MessageRole.User, arguments: JSON.stringify({ - id: 'my-id', + title: 'My Title', text: 'Hello world', is_correction: false, confidence: 'high', @@ -72,28 +71,29 @@ export default function ApiTest({ getService }: FtrProviderContext) { await deleteActionConnector({ supertest, connectorId, log }); await deleteKnowledgeBaseModel(ml); + await clearKnowledgeBase(es); }); it('persists entry in knowledge base', async () => { - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { query: '', - sortBy: 'doc_id', + sortBy: 'title', sortDirection: 'asc', }, }, }); - const { role, public: isPublic, text, type, user, id } = res.body.entries[0]; + const { role, public: isPublic, text, type, user, title } = res.body.entries[0]; expect(role).to.eql('assistant_summarization'); expect(isPublic).to.eql(false); expect(text).to.eql('Hello world'); expect(type).to.eql('contextual'); expect(user?.name).to.eql('editor'); - expect(id).to.eql('my-id'); + expect(title).to.eql('My Title'); expect(res.body.entries).to.have.length(1); }); }); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts index e8363ba41513b..41700b21555fa 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/connectors/connectors.spec.ts @@ -26,14 +26,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('Returns a 2xx for enterprise license', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }) .expect(200); }); it('returns an empty list of connectors', async () => { - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }); @@ -43,7 +43,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it("returns the gen ai connector if it's been created", async () => { const connectorId = await createProxyActionConnector({ supertest, log, port: 1234 }); - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts index 91a418b3000ee..71eb37d357696 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/conversations/conversations.spec.ts @@ -48,7 +48,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('without conversations', () => { it('returns no conversations when listing', async () => { const response = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }) .expect(200); @@ -58,7 +58,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns a 404 for updating conversations', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -74,7 +74,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns a 404 for retrieving a conversation', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -92,7 +92,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { >; before(async () => { createResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/conversation', params: { body: { @@ -105,7 +105,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { after(async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'DELETE /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -116,7 +116,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -148,7 +148,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns a 404 for updating a non-existing conversation', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -164,7 +164,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns a 404 for retrieving a non-existing conversation', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -177,7 +177,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns the conversation that was created', async () => { const response = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -192,7 +192,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns the created conversation when listing', async () => { const response = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }) .expect(200); @@ -210,7 +210,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { updateResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { @@ -234,7 +234,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns the updated conversation after get', async () => { const updateAfterCreateResponse = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}', params: { path: { diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts index c8881e82e43bb..27659f62ad579 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base.spec.ts @@ -6,67 +6,66 @@ */ import expect from '@kbn/expect'; +import { type KnowledgeBaseEntry } from '@kbn/observability-ai-assistant-plugin/common'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { clearKnowledgeBase, createKnowledgeBaseModel, deleteKnowledgeBaseModel } from './helpers'; export default function ApiTest({ getService }: FtrProviderContext) { const ml = getService('ml'); const es = getService('es'); - const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); describe('Knowledge base', () => { before(async () => { await createKnowledgeBaseModel(ml); + + await observabilityAIAssistantAPIClient + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup' }) + .expect(200); }); after(async () => { await deleteKnowledgeBaseModel(ml); + await clearKnowledgeBase(es); }); - it('returns 200 on knowledge base setup', async () => { - const res = await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'POST /internal/observability_ai_assistant/kb/setup', - }) - .expect(200); - expect(res.body).to.eql({}); - }); describe('when managing a single entry', () => { const knowledgeBaseEntry = { id: 'my-doc-id-1', + title: 'My title', text: 'My content', }; it('returns 200 on create', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/save', params: { body: knowledgeBaseEntry }, }) .expect(200); - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { query: '', - sortBy: 'doc_id', + sortBy: 'title', sortDirection: 'asc', }, }, }); const entry = res.body.entries[0]; expect(entry.id).to.equal(knowledgeBaseEntry.id); + expect(entry.title).to.equal(knowledgeBaseEntry.title); expect(entry.text).to.equal(knowledgeBaseEntry.text); }); it('returns 200 on get entries and entry exists', async () => { const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { query: '', - sortBy: 'doc_id', + sortBy: 'title', sortDirection: 'asc', }, }, @@ -74,13 +73,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); const entry = res.body.entries[0]; expect(entry.id).to.equal(knowledgeBaseEntry.id); + expect(entry.title).to.equal(knowledgeBaseEntry.title); expect(entry.text).to.equal(knowledgeBaseEntry.text); }); it('returns 200 on delete', async () => { const entryId = 'my-doc-id-1'; await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'DELETE /internal/observability_ai_assistant/kb/entries/{entryId}', params: { path: { entryId }, @@ -89,12 +89,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { query: { query: '', - sortBy: 'doc_id', + sortBy: 'title', sortDirection: 'asc', }, }, @@ -108,7 +108,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns 500 on delete not found', async () => { const entryId = 'my-doc-id-1'; await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'DELETE /internal/observability_ai_assistant/kb/entries/{entryId}', params: { path: { entryId }, @@ -117,119 +117,88 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(500); }); }); - describe('when managing multiple entries', () => { - before(async () => { - await clearKnowledgeBase(es); - }); - afterEach(async () => { - await clearKnowledgeBase(es); - }); - const knowledgeBaseEntries = [ - { - id: 'my_doc_a', - text: 'My content a', - }, - { - id: 'my_doc_b', - text: 'My content b', - }, - { - id: 'my_doc_c', - text: 'My content c', - }, - ]; - it('returns 200 on create', async () => { - await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', - params: { body: { entries: knowledgeBaseEntries } }, - }) - .expect(200); + describe('when managing multiple entries', () => { + async function getEntries({ + query = '', + sortBy = 'title', + sortDirection = 'asc', + }: { query?: string; sortBy?: string; sortDirection?: 'asc' | 'desc' } = {}) { const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { - query: { - query: '', - sortBy: 'doc_id', - sortDirection: 'asc', - }, + query: { query, sortBy, sortDirection }, }, }) .expect(200); - expect(res.body.entries.filter((entry) => entry.id.startsWith('my_doc')).length).to.eql(3); - }); - it('allows sorting', async () => { + return omitCategories(res.body.entries); + } + + beforeEach(async () => { + await clearKnowledgeBase(es); + await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', - params: { body: { entries: knowledgeBaseEntries } }, - }) - .expect(200); - - const res = await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', params: { - query: { - query: '', - sortBy: 'doc_id', - sortDirection: 'desc', + body: { + entries: [ + { + id: 'my_doc_a', + title: 'My title a', + text: 'My content a', + }, + { + id: 'my_doc_b', + title: 'My title b', + text: 'My content b', + }, + { + id: 'my_doc_c', + title: 'My title c', + text: 'My content c', + }, + ], }, }, }) .expect(200); + }); - const entries = res.body.entries.filter((entry) => entry.id.startsWith('my_doc')); - expect(entries[0].id).to.eql('my_doc_c'); - expect(entries[1].id).to.eql('my_doc_b'); - expect(entries[2].id).to.eql('my_doc_a'); - - // asc - const resAsc = await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', - params: { - query: { - query: '', - sortBy: 'doc_id', - sortDirection: 'asc', - }, - }, - }) - .expect(200); + afterEach(async () => { + await clearKnowledgeBase(es); + }); - const entriesAsc = resAsc.body.entries.filter((entry) => entry.id.startsWith('my_doc')); - expect(entriesAsc[0].id).to.eql('my_doc_a'); - expect(entriesAsc[1].id).to.eql('my_doc_b'); - expect(entriesAsc[2].id).to.eql('my_doc_c'); + it('returns 200 on create', async () => { + const entries = await getEntries(); + expect(omitCategories(entries).length).to.eql(3); }); - it('allows searching', async () => { - await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'POST /internal/observability_ai_assistant/kb/entries/import', - params: { body: { entries: knowledgeBaseEntries } }, - }) - .expect(200); - const res = await observabilityAIAssistantAPIClient - .editorUser({ - endpoint: 'GET /internal/observability_ai_assistant/kb/entries', - params: { - query: { - query: 'my_doc_a', - sortBy: 'doc_id', - sortDirection: 'asc', - }, - }, - }) - .expect(200); + describe('when sorting ', () => { + const ascendingOrder = ['my_doc_a', 'my_doc_b', 'my_doc_c']; + + it('allows sorting ascending', async () => { + const entries = await getEntries({ sortBy: 'title', sortDirection: 'asc' }); + expect(entries.map(({ id }) => id)).to.eql(ascendingOrder); + }); + + it('allows sorting descending', async () => { + const entries = await getEntries({ sortBy: 'title', sortDirection: 'desc' }); + expect(entries.map(({ id }) => id)).to.eql([...ascendingOrder].reverse()); + }); + }); - expect(res.body.entries.length).to.eql(1); - expect(res.body.entries[0].id).to.eql('my_doc_a'); + it('allows searching by title', async () => { + const entries = await getEntries({ query: 'b' }); + expect(entries.length).to.eql(1); + expect(entries[0].title).to.eql('My title b'); }); }); }); } + +function omitCategories(entries: KnowledgeBaseEntry[]) { + return entries.filter((entry) => entry.labels?.category === undefined); +} diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts index 9099eff540d35..77f010d851f3c 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_setup.spec.ts @@ -17,7 +17,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns empty object when successful', async () => { await createKnowledgeBaseModel(ml); const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(200); @@ -27,7 +27,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns bad request if model cannot be installed', async () => { await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(400); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts index 4e9778630a535..6561c416f02cf 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_status.spec.ts @@ -17,7 +17,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { before(async () => { await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup', }) .expect(200); @@ -29,7 +29,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns correct status after knowledge base is setup', async () => { const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/status', }) .expect(200); @@ -41,7 +41,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await ml.api.stopTrainedModelDeploymentES(TINY_ELSER.id, true); const res = await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/status', }) .expect(200); diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts index a93c194c85daa..a9c1f245a1ac3 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts @@ -6,7 +6,6 @@ */ import expect from '@kbn/expect'; -import { kbnTestConfig } from '@kbn/test'; import { sortBy } from 'lodash'; import { Message, MessageRole } from '@kbn/observability-ai-assistant-plugin/common'; import { CONTEXT_FUNCTION_NAME } from '@kbn/observability-ai-assistant-plugin/server/functions/context'; @@ -21,33 +20,27 @@ import { import { getConversationCreatedEvent } from '../conversations/helpers'; import { LlmProxy, createLlmProxy } from '../../common/create_llm_proxy'; import { createProxyActionConnector, deleteActionConnector } from '../../common/action_connectors'; +import { User } from '../../common/users/users'; export default function ApiTest({ getService }: FtrProviderContext) { const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); - const getScopedApiClientForUsername = getService('getScopedApiClientForUsername'); - const security = getService('security'); const supertest = getService('supertest'); const es = getService('es'); const ml = getService('ml'); const log = getService('log'); + const getScopedApiClientForUsername = getService('getScopedApiClientForUsername'); describe('Knowledge base user instructions', () => { - const userJohn = 'john'; - before(async () => { - // create user - const password = kbnTestConfig.getUrlParts().password!; - await security.user.create(userJohn, { password, roles: ['editor'] }); await createKnowledgeBaseModel(ml); await observabilityAIAssistantAPIClient - .editorUser({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup' }) + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup' }) .expect(200); }); after(async () => { await deleteKnowledgeBaseModel(ml); - await security.user.delete(userJohn); await clearKnowledgeBase(es); await clearConversations(es); }); @@ -58,19 +51,19 @@ export default function ApiTest({ getService }: FtrProviderContext) { const promises = [ { - username: 'editor', + username: 'editor' as const, isPublic: true, }, { - username: 'editor', + username: 'editor' as const, isPublic: false, }, { - username: userJohn, + username: 'secondary_editor' as const, isPublic: true, }, { - username: userJohn, + username: 'secondary_editor' as const, isPublic: false, }, ].map(async ({ username, isPublic }) => { @@ -92,61 +85,59 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('"editor" can retrieve their own private instructions and the public instruction', async () => { - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions', }); const instructions = res.body.userInstructions; - const sortByDocId = (data: Array) => - sortBy(data, 'doc_id'); + const sortById = (data: Array) => sortBy(data, 'id'); - expect(sortByDocId(instructions)).to.eql( - sortByDocId([ + expect(sortById(instructions)).to.eql( + sortById([ { - doc_id: 'private-doc-from-editor', + id: 'private-doc-from-editor', public: false, text: 'Private user instruction from "editor"', }, { - doc_id: 'public-doc-from-editor', + id: 'public-doc-from-editor', public: true, text: 'Public user instruction from "editor"', }, { - doc_id: 'public-doc-from-john', + id: 'public-doc-from-secondary_editor', public: true, - text: 'Public user instruction from "john"', + text: 'Public user instruction from "secondary_editor"', }, ]) ); }); - it('"john" can retrieve their own private instructions and the public instruction', async () => { - const res = await getScopedApiClientForUsername(userJohn)({ + it('"secondaryEditor" can retrieve their own private instructions and the public instruction', async () => { + const res = await observabilityAIAssistantAPIClient.secondaryEditor({ endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions', }); const instructions = res.body.userInstructions; - const sortByDocId = (data: Array) => - sortBy(data, 'doc_id'); + const sortById = (data: Array) => sortBy(data, 'id'); - expect(sortByDocId(instructions)).to.eql( - sortByDocId([ + expect(sortById(instructions)).to.eql( + sortById([ { - doc_id: 'public-doc-from-editor', + id: 'public-doc-from-editor', public: true, text: 'Public user instruction from "editor"', }, { - doc_id: 'public-doc-from-john', + id: 'public-doc-from-secondary_editor', public: true, - text: 'Public user instruction from "john"', + text: 'Public user instruction from "secondary_editor"', }, { - doc_id: 'private-doc-from-john', + id: 'private-doc-from-secondary_editor', public: false, - text: 'Private user instruction from "john"', + text: 'Private user instruction from "secondary_editor"', }, ]) ); @@ -158,7 +149,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { await clearKnowledgeBase(es); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions', params: { body: { @@ -171,7 +162,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { .expect(200); await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions', params: { body: { @@ -185,14 +176,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('updates the user instruction', async () => { - const res = await observabilityAIAssistantAPIClient.editorUser({ + const res = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions', }); const instructions = res.body.userInstructions; expect(instructions).to.eql([ { - doc_id: 'doc-to-update', + id: 'doc-to-update', text: 'Updated text', public: false, }, @@ -207,12 +198,12 @@ export default function ApiTest({ getService }: FtrProviderContext) { const userInstructionText = 'Be polite and use language that is easy to understand. Never disagree with the user.'; - async function getConversationForUser(username: string) { + async function getConversationForUser(username: User['username']) { const apiClient = getScopedApiClientForUsername(username); // the user instruction is always created by "editor" user await observabilityAIAssistantAPIClient - .editorUser({ + .editor({ endpoint: 'PUT /internal/observability_ai_assistant/kb/user_instructions', params: { body: { @@ -314,7 +305,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); it('does not add the instruction conversation for other users', async () => { - const conversation = await getConversationForUser('john'); + const conversation = await getConversationForUser('secondary_editor'); const systemMessage = conversation.messages.find( (message) => message.message.role === MessageRole.System )!; diff --git a/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts b/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts index a25a7cf5ef8eb..bb8984256f27c 100644 --- a/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts +++ b/x-pack/test/observability_ai_assistant_api_integration/tests/public_complete/public_complete.spec.ts @@ -70,7 +70,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { (body) => !isFunctionTitleRequest(body) ); - const responsePromise = observabilityAIAssistantAPIClient.adminUser({ + const responsePromise = observabilityAIAssistantAPIClient.admin({ endpoint: 'POST /api/observability_ai_assistant/chat/complete 2023-10-31', params: { query: { format }, diff --git a/x-pack/test/observability_ai_assistant_functional/common/config.ts b/x-pack/test/observability_ai_assistant_functional/common/config.ts index c0f649d51d90d..99213e629e0e3 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/config.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/config.ts @@ -13,8 +13,9 @@ import { KibanaEBTUIProvider, } from '@kbn/test-suites-src/analytics/services/kibana_ebt'; import { - editorUser, - viewerUser, + secondaryEditor, + editor, + viewer, } from '../../observability_ai_assistant_api_integration/common/users/users'; import { ObservabilityAIAssistantFtrConfig, @@ -61,9 +62,10 @@ async function getTestConfig({ ObservabilityAIAssistantUIProvider(context), observabilityAIAssistantAPIClient: async (context: InheritedFtrProviderContext) => { return { - adminUser: await getScopedApiClient(kibanaServer, 'elastic'), - viewerUser: await getScopedApiClient(kibanaServer, viewerUser.username), - editorUser: await getScopedApiClient(kibanaServer, editorUser.username), + admin: getScopedApiClient(kibanaServer, 'elastic'), + viewer: getScopedApiClient(kibanaServer, viewer.username), + editor: getScopedApiClient(kibanaServer, editor.username), + secondaryEditor: getScopedApiClient(kibanaServer, secondaryEditor.username), }; }, kibana_ebt_server: KibanaEBTServerProvider, diff --git a/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts b/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts index 2c6852988cde5..d072cc3777a7d 100644 --- a/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts +++ b/x-pack/test/observability_ai_assistant_functional/common/ui/index.ts @@ -27,6 +27,11 @@ export interface ObservabilityAIAssistantUIService { } const pages = { + kbManagementTab: { + table: 'knowledgeBaseTable', + tableTitleCell: 'knowledgeBaseTableTitleCell', + tableAuthorCell: 'knowledgeBaseTableAuthorCell', + }, conversations: { setupGenAiConnectorsButtonSelector: `observabilityAiAssistantInitialSetupPanelSetUpGenerativeAiConnectorButton`, chatInput: 'observabilityAiAssistantChatPromptEditorTextArea', diff --git a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts index 68b1d97d531dc..de780d2f46b0e 100644 --- a/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts +++ b/x-pack/test/observability_ai_assistant_functional/tests/conversations/index.spec.ts @@ -36,12 +36,12 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte const flyoutService = getService('flyout'); async function deleteConversations() { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }); for (const conversation of response.body.conversations) { - await observabilityAIAssistantAPIClient.editorUser({ + await observabilityAIAssistantAPIClient.editor({ endpoint: `DELETE /internal/observability_ai_assistant/conversation/{conversationId}`, params: { path: { @@ -53,7 +53,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte } async function deleteConnectors() { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }); @@ -66,7 +66,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte } async function createOldConversation() { - await observabilityAIAssistantAPIClient.editorUser({ + await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversation', params: { body: { @@ -204,7 +204,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); it('creates a connector', async () => { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'GET /internal/observability_ai_assistant/connectors', }); @@ -259,7 +259,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); it('creates a conversation and updates the URL', async () => { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }); @@ -325,7 +325,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); it('does not create another conversation', async () => { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }); @@ -333,7 +333,7 @@ export default function ApiTest({ getService, getPageObjects }: FtrProviderConte }); it('appends to the existing one', async () => { - const response = await observabilityAIAssistantAPIClient.editorUser({ + const response = await observabilityAIAssistantAPIClient.editor({ endpoint: 'POST /internal/observability_ai_assistant/conversations', }); diff --git a/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts new file mode 100644 index 0000000000000..7a5a51ae58b6a --- /dev/null +++ b/x-pack/test/observability_ai_assistant_functional/tests/knowledge_base_management/index.spec.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { subj as testSubjSelector } from '@kbn/test-subj-selector'; +import { + clearKnowledgeBase, + createKnowledgeBaseModel, + deleteKnowledgeBaseModel, +} from '../../../observability_ai_assistant_api_integration/tests/knowledge_base/helpers'; +import { ObservabilityAIAssistantApiClient } from '../../../observability_ai_assistant_api_integration/common/observability_ai_assistant_api_client'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ApiTest({ getService, getPageObjects }: FtrProviderContext) { + const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantAPIClient'); + const ui = getService('observabilityAIAssistantUI'); + const testSubjects = getService('testSubjects'); + const log = getService('log'); + const ml = getService('ml'); + const es = getService('es'); + const { common } = getPageObjects(['common']); + + async function saveKbEntry({ + apiClient, + text, + }: { + apiClient: ObservabilityAIAssistantApiClient; + text: string; + }) { + return apiClient({ + endpoint: 'POST /internal/observability_ai_assistant/functions/summarize', + params: { + body: { + title: 'Favourite color', + text, + confidence: 'high', + is_correction: false, + public: false, + labels: {}, + }, + }, + }).expect(200); + } + + describe('Knowledge management tab', () => { + before(async () => { + await clearKnowledgeBase(es); + + // create a knowledge base model + await createKnowledgeBaseModel(ml); + + await Promise.all([ + // setup the knowledge base + observabilityAIAssistantAPIClient + .editor({ endpoint: 'POST /internal/observability_ai_assistant/kb/setup' }) + .expect(200), + + // login as editor + ui.auth.login('editor'), + ]); + }); + + after(async () => { + await Promise.all([deleteKnowledgeBaseModel(ml), clearKnowledgeBase(es), ui.auth.logout()]); + }); + + describe('when the LLM calls the "summarize" function for two different users', () => { + async function getKnowledgeBaseEntries() { + await common.navigateToUrlWithBrowserHistory( + 'management', + '/kibana/observabilityAiAssistantManagement', + 'tab=knowledge_base' + ); + + const entryTitleCells = await testSubjects.findAll(ui.pages.kbManagementTab.tableTitleCell); + + const rows = await Promise.all( + entryTitleCells.map(async (cell) => { + const title = await cell.getVisibleText(); + const parentRow = await cell.findByXpath('ancestor::tr'); + + const authorElm = await parentRow.findByCssSelector( + testSubjSelector(ui.pages.kbManagementTab.tableAuthorCell) + ); + const author = await authorElm.getVisibleText(); + const rowText = (await parentRow.getVisibleText()).split('\n'); + + return { rowText, author, title }; + }) + ); + + log.debug(`Found ${rows.length} rows in the KB management table: ${JSON.stringify(rows)}`); + + return rows.filter(({ title }) => title === 'Favourite color'); + } + + before(async () => { + await saveKbEntry({ + apiClient: observabilityAIAssistantAPIClient.editor, + text: 'My favourite color is red', + }); + + await saveKbEntry({ + apiClient: observabilityAIAssistantAPIClient.secondaryEditor, + text: 'My favourite color is blue', + }); + }); + + it('shows two entries', async () => { + const entries = await getKnowledgeBaseEntries(); + expect(entries.length).to.eql(2); + }); + + it('shows two different authors', async () => { + const entries = await getKnowledgeBaseEntries(); + expect(entries.map(({ author }) => author)).to.eql(['secondary_editor', 'editor']); + }); + }); + }); +} diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 79bdd3ad7df09..091e0fe01e415 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -151,7 +151,6 @@ export default function ({ getService }: FtrProviderContext) { 'fleet:update_agent_tags:retry', 'fleet:upgrade_action:retry', 'logs-data-telemetry', - 'observabilityAIAssistant:indexQueuedDocumentsTaskType', 'osquery:telemetry-configs', 'osquery:telemetry-packs', 'osquery:telemetry-saved-queries', diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts b/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts index b540ee5829e59..ce46939c365be 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/ai_assistant/tests/knowledge_base/knowledge_base.spec.ts @@ -52,6 +52,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { describe('when managing a single entry', () => { const knowledgeBaseEntry = { id: 'my-doc-id-1', + title: 'My title', text: 'My content', }; it('returns 200 on create', async () => { @@ -156,14 +157,17 @@ export default function ApiTest({ getService }: FtrProviderContext) { const knowledgeBaseEntries = [ { id: 'my_doc_a', + title: 'My title a', text: 'My content a', }, { id: 'my_doc_b', + title: 'My title b', text: 'My content b', }, { id: 'my_doc_c', + title: 'My title c', text: 'My content c', }, ]; From a049cbc702b415bd7bce304edfb531d2b276c7e1 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 21:58:31 +1100 Subject: [PATCH 05/12] [8.x] [Observability Onboarding] fix docker integration double detection (#199237) (#199267) # Backport This will backport the following commits from `main` to `8.x`: - [[Observability Onboarding] fix docker integration double detection (#199237)](https://github.com/elastic/kibana/pull/199237) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Joe Reuter --- .../observability_onboarding/public/assets/auto_detect.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh index c13b5cf031e0d..dd3077180d08e 100755 --- a/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh +++ b/x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh @@ -329,6 +329,7 @@ read_open_log_file_list() { "^\/var\/log\/redis\/" "^\/var\/log\/rabbitmq\/" "^\/var\/log\/kafka\/" + "^\/var\/lib\/docker\/" "^\/var\/log\/mongodb\/" "^\/opt\/tomcat\/logs\/" "^\/var\/log\/prometheus\/" From ffaf529f29b9210e917f30296b95bcdf1e1a55f4 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 22:22:27 +1100 Subject: [PATCH 06/12] [8.x] [ML] Migrate ColorRangeLegend from SCSS to emotion. (#199156) (#199274) # Backport This will backport the following commits from `main` to `8.x`: - [[ML] Migrate ColorRangeLegend from SCSS to emotion. (#199156)](https://github.com/elastic/kibana/pull/199156) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Walter Rafelsberger --- .../plugins/ml/public/application/_index.scss | 1 - .../_color_range_legend.scss | 18 ------------- .../components/color_range_legend/_index.scss | 1 - .../color_range_legend/color_range_legend.tsx | 27 +++++++++++++++++-- 4 files changed, 25 insertions(+), 22 deletions(-) delete mode 100644 x-pack/plugins/ml/public/application/components/color_range_legend/_color_range_legend.scss delete mode 100644 x-pack/plugins/ml/public/application/components/color_range_legend/_index.scss diff --git a/x-pack/plugins/ml/public/application/_index.scss b/x-pack/plugins/ml/public/application/_index.scss index 3025b8f7d921b..95fbbf4cb112a 100644 --- a/x-pack/plugins/ml/public/application/_index.scss +++ b/x-pack/plugins/ml/public/application/_index.scss @@ -11,7 +11,6 @@ // Components @import 'components/annotations/annotation_description_list/index'; // SASSTODO: This file overwrites EUI directly @import 'components/anomalies_table/index'; // SASSTODO: This file overwrites EUI directly - @import 'components/color_range_legend/index'; @import 'components/entity_cell/index'; @import 'components/influencers_list/index'; @import 'components/job_selector/index'; diff --git a/x-pack/plugins/ml/public/application/components/color_range_legend/_color_range_legend.scss b/x-pack/plugins/ml/public/application/components/color_range_legend/_color_range_legend.scss deleted file mode 100644 index b164e605a2488..0000000000000 --- a/x-pack/plugins/ml/public/application/components/color_range_legend/_color_range_legend.scss +++ /dev/null @@ -1,18 +0,0 @@ -/* Overrides for d3/svg default styles */ -.mlColorRangeLegend { - text { - @include fontSize($euiFontSizeXS - 2px); - fill: $euiColorDarkShade; - } - - .axis path { - fill: none; - stroke: none; - } - - .axis line { - fill: none; - stroke: $euiColorMediumShade; - shape-rendering: crispEdges; - } -} diff --git a/x-pack/plugins/ml/public/application/components/color_range_legend/_index.scss b/x-pack/plugins/ml/public/application/components/color_range_legend/_index.scss deleted file mode 100644 index c7cd3faac0dcf..0000000000000 --- a/x-pack/plugins/ml/public/application/components/color_range_legend/_index.scss +++ /dev/null @@ -1 +0,0 @@ -@import 'color_range_legend'; diff --git a/x-pack/plugins/ml/public/application/components/color_range_legend/color_range_legend.tsx b/x-pack/plugins/ml/public/application/components/color_range_legend/color_range_legend.tsx index f6a301f5eacce..9c121853cf6b4 100644 --- a/x-pack/plugins/ml/public/application/components/color_range_legend/color_range_legend.tsx +++ b/x-pack/plugins/ml/public/application/components/color_range_legend/color_range_legend.tsx @@ -7,12 +7,36 @@ import type { FC } from 'react'; import React, { useEffect, useRef } from 'react'; +import { css } from '@emotion/react'; import d3 from 'd3'; import { EuiFlexGroup, EuiFlexItem, EuiText } from '@elastic/eui'; +import { euiThemeVars } from '@kbn/ui-theme'; + const COLOR_RANGE_RESOLUTION = 10; +// Overrides for d3/svg default styles +const cssOverride = css({ + // Override default font size and color for axis + text: { + fontSize: `calc(${euiThemeVars.euiFontSizeXS} - 2px)`, + fill: euiThemeVars.euiColorDarkShade, + }, + // Override default styles for axis lines + '.axis': { + path: { + fill: 'none', + stroke: 'none', + }, + line: { + fill: 'none', + stroke: euiThemeVars.euiColorMediumShade, + shapeRendering: 'crispEdges', + }, + }, +}); + interface ColorRangeLegendProps { colorRange: (d: number) => string; justifyTicks?: boolean; @@ -65,7 +89,6 @@ export const ColorRangeLegend: FC = ({ const wrapper = d3 .select(d3Container.current) - .classed('mlColorRangeLegend', true) .attr('width', wrapperWidth) .attr('height', wrapperHeight) .append('g') @@ -144,7 +167,7 @@ export const ColorRangeLegend: FC = ({ - + ); From 3265ee8e9ab7bf3b79c5eaad1e6b44fb68c91c34 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 22:54:59 +1100 Subject: [PATCH 07/12] [8.x] [Synthetics] Refactor bulk delete monitor and params routes !! (#195420) (#199277) # Backport This will backport the following commits from `main` to `8.x`: - [[Synthetics] Refactor bulk delete monitor and params routes !! (#195420)](https://github.com/elastic/kibana/pull/195420) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Shahzad --- .../monitors/delete-monitor-api.asciidoc | 10 +-- .../synthetics/params/delete-param.asciidoc | 44 +++++----- .../monitor_management/synthetics_params.ts | 2 + .../settings/global_params/delete_param.tsx | 73 +++------------- .../settings/global_params/params_list.tsx | 13 ++- .../synthetics/state/global_params/actions.ts | 5 ++ .../synthetics/state/global_params/api.ts | 17 ++-- .../synthetics/state/global_params/effects.ts | 32 ++++++- .../synthetics/state/global_params/index.ts | 19 ++++- .../apps/synthetics/state/monitor_list/api.ts | 9 +- .../apps/synthetics/state/root_effect.ts | 8 +- .../synthetics/server/routes/index.ts | 4 + .../bulk_cruds/add_monitor_bulk.ts | 6 +- .../bulk_cruds/delete_monitor_bulk.ts | 85 +++++++------------ .../routes/monitor_cruds/delete_monitor.ts | 36 +++++--- .../monitor_cruds/delete_monitor_project.ts | 7 +- .../services/delete_monitor_api.ts | 59 +++++++++++-- .../routes/settings/params/delete_param.ts | 41 +++++++-- .../settings/params/delete_params_bulk.ts | 39 +++++++++ .../translations/translations/fr-FR.json | 2 - .../translations/translations/ja-JP.json | 2 - .../translations/translations/zh-CN.json | 2 - .../apis/synthetics/add_edit_params.ts | 40 +++++++++ .../apis/synthetics/delete_monitor.ts | 27 ++++++ .../synthetics_monitor_test_service.ts | 13 +++ .../apis/synthetics/sync_global_params.ts | 5 +- 26 files changed, 403 insertions(+), 197 deletions(-) create mode 100644 x-pack/plugins/observability_solution/synthetics/server/routes/settings/params/delete_params_bulk.ts diff --git a/docs/api/synthetics/monitors/delete-monitor-api.asciidoc b/docs/api/synthetics/monitors/delete-monitor-api.asciidoc index 70861fcd60a36..74798b40830b7 100644 --- a/docs/api/synthetics/monitors/delete-monitor-api.asciidoc +++ b/docs/api/synthetics/monitors/delete-monitor-api.asciidoc @@ -17,9 +17,6 @@ Deletes one or more monitors from the Synthetics app. You must have `all` privileges for the *Synthetics* feature in the *{observability}* section of the <>. -You must have `all` privileges for the *Synthetics* feature in the *{observability}* section of the -<>. - [[delete-monitor-api-path-params]] === {api-path-parms-title} @@ -27,7 +24,6 @@ You must have `all` privileges for the *Synthetics* feature in the *{observabili `config_id`:: (Required, string) The ID of the monitor that you want to delete. - Here is an example of a DELETE request to delete a monitor by ID: [source,sh] @@ -37,7 +33,7 @@ DELETE /api/synthetics/monitors/monitor1-id ==== Bulk Delete Monitors -You can delete multiple monitors by sending a list of config ids to a DELETE request to the `/api/synthetics/monitors` endpoint. +You can delete multiple monitors by sending a list of config ids to a POST request to the `/api/synthetics/monitors/_bulk_delete` endpoint. [[monitors-delete-request-body]] @@ -49,11 +45,11 @@ The request body should contain an array of monitors IDs that you want to delete (Required, array of strings) An array of monitor IDs to delete. -Here is an example of a DELETE request to delete a list of monitors by ID: +Here is an example of a POST request to delete a list of monitors by ID: [source,sh] -------------------------------------------------- -DELETE /api/synthetics/monitors +POST /api/synthetics/monitors/_bulk_delete { "ids": [ "monitor1-id", diff --git a/docs/api/synthetics/params/delete-param.asciidoc b/docs/api/synthetics/params/delete-param.asciidoc index 4c7d7911ec180..031a47501a8a8 100644 --- a/docs/api/synthetics/params/delete-param.asciidoc +++ b/docs/api/synthetics/params/delete-param.asciidoc @@ -8,9 +8,9 @@ Deletes one or more parameters from the Synthetics app. === {api-request-title} -`DELETE :/api/synthetics/params` +`DELETE :/api/synthetics/params/` -`DELETE :/s//api/synthetics/params` +`DELETE :/s//api/synthetics/params/` === {api-prereq-title} @@ -20,26 +20,19 @@ You must have `all` privileges for the *Synthetics* feature in the *{observabili You must have `all` privileges for the *Synthetics* feature in the *{observability}* section of the <>. -[[parameters-delete-request-body]] -==== Request Body +[[parameters-delete-path-param]] +==== Path Parameters The request body should contain an array of parameter IDs that you want to delete. -`ids`:: -(Required, array of strings) An array of parameter IDs to delete. +`param_id`:: +(Required, string) An id of parameter to delete. - -Here is an example of a DELETE request to delete a list of parameters by ID: +Here is an example of a DELETE request to delete a parameter by its ID: [source,sh] -------------------------------------------------- -DELETE /api/synthetics/params -{ - "ids": [ - "param1-id", - "param2-id" - ] -} +DELETE /api/synthetics/params/param_id1 -------------------------------------------------- [[parameters-delete-response-example]] @@ -58,10 +51,21 @@ Here's an example response for deleting multiple parameters: { "id": "param1-id", "deleted": true - }, - { - "id": "param2-id", - "deleted": true } ] --------------------------------------------------- \ No newline at end of file +-------------------------------------------------- + +==== Bulk delete parameters +To delete multiple parameters, you can send a POST request to `/api/synthetics/params/_bulk_delete` with an array of parameter IDs to delete via body. + +Here is an example of a POST request to delete multiple parameters: + +[source,sh] +-------------------------------------------------- +POST /api/synthetics/params/_bulk_delete +{ + "ids": ["param1-id", "param2-id"] +} +-------------------------------------------------- + + diff --git a/x-pack/plugins/observability_solution/synthetics/common/runtime_types/monitor_management/synthetics_params.ts b/x-pack/plugins/observability_solution/synthetics/common/runtime_types/monitor_management/synthetics_params.ts index 5393fed135b7d..4107bc1efbb2a 100644 --- a/x-pack/plugins/observability_solution/synthetics/common/runtime_types/monitor_management/synthetics_params.ts +++ b/x-pack/plugins/observability_solution/synthetics/common/runtime_types/monitor_management/synthetics_params.ts @@ -19,6 +19,8 @@ export const SyntheticsParamsReadonlyCodec = t.intersection([ }), ]); +export const SyntheticsParamsReadonlyCodecList = t.array(SyntheticsParamsReadonlyCodec); + export type SyntheticsParamsReadonly = t.TypeOf; export const SyntheticsParamsCodec = t.intersection([ diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/settings/global_params/delete_param.tsx b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/settings/global_params/delete_param.tsx index 814fb13a99ba9..2615a65ef289c 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/settings/global_params/delete_param.tsx +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/settings/global_params/delete_param.tsx @@ -5,16 +5,17 @@ * 2.0. */ -import React, { useEffect, useState } from 'react'; +import React, { useEffect } from 'react'; import { EuiConfirmModal } from '@elastic/eui'; -import { FETCH_STATUS, useFetcher } from '@kbn/observability-shared-plugin/public'; -import { toMountPoint } from '@kbn/react-kibana-mount'; import { i18n } from '@kbn/i18n'; -import { useDispatch } from 'react-redux'; -import { getGlobalParamAction, deleteGlobalParams } from '../../../state/global_params'; +import { useDispatch, useSelector } from 'react-redux'; +import { + getGlobalParamAction, + deleteGlobalParamsAction, + selectGlobalParamState, +} from '../../../state/global_params'; import { syncGlobalParamsAction } from '../../../state/settings'; -import { kibanaService } from '../../../../../utils/kibana_service'; import { NO_LABEL, YES_LABEL } from '../../monitors_page/management/monitor_list_table/labels'; import { ListParamItem } from './params_list'; @@ -25,19 +26,8 @@ export const DeleteParam = ({ items: ListParamItem[]; setIsDeleteModalVisible: React.Dispatch>; }) => { - const [isDeleting, setIsDeleting] = useState(false); - const dispatch = useDispatch(); - - const handleConfirmDelete = () => { - setIsDeleting(true); - }; - - const { status } = useFetcher(() => { - if (isDeleting) { - return deleteGlobalParams(items.map(({ id }) => id)); - } - }, [items, isDeleting]); + const { isDeleting, listOfParams } = useSelector(selectGlobalParamState); const name = items .map(({ key }) => key) @@ -45,51 +35,12 @@ export const DeleteParam = ({ .slice(0, 50); useEffect(() => { - if (!isDeleting) { - return; - } - const { coreStart, toasts } = kibanaService; - - if (status === FETCH_STATUS.FAILURE) { - toasts.addDanger( - { - title: toMountPoint( -

- {' '} - {i18n.translate('xpack.synthetics.paramManagement.paramDeleteFailuresMessage.name', { - defaultMessage: 'Param {name} failed to delete.', - values: { name }, - })} -

, - coreStart - ), - }, - { toastLifeTimeMs: 3000 } - ); - } else if (status === FETCH_STATUS.SUCCESS) { - toasts.addSuccess( - { - title: toMountPoint( -

- {i18n.translate('xpack.synthetics.paramManagement.paramDeleteSuccessMessage.name', { - defaultMessage: 'Param {name} deleted successfully.', - values: { name }, - })} -

, - coreStart - ), - }, - { toastLifeTimeMs: 3000 } - ); - dispatch(syncGlobalParamsAction.get()); - } - if (status === FETCH_STATUS.SUCCESS || status === FETCH_STATUS.FAILURE) { - setIsDeleting(false); + if (!isDeleting && (listOfParams ?? []).length === 0) { setIsDeleteModalVisible(false); dispatch(getGlobalParamAction.get()); dispatch(syncGlobalParamsAction.get()); } - }, [setIsDeleting, isDeleting, status, setIsDeleteModalVisible, name, dispatch]); + }, [isDeleting, setIsDeleteModalVisible, name, dispatch, listOfParams]); return ( setIsDeleteModalVisible(false)} - onConfirm={handleConfirmDelete} + onConfirm={() => { + dispatch(deleteGlobalParamsAction.get(items.map(({ id }) => id))); + }} cancelButtonText={NO_LABEL} confirmButtonText={YES_LABEL} buttonColor="danger" diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/settings/global_params/params_list.tsx b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/settings/global_params/params_list.tsx index 2ff3ea547ae9f..b16dbcd686d91 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/settings/global_params/params_list.tsx +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/components/settings/global_params/params_list.tsx @@ -83,7 +83,11 @@ export const ParamsList = () => { render: (val: string[]) => { const tags = val ?? []; if (tags.length === 0) { - return --; + return ( + + {i18n.translate('xpack.synthetics.columns.TextLabel', { defaultMessage: '--' })} + + ); } return ( @@ -105,7 +109,11 @@ export const ParamsList = () => { render: (val: string[]) => { const namespaces = val ?? []; if (namespaces.length === 0) { - return --; + return ( + + {i18n.translate('xpack.synthetics.columns.TextLabel', { defaultMessage: '--' })} + + ); } return ( @@ -184,6 +192,7 @@ export const ParamsList = () => { isEditingItem={isEditingItem} setIsEditingItem={setIsEditingItem} items={items} + key="add-param-flyout" />, ]; }; diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/actions.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/actions.ts index b1388bc2674b9..0faef0079657a 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/actions.ts +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/actions.ts @@ -23,3 +23,8 @@ export const editGlobalParamAction = createAsyncAction< }, SyntheticsParams >('EDIT GLOBAL PARAM'); + +export const deleteGlobalParamsAction = createAsyncAction< + string[], + Array<{ id: string; deleted: boolean }> +>('DELETE GLOBAL PARAMS'); diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/api.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/api.ts index 33eb4622bf6c5..1badb74dff26f 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/api.ts +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/api.ts @@ -13,6 +13,7 @@ import { SyntheticsParams, SyntheticsParamsCodec, SyntheticsParamsReadonlyCodec, + SyntheticsParamsReadonlyCodecList, } from '../../../../../common/runtime_types'; import { apiService } from '../../../../utils/api_service/api_service'; @@ -20,14 +21,14 @@ export const getGlobalParams = async (): Promise => { return apiService.get( SYNTHETICS_API_URLS.PARAMS, { version: INITIAL_REST_VERSION }, - SyntheticsParamsReadonlyCodec + SyntheticsParamsReadonlyCodecList ); }; export const addGlobalParam = async ( paramRequest: SyntheticsParamRequest ): Promise => - apiService.post(SYNTHETICS_API_URLS.PARAMS, paramRequest, SyntheticsParamsCodec, { + apiService.post(SYNTHETICS_API_URLS.PARAMS, paramRequest, SyntheticsParamsReadonlyCodec, { version: INITIAL_REST_VERSION, }); @@ -53,11 +54,13 @@ export const editGlobalParam = async ({ ); }; -export const deleteGlobalParams = async (ids: string[]): Promise => - apiService.delete( - SYNTHETICS_API_URLS.PARAMS, - { version: INITIAL_REST_VERSION }, +export const deleteGlobalParams = async (ids: string[]): Promise => { + return await apiService.post( + SYNTHETICS_API_URLS.PARAMS + '/_bulk_delete', { ids, - } + }, + null, + { version: INITIAL_REST_VERSION } ); +}; diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/effects.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/effects.ts index d5249fcfc4519..f5f0c6e4ee951 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/effects.ts +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/effects.ts @@ -8,8 +8,13 @@ import { takeLeading } from 'redux-saga/effects'; import { i18n } from '@kbn/i18n'; import { fetchEffectFactory } from '../utils/fetch_effect'; -import { addGlobalParam, editGlobalParam, getGlobalParams } from './api'; -import { addNewGlobalParamAction, editGlobalParamAction, getGlobalParamAction } from './actions'; +import { addGlobalParam, deleteGlobalParams, editGlobalParam, getGlobalParams } from './api'; +import { + addNewGlobalParamAction, + deleteGlobalParamsAction, + editGlobalParamAction, + getGlobalParamAction, +} from './actions'; export function* getGlobalParamEffect() { yield takeLeading( @@ -69,3 +74,26 @@ const editSuccessMessage = i18n.translate('xpack.synthetics.settings.editParams. const editFailureMessage = i18n.translate('xpack.synthetics.settings.editParams.fail', { defaultMessage: 'Failed to edit global parameter.', }); + +// deleteGlobalParams + +export function* deleteGlobalParamsEffect() { + yield takeLeading( + deleteGlobalParamsAction.get, + fetchEffectFactory( + deleteGlobalParams, + deleteGlobalParamsAction.success, + deleteGlobalParamsAction.fail, + deleteSuccessMessage, + deleteFailureMessage + ) + ); +} + +const deleteSuccessMessage = i18n.translate('xpack.synthetics.settings.deleteParams.success', { + defaultMessage: 'Successfully deleted global parameters.', +}); + +const deleteFailureMessage = i18n.translate('xpack.synthetics.settings.deleteParams.fail', { + defaultMessage: 'Failed to delete global parameters.', +}); diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/index.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/index.ts index 89b3a0b7e1904..a1e2e07ff955f 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/index.ts +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/global_params/index.ts @@ -8,7 +8,12 @@ import { createReducer } from '@reduxjs/toolkit'; import { SyntheticsParams } from '../../../../../common/runtime_types'; import { IHttpSerializedFetchError } from '..'; -import { addNewGlobalParamAction, editGlobalParamAction, getGlobalParamAction } from './actions'; +import { + addNewGlobalParamAction, + deleteGlobalParamsAction, + editGlobalParamAction, + getGlobalParamAction, +} from './actions'; export interface GlobalParamsState { isLoading?: boolean; @@ -16,6 +21,7 @@ export interface GlobalParamsState { addError: IHttpSerializedFetchError | null; editError: IHttpSerializedFetchError | null; isSaving?: boolean; + isDeleting?: boolean; savedData?: SyntheticsParams; } @@ -23,6 +29,7 @@ const initialState: GlobalParamsState = { isLoading: false, addError: null, isSaving: false, + isDeleting: false, editError: null, listOfParams: [], }; @@ -62,6 +69,16 @@ export const globalParamsReducer = createReducer(initialState, (builder) => { .addCase(editGlobalParamAction.fail, (state, action) => { state.isSaving = false; state.editError = action.payload; + }) + .addCase(deleteGlobalParamsAction.get, (state) => { + state.isDeleting = true; + }) + .addCase(deleteGlobalParamsAction.success, (state) => { + state.isDeleting = false; + state.listOfParams = []; + }) + .addCase(deleteGlobalParamsAction.fail, (state) => { + state.isDeleting = false; }); }); diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/monitor_list/api.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/monitor_list/api.ts index 344897dd0eb1d..bef569bf0da39 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/monitor_list/api.ts +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/monitor_list/api.ts @@ -60,12 +60,13 @@ export const fetchDeleteMonitor = async ({ }): Promise => { const baseUrl = SYNTHETICS_API_URLS.SYNTHETICS_MONITORS; - return await apiService.delete( - baseUrl, - { version: INITIAL_REST_VERSION, spaceId }, + return await apiService.post( + baseUrl + '/_bulk_delete', { ids: configIds, - } + }, + undefined, + { version: INITIAL_REST_VERSION, spaceId } ); }; diff --git a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/root_effect.ts b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/root_effect.ts index 1a565fe772aa6..e38a1b5ad918f 100644 --- a/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/root_effect.ts +++ b/x-pack/plugins/observability_solution/synthetics/public/apps/synthetics/state/root_effect.ts @@ -7,7 +7,12 @@ import { all, fork } from 'redux-saga/effects'; import { getCertsListEffect } from './certs'; -import { addGlobalParamEffect, editGlobalParamEffect, getGlobalParamEffect } from './global_params'; +import { + addGlobalParamEffect, + deleteGlobalParamsEffect, + editGlobalParamEffect, + getGlobalParamEffect, +} from './global_params'; import { fetchManualTestRunsEffect } from './manual_test_runs/effects'; import { enableDefaultAlertingEffect, @@ -66,6 +71,7 @@ export const rootEffect = function* root(): Generator { fork(fetchManualTestRunsEffect), fork(addGlobalParamEffect), fork(editGlobalParamEffect), + fork(deleteGlobalParamsEffect), fork(getGlobalParamEffect), fork(getCertsListEffect), fork(getDefaultAlertingEffect), diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/index.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/index.ts index 48e0de1c7fba4..f9d178befeb46 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/index.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/index.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { deleteSyntheticsParamsBulkRoute } from './settings/params/delete_params_bulk'; +import { deleteSyntheticsMonitorBulkRoute } from './monitor_cruds/bulk_cruds/delete_monitor_bulk'; import { createGetDynamicSettingsRoute, createPostDynamicSettingsRoute, @@ -113,4 +115,6 @@ export const syntheticsAppPublicRestApiRoutes: SyntheticsRestApiRouteFactory[] = addSyntheticsMonitorRoute, editSyntheticsMonitorRoute, deleteSyntheticsMonitorRoute, + deleteSyntheticsMonitorBulkRoute, + deleteSyntheticsParamsBulkRoute, ]; diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/bulk_cruds/add_monitor_bulk.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/bulk_cruds/add_monitor_bulk.ts index 2ecbbf83d471c..03c7ede49ceba 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/bulk_cruds/add_monitor_bulk.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/bulk_cruds/add_monitor_bulk.ts @@ -10,7 +10,6 @@ import { SavedObjectsBulkResponse } from '@kbn/core-saved-objects-api-server'; import { v4 as uuidV4 } from 'uuid'; import { NewPackagePolicy } from '@kbn/fleet-plugin/common'; import { SavedObjectError } from '@kbn/core-saved-objects-common'; -import { deleteMonitorBulk } from './delete_monitor_bulk'; import { SyntheticsServerSetup } from '../../../types'; import { RouteContext } from '../../types'; import { formatTelemetryEvent, sendTelemetryEvents } from '../../telemetry/monitor_upgrade_sender'; @@ -190,9 +189,10 @@ export const deleteMonitorIfCreated = async ({ newMonitorId ); if (encryptedMonitor) { - await deleteMonitorBulk({ + const deleteMonitorAPI = new DeleteMonitorAPI(routeContext); + + await deleteMonitorAPI.deleteMonitorBulk({ monitors: [encryptedMonitor], - routeContext, }); } } catch (e) { diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/bulk_cruds/delete_monitor_bulk.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/bulk_cruds/delete_monitor_bulk.ts index 9a031b3e7111a..ba6426de740d3 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/bulk_cruds/delete_monitor_bulk.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/bulk_cruds/delete_monitor_bulk.ts @@ -4,63 +4,40 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { SavedObject } from '@kbn/core-saved-objects-server'; -import { - formatTelemetryDeleteEvent, - sendTelemetryEvents, -} from '../../telemetry/monitor_upgrade_sender'; -import { - ConfigKey, - MonitorFields, - SyntheticsMonitor, - EncryptedSyntheticsMonitorAttributes, - SyntheticsMonitorWithId, -} from '../../../../common/runtime_types'; -import { syntheticsMonitorType } from '../../../../common/types/saved_objects'; -import { RouteContext } from '../../types'; -export const deleteMonitorBulk = async ({ - monitors, - routeContext, -}: { - monitors: Array>; - routeContext: RouteContext; -}) => { - const { savedObjectsClient, server, spaceId, syntheticsMonitorClient } = routeContext; - const { logger, telemetry, stackVersion } = server; +import { schema } from '@kbn/config-schema'; +import { DeleteMonitorAPI } from '../services/delete_monitor_api'; +import { SYNTHETICS_API_URLS } from '../../../../common/constants'; +import { SyntheticsRestApiRouteFactory } from '../../types'; - try { - const deleteSyncPromise = syntheticsMonitorClient.deleteMonitors( - monitors.map((normalizedMonitor) => ({ - ...normalizedMonitor.attributes, - id: normalizedMonitor.attributes[ConfigKey.MONITOR_QUERY_ID], - })) as SyntheticsMonitorWithId[], - savedObjectsClient, - spaceId - ); +export const deleteSyntheticsMonitorBulkRoute: SyntheticsRestApiRouteFactory< + Array<{ id: string; deleted: boolean }>, + Record, + Record, + { ids: string[] } +> = () => ({ + method: 'POST', + path: SYNTHETICS_API_URLS.SYNTHETICS_MONITORS + '/_bulk_delete', + validate: {}, + validation: { + request: { + body: schema.object({ + ids: schema.arrayOf(schema.string(), { + minSize: 1, + }), + }), + }, + }, + handler: async (routeContext): Promise => { + const { request } = routeContext; - const deletePromises = savedObjectsClient.bulkDelete( - monitors.map((monitor) => ({ type: syntheticsMonitorType, id: monitor.id })) - ); + const { ids: idsToDelete } = request.body || {}; + const deleteMonitorAPI = new DeleteMonitorAPI(routeContext); - const [errors, result] = await Promise.all([deleteSyncPromise, deletePromises]); - - monitors.forEach((monitor) => { - sendTelemetryEvents( - logger, - telemetry, - formatTelemetryDeleteEvent( - monitor, - stackVersion, - new Date().toISOString(), - Boolean((monitor.attributes as MonitorFields)[ConfigKey.SOURCE_INLINE]), - errors - ) - ); + const { errors, result } = await deleteMonitorAPI.execute({ + monitorIds: idsToDelete, }); - return { errors, result }; - } catch (e) { - throw e; - } -}; + return { result, errors }; + }, +}); diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor.ts index f40f06f66b1ff..b989d16e4f194 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { i18n } from '@kbn/i18n'; import { schema } from '@kbn/config-schema'; import { DeleteMonitorAPI } from './services/delete_monitor_api'; import { SyntheticsRestApiRouteFactory } from '../types'; @@ -41,30 +42,39 @@ export const deleteSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory< if (ids && queryId) { return response.badRequest({ - body: { message: 'id must be provided either via param or body.' }, + body: { + message: i18n.translate('xpack.synthetics.deleteMonitor.errorMultipleIdsProvided', { + defaultMessage: 'id must be provided either via param or body.', + }), + }, }); } const idsToDelete = [...(ids ?? []), ...(queryId ? [queryId] : [])]; if (idsToDelete.length === 0) { return response.badRequest({ - body: { message: 'id must be provided via param or body.' }, + body: { + message: i18n.translate('xpack.synthetics.deleteMonitor.errorMultipleIdsProvided', { + defaultMessage: 'id must be provided either via param or body.', + }), + }, }); } const deleteMonitorAPI = new DeleteMonitorAPI(routeContext); - try { - const { errors } = await deleteMonitorAPI.execute({ - monitorIds: idsToDelete, - }); + const { errors } = await deleteMonitorAPI.execute({ + monitorIds: idsToDelete, + }); - if (errors && errors.length > 0) { - return response.ok({ - body: { message: 'error pushing monitor to the service', attributes: { errors } }, - }); - } - } catch (getErr) { - throw getErr; + if (errors && errors.length > 0) { + return response.ok({ + body: { + message: i18n.translate('xpack.synthetics.deleteMonitor.errorPushingMonitorToService', { + defaultMessage: 'Error pushing monitor to the service', + }), + attributes: { errors }, + }, + }); } return deleteMonitorAPI.result; diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts index 7b36780937694..a56f66842a703 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts @@ -6,12 +6,12 @@ */ import { schema } from '@kbn/config-schema'; import { i18n } from '@kbn/i18n'; +import { DeleteMonitorAPI } from './services/delete_monitor_api'; import { SyntheticsRestApiRouteFactory } from '../types'; import { syntheticsMonitorType } from '../../../common/types/saved_objects'; import { ConfigKey } from '../../../common/runtime_types'; import { SYNTHETICS_API_URLS } from '../../../common/constants'; import { getMonitors, getSavedObjectKqlFilter } from '../common'; -import { deleteMonitorBulk } from './bulk_cruds/delete_monitor_bulk'; import { validateSpaceId } from './services/validate_space_id'; export const deleteSyntheticsMonitorProjectRoute: SyntheticsRestApiRouteFactory = () => ({ @@ -58,9 +58,10 @@ export const deleteSyntheticsMonitorProjectRoute: SyntheticsRestApiRouteFactory { fields: [] } ); - await deleteMonitorBulk({ + const deleteMonitorAPI = new DeleteMonitorAPI(routeContext); + + await deleteMonitorAPI.deleteMonitorBulk({ monitors, - routeContext, }); return { diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/services/delete_monitor_api.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/services/delete_monitor_api.ts index bd162fc043592..4fc527f930832 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/services/delete_monitor_api.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/services/delete_monitor_api.ts @@ -7,16 +7,22 @@ import pMap from 'p-map'; import { SavedObject, SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server'; -import { deleteMonitorBulk } from '../bulk_cruds/delete_monitor_bulk'; import { validatePermissions } from '../edit_monitor'; import { + ConfigKey, EncryptedSyntheticsMonitorAttributes, + MonitorFields, SyntheticsMonitor, + SyntheticsMonitorWithId, SyntheticsMonitorWithSecretsAttributes, } from '../../../../common/runtime_types'; import { syntheticsMonitorType } from '../../../../common/types/saved_objects'; import { normalizeSecrets } from '../../../synthetics_service/utils'; -import { sendErrorTelemetryEvents } from '../../telemetry/monitor_upgrade_sender'; +import { + formatTelemetryDeleteEvent, + sendErrorTelemetryEvents, + sendTelemetryEvents, +} from '../../telemetry/monitor_upgrade_sender'; import { RouteContext } from '../../types'; export class DeleteMonitorAPI { @@ -100,9 +106,8 @@ export class DeleteMonitorAPI { } try { - const { errors, result } = await deleteMonitorBulk({ + const { errors, result } = await this.deleteMonitorBulk({ monitors, - routeContext: this.routeContext, }); result.statuses?.forEach((res) => { @@ -112,11 +117,55 @@ export class DeleteMonitorAPI { }); }); - return { errors }; + return { errors, result: this.result }; } catch (e) { server.logger.error(`Unable to delete Synthetics monitor with error ${e.message}`); server.logger.error(e); throw e; } } + + async deleteMonitorBulk({ + monitors, + }: { + monitors: Array>; + }) { + const { savedObjectsClient, server, spaceId, syntheticsMonitorClient } = this.routeContext; + const { logger, telemetry, stackVersion } = server; + + try { + const deleteSyncPromise = syntheticsMonitorClient.deleteMonitors( + monitors.map((normalizedMonitor) => ({ + ...normalizedMonitor.attributes, + id: normalizedMonitor.attributes[ConfigKey.MONITOR_QUERY_ID], + })) as SyntheticsMonitorWithId[], + savedObjectsClient, + spaceId + ); + + const deletePromises = savedObjectsClient.bulkDelete( + monitors.map((monitor) => ({ type: syntheticsMonitorType, id: monitor.id })) + ); + + const [errors, result] = await Promise.all([deleteSyncPromise, deletePromises]); + + monitors.forEach((monitor) => { + sendTelemetryEvents( + logger, + telemetry, + formatTelemetryDeleteEvent( + monitor, + stackVersion, + new Date().toISOString(), + Boolean((monitor.attributes as MonitorFields)[ConfigKey.SOURCE_INLINE]), + errors + ) + ); + }); + + return { errors, result }; + } catch (e) { + throw e; + } + } } diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/settings/params/delete_param.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/settings/params/delete_param.ts index 78d24d9452ae9..1a504b263861b 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/settings/params/delete_param.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/settings/params/delete_param.ts @@ -6,6 +6,7 @@ */ import { schema } from '@kbn/config-schema'; +import { i18n } from '@kbn/i18n'; import { SyntheticsRestApiRouteFactory } from '../../types'; import { syntheticsParamType } from '../../../../common/types/saved_objects'; import { SYNTHETICS_API_URLS } from '../../../../common/constants'; @@ -13,25 +14,51 @@ import { DeleteParamsResponse } from '../../../../common/runtime_types'; export const deleteSyntheticsParamsRoute: SyntheticsRestApiRouteFactory< DeleteParamsResponse[], - unknown, + { id?: string }, unknown, { ids: string[] } > = () => ({ method: 'DELETE', - path: SYNTHETICS_API_URLS.PARAMS, + path: SYNTHETICS_API_URLS.PARAMS + '/{id?}', validate: {}, validation: { request: { - body: schema.object({ - ids: schema.arrayOf(schema.string()), + body: schema.nullable( + schema.object({ + ids: schema.arrayOf(schema.string(), { + minSize: 1, + }), + }) + ), + params: schema.object({ + id: schema.maybe(schema.string()), }), }, }, - handler: async ({ savedObjectsClient, request }) => { - const { ids } = request.body; + handler: async ({ savedObjectsClient, request, response }) => { + const { ids } = request.body ?? {}; + const { id: paramId } = request.params ?? {}; + + if (ids && paramId) { + return response.badRequest({ + body: i18n.translate('xpack.synthetics.deleteParam.errorMultipleIdsProvided', { + defaultMessage: `Both param id and body parameters cannot be provided`, + }), + }); + } + + const idsToDelete = ids ?? [paramId]; + + if (idsToDelete.length === 0) { + return response.badRequest({ + body: i18n.translate('xpack.synthetics.deleteParam.errorNoIdsProvided', { + defaultMessage: `No param ids provided`, + }), + }); + } const result = await savedObjectsClient.bulkDelete( - ids.map((id) => ({ type: syntheticsParamType, id })), + idsToDelete.map((id) => ({ type: syntheticsParamType, id })), { force: true } ); return result.statuses.map(({ id, success }) => ({ id, deleted: success })); diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/settings/params/delete_params_bulk.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/settings/params/delete_params_bulk.ts new file mode 100644 index 0000000000000..2cafaf0a1af99 --- /dev/null +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/settings/params/delete_params_bulk.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; +import { SyntheticsRestApiRouteFactory } from '../../types'; +import { syntheticsParamType } from '../../../../common/types/saved_objects'; +import { SYNTHETICS_API_URLS } from '../../../../common/constants'; +import { DeleteParamsResponse } from '../../../../common/runtime_types'; + +export const deleteSyntheticsParamsBulkRoute: SyntheticsRestApiRouteFactory< + DeleteParamsResponse[], + unknown, + unknown, + { ids: string[] } +> = () => ({ + method: 'POST', + path: SYNTHETICS_API_URLS.PARAMS + '/_bulk_delete', + validate: {}, + validation: { + request: { + body: schema.object({ + ids: schema.arrayOf(schema.string()), + }), + }, + }, + handler: async ({ savedObjectsClient, request }) => { + const { ids } = request.body; + + const result = await savedObjectsClient.bulkDelete( + ids.map((id) => ({ type: syntheticsParamType, id })), + { force: true } + ); + return result.statuses.map(({ id, success }) => ({ id, deleted: success })); + }, +}); diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index 7a41f7703eb2a..d7dfcc8ba4072 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -44373,8 +44373,6 @@ "xpack.synthetics.paramForm.namespaces": "Espaces de noms", "xpack.synthetics.paramForm.sharedAcrossSpacesLabel": "Partager entre les espaces", "xpack.synthetics.paramManagement.deleteParamNameLabel": "Supprimer le paramètre \"{name}\" ?", - "xpack.synthetics.paramManagement.paramDeleteFailuresMessage.name": "Impossible de supprimer le paramètre {name}.", - "xpack.synthetics.paramManagement.paramDeleteSuccessMessage.name": "Paramètre {name} supprimé avec succès.", "xpack.synthetics.params.description": "Définissez les variables et paramètres que vous pouvez utiliser dans la configuration du navigateur et des moniteurs légers, tels que des informations d'identification ou des URL. {learnMore}", "xpack.synthetics.params.unprivileged.unprivilegedDescription": "Vous devez disposer de privilèges supplémentaires pour voir les paramètres d'utilisation et de conservation des données des applications Synthetics. {docsLink}", "xpack.synthetics.pingList.collapseRow": "Réduire", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index ccdc4069314e8..656e6f844f0c3 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -44111,8 +44111,6 @@ "xpack.synthetics.paramForm.namespaces": "名前空間", "xpack.synthetics.paramForm.sharedAcrossSpacesLabel": "複数のスペース間で共有", "xpack.synthetics.paramManagement.deleteParamNameLabel": "\"{name}\"パラメーターを削除しますか？", - "xpack.synthetics.paramManagement.paramDeleteFailuresMessage.name": "パラメーター{name}の削除に失敗しました。", - "xpack.synthetics.paramManagement.paramDeleteSuccessMessage.name": "パラメーター\"{name}\"が正常に削除されました。", "xpack.synthetics.params.description": "ブラウザーや軽量モニターの設定に使用できる変数やパラメーター（認証情報やURLなど）を定義します。{learnMore}", "xpack.synthetics.params.unprivileged.unprivilegedDescription": "Syntheticsアプリデータの使用状況と保持設定を表示する追加の権限が必要です。{docsLink}", "xpack.synthetics.pingList.collapseRow": "縮小", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 341eec24125a2..d3341c98103cb 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -44162,8 +44162,6 @@ "xpack.synthetics.paramForm.namespaces": "命名空间", "xpack.synthetics.paramForm.sharedAcrossSpacesLabel": "跨工作区共享", "xpack.synthetics.paramManagement.deleteParamNameLabel": "删除“{name}”参数？", - "xpack.synthetics.paramManagement.paramDeleteFailuresMessage.name": "无法删除参数 {name}。", - "xpack.synthetics.paramManagement.paramDeleteSuccessMessage.name": "已成功删除参数 {name}。", "xpack.synthetics.params.description": "定义可在浏览器和轻量级监测的配置中使用的变量和参数，如凭据或 URL。{learnMore}", "xpack.synthetics.params.unprivileged.unprivilegedDescription": "您需要其他权限才能查看 Synthetics 应用数据使用情况和保留设置。{docsLink}", "xpack.synthetics.pingList.collapseRow": "折叠", diff --git a/x-pack/test/api_integration/apis/synthetics/add_edit_params.ts b/x-pack/test/api_integration/apis/synthetics/add_edit_params.ts index 7b27aaa621f46..0aae85864bf16 100644 --- a/x-pack/test/api_integration/apis/synthetics/add_edit_params.ts +++ b/x-pack/test/api_integration/apis/synthetics/add_edit_params.ts @@ -353,5 +353,45 @@ export default function ({ getService }: FtrProviderContext) { expect(param.key).to.not.empty(); }); }); + + it('should handle bulk deleting params', async () => { + await kServer.savedObjects.clean({ types: [syntheticsParamType] }); + + const params = [ + { key: 'param1', value: 'value1' }, + { key: 'param2', value: 'value2' }, + { key: 'param3', value: 'value3' }, + ]; + + for (const param of params) { + await supertestAPI + .post(SYNTHETICS_API_URLS.PARAMS) + .set('kbn-xsrf', 'true') + .send(param) + .expect(200); + } + + const getResponse = await supertestAPI + .get(SYNTHETICS_API_URLS.PARAMS) + .set('kbn-xsrf', 'true') + .expect(200); + + expect(getResponse.body.length).to.eql(3); + + const ids = getResponse.body.map((param: any) => param.id); + + await supertestAPI + .post(SYNTHETICS_API_URLS.PARAMS + '/_bulk_delete') + .set('kbn-xsrf', 'true') + .send({ ids }) + .expect(200); + + const getResponseAfterDelete = await supertestAPI + .get(SYNTHETICS_API_URLS.PARAMS) + .set('kbn-xsrf', 'true') + .expect(200); + + expect(getResponseAfterDelete.body.length).to.eql(0); + }); }); } diff --git a/x-pack/test/api_integration/apis/synthetics/delete_monitor.ts b/x-pack/test/api_integration/apis/synthetics/delete_monitor.ts index c96175d2982b3..f8781295e8005 100644 --- a/x-pack/test/api_integration/apis/synthetics/delete_monitor.ts +++ b/x-pack/test/api_integration/apis/synthetics/delete_monitor.ts @@ -121,6 +121,33 @@ export default function ({ getService }: FtrProviderContext) { await supertest.get(SYNTHETICS_API_URLS.SYNTHETICS_MONITORS + '/' + monitorId).expect(404); }); + it('deletes multiple monitors by bulk delete', async () => { + const { id: monitorId } = await saveMonitor(httpMonitorJson as MonitorFields); + const { id: monitorId2 } = await saveMonitor({ + ...httpMonitorJson, + name: 'another -2', + } as MonitorFields); + + const deleteResponse = await monitorTestService.deleteMonitorBulk( + [monitorId2, monitorId], + 200 + ); + + expect( + deleteResponse.body.result.sort((a: { id: string }, b: { id: string }) => + a.id > b.id ? 1 : -1 + ) + ).eql( + [ + { id: monitorId2, deleted: true }, + { id: monitorId, deleted: true }, + ].sort((a, b) => (a.id > b.id ? 1 : -1)) + ); + + // Hit get endpoint and expect 404 as well + await supertest.get(SYNTHETICS_API_URLS.SYNTHETICS_MONITORS + '/' + monitorId).expect(404); + }); + it('returns 404 if monitor id is not found', async () => { const invalidMonitorId = 'invalid-id'; const expected404Message = `Monitor id ${invalidMonitorId} not found!`; diff --git a/x-pack/test/api_integration/apis/synthetics/services/synthetics_monitor_test_service.ts b/x-pack/test/api_integration/apis/synthetics/services/synthetics_monitor_test_service.ts index 1c7376e41c4d7..c0c15024b5401 100644 --- a/x-pack/test/api_integration/apis/synthetics/services/synthetics_monitor_test_service.ts +++ b/x-pack/test/api_integration/apis/synthetics/services/synthetics_monitor_test_service.ts @@ -230,4 +230,17 @@ export class SyntheticsMonitorTestService { expect(deleteResponse.status).to.eql(statusCode); return deleteResponse; } + + async deleteMonitorBulk(monitorIds: string[], statusCode = 200, spaceId?: string) { + const deleteResponse = await this.supertest + .post( + spaceId + ? `/s/${spaceId}${SYNTHETICS_API_URLS.SYNTHETICS_MONITORS}/_bulk_delete` + : SYNTHETICS_API_URLS.SYNTHETICS_MONITORS + '/_bulk_delete' + ) + .send({ ids: monitorIds }) + .set('kbn-xsrf', 'true'); + expect(deleteResponse.status).to.eql(statusCode); + return deleteResponse; + } } diff --git a/x-pack/test/api_integration/apis/synthetics/sync_global_params.ts b/x-pack/test/api_integration/apis/synthetics/sync_global_params.ts index 8e0aaeff21580..e0a79a8905ee8 100644 --- a/x-pack/test/api_integration/apis/synthetics/sync_global_params.ts +++ b/x-pack/test/api_integration/apis/synthetics/sync_global_params.ts @@ -287,8 +287,9 @@ export default function ({ getService }: FtrProviderContext) { const deleteResponse = await supertestAPI .delete(SYNTHETICS_API_URLS.PARAMS) .set('kbn-xsrf', 'true') - .send({ ids }) - .expect(200); + .send({ ids }); + + expect(deleteResponse.status).eql(200); expect(deleteResponse.body).to.have.length(2); From ce0e63b2b9255dcbf11f321714e29ca3ff1dad1e Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 22:56:01 +1100 Subject: [PATCH 08/12] [8.x] Fix getAlertSummary returning 400 (Bad Request) (#199116) (#199281) # Backport This will backport the following commits from `main` to `8.x`: - [Fix getAlertSummary returning 400 (Bad Request) (#199116)](https://github.com/elastic/kibana/pull/199116) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Maryam Saeidi --- .../src/hooks/use_alerts_history.ts | 4 +++- .../hooks/use_load_alert_summary.ts | 22 ++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/x-pack/packages/observability/alert_details/src/hooks/use_alerts_history.ts b/x-pack/packages/observability/alert_details/src/hooks/use_alerts_history.ts index 7519fea5f99f8..193acb63f845b 100644 --- a/x-pack/packages/observability/alert_details/src/hooks/use_alerts_history.ts +++ b/x-pack/packages/observability/alert_details/src/hooks/use_alerts_history.ts @@ -55,6 +55,7 @@ export function useAlertsHistory({ http, instanceId, }: Props): UseAlertsHistory { + const enabled = !!featureIds.length; const { isInitialLoading, isLoading, isError, isSuccess, isRefetching, data } = useQuery({ queryKey: ['useAlertsHistory'], queryFn: async ({ signal }) => { @@ -71,10 +72,11 @@ export function useAlertsHistory({ }); }, refetchOnWindowFocus: false, + enabled, }); return { data: isInitialLoading ? EMPTY_ALERTS_HISTORY : data ?? EMPTY_ALERTS_HISTORY, - isLoading: isInitialLoading || isLoading || isRefetching, + isLoading: enabled && (isInitialLoading || isLoading || isRefetching), isSuccess, isError, }; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/hooks/use_load_alert_summary.ts b/x-pack/plugins/triggers_actions_ui/public/application/hooks/use_load_alert_summary.ts index 50fcc6025938a..10ce201885098 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/hooks/use_load_alert_summary.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/hooks/use_load_alert_summary.ts @@ -102,16 +102,18 @@ async function fetchAlertSummary({ timeRange: AlertSummaryTimeRange; filter?: estypes.QueryDslQueryContainer; }): Promise { - const res = await http.post>(`${BASE_RAC_ALERTS_API_PATH}/_alert_summary`, { - signal, - body: JSON.stringify({ - fixed_interval: fixedInterval, - gte: utcFrom, - lte: utcTo, - featureIds, - filter: [filter], - }), - }); + const res = featureIds.length + ? await http.post>(`${BASE_RAC_ALERTS_API_PATH}/_alert_summary`, { + signal, + body: JSON.stringify({ + fixed_interval: fixedInterval, + gte: utcFrom, + lte: utcTo, + featureIds, + filter: [filter], + }), + }) + : {}; const activeAlertCount = res?.activeAlertCount ?? 0; const activeAlerts = res?.activeAlerts ?? []; From 27c3d9a6dccf26bc0fb7551373fecb9999a97b5b Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 23:10:38 +1100 Subject: [PATCH 09/12] [8.x] [Entity Store] Aligning mappings with ECS (#199001) (#199283) # Backport This will backport the following commits from `main` to `8.x`: - [[Entity Store] Aligning mappings with ECS (#199001)](https://github.com/elastic/kibana/pull/199001) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Tiago Vila Verde --- .../entity_store/entities_list.test.tsx | 2 +- .../hooks/use_entities_list_columns.tsx | 2 +- .../united_entity_definitions/constants.ts | 4 +-- .../entity_types/host.ts | 12 +++++++- .../entity_types/user.ts | 12 +++++++- .../get_united_definition.test.ts | 28 ++++++++++++++++--- .../united_entity_definition.ts | 10 +++++++ 7 files changed, 60 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/entities_list.test.tsx b/x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/entities_list.test.tsx index 91f0c42eab385..0f493304e1f87 100644 --- a/x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/entities_list.test.tsx +++ b/x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/entities_list.test.tsx @@ -109,7 +109,7 @@ describe('EntitiesList', () => { fireEvent.click(columnHeader); expect(mockUseEntitiesListQuery).toHaveBeenCalledWith( expect.objectContaining({ - sortField: 'entity.name.text', + sortField: 'entity.name', sortOrder: 'asc', }) ); diff --git a/x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/hooks/use_entities_list_columns.tsx b/x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/hooks/use_entities_list_columns.tsx index 974a80454ee21..e603c95b6604a 100644 --- a/x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/hooks/use_entities_list_columns.tsx +++ b/x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/hooks/use_entities_list_columns.tsx @@ -79,7 +79,7 @@ export const useEntitiesListColumns = (): EntitiesListColumns => { width: '5%', }, { - field: 'entity.name.text', + field: 'entity.name', name: ( { "entity.name": Object { "fields": Object { "text": Object { - "type": "keyword", + "type": "match_only_text", }, }, - "type": "text", + "type": "keyword", }, "entity.source": Object { "type": "keyword", @@ -59,9 +59,19 @@ describe('getUnitedEntityDefinition', () => { "type": "keyword", }, "host.name": Object { + "fields": Object { + "text": Object { + "type": "match_only_text", + }, + }, "type": "keyword", }, "host.os.name": Object { + "fields": Object { + "text": Object { + "type": "match_only_text", + }, + }, "type": "keyword", }, "host.os.type": Object { @@ -335,10 +345,10 @@ describe('getUnitedEntityDefinition', () => { "entity.name": Object { "fields": Object { "text": Object { - "type": "keyword", + "type": "match_only_text", }, }, - "type": "text", + "type": "keyword", }, "entity.source": Object { "type": "keyword", @@ -350,6 +360,11 @@ describe('getUnitedEntityDefinition', () => { "type": "keyword", }, "user.full_name": Object { + "fields": Object { + "text": Object { + "type": "match_only_text", + }, + }, "type": "keyword", }, "user.hash": Object { @@ -359,6 +374,11 @@ describe('getUnitedEntityDefinition', () => { "type": "keyword", }, "user.name": Object { + "fields": Object { + "text": Object { + "type": "match_only_text", + }, + }, "type": "keyword", }, "user.risk.calculated_level": Object { diff --git a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts index eced765c75193..fc7430ebb1806 100644 --- a/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts +++ b/x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/united_entity_definition.ts @@ -94,6 +94,11 @@ export class UnitedEntityDefinition { ...BASE_ENTITY_INDEX_MAPPING, [identityField]: { type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, }, }; @@ -107,6 +112,11 @@ export class UnitedEntityDefinition { properties[identityField] = { type: 'keyword', + fields: { + text: { + type: 'match_only_text', + }, + }, }; return { From 31e0899604fd707762ea454d65e0df7c048b123f Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 23:26:08 +1100 Subject: [PATCH 10/12] [8.x] [Dataset quality] Extracting totalDocs form degradedDocs request (#198757) (#199177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Backport This will backport the following commits from `main` to `8.x`: - [[Dataset quality] Extracting totalDocs form degradedDocs request (#198757)](https://github.com/elastic/kibana/pull/198757) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) --------- Co-authored-by: Yngrid Coello --- .../dataset_quality/common/api_types.ts | 34 +- .../dataset_quality/common/constants.ts | 5 +- .../data_streams_stats/data_stream_stat.ts | 34 +- .../data_streams_stats/malformed_docs_stat.ts | 31 -- .../common/data_streams_stats/types.ts | 12 +- .../dataset_quality_indicator.tsx | 4 +- .../hooks/use_dataset_quality_filters.ts | 2 +- .../hooks/use_dataset_quality_table.tsx | 3 +- .../public/hooks/use_dataset_telemetry.ts | 2 +- .../public/hooks/use_summary_panel.ts | 2 +- .../data_streams_stats_client.ts | 46 ++- .../services/data_streams_stats/types.ts | 7 +- .../src/defaults.ts | 3 +- .../src/notifications.ts | 13 + .../src/state_machine.ts | 106 ++++-- .../dataset_quality_controller/src/types.ts | 17 +- .../public/utils/generate_datasets.test.ts | 335 +++++++++++++----- .../public/utils/generate_datasets.ts | 58 +-- ...et_dataset_aggregated_paginated_results.ts | 94 +++++ .../routes/data_streams/get_degraded_docs.ts | 168 ++------- .../server/routes/data_streams/routes.ts | 47 ++- .../dataset_quality/data_stream_total_docs.ts | 129 +++++++ .../observability/dataset_quality/index.ts | 1 + .../tests/data_streams/degraded_docs.spec.ts | 96 +---- .../tests/data_streams/total_docs.spec.ts | 41 +++ 25 files changed, 806 insertions(+), 484 deletions(-) delete mode 100644 x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/malformed_docs_stat.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_dataset_aggregated_paginated_results.ts create mode 100644 x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_total_docs.ts create mode 100644 x-pack/test/dataset_quality_api_integration/tests/data_streams/total_docs.spec.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index 903d7f0607663..51a1421aec918 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -37,6 +37,25 @@ export const dataStreamStatRt = rt.intersection([ export type DataStreamStat = rt.TypeOf; +export const dataStreamDocsStatRt = rt.type({ + dataset: rt.string, + count: rt.number, +}); + +export type DataStreamDocsStat = rt.TypeOf; + +export const getDataStreamTotalDocsResponseRt = rt.type({ + totalDocs: rt.array(dataStreamDocsStatRt), +}); + +export type DataStreamTotalDocsResponse = rt.TypeOf; + +export const getDataStreamDegradedDocsResponseRt = rt.type({ + degradedDocs: rt.array(dataStreamDocsStatRt), +}); + +export type DataStreamDegradedDocsResponse = rt.TypeOf; + export const integrationDashboardRT = rt.type({ id: rt.string, title: rt.string, @@ -84,15 +103,6 @@ export const getIntegrationsResponseRt = rt.exact( export type IntegrationResponse = rt.TypeOf; -export const degradedDocsRt = rt.type({ - dataset: rt.string, - count: rt.number, - docsCount: rt.number, - percentage: rt.number, -}); - -export type DegradedDocs = rt.TypeOf; - export const degradedFieldRt = rt.type({ name: rt.string, count: rt.number, @@ -188,12 +198,6 @@ export const getDataStreamsStatsResponseRt = rt.exact( }) ); -export const getDataStreamsDegradedDocsStatsResponseRt = rt.exact( - rt.type({ - degradedDocs: rt.array(degradedDocsRt), - }) -); - export const getDataStreamsSettingsResponseRt = rt.exact(dataStreamSettingsRt); export const getDataStreamsDetailsResponseRt = rt.exact(dataStreamDetailsRt); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts index 1b822c6c111d9..74809e0e19420 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts @@ -11,6 +11,7 @@ export const DATASET_QUALITY_APP_ID = 'dataset_quality'; export const DEFAULT_DATASET_TYPE: DataStreamType = 'logs'; export const DEFAULT_LOGS_DATA_VIEW = 'logs-*-*'; +export const DEFAULT_DATASET_QUALITY: QualityIndicators = 'good'; export const POOR_QUALITY_MINIMUM_PERCENTAGE = 3; export const DEGRADED_QUALITY_MINIMUM_PERCENTAGE = 0; @@ -26,10 +27,8 @@ export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' }; export const DEFAULT_DATEPICKER_REFRESH = { value: 60000, pause: false }; export const DEFAULT_DEGRADED_DOCS = { - percentage: 0, count: 0, - docsCount: 0, - quality: 'good' as QualityIndicators, + percentage: 0, }; export const NUMBER_FORMAT = '0,0.[000]'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts index 164a43c625fb1..094d92ff3fea6 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { DEFAULT_DEGRADED_DOCS } from '../constants'; +import { DataStreamDocsStat } from '../api_types'; +import { DEFAULT_DATASET_QUALITY, DEFAULT_DEGRADED_DOCS } from '../constants'; import { DataStreamType, QualityIndicators } from '../types'; import { indexNameToDataStreamParts, mapPercentageToQuality } from '../utils'; import { Integration } from './integration'; -import { DegradedDocsStat } from './malformed_docs_stat'; import { DataStreamStatType } from './types'; export class DataStreamStat { @@ -24,11 +24,11 @@ export class DataStreamStat { userPrivileges?: DataStreamStatType['userPrivileges']; totalDocs?: DataStreamStatType['totalDocs']; // total datastream docs count integration?: Integration; + quality: QualityIndicators; + docsInTimeRange?: number; degradedDocs: { percentage: number; count: number; - docsCount: number; // docs count in the filtered time range - quality: QualityIndicators; }; private constructor(dataStreamStat: DataStreamStat) { @@ -43,12 +43,9 @@ export class DataStreamStat { this.userPrivileges = dataStreamStat.userPrivileges; this.totalDocs = dataStreamStat.totalDocs; this.integration = dataStreamStat.integration; - this.degradedDocs = { - percentage: dataStreamStat.degradedDocs.percentage, - count: dataStreamStat.degradedDocs.count, - docsCount: dataStreamStat.degradedDocs.docsCount, - quality: dataStreamStat.degradedDocs.quality, - }; + this.quality = dataStreamStat.quality; + this.docsInTimeRange = dataStreamStat.docsInTimeRange; + this.degradedDocs = dataStreamStat.degradedDocs; } public static create(dataStreamStat: DataStreamStatType) { @@ -65,6 +62,7 @@ export class DataStreamStat { lastActivity: dataStreamStat.lastActivity, userPrivileges: dataStreamStat.userPrivileges, totalDocs: dataStreamStat.totalDocs, + quality: DEFAULT_DATASET_QUALITY, degradedDocs: DEFAULT_DEGRADED_DOCS, }; @@ -74,9 +72,11 @@ export class DataStreamStat { public static fromDegradedDocStat({ degradedDocStat, datasetIntegrationMap, + totalDocs, }: { - degradedDocStat: DegradedDocsStat; + degradedDocStat: DataStreamDocsStat & { percentage: number }; datasetIntegrationMap: Record; + totalDocs: number; }) { const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset); @@ -87,19 +87,23 @@ export class DataStreamStat { title: datasetIntegrationMap[dataset]?.title || dataset, namespace, integration: datasetIntegrationMap[dataset]?.integration, + quality: mapPercentageToQuality(degradedDocStat.percentage), + docsInTimeRange: totalDocs, degradedDocs: { percentage: degradedDocStat.percentage, count: degradedDocStat.count, - docsCount: degradedDocStat.docsCount, - quality: mapPercentageToQuality(degradedDocStat.percentage), }, }; return new DataStreamStat(dataStreamStatProps); } - public static calculateFilteredSize({ sizeBytes, totalDocs, degradedDocs }: DataStreamStat) { + public static calculateFilteredSize({ sizeBytes, totalDocs, docsInTimeRange }: DataStreamStat) { const avgDocSize = sizeBytes && totalDocs ? sizeBytes / totalDocs : 0; - return avgDocSize * degradedDocs.docsCount; + return avgDocSize * (docsInTimeRange ?? 0); + } + + public static calculatePercentage({ totalDocs, count }: { totalDocs?: number; count?: number }) { + return totalDocs && count ? (count / totalDocs) * 100 : 0; } } diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/malformed_docs_stat.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/malformed_docs_stat.ts deleted file mode 100644 index c86b802ea42da..0000000000000 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/malformed_docs_stat.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { QualityIndicators } from '../types'; -import { mapPercentageToQuality } from '../utils'; -import { DegradedDocsStatType } from './types'; - -export class DegradedDocsStat { - dataset: DegradedDocsStatType['dataset']; - percentage: DegradedDocsStatType['percentage']; - count: DegradedDocsStatType['count']; - docsCount: DegradedDocsStatType['docsCount']; - quality: QualityIndicators; - - private constructor(degradedDocsStat: DegradedDocsStat) { - this.dataset = degradedDocsStat.dataset; - this.percentage = degradedDocsStat.percentage; - this.count = degradedDocsStat.count; - this.docsCount = degradedDocsStat.docsCount; - this.quality = degradedDocsStat.quality; - } - - public static create(degradedDocsStat: DegradedDocsStatType) { - const quality = mapPercentageToQuality(degradedDocsStat.percentage); - return new DegradedDocsStat({ ...degradedDocsStat, quality }); - } -} diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index 1e5adedc20f3a..bc0c12d234d26 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -18,10 +18,14 @@ export type DataStreamStatServiceResponse = GetDataStreamsStatsResponse; export type GetDataStreamsDegradedDocsStatsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params']; export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query']; -export type GetDataStreamsDegradedDocsStatsResponse = - APIReturnType<`GET /internal/dataset_quality/data_streams/degraded_docs`>; -export type DegradedDocsStatType = GetDataStreamsDegradedDocsStatsResponse['degradedDocs'][0]; -export type DataStreamDegradedDocsStatServiceResponse = DegradedDocsStatType[]; + +/* +Types for stats based in documents inside a DataStream +*/ + +export type GetDataStreamsTotalDocsParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/total_docs`>['params']; +export type GetDataStreamsTotalDocsQuery = GetDataStreamsTotalDocsParams['query']; /* Types for Degraded Fields inside a DataStream diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/dataset_quality_indicator.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/dataset_quality_indicator.tsx index 419a13272dbc8..78c6d3bff9331 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/dataset_quality_indicator.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/dataset_quality_indicator.tsx @@ -19,9 +19,7 @@ export const DatasetQualityIndicator = ({ isLoading: boolean; dataStreamStat: DataStreamStat; }) => { - const { - degradedDocs: { quality }, - } = dataStreamStat; + const { quality } = dataStreamStat; const translatedQuality = i18n.translate('xpack.datasetQuality.datasetQualityIdicator', { defaultMessage: '{quality}', diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.ts index e370e7c22d469..056bba2304144 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.ts @@ -49,7 +49,7 @@ export const useDatasetQualityFilters = () => { datasets.reduce( (acc: Filters, dataset) => ({ namespaces: [...new Set([...acc.namespaces, dataset.namespace])], - qualities: [...new Set([...acc.qualities, dataset.degradedDocs.quality])], + qualities: [...new Set([...acc.qualities, dataset.quality])], filteredIntegrations: [ ...new Set([...acc.filteredIntegrations, dataset.integration?.name ?? 'none']), ], diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx index 55265a250bb75..6529ae1841ee3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx @@ -132,8 +132,7 @@ export const useDatasetQualityTable = () => { const passesNamespaceFilter = namespaces.length === 0 || namespaces.includes(dataset.namespace); - const passesQualityFilter = - qualities.length === 0 || qualities.includes(dataset.degradedDocs.quality); + const passesQualityFilter = qualities.length === 0 || qualities.includes(dataset.quality); const passesQueryFilter = !query || dataset.rawName.includes(query); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_telemetry.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_telemetry.ts index 167ebd37fe81a..7d486f94f2607 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_telemetry.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_telemetry.ts @@ -77,7 +77,7 @@ function getDatasetEbtProps( namespace: dataset.namespace, type: dataset.type, }, - data_stream_health: dataset.degradedDocs.quality, + data_stream_health: dataset.quality, data_stream_aggregatable: nonAggregatableDatasets.some( (indexName) => indexName === dataset.rawName ), diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts index a85dc9c21d222..014d9f578eb60 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts @@ -84,7 +84,7 @@ const useSummaryPanel = () => { datasetsActivity, numberOfDatasets: filteredItems.length, - numberOfDocuments: filteredItems.reduce((acc, curr) => acc + curr.degradedDocs.docsCount, 0), + numberOfDocuments: filteredItems.reduce((acc, curr) => acc + curr.docsInTimeRange!, 0), }; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts index 8642a863726df..8e218819315b2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts @@ -10,8 +10,11 @@ import { decodeOrThrow } from '@kbn/io-ts-utils'; import rison from '@kbn/rison'; import { KNOWN_TYPES } from '../../../common/constants'; import { - getDataStreamsDegradedDocsStatsResponseRt, + DataStreamDegradedDocsResponse, + DataStreamTotalDocsResponse, + getDataStreamDegradedDocsResponseRt, getDataStreamsStatsResponseRt, + getDataStreamTotalDocsResponseRt, getIntegrationsResponseRt, getNonAggregatableDatasetsRt, IntegrationResponse, @@ -20,9 +23,9 @@ import { import { DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, - GetDataStreamsDegradedDocsStatsResponse, GetDataStreamsStatsQuery, GetDataStreamsStatsResponse, + GetDataStreamsTotalDocsQuery, GetNonAggregatableDataStreamsParams, } from '../../../common/data_streams_stats'; import { Integration } from '../../../common/data_streams_stats/integration'; @@ -56,16 +59,37 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { return { dataStreamsStats, datasetUserPrivileges }; } + public async getDataStreamsTotalDocs(params: GetDataStreamsTotalDocsQuery) { + const response = await this.http + .get('/internal/dataset_quality/data_streams/total_docs', { + query: { + ...params, + }, + }) + .catch((error) => { + throw new DatasetQualityError(`Failed to fetch data streams total docs: ${error}`, error); + }); + + const { totalDocs } = decodeOrThrow( + getDataStreamTotalDocsResponseRt, + (message: string) => + new DatasetQualityError( + `Failed to decode data streams total docs stats response: ${message}` + ) + )(response); + + return totalDocs; + } + public async getDataStreamsDegradedStats(params: GetDataStreamsDegradedDocsStatsQuery) { + const types = params.types.length === 0 ? KNOWN_TYPES : params.types; const response = await this.http - .get( - '/internal/dataset_quality/data_streams/degraded_docs', - { - query: { - ...params, - }, - } - ) + .get('/internal/dataset_quality/data_streams/degraded_docs', { + query: { + ...params, + types: rison.encodeArray(types), + }, + }) .catch((error) => { throw new DatasetQualityError( `Failed to fetch data streams degraded stats: ${error}`, @@ -74,7 +98,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient { }); const { degradedDocs } = decodeOrThrow( - getDataStreamsDegradedDocsStatsResponseRt, + getDataStreamDegradedDocsResponseRt, (message: string) => new DatasetQualityError( `Failed to decode data streams degraded docs stats response: ${message}` diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts index dd057ee7f3062..240e5519cfc3d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts @@ -7,14 +7,14 @@ import { HttpStart } from '@kbn/core/public'; import { - DataStreamDegradedDocsStatServiceResponse, DataStreamStatServiceResponse, GetDataStreamsDegradedDocsStatsQuery, GetDataStreamsStatsQuery, + GetDataStreamsTotalDocsQuery, GetNonAggregatableDataStreamsParams, } from '../../../common/data_streams_stats'; import { Integration } from '../../../common/data_streams_stats/integration'; -import { NonAggregatableDatasets } from '../../../common/api_types'; +import { DataStreamDocsStat, NonAggregatableDatasets } from '../../../common/api_types'; export type DataStreamsStatsServiceSetup = void; @@ -30,7 +30,8 @@ export interface IDataStreamsStatsClient { getDataStreamsStats(params?: GetDataStreamsStatsQuery): Promise; getDataStreamsDegradedStats( params?: GetDataStreamsDegradedDocsStatsQuery - ): Promise; + ): Promise; + getDataStreamsTotalDocs(params: GetDataStreamsTotalDocsQuery): Promise; getIntegrations(): Promise; getNonAggregatableDatasets( params: GetNonAggregatableDataStreamsParams diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts index 41cfa859ec977..7c77fe9d59422 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts @@ -37,7 +37,8 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = { canViewIntegrations: true, }, dataStreamStats: [], - degradedDocStats: DEFAULT_DICTIONARY_TYPE, + degradedDocStats: [], + totalDocsStats: DEFAULT_DICTIONARY_TYPE, filters: { inactive: true, fullNames: false, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts index a21cc85aac449..0dea80104245f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts @@ -7,6 +7,7 @@ import { IToasts } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; +import { DataStreamType } from '../../../../common/types'; export const fetchDatasetStatsFailedNotifier = (toasts: IToasts, error: Error) => { toasts.addDanger({ @@ -26,6 +27,18 @@ export const fetchDegradedStatsFailedNotifier = (toasts: IToasts, error: Error) }); }; +export const fetchTotalDocsFailedNotifier = (toasts: IToasts, error: Error, meta: any) => { + const dataStreamType = meta._event.origin as DataStreamType; + + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.fetchTotalDocsFailed', { + defaultMessage: "We couldn't get total docs information for {dataStreamType}.", + values: { dataStreamType }, + }), + text: error.message, + }); +}; + export const fetchIntegrationsFailedNotifier = (toasts: IToasts, error: Error) => { toasts.addDanger({ title: i18n.translate('xpack.datasetQuality.fetchIntegrationsFailed', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index a803d73448263..1217e52894ce7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -8,12 +8,13 @@ import { IToasts } from '@kbn/core/public'; import { getDateISORange } from '@kbn/timerange'; import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate'; -import { DataStreamStat, NonAggregatableDatasets } from '../../../../common/api_types'; -import { KNOWN_TYPES } from '../../../../common/constants'; import { - DataStreamDegradedDocsStatServiceResponse, - DataStreamStatServiceResponse, -} from '../../../../common/data_streams_stats'; + DataStreamDocsStat, + DataStreamStat, + NonAggregatableDatasets, +} from '../../../../common/api_types'; +import { KNOWN_TYPES } from '../../../../common/constants'; +import { DataStreamStatServiceResponse } from '../../../../common/data_streams_stats'; import { Integration } from '../../../../common/data_streams_stats/integration'; import { DataStreamType } from '../../../../common/types'; import { IDataStreamsStatsClient } from '../../../services/data_streams_stats'; @@ -24,6 +25,7 @@ import { fetchDatasetStatsFailedNotifier, fetchDegradedStatsFailedNotifier, fetchIntegrationsFailedNotifier, + fetchTotalDocsFailedNotifier, } from './notifications'; import { DatasetQualityControllerContext, @@ -92,34 +94,69 @@ export const createPureDatasetQualityControllerStateMachine = ( initial: 'fetching', states: { fetching: { - ...generateInvokePerType({ + invoke: { src: 'loadDegradedDocs', + onDone: { + target: 'loaded', + actions: ['storeDegradedDocStats', 'storeDatasets'], + }, + onError: [ + { + target: 'unauthorized', + cond: 'checkIfActionForbidden', + }, + { + target: 'loaded', + actions: ['notifyFetchDegradedStatsFailed'], + }, + ], + }, + }, + loaded: {}, + unauthorized: { type: 'final' }, + }, + on: { + UPDATE_TIME_RANGE: { + target: 'degradedDocs.fetching', + actions: ['storeTimeRange'], + }, + REFRESH_DATA: { + target: 'degradedDocs.fetching', + }, + }, + }, + docsStats: { + initial: 'fetching', + states: { + fetching: { + ...generateInvokePerType({ + src: 'loadDataStreamDocsStats', }), }, loaded: {}, unauthorized: { type: 'final' }, }, on: { - SAVE_DEGRADED_DOCS_STATS: { - target: 'degradedDocs.loaded', - actions: ['storeDegradedDocStats', 'storeDatasets'], + SAVE_TOTAL_DOCS_STATS: { + target: 'docsStats.loaded', + actions: ['storeTotalDocStats', 'storeDatasets'], }, - NOTIFY_DEGRADED_DOCS_STATS_FAILED: [ + NOTIFY_TOTAL_DOCS_STATS_FAILED: [ { - target: 'degradedDocs.unauthorized', + target: 'docsStats.unauthorized', cond: 'checkIfActionForbidden', }, { - target: 'degradedDocs.loaded', - actions: ['notifyFetchDegradedStatsFailed'], + target: 'docsStats.loaded', + actions: ['notifyFetchTotalDocsFailed'], }, ], UPDATE_TIME_RANGE: { - target: 'degradedDocs.fetching', + target: 'docsStats.fetching', actions: ['storeTimeRange'], }, REFRESH_DATA: { - target: 'degradedDocs.fetching', + target: 'docsStats.fetching', }, }, }, @@ -329,18 +366,21 @@ export const createPureDatasetQualityControllerStateMachine = ( }; } ), - storeDegradedDocStats: assign( - (context, event: DoneInvokeEvent, meta) => { + storeTotalDocStats: assign( + (context, event: DoneInvokeEvent, meta) => { const type = meta._event.origin as DataStreamType; return { - degradedDocStats: { - ...context.degradedDocStats, + totalDocsStats: { + ...context.totalDocsStats, [type]: event.data, }, }; } ), + storeDegradedDocStats: assign((_context, event: DoneInvokeEvent) => ({ + degradedDocStats: event.data, + })), storeNonAggregatableDatasets: assign( (_context, event: DoneInvokeEvent) => ({ nonAggregatableDatasets: event.data.datasets, @@ -364,7 +404,8 @@ export const createPureDatasetQualityControllerStateMachine = ( datasets: generateDatasets( context.dataStreamStats, context.degradedDocStats, - context.integrations + context.integrations, + context.totalDocsStats ), } : {}; @@ -404,6 +445,8 @@ export const createDatasetQualityControllerStateMachine = ({ fetchNonAggregatableDatasetsFailedNotifier(toasts, event.data), notifyFetchIntegrationsFailed: (_context, event: DoneInvokeEvent) => fetchIntegrationsFailedNotifier(toasts, event.data), + notifyFetchTotalDocsFailed: (_context, event: DoneInvokeEvent, meta) => + fetchTotalDocsFailedNotifier(toasts, event.data, meta), }, services: { loadDataStreamStats: (context, _event) => @@ -411,32 +454,41 @@ export const createDatasetQualityControllerStateMachine = ({ types: context.filters.types as DataStreamType[], datasetQuery: context.filters.query, }), - loadDegradedDocs: + loadDataStreamDocsStats: (context, _event, { data: { type } }) => async (send) => { try { const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); - const degradedDocsStats = await (isTypeSelected(type, context) - ? dataStreamStatsClient.getDataStreamsDegradedStats({ + const totalDocsStats = await (isTypeSelected(type, context) + ? dataStreamStatsClient.getDataStreamsTotalDocs({ type, - datasetQuery: context.filters.query, start, end, }) : Promise.resolve([])); send({ - type: 'SAVE_DEGRADED_DOCS_STATS', - data: degradedDocsStats, + type: 'SAVE_TOTAL_DOCS_STATS', + data: totalDocsStats, }); } catch (e) { send({ - type: 'NOTIFY_DEGRADED_DOCS_STATS_FAILED', + type: 'NOTIFY_TOTAL_DOCS_STATS_FAILED', data: e, }); } }, + loadDegradedDocs: (context) => { + const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); + + return dataStreamStatsClient.getDataStreamsDegradedStats({ + types: context.filters.types as DataStreamType[], + datasetQuery: context.filters.query, + start, + end, + }); + }, loadNonAggregatableDatasets: (context) => { const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index a5e03cfb480ff..de7fdbf9fbd77 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -6,16 +6,18 @@ */ import { DoneInvokeEvent } from 'xstate'; -import { DatasetUserPrivileges, NonAggregatableDatasets } from '../../../../common/api_types'; import { - DataStreamDegradedDocsStatServiceResponse, + DataStreamDocsStat, + DatasetUserPrivileges, + NonAggregatableDatasets, +} from '../../../../common/api_types'; +import { DataStreamDetails, DataStreamStat, DataStreamStatServiceResponse, DataStreamStatType, } from '../../../../common/data_streams_stats'; import { Integration } from '../../../../common/data_streams_stats/integration'; -import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat'; import { DataStreamType, QualityIndicators, @@ -50,8 +52,12 @@ export interface WithDataStreamStats { dataStreamStats: DataStreamStatType[]; } +export interface WithTotalDocs { + totalDocsStats: DictionaryType; +} + export interface WithDegradedDocs { - degradedDocStats: DictionaryType; + degradedDocStats: DataStreamDocsStat[]; } export interface WithNonAggregatableDatasets { @@ -68,6 +74,7 @@ export interface WithIntegrations { export type DefaultDatasetQualityControllerState = WithTableOptions & WithDataStreamStats & + WithTotalDocs & WithDegradedDocs & WithDatasets & WithFilters & @@ -146,7 +153,7 @@ export type DatasetQualityControllerEvent = type: 'UPDATE_TYPES'; types: DataStreamType[]; } - | DoneInvokeEvent + | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts index 6f2e46baacf8c..b75c74c2fd728 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.test.ts @@ -5,11 +5,10 @@ * 2.0. */ -import { indexNameToDataStreamParts } from '../../common/utils'; -import { Integration } from '../../common/data_streams_stats/integration'; -import { generateDatasets } from './generate_datasets'; import { DataStreamStatType } from '../../common/data_streams_stats'; +import { Integration } from '../../common/data_streams_stats/integration'; import { DEFAULT_DICTIONARY_TYPE } from '../state_machines/dataset_quality_controller'; +import { generateDatasets } from './generate_datasets'; describe('generateDatasets', () => { const integrations: Integration[] = [ @@ -41,6 +40,7 @@ describe('generateDatasets', () => { lastActivity: 1712911241117, size: '82.1kb', sizeBytes: 84160, + totalDocs: 100, integration: 'system', userPrivileges: { canMonitor: true, @@ -51,182 +51,337 @@ describe('generateDatasets', () => { lastActivity: 1712911241117, size: '62.5kb', sizeBytes: 64066, + totalDocs: 100, userPrivileges: { canMonitor: true, }, }, ]; - const degradedDocs = { + const totalDocs = { ...DEFAULT_DICTIONARY_TYPE, logs: [ { dataset: 'logs-system.application-default', - percentage: 0, - count: 0, - docsCount: 0, - quality: 'good' as const, + count: 100, }, { dataset: 'logs-synth-default', - percentage: 11.320754716981131, - count: 6, - docsCount: 0, - quality: 'poor' as const, + count: 100, }, ], }; - it('merges integrations information with dataStreamStats', () => { - const datasets = generateDatasets(dataStreamStats, DEFAULT_DICTIONARY_TYPE, integrations); + const degradedDocs = [ + { + dataset: 'logs-system.application-default', + count: 0, + }, + { + dataset: 'logs-synth-default', + count: 6, + }, + ]; + + it('merges integrations information with dataStreamStats and degradedDocs', () => { + const datasets = generateDatasets(dataStreamStats, degradedDocs, integrations, totalDocs); expect(datasets).toEqual([ { - ...dataStreamStats[0], - name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset, - namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace, - title: - integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset], - type: indexNameToDataStreamParts(dataStreamStats[0].name).type, - rawName: dataStreamStats[0].name, + name: 'system.application', + type: 'logs', + namespace: 'default', + title: 'Windows Application Events', + rawName: 'logs-system.application-default', + lastActivity: 1712911241117, + size: '82.1kb', + sizeBytes: 84160, integration: integrations[0], + totalDocs: 100, + userPrivileges: { + canMonitor: true, + }, + docsInTimeRange: 100, + quality: 'good', + degradedDocs: { + percentage: 0, + count: 0, + }, + }, + { + name: 'synth', + type: 'logs', + namespace: 'default', + title: 'synth', + rawName: 'logs-synth-default', + lastActivity: 1712911241117, + size: '62.5kb', + sizeBytes: 64066, + integration: undefined, + totalDocs: 100, + userPrivileges: { + canMonitor: true, + }, + docsInTimeRange: 100, + quality: 'poor', degradedDocs: { - percentage: degradedDocs.logs[0].percentage, - count: degradedDocs.logs[0].count, - docsCount: degradedDocs.logs[0].docsCount, - quality: degradedDocs.logs[0].quality, + count: 6, + percentage: 6, }, }, + ]); + }); + + it('merges integrations information with dataStreamStats and degradedDocs when no docs in timerange', () => { + const datasets = generateDatasets( + dataStreamStats, + degradedDocs, + integrations, + DEFAULT_DICTIONARY_TYPE + ); + + expect(datasets).toEqual([ { - ...dataStreamStats[1], - name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, - namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace, - title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, - type: indexNameToDataStreamParts(dataStreamStats[1].name).type, - rawName: dataStreamStats[1].name, + name: 'system.application', + type: 'logs', + namespace: 'default', + title: 'Windows Application Events', + rawName: 'logs-system.application-default', + lastActivity: 1712911241117, + size: '82.1kb', + sizeBytes: 84160, + integration: integrations[0], + totalDocs: 100, + userPrivileges: { + canMonitor: true, + }, + docsInTimeRange: 0, + quality: 'good', degradedDocs: { + percentage: 0, count: 0, + }, + }, + { + name: 'synth', + type: 'logs', + namespace: 'default', + title: 'synth', + rawName: 'logs-synth-default', + lastActivity: 1712911241117, + size: '62.5kb', + sizeBytes: 64066, + integration: undefined, + totalDocs: 100, + userPrivileges: { + canMonitor: true, + }, + docsInTimeRange: 0, + quality: 'good', + degradedDocs: { + count: 6, percentage: 0, - docsCount: 0, - quality: 'good', }, }, ]); }); it('merges integrations information with degradedDocs', () => { - const datasets = generateDatasets(undefined, degradedDocs, integrations); + const datasets = generateDatasets([], degradedDocs, integrations, totalDocs); expect(datasets).toEqual([ { - rawName: degradedDocs.logs[0].dataset, - name: indexNameToDataStreamParts(degradedDocs.logs[0].dataset).dataset, - type: indexNameToDataStreamParts(degradedDocs.logs[0].dataset).type, + name: 'system.application', + type: 'logs', + namespace: 'default', + title: 'Windows Application Events', + rawName: 'logs-system.application-default', + lastActivity: undefined, + size: undefined, + sizeBytes: undefined, + integration: integrations[0], + totalDocs: undefined, + userPrivileges: undefined, + docsInTimeRange: 100, + quality: 'good', + degradedDocs: { + percentage: 0, + count: 0, + }, + }, + { + name: 'synth', + type: 'logs', + namespace: 'default', + title: 'synth', + rawName: 'logs-synth-default', lastActivity: undefined, size: undefined, sizeBytes: undefined, + integration: undefined, + totalDocs: undefined, userPrivileges: undefined, - namespace: indexNameToDataStreamParts(degradedDocs.logs[0].dataset).namespace, - title: - integrations[0].datasets[ - indexNameToDataStreamParts(degradedDocs.logs[0].dataset).dataset - ], + docsInTimeRange: 100, + quality: 'poor', + degradedDocs: { + count: 6, + percentage: 6, + }, + }, + ]); + }); + + it('merges integrations information with degradedDocs and totalDocs', () => { + const datasets = generateDatasets([], degradedDocs, integrations, { + ...totalDocs, + logs: [...totalDocs.logs, { dataset: 'logs-another-default', count: 100 }], + }); + + expect(datasets).toEqual([ + { + name: 'system.application', + type: 'logs', + namespace: 'default', + title: 'Windows Application Events', + rawName: 'logs-system.application-default', + lastActivity: undefined, + size: undefined, + sizeBytes: undefined, integration: integrations[0], + totalDocs: undefined, + userPrivileges: undefined, + docsInTimeRange: 100, + quality: 'good', degradedDocs: { - percentage: degradedDocs.logs[0].percentage, - count: degradedDocs.logs[0].count, - docsCount: degradedDocs.logs[0].docsCount, - quality: degradedDocs.logs[0].quality, + percentage: 0, + count: 0, }, }, { - rawName: degradedDocs.logs[1].dataset, - name: indexNameToDataStreamParts(degradedDocs.logs[1].dataset).dataset, - type: indexNameToDataStreamParts(degradedDocs.logs[1].dataset).type, + name: 'synth', + type: 'logs', + namespace: 'default', + title: 'synth', + rawName: 'logs-synth-default', lastActivity: undefined, size: undefined, sizeBytes: undefined, + integration: undefined, + totalDocs: undefined, userPrivileges: undefined, - namespace: indexNameToDataStreamParts(degradedDocs.logs[1].dataset).namespace, - title: indexNameToDataStreamParts(degradedDocs.logs[1].dataset).dataset, + docsInTimeRange: 100, + quality: 'poor', + degradedDocs: { + count: 6, + percentage: 6, + }, + }, + { + name: 'another', + type: 'logs', + namespace: 'default', + title: 'another', + rawName: 'logs-another-default', + lastActivity: undefined, + size: undefined, + sizeBytes: undefined, integration: undefined, + totalDocs: undefined, + userPrivileges: undefined, + docsInTimeRange: 100, + quality: 'good', degradedDocs: { - percentage: degradedDocs.logs[1].percentage, - count: degradedDocs.logs[1].count, - docsCount: degradedDocs.logs[1].docsCount, - quality: degradedDocs.logs[1].quality, + percentage: 0, + count: 0, }, }, ]); }); - it('merges integrations information with dataStreamStats and degradedDocs', () => { - const datasets = generateDatasets(dataStreamStats, degradedDocs, integrations); + it('merges integrations information with dataStreamStats', () => { + const datasets = generateDatasets(dataStreamStats, [], integrations, totalDocs); expect(datasets).toEqual([ { - ...dataStreamStats[0], - name: indexNameToDataStreamParts(dataStreamStats[0].name).dataset, - namespace: indexNameToDataStreamParts(dataStreamStats[0].name).namespace, - title: - integrations[0].datasets[indexNameToDataStreamParts(dataStreamStats[0].name).dataset], - type: indexNameToDataStreamParts(dataStreamStats[0].name).type, - rawName: dataStreamStats[0].name, + name: 'system.application', + type: 'logs', + namespace: 'default', + title: 'Windows Application Events', + rawName: 'logs-system.application-default', + lastActivity: 1712911241117, + size: '82.1kb', + sizeBytes: 84160, integration: integrations[0], + totalDocs: 100, + userPrivileges: { + canMonitor: true, + }, + quality: 'good', + docsInTimeRange: 100, degradedDocs: { - percentage: degradedDocs.logs[0].percentage, - count: degradedDocs.logs[0].count, - docsCount: degradedDocs.logs[0].docsCount, - quality: degradedDocs.logs[0].quality, + count: 0, + percentage: 0, }, }, { - ...dataStreamStats[1], - name: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, - namespace: indexNameToDataStreamParts(dataStreamStats[1].name).namespace, - title: indexNameToDataStreamParts(dataStreamStats[1].name).dataset, - type: indexNameToDataStreamParts(dataStreamStats[1].name).type, - rawName: dataStreamStats[1].name, + name: 'synth', + type: 'logs', + namespace: 'default', + title: 'synth', + rawName: 'logs-synth-default', + lastActivity: 1712911241117, + size: '62.5kb', + sizeBytes: 64066, + integration: undefined, + totalDocs: 100, + userPrivileges: { + canMonitor: true, + }, + quality: 'good', + docsInTimeRange: 100, degradedDocs: { - percentage: degradedDocs.logs[1].percentage, - count: degradedDocs.logs[1].count, - docsCount: degradedDocs.logs[1].docsCount, - quality: degradedDocs.logs[1].quality, + count: 0, + percentage: 0, }, }, ]); }); it('merges integration information with dataStreamStats when dataset is not an integration default one', () => { - const dataset = 'logs-system.custom-default'; - const nonDefaultDataset = { - name: dataset, + name: 'logs-system.custom-default', lastActivity: 1712911241117, size: '82.1kb', sizeBytes: 84160, + totalDocs: 100, integration: 'system', userPrivileges: { canMonitor: true, }, }; - const datasets = generateDatasets([nonDefaultDataset], DEFAULT_DICTIONARY_TYPE, integrations); + const datasets = generateDatasets([nonDefaultDataset], [], integrations, totalDocs); expect(datasets).toEqual([ { - ...nonDefaultDataset, - title: indexNameToDataStreamParts(dataset).dataset, - name: indexNameToDataStreamParts(dataset).dataset, - namespace: indexNameToDataStreamParts(dataset).namespace, - type: indexNameToDataStreamParts(dataset).type, - rawName: nonDefaultDataset.name, + name: 'system.custom', + type: 'logs', + namespace: 'default', + title: 'system.custom', + rawName: 'logs-system.custom-default', + lastActivity: 1712911241117, + size: '82.1kb', + sizeBytes: 84160, integration: integrations[0], + userPrivileges: { + canMonitor: true, + }, + quality: 'good', + totalDocs: 100, + docsInTimeRange: 0, degradedDocs: { count: 0, percentage: 0, - docsCount: 0, - quality: 'good', }, }, ]); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts index fb479198bbac3..8e9f2f3db7083 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/utils/generate_datasets.ts @@ -5,23 +5,20 @@ * 2.0. */ +import { DEFAULT_DEGRADED_DOCS } from '../../common/constants'; +import { DataStreamDocsStat } from '../../common/api_types'; import { DataStreamStatType } from '../../common/data_streams_stats/types'; import { mapPercentageToQuality } from '../../common/utils'; import { Integration } from '../../common/data_streams_stats/integration'; import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat'; -import { DegradedDocsStat } from '../../common/data_streams_stats/malformed_docs_stat'; import { DictionaryType } from '../state_machines/dataset_quality_controller/src/types'; import { flattenStats } from './flatten_stats'; - export function generateDatasets( dataStreamStats: DataStreamStatType[] = [], - degradedDocStats: DictionaryType, - integrations: Integration[] + degradedDocStats: DataStreamDocsStat[] = [], + integrations: Integration[], + totalDocsStats: DictionaryType ): DataStreamStat[] { - if (!dataStreamStats.length && !integrations.length) { - return []; - } - const { datasetIntegrationMap, integrationsMap, @@ -50,35 +47,42 @@ export function generateDatasets( { datasetIntegrationMap: {}, integrationsMap: {} } ); - const degradedDocs = flattenStats(degradedDocStats); - - if (!dataStreamStats.length) { - return degradedDocs.map((degradedDocStat) => - DataStreamStat.fromDegradedDocStat({ degradedDocStat, datasetIntegrationMap }) - ); - } + const totalDocs = flattenStats(totalDocsStats); + const totalDocsMap: Record = + Object.fromEntries(totalDocs.map(({ dataset, count }) => [dataset, count])); const degradedMap: Record< - DegradedDocsStat['dataset'], + DataStreamDocsStat['dataset'], { - percentage: DegradedDocsStat['percentage']; - count: DegradedDocsStat['count']; - docsCount: DegradedDocsStat['docsCount']; - quality: DegradedDocsStat['quality']; + percentage: number; + count: DataStreamDocsStat['count']; } - > = degradedDocs.reduce( - (degradedMapAcc, { dataset, percentage, count, docsCount }) => + > = degradedDocStats.reduce( + (degradedMapAcc, { dataset, count }) => Object.assign(degradedMapAcc, { [dataset]: { - percentage, count, - docsCount, - quality: mapPercentageToQuality(percentage), + percentage: DataStreamStat.calculatePercentage({ + totalDocs: totalDocsMap[dataset], + count, + }), }, }), {} ); + if (!dataStreamStats.length) { + // We want to pick up all datasets even when they don't have degraded docs + const dataStreams = [...new Set([...Object.keys(totalDocsMap), ...Object.keys(degradedMap)])]; + return dataStreams.map((dataset) => + DataStreamStat.fromDegradedDocStat({ + degradedDocStat: { dataset, ...(degradedMap[dataset] || DEFAULT_DEGRADED_DOCS) }, + datasetIntegrationMap, + totalDocs: totalDocsMap[dataset] ?? 0, + }) + ); + } + return dataStreamStats?.map((dataStream) => { const dataset = DataStreamStat.create(dataStream); @@ -89,6 +93,10 @@ export function generateDatasets( datasetIntegrationMap[dataset.name]?.integration ?? integrationsMap[dataStream.integration ?? ''], degradedDocs: degradedMap[dataset.rawName] || dataset.degradedDocs, + docsInTimeRange: totalDocsMap[dataset.rawName] ?? 0, + quality: mapPercentageToQuality( + (degradedMap[dataset.rawName] || dataset.degradedDocs).percentage + ), }; }); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_dataset_aggregated_paginated_results.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_dataset_aggregated_paginated_results.ts new file mode 100644 index 0000000000000..062dcd2f16cf7 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_dataset_aggregated_paginated_results.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient } from '@kbn/core/server'; +import { rangeQuery } from '@kbn/observability-plugin/server'; +import { QueryDslBoolQuery } from '@elastic/elasticsearch/lib/api/types'; +import { DataStreamDocsStat } from '../../../common/api_types'; +import { createDatasetQualityESClient } from '../../utils'; + +interface Dataset { + type: string; + dataset: string; + namespace: string; +} + +const SIZE_LIMIT = 10000; + +export async function getAggregatedDatasetPaginatedResults(options: { + esClient: ElasticsearchClient; + index: string; + start: number; + end: number; + query?: QueryDslBoolQuery; + after?: Dataset; + prevResults?: DataStreamDocsStat[]; +}): Promise { + const { esClient, index, query, start, end, after, prevResults = [] } = options; + + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const aggs = (afterKey?: Dataset) => ({ + datasets: { + composite: { + ...(afterKey ? { after: afterKey } : {}), + size: SIZE_LIMIT, + sources: [ + { type: { terms: { field: 'data_stream.type' } } }, + { dataset: { terms: { field: 'data_stream.dataset' } } }, + { namespace: { terms: { field: 'data_stream.namespace' } } }, + ], + }, + }, + }); + + const bool = { + ...query, + filter: [ + ...(query?.filter ? (Array.isArray(query.filter) ? query.filter : [query.filter]) : []), + ...[...rangeQuery(start, end)], + ], + }; + + const response = await datasetQualityESClient.search({ + index, + size: 0, + query: { + bool, + }, + aggs: aggs(after), + }); + + const currResults = + response.aggregations?.datasets.buckets.map((bucket) => ({ + dataset: `${bucket.key.type}-${bucket.key.dataset}-${bucket.key.namespace}`, + count: bucket.doc_count, + })) ?? []; + + const results = [...prevResults, ...currResults]; + + if ( + response.aggregations?.datasets.after_key && + response.aggregations?.datasets.buckets.length === SIZE_LIMIT + ) { + return getAggregatedDatasetPaginatedResults({ + esClient, + index, + start, + end, + after: + (response.aggregations?.datasets.after_key as { + type: string; + dataset: string; + namespace: string; + }) || after, + prevResults: results, + }); + } + + return results; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_docs.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_docs.ts index 454fdb7e1a8b8..48b50c4b8680d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_docs.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_docs.ts @@ -6,161 +6,37 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; -import { rangeQuery, termQuery } from '@kbn/observability-plugin/server'; -import { DEFAULT_DATASET_TYPE } from '../../../common/constants'; +import { streamPartsToIndexPattern } from '../../../common/utils'; import { DataStreamType } from '../../../common/types'; -import { DegradedDocs } from '../../../common/api_types'; -import { - DATA_STREAM_DATASET, - DATA_STREAM_NAMESPACE, - DATA_STREAM_TYPE, - _IGNORED, -} from '../../../common/es_fields'; -import { createDatasetQualityESClient, wildcardQuery } from '../../utils'; - -interface ResultBucket { - dataset: string; - count: number; -} - -const SIZE_LIMIT = 10000; +import { DataStreamDocsStat } from '../../../common/api_types'; +import { _IGNORED } from '../../../common/es_fields'; +import { getAggregatedDatasetPaginatedResults } from './get_dataset_aggregated_paginated_results'; export async function getDegradedDocsPaginated(options: { esClient: ElasticsearchClient; - type?: DataStreamType; + types: DataStreamType[]; + datasetQuery?: string; start: number; end: number; - datasetQuery?: string; - after?: { - degradedDocs?: { dataset: string; namespace: string }; - docsCount?: { dataset: string; namespace: string }; - }; - prevResults?: { degradedDocs: ResultBucket[]; docsCount: ResultBucket[] }; -}): Promise { - const { +}): Promise { + const { esClient, types, datasetQuery, start, end } = options; + + const datasetNames = datasetQuery + ? [datasetQuery] + : types.map((type) => + streamPartsToIndexPattern({ + typePattern: type, + datasetPattern: '*-*', + }) + ); + + return await getAggregatedDatasetPaginatedResults({ esClient, - type = DEFAULT_DATASET_TYPE, - datasetQuery, start, end, - after, - prevResults = { degradedDocs: [], docsCount: [] }, - } = options; - - const datasetQualityESClient = createDatasetQualityESClient(esClient); - - const datasetFilter = { - ...(datasetQuery - ? { - should: [ - ...wildcardQuery(DATA_STREAM_DATASET, datasetQuery), - ...wildcardQuery(DATA_STREAM_NAMESPACE, datasetQuery), - ], - minimum_should_match: 1, - } - : {}), - }; - - const otherFilters = [...rangeQuery(start, end), ...termQuery(DATA_STREAM_TYPE, type)]; - - const aggs = (afterKey?: { dataset: string; namespace: string }) => ({ - datasets: { - composite: { - ...(afterKey ? { after: afterKey } : {}), - size: SIZE_LIMIT, - sources: [ - { dataset: { terms: { field: 'data_stream.dataset' } } }, - { namespace: { terms: { field: 'data_stream.namespace' } } }, - ], - }, + index: datasetNames.join(','), + query: { + must: { exists: { field: _IGNORED } }, }, }); - - const response = await datasetQualityESClient.msearch({ index: `${type}-*-*` }, [ - // degraded docs per dataset - { - size: 0, - query: { - bool: { - ...datasetFilter, - filter: otherFilters, - must: { exists: { field: _IGNORED } }, - }, - }, - aggs: aggs(after?.degradedDocs), - }, - // total docs per dataset - { - size: 0, - query: { - bool: { - ...datasetFilter, - filter: otherFilters, - }, - }, - aggs: aggs(after?.docsCount), - }, - ]); - const [degradedDocsResponse, totalDocsResponse] = response.responses; - - const currDegradedDocs = - degradedDocsResponse.aggregations?.datasets.buckets.map((bucket) => ({ - dataset: `${type}-${bucket.key.dataset}-${bucket.key.namespace}`, - count: bucket.doc_count, - })) ?? []; - - const degradedDocs = [...prevResults.degradedDocs, ...currDegradedDocs]; - - const currTotalDocs = - totalDocsResponse.aggregations?.datasets.buckets.map((bucket) => ({ - dataset: `${type}-${bucket.key.dataset}-${bucket.key.namespace}`, - count: bucket.doc_count, - })) ?? []; - - const docsCount = [...prevResults.docsCount, ...currTotalDocs]; - - if ( - totalDocsResponse.aggregations?.datasets.after_key && - totalDocsResponse.aggregations?.datasets.buckets.length === SIZE_LIMIT - ) { - return getDegradedDocsPaginated({ - esClient, - type, - start, - end, - datasetQuery, - after: { - degradedDocs: - (degradedDocsResponse.aggregations?.datasets.after_key as { - dataset: string; - namespace: string; - }) || after?.degradedDocs, - docsCount: - (totalDocsResponse.aggregations?.datasets.after_key as { - dataset: string; - namespace: string; - }) || after?.docsCount, - }, - prevResults: { degradedDocs, docsCount }, - }); - } - - const degradedDocsMap = degradedDocs.reduce( - (acc, curr) => ({ - ...acc, - [curr.dataset]: curr.count, - }), - {} - ); - - return docsCount.map((curr) => { - const degradedDocsCount = degradedDocsMap[curr.dataset as keyof typeof degradedDocsMap] || 0; - - return { - ...curr, - docsCount: curr.count, - count: degradedDocsCount, - percentage: (degradedDocsCount / curr.count) * 100, - }; - }); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 41ba3ee8c7299..3a60f0b9a8ef3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -10,12 +10,12 @@ import { DataStreamDetails, DataStreamSettings, DataStreamStat, - DegradedDocs, NonAggregatableDatasets, DegradedFieldResponse, DatasetUserPrivileges, DegradedFieldValues, DegradedFieldAnalysis, + DataStreamDocsStat, UpdateFieldLimitResponse, DataStreamRolloverResponse, } from '../../../common/api_types'; @@ -31,6 +31,7 @@ import { getDegradedFields } from './get_degraded_fields'; import { getDegradedFieldValues } from './get_degraded_field_values'; import { analyzeDegradedField } from './get_degraded_field_analysis'; import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats'; +import { getAggregatedDatasetPaginatedResults } from './get_dataset_aggregated_paginated_results'; import { updateFieldLimit } from './update_field_limit'; import { createDatasetQualityESClient } from '../../utils'; @@ -97,7 +98,7 @@ const degradedDocsRoute = createDatasetQualityServerRoute({ params: t.type({ query: t.intersection([ rangeRt, - typeRt, + t.type({ types: typesRt }), t.partial({ datasetQuery: t.string, }), @@ -107,19 +108,13 @@ const degradedDocsRoute = createDatasetQualityServerRoute({ tags: [], }, async handler(resources): Promise<{ - degradedDocs: DegradedDocs[]; + degradedDocs: DataStreamDocsStat[]; }> { const { context, params } = resources; const coreContext = await context.core; const esClient = coreContext.elasticsearch.client.asCurrentUser; - await datasetQualityPrivileges.throwIfCannotReadDataset( - esClient, - params.query.type, - params.query.datasetQuery - ); - const degradedDocs = await getDegradedDocsPaginated({ esClient, ...params.query, @@ -131,6 +126,39 @@ const degradedDocsRoute = createDatasetQualityServerRoute({ }, }); +const totalDocsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/total_docs', + params: t.type({ + query: t.intersection([rangeRt, typeRt]), + }), + options: { + tags: [], + }, + async handler(resources): Promise<{ + totalDocs: DataStreamDocsStat[]; + }> { + const { context, params } = resources; + const coreContext = await context.core; + + const esClient = coreContext.elasticsearch.client.asCurrentUser; + + await datasetQualityPrivileges.throwIfCannotReadDataset(esClient, params.query.type); + + const { type, start, end } = params.query; + + const totalDocs = await getAggregatedDatasetPaginatedResults({ + esClient, + start, + end, + index: `${type}-*-*`, + }); + + return { + totalDocs, + }; + }, +}); + const nonAggregatableDatasetsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/non_aggregatable', params: t.type({ @@ -383,6 +411,7 @@ const rolloverDataStream = createDatasetQualityServerRoute({ export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, + ...totalDocsRoute, ...nonAggregatableDatasetsRoute, ...nonAggregatableDatasetRoute, ...degradedFieldsRoute, diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_total_docs.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_total_docs.ts new file mode 100644 index 0000000000000..c513f3519a30a --- /dev/null +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_stream_total_docs.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; + +import { APIClientRequestParamsOf } from '@kbn/dataset-quality-plugin/common/rest'; +import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context'; +import { RoleCredentials, SupertestWithRoleScopeType } from '../../../services'; + +export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { + const samlAuth = getService('samlAuth'); + const roleScopedSupertest = getService('roleScopedSupertest'); + const synthtrace = getService('logsSynthtraceEsClient'); + const from = '2024-09-20T11:00:00.000Z'; + const to = '2024-09-20T11:01:00.000Z'; + const dataStreamType = 'logs'; + const dataset = 'synth'; + const syntheticsDataset = 'synthetics'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + const dataStreamName = `${dataStreamType}-${dataset}-${namespace}`; + const syntheticsDataStreamName = `${dataStreamType}-${syntheticsDataset}-${namespace}`; + + const endpoint = 'GET /internal/dataset_quality/data_streams/total_docs'; + type ApiParams = APIClientRequestParamsOf['params']['query']; + + async function callApiAs({ + roleScopedSupertestWithCookieCredentials, + apiParams: { type, start, end }, + }: { + roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType; + apiParams: ApiParams; + }) { + return roleScopedSupertestWithCookieCredentials + .get(`/internal/dataset_quality/data_streams/total_docs`) + .query({ + type, + start, + end, + }); + } + + describe('DataStream total docs', function () { + let adminRoleAuthc: RoleCredentials; + let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType; + + before(async () => { + adminRoleAuthc = await samlAuth.createM2mApiKeyWithRoleScope('admin'); + supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( + 'admin', + { + useCookieHeader: true, + withInternalHeaders: true, + } + ); + + await synthtrace.index([ + timerange(from, to) + .interval('1m') + .rate(1) + .generator((timestamp) => [ + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }), + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(syntheticsDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName, + 'host.name': hostName, + }), + ]), + ]); + }); + + after(async () => { + await synthtrace.clean(); + await samlAuth.invalidateM2mApiKeyWithRoleScope(adminRoleAuthc); + }); + + it('returns number of documents per DataStream', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + type: dataStreamType, + start: from, + end: to, + }, + }); + + expect(resp.body.totalDocs.length).to.be(2); + expect(resp.body.totalDocs[0].dataset).to.be(dataStreamName); + expect(resp.body.totalDocs[0].count).to.be(1); + expect(resp.body.totalDocs[1].dataset).to.be(syntheticsDataStreamName); + expect(resp.body.totalDocs[1].count).to.be(1); + }); + + it('returns empty when all documents are outside timeRange', async () => { + const resp = await callApiAs({ + roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials, + apiParams: { + type: dataStreamType, + start: '2024-09-21T11:00:00.000Z', + end: '2024-09-21T11:01:00.000Z', + }, + }); + + expect(resp.body.totalDocs.length).to.be(0); + }); + }); +} diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts index 7e555b7a310e1..28133d6c8e613 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts @@ -14,5 +14,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) loadTestFile(require.resolve('./data_stream_settings')); loadTestFile(require.resolve('./data_stream_rollover')); loadTestFile(require.resolve('./update_field_limit')); + loadTestFile(require.resolve('./data_stream_total_docs')); }); } diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_docs.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_docs.spec.ts index 92aa69610a66d..60aeef1af9c93 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_docs.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_docs.spec.ts @@ -7,8 +7,7 @@ import { log, timerange } from '@kbn/apm-synthtrace-client'; import expect from '@kbn/expect'; -import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest'; -import { expectToReject } from '../../utils'; +import rison from '@kbn/rison'; import { DatasetQualityApiClientKey } from '../../common/config'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -24,7 +23,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { endpoint: 'GET /internal/dataset_quality/data_streams/degraded_docs', params: { query: { - type: 'logs', + types: rison.encodeArray(['logs']), start, end, }, @@ -33,13 +32,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { } registry.when('Degraded docs', { config: 'basic' }, () => { - describe('authorization', () => { - it('should return a 403 when the user does not have sufficient privileges', async () => { - const err = await expectToReject(() => callApiAs('noAccessUser')); - expect(err.res.status).to.be(403); - }); - }); - describe('and there are log documents', () => { before(async () => { await synthtrace.index([ @@ -75,25 +67,19 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns stats correctly', async () => { const stats = await callApiAs('datasetQualityMonitorUser'); - expect(stats.body.degradedDocs.length).to.be(2); + expect(stats.body.degradedDocs.length).to.be(1); const degradedDocsStats = stats.body.degradedDocs.reduce( (acc, curr) => ({ ...acc, [curr.dataset]: { - percentage: curr.percentage, count: curr.count, }, }), - {} as Record + {} as Record ); - expect(degradedDocsStats['logs-synth.1-default']).to.eql({ - percentage: 0, - count: 0, - }); expect(degradedDocsStats['logs-synth.2-default']).to.eql({ - percentage: 100, count: 1, }); }); @@ -155,117 +141,45 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns counts and list of datasets correctly', async () => { const stats = await callApiAs('datasetQualityMonitorUser'); - expect(stats.body.degradedDocs.length).to.be(18); + expect(stats.body.degradedDocs.length).to.be(9); const expected = { degradedDocs: [ - { - dataset: 'logs-apache.access-default', - count: 0, - docsCount: 1, - percentage: 0, - }, - { - dataset: 'logs-apache.access-space1', - count: 0, - docsCount: 1, - percentage: 0, - }, - { - dataset: 'logs-apache.access-space2', - count: 0, - docsCount: 1, - percentage: 0, - }, { dataset: 'logs-apache.error-default', count: 1, - docsCount: 2, - percentage: 50, }, { dataset: 'logs-apache.error-space1', count: 1, - docsCount: 2, - percentage: 50, }, { dataset: 'logs-apache.error-space2', count: 1, - docsCount: 2, - percentage: 50, - }, - { - dataset: 'logs-mysql.access-default', - count: 0, - docsCount: 1, - percentage: 0, - }, - { - dataset: 'logs-mysql.access-space1', - count: 0, - docsCount: 1, - percentage: 0, - }, - { - dataset: 'logs-mysql.access-space2', - count: 0, - docsCount: 1, - percentage: 0, }, { dataset: 'logs-mysql.error-default', count: 1, - docsCount: 2, - percentage: 50, }, { dataset: 'logs-mysql.error-space1', count: 1, - docsCount: 2, - percentage: 50, }, { dataset: 'logs-mysql.error-space2', count: 1, - docsCount: 2, - percentage: 50, - }, - { - dataset: 'logs-nginx.access-default', - count: 0, - docsCount: 1, - percentage: 0, - }, - { - dataset: 'logs-nginx.access-space1', - count: 0, - docsCount: 1, - percentage: 0, - }, - { - dataset: 'logs-nginx.access-space2', - count: 0, - docsCount: 1, - percentage: 0, }, { dataset: 'logs-nginx.error-default', count: 1, - docsCount: 2, - percentage: 50, }, { dataset: 'logs-nginx.error-space1', count: 1, - docsCount: 2, - percentage: 50, }, { dataset: 'logs-nginx.error-space2', count: 1, - docsCount: 2, - percentage: 50, }, ], }; diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/total_docs.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/total_docs.spec.ts new file mode 100644 index 0000000000000..71442e1300a2b --- /dev/null +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/total_docs.spec.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { DatasetQualityApiClientKey } from '../../common/config'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { expectToReject } from '../../utils'; +import { DatasetQualityApiError } from '../../common/dataset_quality_api_supertest'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const datasetQualityApiClient = getService('datasetQualityApiClient'); + const start = '2023-12-11T18:00:00.000Z'; + const end = '2023-12-11T18:01:00.000Z'; + + async function callApiAs(user: DatasetQualityApiClientKey) { + return await datasetQualityApiClient[user]({ + endpoint: 'GET /internal/dataset_quality/data_streams/total_docs', + params: { + query: { + type: 'logs', + start, + end, + }, + }, + }); + } + + registry.when('Total docs', { config: 'basic' }, () => { + describe('authorization', () => { + it('should return a 403 when the user does not have sufficient privileges', async () => { + const err = await expectToReject(() => callApiAs('noAccessUser')); + expect(err.res.status).to.be(403); + }); + }); + }); +} From daa1b3c82941718557fc5dcf55b36428c7a31212 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 23:33:54 +1100 Subject: [PATCH 11/12] [8.x] [Response Ops][Connectors] Refactor Jira Connector to use latest API only (#197787) (#199289) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Backport This will backport the following commits from `main` to `8.x`: - [[Response Ops][Connectors] Refactor Jira Connector to use latest API only (#197787)](https://github.com/elastic/kibana/pull/197787) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Julian Gernun <17549662+jcger@users.noreply.github.com> --- .../connectors/action-types/jira.asciidoc | 4 +- .../actions/server/lib/axios_utils.test.ts | 8 + .../plugins/actions/server/lib/axios_utils.ts | 10 + .../connector_types/jira/service.test.ts | 667 +++++------------- .../server/connector_types/jira/service.ts | 130 +--- .../server/connector_types/jira/types.ts | 1 - .../server/jira_simulation.ts | 39 +- 7 files changed, 236 insertions(+), 623 deletions(-) diff --git a/docs/management/connectors/action-types/jira.asciidoc b/docs/management/connectors/action-types/jira.asciidoc index 906a2945d82de..2111de7a77ce6 100644 --- a/docs/management/connectors/action-types/jira.asciidoc +++ b/docs/management/connectors/action-types/jira.asciidoc @@ -14,7 +14,7 @@ The Jira connector uses the https://developer.atlassian.com/cloud/jira/platform/ [[jira-compatibility]] === Compatibility -Jira on-premise deployments (Server and Data Center) are not supported. +Jira Cloud and Jira Data Center are supported. Jira on-premise deployments are not supported. [float] [[define-jira-ui]] @@ -37,7 +37,7 @@ Name:: The name of the connector. URL:: Jira instance URL. Project key:: Jira project key. Email:: The account email for HTTP Basic authentication. -API token:: Jira API authentication token for HTTP Basic authentication. +API token:: Jira API authentication token for HTTP Basic authentication. For Jira Data Center, this value should be the password associated with the email owner. [float] [[jira-action-configuration]] diff --git a/x-pack/plugins/actions/server/lib/axios_utils.test.ts b/x-pack/plugins/actions/server/lib/axios_utils.test.ts index bee09a90ed27b..b7bb7548b9052 100644 --- a/x-pack/plugins/actions/server/lib/axios_utils.test.ts +++ b/x-pack/plugins/actions/server/lib/axios_utils.test.ts @@ -577,4 +577,12 @@ describe('throwIfResponseIsNotValid', () => { }) ).not.toThrow(); }); + + test('it does NOT throw if HTTP status code is 204 even if the content type is not supported', () => { + expect(() => + throwIfResponseIsNotValid({ + res: { ...res, status: 204, headers: { ['content-type']: 'text/html' } }, + }) + ).not.toThrow(); + }); }); diff --git a/x-pack/plugins/actions/server/lib/axios_utils.ts b/x-pack/plugins/actions/server/lib/axios_utils.ts index 254ad1a36f6e2..78abebf48022f 100644 --- a/x-pack/plugins/actions/server/lib/axios_utils.ts +++ b/x-pack/plugins/actions/server/lib/axios_utils.ts @@ -137,6 +137,16 @@ export const throwIfResponseIsNotValid = ({ const requiredContentType = 'application/json'; const contentType = res.headers['content-type'] ?? 'undefined'; const data = res.data; + const statusCode = res.status; + + /** + * Some external services may return a 204 + * status code but with unsupported content type like text/html. + * To avoid throwing on valid requests we return. + */ + if (statusCode === 204) { + return; + } /** * Check that the content-type of the response is application/json. diff --git a/x-pack/plugins/stack_connectors/server/connector_types/jira/service.test.ts b/x-pack/plugins/stack_connectors/server/connector_types/jira/service.test.ts index 5e98bdc96c0ee..0001d7cf13284 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/jira/service.test.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/jira/service.test.ts @@ -36,67 +36,14 @@ const configurationUtilities = actionsConfigMock.create(); const issueTypesResponse = createAxiosResponse({ data: { - projects: [ + issueTypes: [ { - issuetypes: [ - { - id: '10006', - name: 'Task', - }, - { - id: '10007', - name: 'Bug', - }, - ], + id: '10006', + name: 'Task', }, - ], - }, -}); - -const fieldsResponse = createAxiosResponse({ - data: { - projects: [ { - issuetypes: [ - { - id: '10006', - name: 'Task', - fields: { - summary: { required: true, schema: { type: 'string' }, fieldId: 'summary' }, - priority: { - required: false, - schema: { type: 'string' }, - fieldId: 'priority', - allowedValues: [ - { - name: 'Highest', - id: '1', - }, - { - name: 'High', - id: '2', - }, - { - name: 'Medium', - id: '3', - }, - { - name: 'Low', - id: '4', - }, - { - name: 'Lowest', - id: '5', - }, - ], - defaultValue: { - name: 'Medium', - id: '3', - }, - }, - }, - }, - ], + id: '10007', + name: 'Bug', }, ], }, @@ -110,30 +57,6 @@ const issueResponse = { const issuesResponse = [issueResponse]; -const mockNewAPI = () => - requestMock.mockImplementationOnce(() => - createAxiosResponse({ - data: { - capabilities: { - 'list-project-issuetypes': - 'https://coolsite.net/rest/capabilities/list-project-issuetypes', - 'list-issuetype-fields': 'https://coolsite.net/rest/capabilities/list-issuetype-fields', - }, - }, - }) - ); - -const mockOldAPI = () => - requestMock.mockImplementationOnce(() => - createAxiosResponse({ - data: { - capabilities: { - navigation: 'https://coolsite.net/rest/capabilities/navigation', - }, - }, - }) - ); - describe('Jira service', () => { let service: ExternalService; let connectorUsageCollector: ConnectorUsageCollector; @@ -347,23 +270,6 @@ describe('Jira service', () => { }); test('it creates the incident correctly without issue type', async () => { - /* The response from Jira when creating an issue contains only the key and the id. - The function makes the following calls when creating an issue: - 1. Get issueTypes to set a default ONLY when incident.issueType is missing - 2. Create the issue. - 3. Get the created issue with all the necessary fields. - */ - // getIssueType mocks - requestMock.mockImplementationOnce(() => - createAxiosResponse({ - data: { - capabilities: { - navigation: 'https://coolsite.net/rest/capabilities/navigation', - }, - }, - }) - ); - // getIssueType mocks requestMock.mockImplementationOnce(() => issueTypesResponse); @@ -419,16 +325,6 @@ describe('Jira service', () => { }); test('removes newline characters and trialing spaces from summary', async () => { - requestMock.mockImplementationOnce(() => - createAxiosResponse({ - data: { - capabilities: { - navigation: 'https://coolsite.net/rest/capabilities/navigation', - }, - }, - }) - ); - // getIssueType mocks requestMock.mockImplementationOnce(() => issueTypesResponse); @@ -800,28 +696,47 @@ describe('Jira service', () => { }); }); - describe('getCapabilities', () => { - test('it should return the capabilities', async () => { - mockOldAPI(); - const res = await service.getCapabilities(); - expect(res).toEqual({ - capabilities: { - navigation: 'https://coolsite.net/rest/capabilities/navigation', + describe('getIssueTypes', () => { + test('it should return the issue types', async () => { + requestMock.mockImplementationOnce(() => + createAxiosResponse({ + data: { + issueTypes: issueTypesResponse.data.issueTypes, + }, + }) + ); + + const res = await service.getIssueTypes(); + + expect(res).toEqual([ + { + id: '10006', + name: 'Task', }, - }); + { + id: '10007', + name: 'Bug', + }, + ]); }); test('it should call request with correct arguments', async () => { - mockOldAPI(); + requestMock.mockImplementationOnce(() => + createAxiosResponse({ + data: { + issueTypes: issueTypesResponse.data.issueTypes, + }, + }) + ); - await service.getCapabilities(); + await service.getIssueTypes(); - expect(requestMock).toHaveBeenCalledWith({ + expect(requestMock).toHaveBeenLastCalledWith({ axios, logger, method: 'get', configurationUtilities, - url: 'https://coolsite.net/rest/capabilities', + url: 'https://coolsite.net/rest/api/2/issue/createmeta/CK/issuetypes', connectorUsageCollector, }); }); @@ -829,25 +744,12 @@ describe('Jira service', () => { test('it should throw an error', async () => { requestMock.mockImplementation(() => { const error: ResponseError = new Error('An error has occurred'); - error.response = { data: { errors: { capabilities: 'Could not get capabilities' } } }; - throw error; - }); - - await expect(service.getCapabilities()).rejects.toThrow( - '[Action][Jira]: Unable to get capabilities. Error: An error has occurred. Reason: Could not get capabilities' - ); - }); - - test('it should return unknown if the error is a string', async () => { - requestMock.mockImplementation(() => { - const error = new Error('An error has occurred'); - // @ts-ignore - error.response = { data: 'Unauthorized' }; + error.response = { data: { errors: { issuetypes: 'Could not get issue types' } } }; throw error; }); - await expect(service.getCapabilities()).rejects.toThrow( - '[Action][Jira]: Unable to get capabilities. Error: An error has occurred. Reason: unknown: errorResponse.errors was null' + await expect(service.getIssueTypes()).rejects.toThrow( + '[Action][Jira]: Unable to get issue types. Error: An error has occurred. Reason: Could not get issue types' ); }); @@ -856,346 +758,178 @@ describe('Jira service', () => { createAxiosResponse({ data: { id: '1' }, headers: { ['content-type']: 'text/html' } }) ); - await expect(service.getCapabilities()).rejects.toThrow( - '[Action][Jira]: Unable to get capabilities. Error: Unsupported content type: text/html in GET https://example.com. Supported content types: application/json. Reason: unknown: errorResponse was null' - ); - }); - - test('it should throw if the required attributes are not there', async () => { - requestMock.mockImplementation(() => createAxiosResponse({ data: { notRequired: 'test' } })); - - await expect(service.getCapabilities()).rejects.toThrow( - '[Action][Jira]: Unable to get capabilities. Error: Response is missing at least one of the expected fields: capabilities. Reason: unknown: errorResponse was null' + await expect(service.getIssueTypes()).rejects.toThrow( + '[Action][Jira]: Unable to get issue types. Error: Unsupported content type: text/html in GET https://example.com. Supported content types: application/json. Reason: unknown: errorResponse was null' ); }); - }); - - describe('getIssueTypes', () => { - describe('Old API', () => { - test('it should return the issue types', async () => { - mockOldAPI(); - - requestMock.mockImplementationOnce(() => issueTypesResponse); - const res = await service.getIssueTypes(); - - expect(res).toEqual([ - { - id: '10006', - name: 'Task', - }, - { - id: '10007', - name: 'Bug', + test('it should work with data center response - issueTypes returned in data.values', async () => { + requestMock.mockImplementationOnce(() => + createAxiosResponse({ + data: { + values: issueTypesResponse.data.issueTypes, }, - ]); - }); - - test('it should call request with correct arguments', async () => { - mockOldAPI(); - - requestMock.mockImplementationOnce(() => issueTypesResponse); - - await service.getIssueTypes(); - - expect(requestMock).toHaveBeenLastCalledWith({ - axios, - logger, - method: 'get', - configurationUtilities, - url: 'https://coolsite.net/rest/api/2/issue/createmeta?projectKeys=CK&expand=projects.issuetypes.fields', - connectorUsageCollector, - }); - }); - - test('it should throw an error', async () => { - mockOldAPI(); - - requestMock.mockImplementation(() => { - const error: ResponseError = new Error('An error has occurred'); - error.response = { data: { errors: { issuetypes: 'Could not get issue types' } } }; - throw error; - }); - - await expect(service.getIssueTypes()).rejects.toThrow( - '[Action][Jira]: Unable to get issue types. Error: An error has occurred. Reason: Could not get issue types' - ); - }); - - test('it should throw if the request is not a JSON', async () => { - mockOldAPI(); + }) + ); - requestMock.mockImplementation(() => - createAxiosResponse({ data: { id: '1' }, headers: { ['content-type']: 'text/html' } }) - ); + await service.getIssueTypes(); - await expect(service.getIssueTypes()).rejects.toThrow( - '[Action][Jira]: Unable to get issue types. Error: Unsupported content type: text/html in GET https://example.com. Supported content types: application/json. Reason: unknown: errorResponse was null' - ); + expect(requestMock).toHaveBeenLastCalledWith({ + axios, + logger, + method: 'get', + configurationUtilities, + url: 'https://coolsite.net/rest/api/2/issue/createmeta/CK/issuetypes', + connectorUsageCollector, }); }); - describe('New API', () => { - test('it should return the issue types', async () => { - mockNewAPI(); - - requestMock.mockImplementationOnce(() => - createAxiosResponse({ - data: { - values: issueTypesResponse.data.projects[0].issuetypes, - }, - }) - ); - - const res = await service.getIssueTypes(); + }); - expect(res).toEqual([ - { - id: '10006', - name: 'Task', - }, - { - id: '10007', - name: 'Bug', + describe('getFieldsByIssueType', () => { + test('it should return the fields', async () => { + requestMock.mockImplementationOnce(() => + createAxiosResponse({ + data: { + fields: [ + { required: true, schema: { type: 'string' }, fieldId: 'summary' }, + { + required: false, + schema: { type: 'string' }, + fieldId: 'priority', + allowedValues: [ + { + name: 'Medium', + id: '3', + }, + ], + defaultValue: { + name: 'Medium', + id: '3', + }, + }, + ], }, - ]); - }); - - test('it should call request with correct arguments', async () => { - mockNewAPI(); - - requestMock.mockImplementationOnce(() => - createAxiosResponse({ - data: { - values: issueTypesResponse.data.projects[0].issuetypes, - }, - }) - ); - - await service.getIssueTypes(); - - expect(requestMock).toHaveBeenLastCalledWith({ - axios, - logger, - method: 'get', - configurationUtilities, - url: 'https://coolsite.net/rest/api/2/issue/createmeta/CK/issuetypes', - connectorUsageCollector, - }); - }); - - test('it should throw an error', async () => { - mockNewAPI(); - - requestMock.mockImplementation(() => { - const error: ResponseError = new Error('An error has occurred'); - error.response = { data: { errors: { issuetypes: 'Could not get issue types' } } }; - throw error; - }); - - await expect(service.getIssueTypes()).rejects.toThrow( - '[Action][Jira]: Unable to get issue types. Error: An error has occurred. Reason: Could not get issue types' - ); - }); - - test('it should throw if the request is not a JSON', async () => { - mockNewAPI(); + }) + ); - requestMock.mockImplementation(() => - createAxiosResponse({ data: { id: '1' }, headers: { ['content-type']: 'text/html' } }) - ); + const res = await service.getFieldsByIssueType('10006'); - await expect(service.getIssueTypes()).rejects.toThrow( - '[Action][Jira]: Unable to get issue types. Error: Unsupported content type: text/html in GET https://example.com. Supported content types: application/json. Reason: unknown: errorResponse was null' - ); + expect(res).toEqual({ + priority: { + required: false, + schema: { type: 'string' }, + allowedValues: [{ id: '3', name: 'Medium' }], + defaultValue: { id: '3', name: 'Medium' }, + }, + summary: { + required: true, + schema: { type: 'string' }, + allowedValues: [], + defaultValue: {}, + }, }); }); - }); - describe('getFieldsByIssueType', () => { - describe('Old API', () => { - test('it should return the fields', async () => { - mockOldAPI(); - - requestMock.mockImplementationOnce(() => fieldsResponse); - - const res = await service.getFieldsByIssueType('10006'); - - expect(res).toEqual({ - priority: { - required: false, - schema: { type: 'string' }, - allowedValues: [ - { id: '1', name: 'Highest' }, - { id: '2', name: 'High' }, - { id: '3', name: 'Medium' }, - { id: '4', name: 'Low' }, - { id: '5', name: 'Lowest' }, + test('it should call request with correct arguments', async () => { + requestMock.mockImplementationOnce(() => + createAxiosResponse({ + data: { + fields: [ + { required: true, schema: { type: 'string' }, fieldId: 'summary' }, + { + required: true, + schema: { type: 'string' }, + fieldId: 'priority', + allowedValues: [ + { + name: 'Medium', + id: '3', + }, + ], + defaultValue: { + name: 'Medium', + id: '3', + }, + }, ], - defaultValue: { id: '3', name: 'Medium' }, - }, - summary: { - required: true, - schema: { type: 'string' }, - allowedValues: [], - defaultValue: {}, }, - }); - }); - - test('it should call request with correct arguments', async () => { - mockOldAPI(); - - requestMock.mockImplementationOnce(() => fieldsResponse); + }) + ); - await service.getFieldsByIssueType('10006'); + await service.getFieldsByIssueType('10006'); - expect(requestMock).toHaveBeenLastCalledWith({ - axios, - logger, - method: 'get', - configurationUtilities, - url: 'https://coolsite.net/rest/api/2/issue/createmeta?projectKeys=CK&issuetypeIds=10006&expand=projects.issuetypes.fields', - connectorUsageCollector, - }); + expect(requestMock).toHaveBeenLastCalledWith({ + axios, + logger, + method: 'get', + configurationUtilities, + url: 'https://coolsite.net/rest/api/2/issue/createmeta/CK/issuetypes/10006', }); + }); - test('it should throw an error', async () => { - mockOldAPI(); - - requestMock.mockImplementation(() => { - const error: ResponseError = new Error('An error has occurred'); - error.response = { data: { errors: { fields: 'Could not get fields' } } }; - throw error; - }); - - await expect(service.getFieldsByIssueType('10006')).rejects.toThrow( - '[Action][Jira]: Unable to get fields. Error: An error has occurred. Reason: Could not get fields' - ); + test('it should throw an error', async () => { + requestMock.mockImplementation(() => { + const error: ResponseError = new Error('An error has occurred'); + error.response = { data: { errors: { issuetypes: 'Could not get issue types' } } }; + throw error; }); - test('it should throw if the request is not a JSON', async () => { - mockOldAPI(); + await expect(service.getFieldsByIssueType('10006')).rejects.toThrowError( + '[Action][Jira]: Unable to get fields. Error: An error has occurred. Reason: Could not get issue types' + ); + }); - requestMock.mockImplementation(() => - createAxiosResponse({ data: { id: '1' }, headers: { ['content-type']: 'text/html' } }) - ); + test('it should throw if the request is not a JSON', async () => { + requestMock.mockImplementation(() => + createAxiosResponse({ data: { id: '1' }, headers: { ['content-type']: 'text/html' } }) + ); - await expect(service.getFieldsByIssueType('10006')).rejects.toThrow( - '[Action][Jira]: Unable to get fields. Error: Unsupported content type: text/html in GET https://example.com. Supported content types: application/json. Reason: unknown: errorResponse was null' - ); - }); + await expect(service.getFieldsByIssueType('10006')).rejects.toThrow( + '[Action][Jira]: Unable to get fields. Error: Unsupported content type: text/html in GET https://example.com. Supported content types: application/json. Reason: unknown: errorResponse was null' + ); }); - describe('New API', () => { - test('it should return the fields', async () => { - mockNewAPI(); - - requestMock.mockImplementationOnce(() => - createAxiosResponse({ - data: { - values: [ - { required: true, schema: { type: 'string' }, fieldId: 'summary' }, - { - required: false, - schema: { type: 'string' }, - fieldId: 'priority', - allowedValues: [ - { - name: 'Medium', - id: '3', - }, - ], - defaultValue: { + test('it should work with data center response - issueTypes returned in data.values', async () => { + requestMock.mockImplementationOnce(() => + createAxiosResponse({ + data: { + values: [ + { required: true, schema: { type: 'string' }, fieldId: 'summary' }, + { + required: false, + schema: { type: 'string' }, + fieldId: 'priority', + allowedValues: [ + { name: 'Medium', id: '3', }, + ], + defaultValue: { + name: 'Medium', + id: '3', }, - ], - }, - }) - ); - - const res = await service.getFieldsByIssueType('10006'); - - expect(res).toEqual({ - priority: { - required: false, - schema: { type: 'string' }, - allowedValues: [{ id: '3', name: 'Medium' }], - defaultValue: { id: '3', name: 'Medium' }, - }, - summary: { - required: true, - schema: { type: 'string' }, - allowedValues: [], - defaultValue: {}, + }, + ], }, - }); - }); - - test('it should call request with correct arguments', async () => { - mockNewAPI(); - - requestMock.mockImplementationOnce(() => - createAxiosResponse({ - data: { - values: [ - { required: true, schema: { type: 'string' }, fieldId: 'summary' }, - { - required: true, - schema: { type: 'string' }, - fieldId: 'priority', - allowedValues: [ - { - name: 'Medium', - id: '3', - }, - ], - defaultValue: { - name: 'Medium', - id: '3', - }, - }, - ], - }, - }) - ); - - await service.getFieldsByIssueType('10006'); - - expect(requestMock).toHaveBeenLastCalledWith({ - axios, - logger, - method: 'get', - configurationUtilities, - url: 'https://coolsite.net/rest/api/2/issue/createmeta/CK/issuetypes/10006', - }); - }); - - test('it should throw an error', async () => { - mockNewAPI(); - - requestMock.mockImplementation(() => { - const error: ResponseError = new Error('An error has occurred'); - error.response = { data: { errors: { issuetypes: 'Could not get issue types' } } }; - throw error; - }); - - await expect(service.getFieldsByIssueType('10006')).rejects.toThrowError( - '[Action][Jira]: Unable to get fields. Error: An error has occurred. Reason: Could not get issue types' - ); - }); - - test('it should throw if the request is not a JSON', async () => { - mockNewAPI(); + }) + ); - requestMock.mockImplementation(() => - createAxiosResponse({ data: { id: '1' }, headers: { ['content-type']: 'text/html' } }) - ); + const res = await service.getFieldsByIssueType('10006'); - await expect(service.getFieldsByIssueType('10006')).rejects.toThrow( - '[Action][Jira]: Unable to get fields. Error: Unsupported content type: text/html in GET https://example.com. Supported content types: application/json. Reason: unknown: errorResponse was null' - ); + expect(res).toEqual({ + priority: { + required: false, + schema: { type: 'string' }, + allowedValues: [{ id: '3', name: 'Medium' }], + defaultValue: { id: '3', name: 'Medium' }, + }, + summary: { + required: true, + schema: { type: 'string' }, + allowedValues: [], + defaultValue: {}, + }, }); }); }); @@ -1403,50 +1137,14 @@ describe('Jira service', () => { .mockImplementationOnce(() => createAxiosResponse({ data: { - capabilities: { - 'list-project-issuetypes': - 'https://coolsite.net/rest/capabilities/list-project-issuetypes', - 'list-issuetype-fields': - 'https://coolsite.net/rest/capabilities/list-issuetype-fields', - }, - }, - }) - ) - .mockImplementationOnce(() => - createAxiosResponse({ - data: { - values: issueTypesResponse.data.projects[0].issuetypes, - }, - }) - ) - .mockImplementationOnce(() => - createAxiosResponse({ - data: { - capabilities: { - 'list-project-issuetypes': - 'https://coolsite.net/rest/capabilities/list-project-issuetypes', - 'list-issuetype-fields': - 'https://coolsite.net/rest/capabilities/list-issuetype-fields', - }, - }, - }) - ) - .mockImplementationOnce(() => - createAxiosResponse({ - data: { - capabilities: { - 'list-project-issuetypes': - 'https://coolsite.net/rest/capabilities/list-project-issuetypes', - 'list-issuetype-fields': - 'https://coolsite.net/rest/capabilities/list-issuetype-fields', - }, + issueTypes: issueTypesResponse.data.issueTypes, }, }) ) .mockImplementationOnce(() => createAxiosResponse({ data: { - values: [ + fields: [ { required: true, schema: { type: 'string' }, fieldId: 'summary' }, { required: true, schema: { type: 'string' }, fieldId: 'description' }, { @@ -1471,7 +1169,7 @@ describe('Jira service', () => { .mockImplementationOnce(() => createAxiosResponse({ data: { - values: [ + fields: [ { required: true, schema: { type: 'string' }, fieldId: 'summary' }, { required: true, schema: { type: 'string' }, fieldId: 'description' }, ], @@ -1488,10 +1186,7 @@ describe('Jira service', () => { callMocks(); await service.getFields(); const callUrls = [ - 'https://coolsite.net/rest/capabilities', 'https://coolsite.net/rest/api/2/issue/createmeta/CK/issuetypes', - 'https://coolsite.net/rest/capabilities', - 'https://coolsite.net/rest/capabilities', 'https://coolsite.net/rest/api/2/issue/createmeta/CK/issuetypes/10006', 'https://coolsite.net/rest/api/2/issue/createmeta/CK/issuetypes/10007', ]; @@ -1525,7 +1220,7 @@ describe('Jira service', () => { throw error; }); await expect(service.getFields()).rejects.toThrow( - '[Action][Jira]: Unable to get capabilities. Error: An error has occurred. Reason: Required field' + '[Action][Jira]: Unable to get issue types. Error: An error has occurred. Reason: Required field' ); }); }); diff --git a/x-pack/plugins/stack_connectors/server/connector_types/jira/service.ts b/x-pack/plugins/stack_connectors/server/connector_types/jira/service.ts index 064667558b37e..f8929ce67b68a 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/jira/service.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/jira/service.ts @@ -39,12 +39,9 @@ import * as i18n from './translations'; const VERSION = '2'; const BASE_URL = `rest/api/${VERSION}`; -const CAPABILITIES_URL = `rest/capabilities`; const VIEW_INCIDENT_URL = `browse`; -const createMetaCapabilities = ['list-project-issuetypes', 'list-issuetype-fields']; - export const createExternalService = ( { config, secrets }: ExternalServiceCredentials, logger: Logger, @@ -60,10 +57,7 @@ export const createExternalService = ( const urlWithoutTrailingSlash = url.endsWith('/') ? url.slice(0, -1) : url; const incidentUrl = `${urlWithoutTrailingSlash}/${BASE_URL}/issue`; - const capabilitiesUrl = `${urlWithoutTrailingSlash}/${CAPABILITIES_URL}`; const commentUrl = `${incidentUrl}/{issueId}/comment`; - const getIssueTypesOldAPIURL = `${urlWithoutTrailingSlash}/${BASE_URL}/issue/createmeta?projectKeys=${projectKey}&expand=projects.issuetypes.fields`; - const getIssueTypeFieldsOldAPIURL = `${urlWithoutTrailingSlash}/${BASE_URL}/issue/createmeta?projectKeys=${projectKey}&issuetypeIds={issueTypeId}&expand=projects.issuetypes.fields`; const getIssueTypesUrl = `${urlWithoutTrailingSlash}/${BASE_URL}/issue/createmeta/${projectKey}/issuetypes`; const getIssueTypeFieldsUrl = `${urlWithoutTrailingSlash}/${BASE_URL}/issue/createmeta/${projectKey}/issuetypes/{issueTypeId}`; const searchUrl = `${urlWithoutTrailingSlash}/${BASE_URL}/search`; @@ -144,9 +138,6 @@ export const createExternalService = ( }, ''); }; - const hasSupportForNewAPI = (capabilities: { capabilities?: {} }) => - createMetaCapabilities.every((c) => Object.keys(capabilities?.capabilities ?? {}).includes(c)); - const normalizeIssueTypes = (issueTypes: Array<{ id: string; name: string }>) => issueTypes.map((type) => ({ id: type.id, name: type.name })); @@ -356,12 +347,12 @@ export const createExternalService = ( } }; - const getCapabilities = async () => { + const getIssueTypes = async () => { try { const res = await request({ axios: axiosInstance, method: 'get', - url: capabilitiesUrl, + url: getIssueTypesUrl, logger, configurationUtilities, connectorUsageCollector, @@ -369,59 +360,11 @@ export const createExternalService = ( throwIfResponseIsNotValid({ res, - requiredAttributesToBeInTheResponse: ['capabilities'], }); - return { ...res.data }; - } catch (error) { - throw new Error( - getErrorMessage( - i18n.NAME, - `Unable to get capabilities. Error: ${error.message}. Reason: ${createErrorMessage( - error.response?.data - )}` - ) - ); - } - }; - - const getIssueTypes = async () => { - const capabilitiesResponse = await getCapabilities(); - const supportsNewAPI = hasSupportForNewAPI(capabilitiesResponse); - try { - if (!supportsNewAPI) { - const res = await request({ - axios: axiosInstance, - method: 'get', - url: getIssueTypesOldAPIURL, - logger, - configurationUtilities, - connectorUsageCollector, - }); - - throwIfResponseIsNotValid({ - res, - }); - - const issueTypes = res.data.projects[0]?.issuetypes ?? []; - return normalizeIssueTypes(issueTypes); - } else { - const res = await request({ - axios: axiosInstance, - method: 'get', - url: getIssueTypesUrl, - logger, - configurationUtilities, - connectorUsageCollector, - }); - - throwIfResponseIsNotValid({ - res, - }); - - const issueTypes = res.data.values; - return normalizeIssueTypes(issueTypes); - } + // Cloud returns issueTypes and Data Center returns values + const { issueTypes, values } = res.data; + return normalizeIssueTypes(issueTypes || values); } catch (error) { throw new Error( getErrorMessage( @@ -435,47 +378,29 @@ export const createExternalService = ( }; const getFieldsByIssueType = async (issueTypeId: string) => { - const capabilitiesResponse = await getCapabilities(); - const supportsNewAPI = hasSupportForNewAPI(capabilitiesResponse); try { - if (!supportsNewAPI) { - const res = await request({ - axios: axiosInstance, - method: 'get', - url: createGetIssueTypeFieldsUrl(getIssueTypeFieldsOldAPIURL, issueTypeId), - logger, - configurationUtilities, - connectorUsageCollector, - }); - - throwIfResponseIsNotValid({ - res, - }); - - const fields = res.data.projects[0]?.issuetypes[0]?.fields || {}; - return normalizeFields(fields); - } else { - const res = await request({ - axios: axiosInstance, - method: 'get', - url: createGetIssueTypeFieldsUrl(getIssueTypeFieldsUrl, issueTypeId), - logger, - configurationUtilities, - }); - - throwIfResponseIsNotValid({ - res, - }); - - const fields = res.data.values.reduce( - (acc: { [x: string]: {} }, value: { fieldId: string }) => ({ - ...acc, - [value.fieldId]: { ...value }, - }), - {} - ); - return normalizeFields(fields); - } + const res = await request({ + axios: axiosInstance, + method: 'get', + url: createGetIssueTypeFieldsUrl(getIssueTypeFieldsUrl, issueTypeId), + logger, + configurationUtilities, + }); + + throwIfResponseIsNotValid({ + res, + }); + + // Cloud returns fields and Data Center returns values + const { fields: rawFields, values } = res.data; + const fields = (rawFields || values).reduce( + (acc: { [x: string]: {} }, value: { fieldId: string }) => ({ + ...acc, + [value.fieldId]: { ...value }, + }), + {} + ); + return normalizeFields(fields); } catch (error) { throw new Error( getErrorMessage( @@ -580,7 +505,6 @@ export const createExternalService = ( createIncident, updateIncident, createComment, - getCapabilities, getIssueTypes, getFieldsByIssueType, getIssues, diff --git a/x-pack/plugins/stack_connectors/server/connector_types/jira/types.ts b/x-pack/plugins/stack_connectors/server/connector_types/jira/types.ts index c975e23b1b783..755726137e412 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/jira/types.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/jira/types.ts @@ -101,7 +101,6 @@ export interface ExternalService { createComment: (params: CreateCommentParams) => Promise; createIncident: (params: CreateIncidentParams) => Promise; getFields: () => Promise; - getCapabilities: () => Promise; getFieldsByIssueType: (issueTypeId: string) => Promise; getIncident: (id: string) => Promise; getIssue: (id: string) => Promise; diff --git a/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/jira_simulation.ts b/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/jira_simulation.ts index 31ae7f1507ca8..ee559ca91b9ec 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/jira_simulation.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/actions_simulators/server/jira_simulation.ts @@ -100,7 +100,7 @@ export function initPlugin(router: IRouter, path: string) { router.get( { - path: `${path}/rest/capabilities`, + path: `${path}/rest/api/2/issue/createmeta/{projectId}/issuetypes`, options: { authRequired: false, }, @@ -112,37 +112,14 @@ export function initPlugin(router: IRouter, path: string) { res: KibanaResponseFactory ): Promise> { return jsonResponse(res, 200, { - capabilities: {}, - }); - } - ); - - router.get( - { - path: `${path}/rest/api/2/issue/createmeta`, - options: { - authRequired: false, - }, - validate: {}, - }, - async function ( - context: RequestHandlerContext, - req: KibanaRequest, - res: KibanaResponseFactory - ): Promise> { - return jsonResponse(res, 200, { - projects: [ + issueTypes: [ + { + id: '10006', + name: 'Task', + }, { - issuetypes: [ - { - id: '10006', - name: 'Task', - }, - { - id: '10007', - name: 'Sub-task', - }, - ], + id: '10007', + name: 'Sub-task', }, ], }); From c3bf0b4c9432b553c6bdaa7b82b869160ea4e1d9 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 7 Nov 2024 23:51:17 +1100 Subject: [PATCH 12/12] [8.x] Use getUrlPartsWithStrippedDefaultPort instead of getUrlParts (#199264) (#199288) # Backport This will backport the following commits from `main` to `8.x`: - [Use getUrlPartsWithStrippedDefaultPort instead of getUrlParts (#199264)](https://github.com/elastic/kibana/pull/199264) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Maryam Saeidi --- .../alerting/custom_threshold/custom_eq_avg_bytes_fired.ts | 2 +- .../alerting/custom_threshold/documents_count_fired.ts | 2 +- .../observability/alerting/custom_threshold/group_by_fired.ts | 2 +- .../observability/alerting/custom_threshold/p99_pct_fired.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/custom_eq_avg_bytes_fired.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/custom_eq_avg_bytes_fired.ts index 4163c85c849f4..5eb9f4cc21738 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/custom_eq_avg_bytes_fired.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/custom_eq_avg_bytes_fired.ts @@ -249,7 +249,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { docCountTarget: 1, }); - const { protocol, hostname, port } = kbnTestConfig.getUrlParts(); + const { protocol, hostname, port } = kbnTestConfig.getUrlPartsWithStrippedDefaultPort(); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql( `${protocol}://${hostname}${port ? `:${port}` : ''}/app/observability/alerts/${alertId}` diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/documents_count_fired.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/documents_count_fired.ts index f9d7067f0e0a4..97451b1930afc 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/documents_count_fired.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/documents_count_fired.ts @@ -247,7 +247,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { docCountTarget: 1, }); - const { protocol, hostname, port } = kbnTestConfig.getUrlParts(); + const { protocol, hostname, port } = kbnTestConfig.getUrlPartsWithStrippedDefaultPort(); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql( diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/group_by_fired.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/group_by_fired.ts index 3a554a16c5cec..457b8bdceae44 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/group_by_fired.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/group_by_fired.ts @@ -272,7 +272,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { const resp = await alertingApi.waitForDocumentInIndex({ indexName: ALERT_ACTION_INDEX, }); - const { protocol, hostname, port } = kbnTestConfig.getUrlParts(); + const { protocol, hostname, port } = kbnTestConfig.getUrlPartsWithStrippedDefaultPort(); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql( diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/p99_pct_fired.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/p99_pct_fired.ts index 743bca2683895..5d6cfb49f86f0 100644 --- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/p99_pct_fired.ts +++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/alerting/custom_threshold/p99_pct_fired.ts @@ -244,7 +244,7 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) { docCountTarget: 1, }); - const { protocol, hostname, port } = kbnTestConfig.getUrlParts(); + const { protocol, hostname, port } = kbnTestConfig.getUrlPartsWithStrippedDefaultPort(); expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql( `${protocol}://${hostname}${port ? `:${port}` : ''}/app/observability/alerts/${alertId}`