From 88464e5b6da3a469435bf1b510286c5a93dd6fa8 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Fri, 19 Jul 2024 15:00:53 +0200 Subject: [PATCH] [FTR] split configs by target into multiple manifest files (#187440) ## Summary Part of #186515 Split FTR configs manifest into multiple files based on distro (serverless/stateful) and area of testing (platform/solutions) Update the CI scripts to support the change, but without logic modification More context: With this change we will have a clear split of FTR test configs owned by Platform and Solutions. It is a starting point to make configs discoverable, our test pipelines be flexible and run tests based on distro/solution. --- .buildkite/ftr_base_serverless_configs.yml | 7 + .buildkite/ftr_configs.yml | 597 ------------------ .buildkite/ftr_configs_manifests.json | 14 + .buildkite/ftr_oblt_serverless_configs.yml | 27 + .buildkite/ftr_oblt_stateful_configs.yml | 44 ++ .buildkite/ftr_platform_stateful_configs.yml | 360 +++++++++++ .buildkite/ftr_search_serverless_configs.yml | 18 + .buildkite/ftr_search_stateful_configs.yml | 9 + .../ftr_security_serverless_configs.yml | 97 +++ .buildkite/ftr_security_stateful_configs.yml | 84 +++ .../ci-stats/pick_test_group_run_order.ts | 51 +- .../development-functional-tests.asciidoc | 14 +- .../lib/config/config_loading.ts | 8 +- .../lib/config/ftr_configs_manifest.ts | 54 +- .../lib/config/run_check_ftr_configs_cli.ts | 12 +- scripts/enabled_ftr_configs.js | 19 +- .../README.md | 2 +- 17 files changed, 785 insertions(+), 632 deletions(-) create mode 100644 .buildkite/ftr_base_serverless_configs.yml delete mode 100644 .buildkite/ftr_configs.yml create mode 100644 .buildkite/ftr_configs_manifests.json create mode 100644 .buildkite/ftr_oblt_serverless_configs.yml create mode 100644 .buildkite/ftr_oblt_stateful_configs.yml create mode 100644 .buildkite/ftr_platform_stateful_configs.yml create mode 100644 .buildkite/ftr_search_serverless_configs.yml create mode 100644 .buildkite/ftr_search_stateful_configs.yml create mode 100644 .buildkite/ftr_security_serverless_configs.yml create mode 100644 .buildkite/ftr_security_stateful_configs.yml diff --git a/.buildkite/ftr_base_serverless_configs.yml b/.buildkite/ftr_base_serverless_configs.yml new file mode 100644 index 0000000000000..5e7baeb3c3aea --- /dev/null +++ b/.buildkite/ftr_base_serverless_configs.yml @@ -0,0 +1,7 @@ +disabled: + # Base config files, only necessary to inform config finding script + + # Serverless base config files + - x-pack/test_serverless/api_integration/config.base.ts + - x-pack/test_serverless/functional/config.base.ts + - x-pack/test_serverless/shared/config.base.ts diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml deleted file mode 100644 index 1d1ee79deee4a..0000000000000 --- a/.buildkite/ftr_configs.yml +++ /dev/null @@ -1,597 +0,0 @@ -disabled: - # Base config files, only necessary to inform config finding script - - test/functional/config.base.js - - test/functional/firefox/config.base.ts - - x-pack/test/functional/config.base.js - - x-pack/test/functional_enterprise_search/base_config.ts - - x-pack/test/localization/config.base.ts - - test/server_integration/config.base.js - - x-pack/test/functional_with_es_ssl/config.base.ts - - x-pack/test/api_integration/config.ts - - x-pack/test/fleet_api_integration/config.base.ts - - x-pack/test/security_solution_api_integration/config/ess/config.base.ts - - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts - - x-pack/test/security_solution_api_integration/config/ess/config.base.edr_workflows.trial.ts - - x-pack/test/security_solution_api_integration/config/ess/config.base.edr_workflows.ts - - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts - - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts - - x-pack/test/security_solution_api_integration/config/serverless/config.base.edr_workflows.ts - - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts - - x-pack/test/security_solution_endpoint/configs/config.base.ts - - # QA suites that are run out-of-band - - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js - - x-pack/test/upgrade/config.ts - - test/functional/config.edge.js - - x-pack/test/functional/config.edge.js - - x-pack/test/cloud_security_posture_functional/config.cloud.ts - - # Cypress configs, for now these are still run manually - - x-pack/test/fleet_cypress/cli_config.ts - - x-pack/test/fleet_cypress/config.ts - - x-pack/test/fleet_cypress/visual_config.ts - - x-pack/test/functional_enterprise_search/cypress.config.ts - - x-pack/test/defend_workflows_cypress/cli_config.ts - - x-pack/test/defend_workflows_cypress/config.ts - - x-pack/test/defend_workflows_cypress/serverless_config.ts - - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_open.ts - - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_runner.ts - - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config.ts - - x-pack/test/osquery_cypress/cli_config.ts - - x-pack/test/osquery_cypress/serverless_cli_config.ts - - x-pack/test/osquery_cypress/config.ts - - x-pack/test/osquery_cypress/visual_config.ts - - x-pack/test/security_solution_cypress/cli_config.ts - - x-pack/test/security_solution_cypress/config.ts - - x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts - - x-pack/test/threat_intelligence_cypress/config.ts - - x-pack/test/functional_enterprise_search/visual_config.ts - - x-pack/test/functional_enterprise_search/cli_config.ts - - x-pack/test_serverless/functional/test_suites/security/cypress/security_config.ts - - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_open.ts - - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts - - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts - - x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts - - x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts - - x-pack/test/security_solution_cypress/serverless_config.ts - - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts - - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts - - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts - - # Elastic Synthetics configs - - x-pack/plugins/observability_solution/uptime/e2e/uptime/synthetics_run.ts - - x-pack/plugins/observability_solution/synthetics/e2e/config.ts - - x-pack/plugins/observability_solution/synthetics/e2e/synthetics/synthetics_run.ts - - x-pack/plugins/observability_solution/exploratory_view/e2e/synthetics_run.ts - - x-pack/plugins/observability_solution/ux/e2e/synthetics_run.ts - - x-pack/plugins/observability_solution/slo/e2e/synthetics_run.ts - - # Configs that exist but weren't running in CI when this file was introduced - - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts - - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/config.ts - - x-pack/test/alerting_api_integration/spaces_only_legacy/config.ts - - x-pack/test/cloud_integration/config.ts - - x-pack/test/performance/config.playwright.ts - - x-pack/test/load/config.ts - - x-pack/test/plugin_api_perf/config.js - - x-pack/test/screenshot_creation/config.ts - - x-pack/test/fleet_packages/config.ts - - # Scalability testing config that we run in its own pipeline - - x-pack/test/scalability/config.ts - - # Serverless base config files - - x-pack/test_serverless/api_integration/config.base.ts - - x-pack/test_serverless/functional/config.base.ts - - x-pack/test_serverless/shared/config.base.ts - - # Serverless feature flag config files (move to enabled after tests are added) - - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts - - x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts - - x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts - - x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts - - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts - - # http/2 security muted tests - - x-pack/test/security_functional/saml.http2.config.ts - - x-pack/test/security_functional/oidc.http2.config.ts - -defaultQueue: 'n2-4-spot' -enabled: - - test/accessibility/config.ts - - test/analytics/config.ts - - test/api_integration/config.js - - test/examples/config.js - - test/functional/apps/bundles/config.ts - - test/functional/apps/console/monaco/config.ts - - test/functional/apps/console/ace/config.ts - - test/functional/apps/context/config.ts - - test/functional/apps/dashboard_elements/controls/common/config.ts - - test/functional/apps/dashboard_elements/controls/options_list/config.ts - - test/functional/apps/dashboard_elements/image_embeddable/config.ts - - test/functional/apps/dashboard_elements/input_control_vis/config.ts - - test/functional/apps/dashboard_elements/links/config.ts - - test/functional/apps/dashboard_elements/markdown/config.ts - - test/functional/apps/dashboard/group1/config.ts - - test/functional/apps/dashboard/group2/config.ts - - test/functional/apps/dashboard/group3/config.ts - - test/functional/apps/dashboard/group4/config.ts - - test/functional/apps/dashboard/group5/config.ts - - test/functional/apps/dashboard/group6/config.ts - - test/functional/apps/discover/ccs_compatibility/config.ts - - test/functional/apps/discover/classic/config.ts - - test/functional/apps/discover/embeddable/config.ts - - test/functional/apps/discover/esql/config.ts - - test/functional/apps/discover/group1/config.ts - - test/functional/apps/discover/group2_data_grid1/config.ts - - test/functional/apps/discover/group2_data_grid2/config.ts - - test/functional/apps/discover/group2_data_grid3/config.ts - - test/functional/apps/discover/group3/config.ts - - test/functional/apps/discover/group4/config.ts - - test/functional/apps/discover/group5/config.ts - - test/functional/apps/discover/group6/config.ts - - test/functional/apps/discover/group7/config.ts - - test/functional/apps/discover/group8/config.ts - - test/functional/apps/discover/context_awareness/config.ts - - test/functional/apps/getting_started/config.ts - - test/functional/apps/home/config.ts - - test/functional/apps/kibana_overview/config.ts - - test/functional/apps/management/config.ts - - test/functional/apps/saved_objects_management/config.ts - - test/functional/apps/sharing/config.ts - - test/functional/apps/status_page/config.ts - - test/functional/apps/visualize/group1/config.ts - - test/functional/apps/visualize/group2/config.ts - - test/functional/apps/visualize/group3/config.ts - - test/functional/apps/visualize/group4/config.ts - - test/functional/apps/visualize/group5/config.ts - - test/functional/apps/visualize/group6/config.ts - - test/functional/apps/visualize/replaced_vislib_chart_types/config.ts - - test/functional/config.ccs.ts - - test/functional/firefox/console.config.ts - - test/functional/firefox/dashboard.config.ts - - test/functional/firefox/discover.config.ts - - test/functional/firefox/home.config.ts - - test/functional/firefox/visualize.config.ts - - test/health_gateway/config.ts - - test/interactive_setup_api_integration/enrollment_flow.config.ts - - test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts - - test/interactive_setup_api_integration/manual_configuration_flow.config.ts - - test/interactive_setup_functional/enrollment_token.config.ts - - test/interactive_setup_functional/manual_configuration_without_security.config.ts - - test/interactive_setup_functional/manual_configuration_without_tls.config.ts - - test/interactive_setup_functional/manual_configuration.config.ts - - test/interpreter_functional/config.ts - - test/node_roles_functional/all.config.ts - - test/node_roles_functional/background_tasks.config.ts - - test/node_roles_functional/ui.config.ts - - test/plugin_functional/config.ts - - test/server_integration/http/platform/config.status.ts - - test/server_integration/http/platform/config.ts - - test/server_integration/http/ssl_redirect/config.ts - - test/server_integration/http/ssl_with_p12_intermediate/config.js - - test/server_integration/http/ssl_with_p12/config.js - - test/server_integration/http/ssl/config.js - - test/ui_capabilities/newsfeed_err/config.ts - - x-pack/test/accessibility/apps/group1/config.ts - - x-pack/test/accessibility/apps/group2/config.ts - - x-pack/test/accessibility/apps/group3/config.ts - - x-pack/test/localization/config.ja_jp.ts - - x-pack/test/localization/config.fr_fr.ts - - x-pack/test/localization/config.zh_cn.ts - - x-pack/test/alerting_api_integration/basic/config.ts - - x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts - - x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts - - x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts - - x-pack/test/alerting_api_integration/security_and_spaces/group4/config.ts - - x-pack/test/alerting_api_integration/security_and_spaces/group3/config_with_schedule_circuit_breaker.ts - - x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts - - x-pack/test/alerting_api_integration/security_and_spaces/group4/config_non_dedicated_task_runner.ts - - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/config.ts - - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group2/config.ts - - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/config.ts - - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/config.ts - - x-pack/test/alerting_api_integration/spaces_only/tests/actions/config.ts - - x-pack/test/alerting_api_integration/spaces_only/tests/action_task_params/config.ts - - x-pack/test/alerting_api_integration/observability/config.ts - - x-pack/test/api_integration_basic/config.ts - - x-pack/test/api_integration/config_security_basic.ts - - x-pack/test/api_integration/config_security_trial.ts - - x-pack/test/api_integration/apis/aiops/config.ts - - x-pack/test/api_integration/apis/cases/config.ts - - x-pack/test/api_integration/apis/content_management/config.ts - - x-pack/test/api_integration/apis/cloud_security_posture/config.ts - - x-pack/test/api_integration/apis/console/config.ts - - x-pack/test/api_integration/apis/es/config.ts - - x-pack/test/api_integration/apis/features/config.ts - - x-pack/test/api_integration/apis/file_upload/config.ts - - x-pack/test/api_integration/apis/grok_debugger/config.ts - - x-pack/test/api_integration/apis/kibana/config.ts - - x-pack/test/api_integration/apis/lists/config.ts - - x-pack/test/api_integration/apis/logs_ui/config.ts - - x-pack/test/api_integration/apis/logstash/config.ts - - x-pack/test/api_integration/apis/management/config.ts - - x-pack/test/api_integration/apis/management/index_management/disabled_data_enrichers/config.ts - - x-pack/test/api_integration/apis/maps/config.ts - - x-pack/test/api_integration/apis/metrics_ui/config.ts - - x-pack/test/api_integration/apis/ml/config.ts - - x-pack/test/api_integration/apis/monitoring/config.ts - - x-pack/test/api_integration/apis/monitoring_collection/config.ts - - x-pack/test/api_integration/apis/osquery/config.ts - - x-pack/test/api_integration/apis/painless_lab/config.ts - - x-pack/test/api_integration/apis/search/config.ts - - x-pack/test/api_integration/apis/searchprofiler/config.ts - - x-pack/test/api_integration/apis/security/config.ts - - x-pack/test/api_integration/apis/spaces/config.ts - - x-pack/test/api_integration/apis/stats/config.ts - - x-pack/test/api_integration/apis/status/config.ts - - x-pack/test/api_integration/apis/synthetics/config.ts - - x-pack/test/api_integration/apis/slos/config.ts - - x-pack/test/api_integration/apis/telemetry/config.ts - - x-pack/test/api_integration/apis/transform/config.ts - - x-pack/test/api_integration/apis/upgrade_assistant/config.ts - - x-pack/test/api_integration/apis/uptime/config.ts - - x-pack/test/api_integration/apis/watcher/config.ts - - x-pack/test/apm_api_integration/basic/config.ts - - x-pack/test/apm_api_integration/cloud/config.ts - - x-pack/test/apm_api_integration/rules/config.ts - - x-pack/test/apm_api_integration/trial/config.ts - - x-pack/test/banners_functional/config.ts - - x-pack/test/cases_api_integration/security_and_spaces/config_basic.ts - - x-pack/test/cases_api_integration/security_and_spaces/config_trial.ts - - x-pack/test/cases_api_integration/security_and_spaces/config_no_public_base_url.ts - - x-pack/test/cases_api_integration/spaces_only/config.ts - - x-pack/test/cloud_security_posture_functional/config.ts - - x-pack/test/cloud_security_posture_api/config.ts - - x-pack/test/dataset_quality_api_integration/basic/config.ts - - x-pack/test/disable_ems/config.ts - - x-pack/test/encrypted_saved_objects_api_integration/config.ts - - x-pack/test/examples/config.ts - - x-pack/test/fleet_api_integration/config.agent.ts - - x-pack/test/fleet_api_integration/config.agent_policy.ts - - x-pack/test/fleet_api_integration/config.epm.ts - - x-pack/test/fleet_api_integration/config.fleet.ts - - x-pack/test/fleet_api_integration/config.package_policy.ts - - x-pack/test/fleet_api_integration/config.space_awareness.ts - - x-pack/test/fleet_functional/config.ts - - x-pack/test/ftr_apis/security_and_spaces/config.ts - - x-pack/test/functional_basic/apps/ml/permissions/config.ts - - x-pack/test/functional_basic/apps/ml/data_visualizer/group1/config.ts - - x-pack/test/functional_basic/apps/ml/data_visualizer/group2/config.ts - - x-pack/test/functional_basic/apps/ml/data_visualizer/group3/config.ts - - x-pack/test/functional_basic/apps/transform/creation/index_pattern/config.ts - - x-pack/test/functional_basic/apps/transform/actions/config.ts - - x-pack/test/functional_basic/apps/transform/edit_clone/config.ts - - x-pack/test/functional_basic/apps/transform/creation/runtime_mappings_saved_search/config.ts - - x-pack/test/functional_basic/apps/transform/permissions/config.ts - - x-pack/test/functional_basic/apps/transform/feature_controls/config.ts - - x-pack/test/functional_cors/config.ts - - x-pack/test/functional_embedded/config.ts - - x-pack/test/functional_execution_context/config.ts - - x-pack/test/functional_with_es_ssl/apps/cases/group1/config.ts - - x-pack/test/functional_with_es_ssl/apps/cases/group2/config.ts - - x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/config.ts - - x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/config.ts - - x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/shared/config.ts - - x-pack/test/functional/apps/advanced_settings/config.ts - - x-pack/test/functional/apps/aiops/config.ts - - x-pack/test/functional/apps/api_keys/config.ts - - x-pack/test/functional/apps/apm/config.ts - - x-pack/test/functional/apps/canvas/config.ts - - x-pack/test/functional/apps/cross_cluster_replication/config.ts - - x-pack/test/functional/apps/dashboard/group1/config.ts - - x-pack/test/functional/apps/dashboard/group2/config.ts - - x-pack/test/functional/apps/dashboard/group3/config.ts - - x-pack/test/functional/apps/data_views/config.ts - - x-pack/test/functional/apps/dev_tools/config.ts - - x-pack/test/functional/apps/discover/config.ts - - x-pack/test/functional/apps/graph/config.ts - - x-pack/test/functional/apps/grok_debugger/config.ts - - x-pack/test/functional/apps/home/config.ts - - x-pack/test/functional/apps/index_lifecycle_management/config.ts - - x-pack/test/functional/apps/index_management/config.ts - - x-pack/test/functional/apps/infra/config.ts - - x-pack/test/functional/apps/ingest_pipelines/config.ts - - x-pack/test/functional/apps/lens/group1/config.ts - - x-pack/test/functional/apps/lens/group2/config.ts - - x-pack/test/functional/apps/lens/group3/config.ts - - x-pack/test/functional/apps/lens/group4/config.ts - - x-pack/test/functional/apps/lens/group5/config.ts - - x-pack/test/functional/apps/lens/group6/config.ts - - x-pack/test/functional/apps/lens/open_in_lens/tsvb/config.ts - - x-pack/test/functional/apps/lens/open_in_lens/agg_based/config.ts - - x-pack/test/functional/apps/lens/open_in_lens/dashboard/config.ts - - x-pack/test/functional/apps/license_management/config.ts - - x-pack/test/functional/apps/logstash/config.ts - - x-pack/test/functional/apps/managed_content/config.ts - - x-pack/test/functional/apps/management/config.ts - - x-pack/test/functional/apps/maps/group1/config.ts - - x-pack/test/functional/apps/maps/group2/config.ts - - x-pack/test/functional/apps/maps/group3/config.ts - - x-pack/test/functional/apps/maps/group4/config.ts - - x-pack/test/functional/apps/ml/anomaly_detection_jobs/config.ts - - x-pack/test/functional/apps/ml/anomaly_detection_integrations/config.ts - - x-pack/test/functional/apps/ml/anomaly_detection_result_views/config.ts - - x-pack/test/functional/apps/ml/data_frame_analytics/config.ts - - x-pack/test/functional/apps/ml/data_visualizer/config.ts - - x-pack/test/functional/apps/ml/permissions/config.ts - - x-pack/test/functional/apps/ml/short_tests/config.ts - - x-pack/test/functional/apps/ml/stack_management_jobs/config.ts - - x-pack/test/functional/apps/monitoring/config.ts - - x-pack/test/functional/apps/observability_logs_explorer/config.ts - - x-pack/test/functional/apps/dataset_quality/config.ts - - x-pack/test/functional/apps/painless_lab/config.ts - - x-pack/test/functional/apps/remote_clusters/config.ts - - x-pack/test/functional/apps/reporting_management/config.ts - - x-pack/test/functional/apps/rollup_job/config.ts - - x-pack/test/functional/apps/saved_objects_management/config.ts - - x-pack/test/functional/apps/saved_query_management/config.ts - - x-pack/test/functional/apps/security/config.ts - - x-pack/test/functional/apps/slo/embeddables/config.ts - - x-pack/test/functional/apps/search_playground/config.ts - - x-pack/test/functional/apps/snapshot_restore/config.ts - - x-pack/test/functional/apps/spaces/config.ts - - x-pack/test/functional/apps/spaces/solution_view_flag_enabled/config.ts - - x-pack/test/functional/apps/status_page/config.ts - - x-pack/test/functional/apps/transform/creation/index_pattern/config.ts - - x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/config.ts - - x-pack/test/functional/apps/transform/actions/config.ts - - x-pack/test/functional/apps/transform/edit_clone/config.ts - - x-pack/test/functional/apps/transform/permissions/config.ts - - x-pack/test/functional/apps/transform/feature_controls/config.ts - - x-pack/test/functional/apps/upgrade_assistant/config.ts - - x-pack/test/functional/apps/uptime/config.ts - - x-pack/test/functional/apps/user_profiles/config.ts - - x-pack/test/functional/apps/visualize/config.ts - - x-pack/test/functional/apps/watcher/config.ts - - x-pack/test/functional/config_security_basic.ts - - x-pack/test/functional/config.ccs.ts - - x-pack/test/functional/config.firefox.js - - x-pack/test/functional/config.upgrade_assistant.ts - - x-pack/test/functional_cloud/config.ts - - x-pack/test/kubernetes_security/basic/config.ts - - x-pack/test/licensing_plugin/config.public.ts - - x-pack/test/licensing_plugin/config.ts - - x-pack/test/monitoring_api_integration/config.ts - - x-pack/test/observability_api_integration/basic/config.ts - - x-pack/test/observability_api_integration/trial/config.ts - - x-pack/test/observability_functional/with_rac_write.config.ts - - x-pack/test/observability_onboarding_api_integration/basic/config.ts - - x-pack/test/observability_onboarding_api_integration/cloud/config.ts - - x-pack/test/observability_ai_assistant_api_integration/enterprise/config.ts - - x-pack/test/observability_ai_assistant_functional/enterprise/config.ts - - x-pack/test/plugin_api_integration/config.ts - - x-pack/test/plugin_functional/config.ts - - x-pack/test/reporting_api_integration/reporting_and_security.config.ts - - x-pack/test/reporting_api_integration/reporting_without_security.config.ts - - x-pack/test/reporting_functional/reporting_and_deprecated_security.config.ts - - x-pack/test/reporting_functional/reporting_and_security.config.ts - - x-pack/test/reporting_functional/reporting_without_security.config.ts - - x-pack/test/rule_registry/security_and_spaces/config_basic.ts - - x-pack/test/rule_registry/security_and_spaces/config_trial.ts - - x-pack/test/rule_registry/spaces_only/config_basic.ts - - x-pack/test/rule_registry/spaces_only/config_trial.ts - - x-pack/test/saved_object_api_integration/security_and_spaces/config_basic.ts - - x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts - - x-pack/test/saved_object_api_integration/spaces_only/config.ts - - x-pack/test/saved_object_api_integration/user_profiles/config.ts - - x-pack/test/saved_object_tagging/api_integration/security_and_spaces/config.ts - - x-pack/test/saved_object_tagging/api_integration/tagging_api/config.ts - - x-pack/test/saved_object_tagging/api_integration/tagging_usage_collection/config.ts - - x-pack/test/saved_object_tagging/functional/config.ts - - x-pack/test/saved_objects_field_count/config.ts - - x-pack/test/search_sessions_integration/config.ts - - x-pack/test/security_api_integration/anonymous_es_anonymous.config.ts - - x-pack/test/security_api_integration/anonymous.config.ts - - x-pack/test/security_api_integration/api_keys.config.ts - - x-pack/test/security_api_integration/audit.config.ts - - x-pack/test/security_api_integration/http_bearer.config.ts - - x-pack/test/security_api_integration/http_no_auth_providers.config.ts - - x-pack/test/security_api_integration/kerberos_anonymous_access.config.ts - - x-pack/test/security_api_integration/kerberos.config.ts - - x-pack/test/security_api_integration/login_selector.config.ts - - x-pack/test/security_api_integration/oidc_implicit_flow.config.ts - - x-pack/test/security_api_integration/oidc.config.ts - - x-pack/test/security_api_integration/oidc.http2.config.ts - - x-pack/test/security_api_integration/pki.config.ts - - x-pack/test/security_api_integration/saml.config.ts - - x-pack/test/security_api_integration/saml.http2.config.ts - - x-pack/test/security_api_integration/saml_cloud.config.ts - - x-pack/test/security_api_integration/session_idle.config.ts - - x-pack/test/security_api_integration/session_invalidate.config.ts - - x-pack/test/security_api_integration/session_lifespan.config.ts - - x-pack/test/security_api_integration/session_concurrent_limit.config.ts - - x-pack/test/security_api_integration/token.config.ts - - x-pack/test/security_api_integration/user_profiles.config.ts - - x-pack/test/security_functional/login_selector.config.ts - - x-pack/test/security_functional/oidc.config.ts - - x-pack/test/security_functional/saml.config.ts - - x-pack/test/security_functional/insecure_cluster_warning.config.ts - - x-pack/test/security_functional/user_profiles.config.ts - - x-pack/test/security_functional/expired_session.config.ts - - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts - - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts - - x-pack/test/security_solution_endpoint/configs/integrations.config.ts - - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts - - x-pack/test/session_view/basic/config.ts - - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts - - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts - - x-pack/test/spaces_api_integration/security_and_spaces/config_trial.ts - - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_trial.ts - - x-pack/test/spaces_api_integration/spaces_only/config.ts - - x-pack/test/task_manager_claimer_mget/config.ts - - x-pack/test/ui_capabilities/security_and_spaces/config.ts - - x-pack/test/ui_capabilities/spaces_only/config.ts - - x-pack/test/upgrade_assistant_integration/config.js - - x-pack/test/usage_collection/config.ts - - x-pack/test_serverless/api_integration/test_suites/observability/config.ts - - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts - - x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts - - x-pack/test_serverless/api_integration/test_suites/observability/fleet/config.ts - - x-pack/test_serverless/api_integration/test_suites/search/config.ts - - x-pack/test_serverless/api_integration/test_suites/search/common_configs/config.group1.ts - - x-pack/test_serverless/api_integration/test_suites/security/config.ts - - x-pack/test_serverless/api_integration/test_suites/security/common_configs/config.group1.ts - - x-pack/test_serverless/api_integration/test_suites/security/fleet/config.ts - - x-pack/test_serverless/functional/test_suites/observability/config.ts - - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts - - x-pack/test_serverless/functional/test_suites/observability/config.saved_objects_management.ts - - x-pack/test_serverless/functional/test_suites/observability/config.context_awareness.ts - - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group1.ts - - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group2.ts - - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group3.ts - - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group4.ts - - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts - - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts - - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts - - x-pack/test_serverless/functional/test_suites/security/config.screenshots.ts - - x-pack/test_serverless/functional/test_suites/search/config.ts - - x-pack/test_serverless/functional/test_suites/search/config.examples.ts - - x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts - - x-pack/test_serverless/functional/test_suites/search/config.saved_objects_management.ts - - x-pack/test_serverless/functional/test_suites/search/config.context_awareness.ts - - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group1.ts - - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group2.ts - - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group3.ts - - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts - - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts - - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts - - x-pack/test_serverless/functional/test_suites/security/config.ts - - x-pack/test_serverless/functional/test_suites/security/config.examples.ts - - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.basic.ts - - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.essentials.ts - - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.agentless.ts - - x-pack/test_serverless/functional/test_suites/security/config.saved_objects_management.ts - - x-pack/test_serverless/functional/test_suites/security/config.context_awareness.ts - - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts - - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group2.ts - - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group3.ts - - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group4.ts - - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group5.ts - - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group6.ts - - x-pack/performance/journeys_e2e/aiops_log_rate_analysis.ts - - x-pack/performance/journeys_e2e/ecommerce_dashboard.ts - - x-pack/performance/journeys_e2e/ecommerce_dashboard_map_only.ts - - x-pack/performance/journeys_e2e/flight_dashboard.ts - - x-pack/performance/journeys_e2e/login.ts - - x-pack/performance/journeys_e2e/many_fields_discover.ts - - x-pack/performance/journeys_e2e/many_fields_discover_esql.ts - - x-pack/performance/journeys_e2e/many_fields_lens_editor.ts - - x-pack/performance/journeys_e2e/many_fields_transform.ts - - x-pack/performance/journeys_e2e/tsdb_logs_data_visualizer.ts - - x-pack/performance/journeys_e2e/promotion_tracking_dashboard.ts - - x-pack/performance/journeys_e2e/web_logs_dashboard.ts - - x-pack/performance/journeys_e2e/data_stress_test_lens.ts - - x-pack/performance/journeys_e2e/ecommerce_dashboard_saved_search_only.ts - - x-pack/performance/journeys_e2e/ecommerce_dashboard_tsvb_gauge_only.ts - - x-pack/performance/journeys_e2e/dashboard_listing_page.ts - - x-pack/performance/journeys_e2e/tags_listing_page.ts - - x-pack/performance/journeys_e2e/cloud_security_dashboard.ts - - x-pack/performance/journeys_e2e/apm_service_inventory.ts - - x-pack/performance/journeys_e2e/infra_hosts_view.ts - - x-pack/test/custom_branding/config.ts - - x-pack/test/profiling_api_integration/cloud/config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/actions/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/actions/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/date_numeric_types/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/date_numeric_types/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/long/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/long/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_gaps/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_gaps/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/large_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/large_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/explore/hosts/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/explore/hosts/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/explore/network/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/explore/network/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/explore/users/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/explore/users/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/explore/overview/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/explore/overview/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/security_and_spaces/configs/ess.basic.config.ts - - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/security_and_spaces/configs/ess.trial.config.ts - - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/artifacts/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/artifacts/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/authentication/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/authentication/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts - - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts diff --git a/.buildkite/ftr_configs_manifests.json b/.buildkite/ftr_configs_manifests.json new file mode 100644 index 0000000000000..fae4ee580b48d --- /dev/null +++ b/.buildkite/ftr_configs_manifests.json @@ -0,0 +1,14 @@ +{ + "stateful": [ + ".buildkite/ftr_platform_stateful_configs.yml", + ".buildkite/ftr_oblt_stateful_configs.yml", + ".buildkite/ftr_security_stateful_configs.yml", + ".buildkite/ftr_search_stateful_configs.yml" + ], + "serverless" : [ + ".buildkite/ftr_base_serverless_configs.yml", + ".buildkite/ftr_oblt_serverless_configs.yml", + ".buildkite/ftr_security_serverless_configs.yml", + ".buildkite/ftr_search_serverless_configs.yml" + ] +} \ No newline at end of file diff --git a/.buildkite/ftr_oblt_serverless_configs.yml b/.buildkite/ftr_oblt_serverless_configs.yml new file mode 100644 index 0000000000000..9534e62926f06 --- /dev/null +++ b/.buildkite/ftr_oblt_serverless_configs.yml @@ -0,0 +1,27 @@ +disabled: + # Base config files, only necessary to inform config finding script + + # Cypress configs, for now these are still run manually + - x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts + - x-pack/test_serverless/functional/test_suites/observability/cypress/config_runner.ts + + # Serverless feature flag config files (move to enabled after tests are added) + - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts + +defaultQueue: 'n2-4-spot' +enabled: + - x-pack/test_serverless/api_integration/test_suites/observability/config.ts + - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts + - x-pack/test_serverless/api_integration/test_suites/observability/common_configs/config.group1.ts + - x-pack/test_serverless/api_integration/test_suites/observability/fleet/config.ts + - x-pack/test_serverless/functional/test_suites/observability/config.ts + - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts + - x-pack/test_serverless/functional/test_suites/observability/config.saved_objects_management.ts + - x-pack/test_serverless/functional/test_suites/observability/config.context_awareness.ts + - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group1.ts + - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group2.ts + - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group3.ts + - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group4.ts + - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts + - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts + - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts diff --git a/.buildkite/ftr_oblt_stateful_configs.yml b/.buildkite/ftr_oblt_stateful_configs.yml new file mode 100644 index 0000000000000..7cecfd5e8d665 --- /dev/null +++ b/.buildkite/ftr_oblt_stateful_configs.yml @@ -0,0 +1,44 @@ +disabled: + # Cypress configs, for now these are still run manually + - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_open.ts + - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_runner.ts + - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config.ts + - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_open.ts + - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts + - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts + - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts + - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts + - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts + + # Elastic Synthetics configs + - x-pack/plugins/observability_solution/uptime/e2e/uptime/synthetics_run.ts + - x-pack/plugins/observability_solution/synthetics/e2e/config.ts + - x-pack/plugins/observability_solution/synthetics/e2e/synthetics/synthetics_run.ts + - x-pack/plugins/observability_solution/exploratory_view/e2e/synthetics_run.ts + - x-pack/plugins/observability_solution/ux/e2e/synthetics_run.ts + - x-pack/plugins/observability_solution/slo/e2e/synthetics_run.ts + +defaultQueue: 'n2-4-spot' +enabled: + - x-pack/test/alerting_api_integration/observability/config.ts + - x-pack/test/api_integration/apis/logs_ui/config.ts + - x-pack/test/api_integration/apis/metrics_ui/config.ts + - x-pack/test/api_integration/apis/osquery/config.ts + - x-pack/test/api_integration/apis/synthetics/config.ts + - x-pack/test/api_integration/apis/slos/config.ts + - x-pack/test/api_integration/apis/uptime/config.ts + - x-pack/test/dataset_quality_api_integration/basic/config.ts + - x-pack/test/functional/apps/observability_logs_explorer/config.ts + - x-pack/test/functional/apps/dataset_quality/config.ts + - x-pack/test/functional/apps/slo/embeddables/config.ts + - x-pack/test/functional/apps/uptime/config.ts + - x-pack/test/observability_api_integration/basic/config.ts + - x-pack/test/observability_api_integration/trial/config.ts + - x-pack/test/observability_functional/with_rac_write.config.ts + - x-pack/test/observability_onboarding_api_integration/basic/config.ts + - x-pack/test/observability_onboarding_api_integration/cloud/config.ts + - x-pack/test/observability_ai_assistant_api_integration/enterprise/config.ts + - x-pack/test/observability_ai_assistant_functional/enterprise/config.ts + - x-pack/test/profiling_api_integration/cloud/config.ts + - x-pack/test/functional/apps/apm/config.ts + \ No newline at end of file diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml new file mode 100644 index 0000000000000..f25eaa634e5a3 --- /dev/null +++ b/.buildkite/ftr_platform_stateful_configs.yml @@ -0,0 +1,360 @@ +disabled: + # Base config files, only necessary to inform config finding script + - test/functional/config.base.js + - test/functional/firefox/config.base.ts + - x-pack/test/functional/config.base.js + - x-pack/test/localization/config.base.ts + - test/server_integration/config.base.js + - x-pack/test/functional_with_es_ssl/config.base.ts + - x-pack/test/api_integration/config.ts + - x-pack/test/fleet_api_integration/config.base.ts + + # QA suites that are run out-of-band + - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js + - x-pack/test/upgrade/config.ts + - test/functional/config.edge.js + - x-pack/test/functional/config.edge.js + + # Configs that exist but weren't running in CI when this file was introduced + - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts + - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/config.ts + - x-pack/test/alerting_api_integration/spaces_only_legacy/config.ts + - x-pack/test/cloud_integration/config.ts + - x-pack/test/load/config.ts + - x-pack/test/plugin_api_perf/config.js + - x-pack/test/screenshot_creation/config.ts + - x-pack/test/fleet_packages/config.ts + + # Scalability testing config that we run in its own pipeline + - x-pack/test/scalability/config.ts + + # Cypress configs, for now these are still run manually + - x-pack/test/fleet_cypress/cli_config.ts + - x-pack/test/fleet_cypress/config.ts + - x-pack/test/fleet_cypress/visual_config.ts + + # http/2 security muted tests + - x-pack/test/security_functional/saml.http2.config.ts + - x-pack/test/security_functional/oidc.http2.config.ts + +defaultQueue: 'n2-4-spot' +enabled: + - test/accessibility/config.ts + - test/analytics/config.ts + - test/api_integration/config.js + - test/examples/config.js + - test/functional/apps/bundles/config.ts + - test/functional/apps/console/monaco/config.ts + - test/functional/apps/console/ace/config.ts + - test/functional/apps/context/config.ts + - test/functional/apps/dashboard_elements/controls/common/config.ts + - test/functional/apps/dashboard_elements/controls/options_list/config.ts + - test/functional/apps/dashboard_elements/image_embeddable/config.ts + - test/functional/apps/dashboard_elements/input_control_vis/config.ts + - test/functional/apps/dashboard_elements/links/config.ts + - test/functional/apps/dashboard_elements/markdown/config.ts + - test/functional/apps/dashboard/group1/config.ts + - test/functional/apps/dashboard/group2/config.ts + - test/functional/apps/dashboard/group3/config.ts + - test/functional/apps/dashboard/group4/config.ts + - test/functional/apps/dashboard/group5/config.ts + - test/functional/apps/dashboard/group6/config.ts + - test/functional/apps/discover/ccs_compatibility/config.ts + - test/functional/apps/discover/classic/config.ts + - test/functional/apps/discover/embeddable/config.ts + - test/functional/apps/discover/esql/config.ts + - test/functional/apps/discover/group1/config.ts + - test/functional/apps/discover/group2_data_grid1/config.ts + - test/functional/apps/discover/group2_data_grid2/config.ts + - test/functional/apps/discover/group2_data_grid3/config.ts + - test/functional/apps/discover/group3/config.ts + - test/functional/apps/discover/group4/config.ts + - test/functional/apps/discover/group5/config.ts + - test/functional/apps/discover/group6/config.ts + - test/functional/apps/discover/group7/config.ts + - test/functional/apps/discover/group8/config.ts + - test/functional/apps/discover/context_awareness/config.ts + - test/functional/apps/getting_started/config.ts + - test/functional/apps/home/config.ts + - test/functional/apps/kibana_overview/config.ts + - test/functional/apps/management/config.ts + - test/functional/apps/saved_objects_management/config.ts + - test/functional/apps/sharing/config.ts + - test/functional/apps/status_page/config.ts + - test/functional/apps/visualize/group1/config.ts + - test/functional/apps/visualize/group2/config.ts + - test/functional/apps/visualize/group3/config.ts + - test/functional/apps/visualize/group4/config.ts + - test/functional/apps/visualize/group5/config.ts + - test/functional/apps/visualize/group6/config.ts + - test/functional/apps/visualize/replaced_vislib_chart_types/config.ts + - test/functional/config.ccs.ts + - test/functional/firefox/console.config.ts + - test/functional/firefox/dashboard.config.ts + - test/functional/firefox/discover.config.ts + - test/functional/firefox/home.config.ts + - test/functional/firefox/visualize.config.ts + - test/health_gateway/config.ts + - test/interactive_setup_api_integration/enrollment_flow.config.ts + - test/interactive_setup_api_integration/manual_configuration_flow_without_tls.config.ts + - test/interactive_setup_api_integration/manual_configuration_flow.config.ts + - test/interactive_setup_functional/enrollment_token.config.ts + - test/interactive_setup_functional/manual_configuration_without_security.config.ts + - test/interactive_setup_functional/manual_configuration_without_tls.config.ts + - test/interactive_setup_functional/manual_configuration.config.ts + - test/interpreter_functional/config.ts + - test/node_roles_functional/all.config.ts + - test/node_roles_functional/background_tasks.config.ts + - test/node_roles_functional/ui.config.ts + - test/plugin_functional/config.ts + - test/server_integration/http/platform/config.status.ts + - test/server_integration/http/platform/config.ts + - test/server_integration/http/ssl_redirect/config.ts + - test/server_integration/http/ssl_with_p12_intermediate/config.js + - test/server_integration/http/ssl_with_p12/config.js + - test/server_integration/http/ssl/config.js + - test/ui_capabilities/newsfeed_err/config.ts + - x-pack/test/accessibility/apps/group1/config.ts + - x-pack/test/accessibility/apps/group2/config.ts + - x-pack/test/accessibility/apps/group3/config.ts + - x-pack/test/localization/config.ja_jp.ts + - x-pack/test/localization/config.fr_fr.ts + - x-pack/test/localization/config.zh_cn.ts + - x-pack/test/alerting_api_integration/basic/config.ts + - x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts + - x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts + - x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts + - x-pack/test/alerting_api_integration/security_and_spaces/group4/config.ts + - x-pack/test/alerting_api_integration/security_and_spaces/group3/config_with_schedule_circuit_breaker.ts + - x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts + - x-pack/test/alerting_api_integration/security_and_spaces/group4/config_non_dedicated_task_runner.ts + - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/config.ts + - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group2/config.ts + - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group3/config.ts + - x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/config.ts + - x-pack/test/alerting_api_integration/spaces_only/tests/actions/config.ts + - x-pack/test/alerting_api_integration/spaces_only/tests/action_task_params/config.ts + - x-pack/test/api_integration_basic/config.ts + - x-pack/test/api_integration/config_security_basic.ts + - x-pack/test/api_integration/config_security_trial.ts + - x-pack/test/api_integration/apis/aiops/config.ts + - x-pack/test/api_integration/apis/cases/config.ts + - x-pack/test/api_integration/apis/content_management/config.ts + - x-pack/test/api_integration/apis/console/config.ts + - x-pack/test/api_integration/apis/es/config.ts + - x-pack/test/api_integration/apis/features/config.ts + - x-pack/test/api_integration/apis/file_upload/config.ts + - x-pack/test/api_integration/apis/grok_debugger/config.ts + - x-pack/test/api_integration/apis/kibana/config.ts + - x-pack/test/api_integration/apis/lists/config.ts + - x-pack/test/api_integration/apis/logs_ui/config.ts + - x-pack/test/api_integration/apis/logstash/config.ts + - x-pack/test/api_integration/apis/management/config.ts + - x-pack/test/api_integration/apis/management/index_management/disabled_data_enrichers/config.ts + - x-pack/test/api_integration/apis/maps/config.ts + - x-pack/test/api_integration/apis/metrics_ui/config.ts + - x-pack/test/api_integration/apis/ml/config.ts + - x-pack/test/api_integration/apis/monitoring/config.ts + - x-pack/test/api_integration/apis/monitoring_collection/config.ts + - x-pack/test/api_integration/apis/osquery/config.ts + - x-pack/test/api_integration/apis/painless_lab/config.ts + - x-pack/test/api_integration/apis/search/config.ts + - x-pack/test/api_integration/apis/searchprofiler/config.ts + - x-pack/test/api_integration/apis/security/config.ts + - x-pack/test/api_integration/apis/spaces/config.ts + - x-pack/test/api_integration/apis/stats/config.ts + - x-pack/test/api_integration/apis/status/config.ts + - x-pack/test/api_integration/apis/synthetics/config.ts + - x-pack/test/api_integration/apis/telemetry/config.ts + - x-pack/test/api_integration/apis/transform/config.ts + - x-pack/test/api_integration/apis/upgrade_assistant/config.ts + - x-pack/test/api_integration/apis/watcher/config.ts + - x-pack/test/banners_functional/config.ts + - x-pack/test/cases_api_integration/security_and_spaces/config_basic.ts + - x-pack/test/cases_api_integration/security_and_spaces/config_trial.ts + - x-pack/test/cases_api_integration/security_and_spaces/config_no_public_base_url.ts + - x-pack/test/cases_api_integration/spaces_only/config.ts + - x-pack/test/disable_ems/config.ts + - x-pack/test/encrypted_saved_objects_api_integration/config.ts + - x-pack/test/examples/config.ts + - x-pack/test/fleet_api_integration/config.agent.ts + - x-pack/test/fleet_api_integration/config.agent_policy.ts + - x-pack/test/fleet_api_integration/config.epm.ts + - x-pack/test/fleet_api_integration/config.fleet.ts + - x-pack/test/fleet_api_integration/config.package_policy.ts + - x-pack/test/fleet_api_integration/config.space_awareness.ts + - x-pack/test/fleet_functional/config.ts + - x-pack/test/ftr_apis/security_and_spaces/config.ts + - x-pack/test/functional_basic/apps/ml/permissions/config.ts + - x-pack/test/functional_basic/apps/ml/data_visualizer/group1/config.ts + - x-pack/test/functional_basic/apps/ml/data_visualizer/group2/config.ts + - x-pack/test/functional_basic/apps/ml/data_visualizer/group3/config.ts + - x-pack/test/functional_basic/apps/transform/creation/index_pattern/config.ts + - x-pack/test/functional_basic/apps/transform/actions/config.ts + - x-pack/test/functional_basic/apps/transform/edit_clone/config.ts + - x-pack/test/functional_basic/apps/transform/creation/runtime_mappings_saved_search/config.ts + - x-pack/test/functional_basic/apps/transform/permissions/config.ts + - x-pack/test/functional_basic/apps/transform/feature_controls/config.ts + - x-pack/test/functional_cors/config.ts + - x-pack/test/functional_embedded/config.ts + - x-pack/test/functional_execution_context/config.ts + - x-pack/test/functional_with_es_ssl/apps/cases/group1/config.ts + - x-pack/test/functional_with_es_ssl/apps/cases/group2/config.ts + - x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/config.ts + - x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/config.ts + - x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/shared/config.ts + - x-pack/test/functional/apps/advanced_settings/config.ts + - x-pack/test/functional/apps/aiops/config.ts + - x-pack/test/functional/apps/api_keys/config.ts + - x-pack/test/functional/apps/canvas/config.ts + - x-pack/test/functional/apps/cross_cluster_replication/config.ts + - x-pack/test/functional/apps/dashboard/group1/config.ts + - x-pack/test/functional/apps/dashboard/group2/config.ts + - x-pack/test/functional/apps/dashboard/group3/config.ts + - x-pack/test/functional/apps/data_views/config.ts + - x-pack/test/functional/apps/dev_tools/config.ts + - x-pack/test/functional/apps/discover/config.ts + - x-pack/test/functional/apps/graph/config.ts + - x-pack/test/functional/apps/grok_debugger/config.ts + - x-pack/test/functional/apps/home/config.ts + - x-pack/test/functional/apps/index_lifecycle_management/config.ts + - x-pack/test/functional/apps/index_management/config.ts + - x-pack/test/functional/apps/infra/config.ts + - x-pack/test/functional/apps/ingest_pipelines/config.ts + - x-pack/test/functional/apps/lens/group1/config.ts + - x-pack/test/functional/apps/lens/group2/config.ts + - x-pack/test/functional/apps/lens/group3/config.ts + - x-pack/test/functional/apps/lens/group4/config.ts + - x-pack/test/functional/apps/lens/group5/config.ts + - x-pack/test/functional/apps/lens/group6/config.ts + - x-pack/test/functional/apps/lens/open_in_lens/tsvb/config.ts + - x-pack/test/functional/apps/lens/open_in_lens/agg_based/config.ts + - x-pack/test/functional/apps/lens/open_in_lens/dashboard/config.ts + - x-pack/test/functional/apps/license_management/config.ts + - x-pack/test/functional/apps/logstash/config.ts + - x-pack/test/functional/apps/managed_content/config.ts + - x-pack/test/functional/apps/management/config.ts + - x-pack/test/functional/apps/maps/group1/config.ts + - x-pack/test/functional/apps/maps/group2/config.ts + - x-pack/test/functional/apps/maps/group3/config.ts + - x-pack/test/functional/apps/maps/group4/config.ts + - x-pack/test/functional/apps/ml/anomaly_detection_jobs/config.ts + - x-pack/test/functional/apps/ml/anomaly_detection_integrations/config.ts + - x-pack/test/functional/apps/ml/anomaly_detection_result_views/config.ts + - x-pack/test/functional/apps/ml/data_frame_analytics/config.ts + - x-pack/test/functional/apps/ml/data_visualizer/config.ts + - x-pack/test/functional/apps/ml/permissions/config.ts + - x-pack/test/functional/apps/ml/short_tests/config.ts + - x-pack/test/functional/apps/ml/stack_management_jobs/config.ts + - x-pack/test/functional/apps/monitoring/config.ts + - x-pack/test/functional/apps/painless_lab/config.ts + - x-pack/test/functional/apps/remote_clusters/config.ts + - x-pack/test/functional/apps/reporting_management/config.ts + - x-pack/test/functional/apps/rollup_job/config.ts + - x-pack/test/functional/apps/saved_objects_management/config.ts + - x-pack/test/functional/apps/saved_query_management/config.ts + - x-pack/test/functional/apps/security/config.ts + - x-pack/test/functional/apps/snapshot_restore/config.ts + - x-pack/test/functional/apps/spaces/config.ts + - x-pack/test/functional/apps/spaces/solution_view_flag_enabled/config.ts + - x-pack/test/functional/apps/status_page/config.ts + - x-pack/test/functional/apps/transform/creation/index_pattern/config.ts + - x-pack/test/functional/apps/transform/creation/runtime_mappings_saved_search/config.ts + - x-pack/test/functional/apps/transform/actions/config.ts + - x-pack/test/functional/apps/transform/edit_clone/config.ts + - x-pack/test/functional/apps/transform/permissions/config.ts + - x-pack/test/functional/apps/transform/feature_controls/config.ts + - x-pack/test/functional/apps/upgrade_assistant/config.ts + - x-pack/test/functional/apps/user_profiles/config.ts + - x-pack/test/functional/apps/visualize/config.ts + - x-pack/test/functional/apps/watcher/config.ts + - x-pack/test/functional/config_security_basic.ts + - x-pack/test/functional/config.ccs.ts + - x-pack/test/functional/config.firefox.js + - x-pack/test/functional/config.upgrade_assistant.ts + - x-pack/test/functional_cloud/config.ts + - x-pack/test/kubernetes_security/basic/config.ts + - x-pack/test/licensing_plugin/config.public.ts + - x-pack/test/licensing_plugin/config.ts + - x-pack/test/monitoring_api_integration/config.ts + - x-pack/test/plugin_api_integration/config.ts + - x-pack/test/plugin_functional/config.ts + - x-pack/test/reporting_api_integration/reporting_and_security.config.ts + - x-pack/test/reporting_api_integration/reporting_without_security.config.ts + - x-pack/test/reporting_functional/reporting_and_deprecated_security.config.ts + - x-pack/test/reporting_functional/reporting_and_security.config.ts + - x-pack/test/reporting_functional/reporting_without_security.config.ts + - x-pack/test/rule_registry/security_and_spaces/config_basic.ts + - x-pack/test/rule_registry/security_and_spaces/config_trial.ts + - x-pack/test/rule_registry/spaces_only/config_basic.ts + - x-pack/test/rule_registry/spaces_only/config_trial.ts + - x-pack/test/saved_object_api_integration/security_and_spaces/config_basic.ts + - x-pack/test/saved_object_api_integration/security_and_spaces/config_trial.ts + - x-pack/test/saved_object_api_integration/spaces_only/config.ts + - x-pack/test/saved_object_api_integration/user_profiles/config.ts + - x-pack/test/saved_object_tagging/api_integration/security_and_spaces/config.ts + - x-pack/test/saved_object_tagging/api_integration/tagging_api/config.ts + - x-pack/test/saved_object_tagging/api_integration/tagging_usage_collection/config.ts + - x-pack/test/saved_object_tagging/functional/config.ts + - x-pack/test/saved_objects_field_count/config.ts + - x-pack/test/search_sessions_integration/config.ts + - x-pack/test/security_api_integration/anonymous_es_anonymous.config.ts + - x-pack/test/security_api_integration/anonymous.config.ts + - x-pack/test/security_api_integration/api_keys.config.ts + - x-pack/test/security_api_integration/audit.config.ts + - x-pack/test/security_api_integration/http_bearer.config.ts + - x-pack/test/security_api_integration/http_no_auth_providers.config.ts + - x-pack/test/security_api_integration/kerberos_anonymous_access.config.ts + - x-pack/test/security_api_integration/kerberos.config.ts + - x-pack/test/security_api_integration/login_selector.config.ts + - x-pack/test/security_api_integration/oidc_implicit_flow.config.ts + - x-pack/test/security_api_integration/oidc.config.ts + - x-pack/test/security_api_integration/oidc.http2.config.ts + - x-pack/test/security_api_integration/pki.config.ts + - x-pack/test/security_api_integration/saml.config.ts + - x-pack/test/security_api_integration/saml.http2.config.ts + - x-pack/test/security_api_integration/saml_cloud.config.ts + - x-pack/test/security_api_integration/session_idle.config.ts + - x-pack/test/security_api_integration/session_invalidate.config.ts + - x-pack/test/security_api_integration/session_lifespan.config.ts + - x-pack/test/security_api_integration/session_concurrent_limit.config.ts + - x-pack/test/security_api_integration/token.config.ts + - x-pack/test/security_api_integration/user_profiles.config.ts + - x-pack/test/security_functional/login_selector.config.ts + - x-pack/test/security_functional/oidc.config.ts + - x-pack/test/security_functional/saml.config.ts + - x-pack/test/security_functional/insecure_cluster_warning.config.ts + - x-pack/test/security_functional/user_profiles.config.ts + - x-pack/test/security_functional/expired_session.config.ts + - x-pack/test/session_view/basic/config.ts + - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts + - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts + - x-pack/test/spaces_api_integration/security_and_spaces/config_trial.ts + - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_trial.ts + - x-pack/test/spaces_api_integration/spaces_only/config.ts + - x-pack/test/task_manager_claimer_mget/config.ts + - x-pack/test/ui_capabilities/security_and_spaces/config.ts + - x-pack/test/ui_capabilities/spaces_only/config.ts + - x-pack/test/upgrade_assistant_integration/config.js + - x-pack/test/usage_collection/config.ts + - x-pack/performance/journeys_e2e/aiops_log_rate_analysis.ts + - x-pack/performance/journeys_e2e/ecommerce_dashboard.ts + - x-pack/performance/journeys_e2e/ecommerce_dashboard_map_only.ts + - x-pack/performance/journeys_e2e/flight_dashboard.ts + - x-pack/performance/journeys_e2e/login.ts + - x-pack/performance/journeys_e2e/many_fields_discover.ts + - x-pack/performance/journeys_e2e/many_fields_discover_esql.ts + - x-pack/performance/journeys_e2e/many_fields_lens_editor.ts + - x-pack/performance/journeys_e2e/many_fields_transform.ts + - x-pack/performance/journeys_e2e/tsdb_logs_data_visualizer.ts + - x-pack/performance/journeys_e2e/promotion_tracking_dashboard.ts + - x-pack/performance/journeys_e2e/web_logs_dashboard.ts + - x-pack/performance/journeys_e2e/data_stress_test_lens.ts + - x-pack/performance/journeys_e2e/ecommerce_dashboard_saved_search_only.ts + - x-pack/performance/journeys_e2e/ecommerce_dashboard_tsvb_gauge_only.ts + - x-pack/performance/journeys_e2e/dashboard_listing_page.ts + - x-pack/performance/journeys_e2e/tags_listing_page.ts + - x-pack/performance/journeys_e2e/cloud_security_dashboard.ts + - x-pack/performance/journeys_e2e/apm_service_inventory.ts + - x-pack/performance/journeys_e2e/infra_hosts_view.ts + - x-pack/test/custom_branding/config.ts diff --git a/.buildkite/ftr_search_serverless_configs.yml b/.buildkite/ftr_search_serverless_configs.yml new file mode 100644 index 0000000000000..73b6238027bce --- /dev/null +++ b/.buildkite/ftr_search_serverless_configs.yml @@ -0,0 +1,18 @@ +disabled: + # Base config files, only necessary to inform config finding script + +defaultQueue: 'n2-4-spot' +enabled: + - x-pack/test_serverless/api_integration/test_suites/search/config.ts + - x-pack/test_serverless/api_integration/test_suites/search/common_configs/config.group1.ts + - x-pack/test_serverless/functional/test_suites/search/config.ts + - x-pack/test_serverless/functional/test_suites/search/config.examples.ts + - x-pack/test_serverless/functional/test_suites/search/config.screenshots.ts + - x-pack/test_serverless/functional/test_suites/search/config.saved_objects_management.ts + - x-pack/test_serverless/functional/test_suites/search/config.context_awareness.ts + - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group1.ts + - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group2.ts + - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group3.ts + - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts + - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts + - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts diff --git a/.buildkite/ftr_search_stateful_configs.yml b/.buildkite/ftr_search_stateful_configs.yml new file mode 100644 index 0000000000000..f86df553cdfc3 --- /dev/null +++ b/.buildkite/ftr_search_stateful_configs.yml @@ -0,0 +1,9 @@ +disabled: + # Base config files, only necessary to inform config finding script + - x-pack/test/functional_enterprise_search/base_config.ts + + # Cypress configs, for now these are still run manually + - x-pack/test/functional_enterprise_search/cypress.config.ts + - x-pack/test/functional_enterprise_search/visual_config.ts + - x-pack/test/functional_enterprise_search/cli_config.ts + - x-pack/test/functional/apps/search_playground/config.ts diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml new file mode 100644 index 0000000000000..51e3eba941c6b --- /dev/null +++ b/.buildkite/ftr_security_serverless_configs.yml @@ -0,0 +1,97 @@ +disabled: + # Base config files, only necessary to inform config finding script + - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts + - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts + - x-pack/test/security_solution_api_integration/config/serverless/config.base.edr_workflows.ts + + # Cypress configs, for now these are still run manually + - x-pack/test/defend_workflows_cypress/serverless_config.ts + - x-pack/test/osquery_cypress/serverless_cli_config.ts + - x-pack/test_serverless/functional/test_suites/security/cypress/security_config.ts + - x-pack/test/security_solution_cypress/serverless_config.ts + + # Serverless base config files + - x-pack/test_serverless/api_integration/config.base.ts + - x-pack/test_serverless/functional/config.base.ts + - x-pack/test_serverless/shared/config.base.ts + + # Serverless feature flag config files (move to enabled after tests are added) + - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts + - x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts + - x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts + - x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts + - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts + +defaultQueue: 'n2-4-spot' +enabled: + - x-pack/test_serverless/api_integration/test_suites/security/config.ts + - x-pack/test_serverless/api_integration/test_suites/security/common_configs/config.group1.ts + - x-pack/test_serverless/api_integration/test_suites/security/fleet/config.ts + - x-pack/test_serverless/functional/test_suites/security/config.screenshots.ts + - x-pack/test_serverless/functional/test_suites/security/config.ts + - x-pack/test_serverless/functional/test_suites/security/config.examples.ts + - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.basic.ts + - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.essentials.ts + - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.agentless.ts + - x-pack/test_serverless/functional/test_suites/security/config.saved_objects_management.ts + - x-pack/test_serverless/functional/test_suites/security/config.context_awareness.ts + - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts + - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group2.ts + - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group3.ts + - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group4.ts + - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group5.ts + - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group6.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/actions/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/date_numeric_types/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/long/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_gaps/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/large_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/explore/hosts/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/explore/network/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/explore/users/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/explore/overview/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/artifacts/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/authentication/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts + - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts + - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts diff --git a/.buildkite/ftr_security_stateful_configs.yml b/.buildkite/ftr_security_stateful_configs.yml new file mode 100644 index 0000000000000..8f1605b363e3d --- /dev/null +++ b/.buildkite/ftr_security_stateful_configs.yml @@ -0,0 +1,84 @@ +disabled: + # Base config files, only necessary to inform config finding script + - x-pack/test/security_solution_api_integration/config/ess/config.base.ts + - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts + - x-pack/test/security_solution_api_integration/config/ess/config.base.edr_workflows.trial.ts + - x-pack/test/security_solution_api_integration/config/ess/config.base.edr_workflows.ts + - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts + - x-pack/test/security_solution_endpoint/configs/config.base.ts + - x-pack/test/security_solution_endpoint/config.base.ts + - x-pack/test/security_solution_endpoint_api_int/config.base.ts + + # QA suites that are run out-of-band + - x-pack/test/cloud_security_posture_functional/config.cloud.ts + + # Cypress configs, for now these are still run manually + - x-pack/test/defend_workflows_cypress/cli_config.ts + - x-pack/test/defend_workflows_cypress/config.ts + - x-pack/test/osquery_cypress/cli_config.ts + - x-pack/test/osquery_cypress/config.ts + - x-pack/test/osquery_cypress/visual_config.ts + - x-pack/test/security_solution_cypress/cli_config.ts + - x-pack/test/security_solution_cypress/config.ts + - x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts + - x-pack/test/threat_intelligence_cypress/config.ts + +defaultQueue: 'n2-4-spot' +enabled: + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/actions/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/alerts/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/date_numeric_types/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/ips/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/keyword/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/long/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/operators_data_types/text/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_gaps/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_creation/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_patch/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/bundled_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/large_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/management/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/update_prebuilt_rules_package/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_bulk_actions/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_delete/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_import_export/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_management/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/explore/hosts/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/explore/network/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/explore/users/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/explore/overview/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/security_and_spaces/configs/ess.basic.config.ts + - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/security_and_spaces/configs/ess.trial.config.ts + - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/artifacts/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/authentication/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts + - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts + - x-pack/test/security_solution_endpoint/configs/integrations.config.ts + - x-pack/test/api_integration/apis/cloud_security_posture/config.ts + - x-pack/test/cloud_security_posture_api/config.ts + - x-pack/test/cloud_security_posture_functional/config.ts \ No newline at end of file diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts index a24214b1e62b0..af77cd3138c46 100644 --- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts +++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts @@ -16,8 +16,11 @@ import { BuildkiteClient, BuildkiteStep } from '../buildkite'; import { CiStatsClient, TestGroupRunOrderResponse } from './client'; import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json'; +import { serverless, stateful } from '../../ftr_configs_manifests.json'; import { expandAgentQueue } from '#pipeline-utils'; +const ALL_FTR_MANIFEST_REL_PATHS = serverless.concat(stateful); + type RunGroup = TestGroupRunOrderResponse['types'][0]; const getRequiredEnv = (name: string) => { @@ -107,15 +110,50 @@ function isObj(x: unknown): x is Record { return typeof x === 'object' && x !== null; } +interface FtrConfigsManifest { + defaultQueue?: string; + disabled?: string[]; + enabled?: Array; +} + function getEnabledFtrConfigs(patterns?: string[]) { + const configs: { + enabled: Array; + defaultQueue: string | undefined; + } = { enabled: [], defaultQueue: undefined }; + const uniqueQueues = new Set(); + + for (const manifestRelPath of ALL_FTR_MANIFEST_REL_PATHS) { + try { + const ymlData = loadYaml(Fs.readFileSync(manifestRelPath, 'utf8')); + if (!isObj(ymlData)) { + throw new Error('expected yaml file to parse to an object'); + } + const manifest = ymlData as FtrConfigsManifest; + + configs.enabled.push(...(manifest?.enabled ?? [])); + if (manifest.defaultQueue) { + uniqueQueues.add(manifest.defaultQueue); + } + } catch (_) { + const error = _ instanceof Error ? _ : new Error(`${_} thrown`); + throw new Error(`unable to parse ${manifestRelPath} file: ${error.message}`); + } + } + try { - const configs = loadYaml(Fs.readFileSync('.buildkite/ftr_configs.yml', 'utf8')); - if (!isObj(configs)) { - throw new Error('expected yaml file to parse to an object'); + if (configs.enabled.length === 0) { + throw new Error('expected yaml files to have at least 1 "enabled" key'); } - if (!configs.enabled) { - throw new Error('expected yaml file to have an "enabled" key'); + if (uniqueQueues.size !== 1) { + throw Error( + `FTR manifest yml files should define the same 'defaultQueue', but found different ones: ${[ + ...uniqueQueues, + ].join(' ')}` + ); } + configs.defaultQueue = uniqueQueues.values().next().value; + if ( !Array.isArray(configs.enabled) || !configs.enabled.every( @@ -149,11 +187,10 @@ function getEnabledFtrConfigs(patterns?: string[]) { ftrConfigsByQueue.set(queue, [path]); } } - return { defaultQueue, ftrConfigsByQueue }; } catch (_) { const error = _ instanceof Error ? _ : new Error(`${_} thrown`); - throw new Error(`unable to parse ftr_configs.yml file: ${error.message}`); + throw new Error(`unable to collect enabled FTR configs: ${error.message}`); } } diff --git a/docs/developer/contributing/development-functional-tests.asciidoc b/docs/developer/contributing/development-functional-tests.asciidoc index 9dcb9785cf045..23d43480eb090 100644 --- a/docs/developer/contributing/development-functional-tests.asciidoc +++ b/docs/developer/contributing/development-functional-tests.asciidoc @@ -6,7 +6,19 @@ We use functional tests to make sure the {kib} UI works as expected. It replaces [discrete] === Running functional tests -The `FunctionalTestRunner` (FTR) is very bare bones and gets most of its functionality from its config file. The {kib} repo contains many FTR config files which use slightly different configurations for the {kib} server or {es}, have different test files, and potentially other config differences. You can find a manifest of all the FTR config files in `.buildkite/ftr_configs.yml`. If you’re writing a plugin outside the {kib} repo, you will have your own config file. +The `FunctionalTestRunner` (FTR) is very bare bones and gets most of its functionality from its config file. The {kib} repo contains many FTR config files which use slightly different configurations for the {kib} server or {es}, have different test files, and potentially other config differences. +FTR config files are organised in manifest files, based on testing area and type of distribution: +serverless: +- ftr_base_serverless_configs.yml +- ftr_oblt_serverless_configs.yml +- ftr_security_serverless_configs.yml +- ftr_search_serverless_configs.yml +stateful: +- ftr_platform_stateful_configs.yml +- ftr_oblt_stateful_configs.yml +- ftr_security_stateful_configs.yml +- ftr_search_stateful_configs.yml +If you’re writing a plugin outside the {kib} repo, you will have your own config file. See <> for more info. There are three ways to run the tests depending on your goals: diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/config_loading.ts b/packages/kbn-test/src/functional_test_runner/lib/config/config_loading.ts index ad03d6d1c76ba..3e4bf21df903d 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/config_loading.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/config_loading.ts @@ -15,7 +15,7 @@ import { REPO_ROOT } from '@kbn/repo-info'; import { FtrConfigProvider, GenericFtrProviderContext } from '../../public_types'; import { Config } from './config'; import { EsVersion } from '../es_version'; -import { FTR_CONFIGS_MANIFEST_REL, FTR_CONFIGS_MANIFEST_PATHS } from './ftr_configs_manifest'; +import { getAllFtrConfigsAndManifests } from './ftr_configs_manifest'; interface LoadSettingsOptions { path: string; @@ -60,14 +60,16 @@ async function getConfigModule({ throw error; } + const { allFtrConfigs, manifestPaths } = getAllFtrConfigsAndManifests(); + if ( primary && - !FTR_CONFIGS_MANIFEST_PATHS.includes(resolvedPath) && + !allFtrConfigs.includes(resolvedPath) && !resolvedPath.includes(`${Path.sep}__fixtures__${Path.sep}`) ) { const rel = Path.relative(REPO_ROOT, resolvedPath); throw createFlagError( - `Refusing to load FTR Config at [${rel}] which is not listed in [${FTR_CONFIGS_MANIFEST_REL}]. All FTR Config files must be listed there, use the "enabled" key if the FTR Config should be run on automatically on PR CI, or the "disabled" key if it is run manually or by a special job.` + `Refusing to load FTR Config at [${rel}] which is not listed in [${manifestPaths.all}]. All FTR Config files must be listed in one of manifest files, use the "enabled" key if the FTR Config should be run on automatically on PR CI, or the "disabled" key if it is run manually or by a special job.` ); } diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/ftr_configs_manifest.ts b/packages/kbn-test/src/functional_test_runner/lib/config/ftr_configs_manifest.ts index f45f4e7a5736d..730b6a612ab36 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/ftr_configs_manifest.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/ftr_configs_manifest.ts @@ -12,8 +12,6 @@ import Fs from 'fs'; import { REPO_ROOT } from '@kbn/repo-info'; import JsYaml from 'js-yaml'; -export const FTR_CONFIGS_MANIFEST_REL = '.buildkite/ftr_configs.yml'; - interface FtrConfigWithOptions { [configPath: string]: { queue: string; @@ -26,16 +24,42 @@ interface FtrConfigsManifest { enabled: Array; } -const ftrConfigsManifest: FtrConfigsManifest = JsYaml.safeLoad( - Fs.readFileSync(Path.resolve(REPO_ROOT, FTR_CONFIGS_MANIFEST_REL), 'utf8') -); - -export const FTR_CONFIGS_MANIFEST_PATHS = [ - Object.values(ftrConfigsManifest.enabled), - Object.values(ftrConfigsManifest.disabled), -] - .flat() - .map((config) => { - const rel = typeof config === 'string' ? config : Object.keys(config)[0]; - return Path.resolve(REPO_ROOT, rel); - }); +const FTR_CONFIGS_MANIFEST_SOURCE_REL = '.buildkite/ftr_configs_manifests.json'; + +const getAllFtrConfigsManifests = () => { + const ftrConfigsManifestsSourcePath = Path.resolve(REPO_ROOT, FTR_CONFIGS_MANIFEST_SOURCE_REL); + const manifestPaths: { + serverless: string[]; + stateful: string[]; + } = JSON.parse(Fs.readFileSync(ftrConfigsManifestsSourcePath, 'utf8')); + return { + stateful: manifestPaths.stateful, + serverless: manifestPaths.serverless, + all: manifestPaths.stateful.concat(manifestPaths.serverless), + }; +}; + +export const getAllFtrConfigsAndManifests = () => { + const manifestPaths = getAllFtrConfigsManifests(); + const allFtrConfigs: string[] = []; + + for (const manifestRelPath of manifestPaths.all) { + const manifest: FtrConfigsManifest = JsYaml.safeLoad( + Fs.readFileSync(Path.resolve(REPO_ROOT, manifestRelPath), 'utf8') + ); + + const ftrConfigsInManifest = [ + Object.values(manifest.enabled ?? []), + Object.values(manifest.disabled ?? []), + ] + .flat() + .map((config) => { + const rel = typeof config === 'string' ? config : Object.keys(config)[0]; + return Path.resolve(REPO_ROOT, rel); + }); + + allFtrConfigs.push(...ftrConfigsInManifest); + } + + return { allFtrConfigs, manifestPaths }; +}; diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts b/packages/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts index bfc727537fe22..31afcac759357 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/run_check_ftr_configs_cli.ts @@ -12,7 +12,7 @@ import { REPO_ROOT } from '@kbn/repo-info'; import { run } from '@kbn/dev-cli-runner'; import { createFailError } from '@kbn/dev-cli-errors'; -import { FTR_CONFIGS_MANIFEST_PATHS } from './ftr_configs_manifest'; +import { getAllFtrConfigsAndManifests } from './ftr_configs_manifest'; const THIS_PATH = Path.resolve( REPO_ROOT, @@ -71,19 +71,21 @@ export async function runCheckFtrConfigsCli() { .match(/(testRunner)|(testFiles)/); }); - const invalid = possibleConfigs.filter((path) => !FTR_CONFIGS_MANIFEST_PATHS.includes(path)); + const { allFtrConfigs, manifestPaths } = getAllFtrConfigsAndManifests(); + + const invalid = possibleConfigs.filter((path) => !allFtrConfigs.includes(path)); if (invalid.length) { const invalidList = invalid.map((path) => Path.relative(REPO_ROOT, path)).join('\n - '); log.error( - `The following files look like FTR configs which are not listed in .buildkite/ftr_configs.yml:\n - ${invalidList}` + `The following files look like FTR configs which are not listed in one of manifest files:\nstateful: ${manifestPaths.stateful}\nserverless: ${manifestPaths.serverless}\n - ${invalidList}` ); throw createFailError( - `Please add the listed paths to .buildkite/ftr_configs.yml. If it's not an FTR config, you can add it to the IGNORED_PATHS in ${THIS_REL} or contact #kibana-operations` + `Please add the listed paths to the correct manifest file. If it's not an FTR config, you can add it to the IGNORED_PATHS in ${THIS_REL} or contact #kibana-operations` ); } }, { - description: 'Check that all FTR configs are in .buildkite/ftr_configs.yml', + description: 'Check that all FTR configs are listed in manifest files', } ); } diff --git a/scripts/enabled_ftr_configs.js b/scripts/enabled_ftr_configs.js index 41ad21a464fec..ccece9f370115 100644 --- a/scripts/enabled_ftr_configs.js +++ b/scripts/enabled_ftr_configs.js @@ -10,11 +10,24 @@ require('../src/setup_node_env'); var yaml = require('js-yaml'); var fs = require('fs'); +var path = require('path'); + +var manifestsJsonPath = path.resolve(__dirname, '../.buildkite/ftr_configs_manifests.json'); +console.log(manifestsJsonPath); +var manifestsSource = JSON.parse(fs.readFileSync(manifestsJsonPath, 'utf8')); +var allManifestPaths = Object.values(manifestsSource).flat(); try { - yaml.load(fs.readFileSync('.buildkite/ftr_configs.yml', 'utf8')).enabled.forEach(function (x) { - console.log(x); - }); + for (var manifestRelPath of allManifestPaths) { + var manifest = yaml.load(fs.readFileSync(manifestRelPath, 'utf8')); + if (manifest.enabled) { + manifest.enabled.forEach(function (x) { + console.log(x); + }); + } else { + console.log(`${manifestRelPath} has no enabled FTR configs`); + } + } } catch (e) { console.log(e); } diff --git a/x-pack/test/security_solution_api_integration/README.md b/x-pack/test/security_solution_api_integration/README.md index cc547b20469c6..8f5aafdf15f94 100644 --- a/x-pack/test/security_solution_api_integration/README.md +++ b/x-pack/test/security_solution_api_integration/README.md @@ -45,7 +45,7 @@ ex: 1. Within the `test_suites` directory, create a new area folder. 2. Introduce `ess.config` and `serverless.config` files to reference the new test files and incorporate any additional custom properties defined in the `CreateTestConfigOptions` interface. 3. In these new configuration files, include references to the base configurations located under the config directory to inherit CI configurations, environment variables, and other settings. -4. Append a new entry in the `ftr_configs.yml` file to enable the execution of the newly added tests within the CI pipeline. +4. Append a new entry in the `.buildkite/ftr_security_stateful_configs.yml` / `.buildkite/ftr_security_serverless_configs.yml` file to enable the execution of the newly added tests within the CI pipeline. ## Adding tests for MKI which rely onto NON default project configuration