[Data Discovery] Run example plugin functional tests in Serverless (#…

…163411) ## Summary > [!IMPORTANT] > These tests currently do not run in "real" Serverless and only run in "local" Serverless (what we use in CI). Additional work will need to be done to allow these tests to be run in an actual Serverless environment, since they required `config.ts` changes which are only loaded when running locally. This PR copies the Data Discovery example plugin functional tests to `test_serverless` and adds support for running them against the Search project. It also adds support for running functional tests against example plugins in general in Serverless ("local" Serverless only currently). In the future these should run as deployment-agnostic tests, but support does not yet exist (see #161574), so in the meantime they've been duplicated and modified in place. I've left `TODO` comments where test files have been modified so we know what needs to be addressed once they are converted to deployment-agnostic tests. Part of #162347. ### Checklist - [ ] ~Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)~ - [ ] ~[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials~ - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] ~Any UI touched in this PR is usable by keyboard only (learn more about [keyboard accessibility](https://webaim.org/techniques/keyboard/))~ - [ ] ~Any UI touched in this PR does not create any new axe failures (run axe in browser: [FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/), [Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))~ - [ ] ~If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)~ - [ ] ~This renders correctly on smaller devices using a responsive layout. (You can test this [in your browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))~ - [ ] ~This was checked for [cross-browser compatibility](https://www.elastic.co/support/matrix#matrix_browsers)~ ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Dzmitry Lemechko <[email protected]>
elastic · Aug 31, 2023 · b4bfb2a · b4bfb2a
1 parent a62d9a9
commit b4bfb2a
Show file tree

Hide file tree

Showing 27 changed files with 1,502 additions and 0 deletions.
diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml
@@ -90,10 +90,13 @@ disabled:
   - x-pack/test_serverless/functional/test_suites/common/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts
+  - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/search/config.ts
   - x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts
+  - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/security/config.ts
   - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.examples.ts
 
 defaultQueue: 'n2-4-spot'
 enabled:

diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml
@@ -69,6 +69,19 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: SERVERLESS_ENVIRONMENT=observability.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Observability Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    soft_fail:
+      - exit_status: 10
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - command: SERVERLESS_ENVIRONMENT=search .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Search Tests'
     agents:
@@ -82,6 +95,19 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: SERVERLESS_ENVIRONMENT=search.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Search Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    soft_fail:
+      - exit_status: 10
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - command: SERVERLESS_ENVIRONMENT=security .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Security Tests'
     agents:
@@ -95,6 +121,19 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: SERVERLESS_ENVIRONMENT=security.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Security Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    soft_fail:
+      - exit_status: 10
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - command: .buildkite/scripts/steps/functional/security_serverless.sh
     label: 'Serverless Security Cypress Tests'
     agents:

diff --git a/.buildkite/pipelines/serverless.yml b/.buildkite/pipelines/serverless.yml
@@ -34,6 +34,19 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: SERVERLESS_ENVIRONMENT=observability.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Observability Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
   - command: SERVERLESS_ENVIRONMENT=search .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Search Tests'
     agents:
@@ -47,6 +60,19 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: SERVERLESS_ENVIRONMENT=search.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Search Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
   - command: SERVERLESS_ENVIRONMENT=security .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Security Tests'
     agents:
@@ -60,6 +86,19 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: SERVERLESS_ENVIRONMENT=security.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Security Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_serverless.sh
     label: 'Serverless Security Cypress Tests'
     agents:

diff --git a/.buildkite/scripts/steps/functional/serverless_ftr.sh b/.buildkite/scripts/steps/functional/serverless_ftr.sh
@@ -12,19 +12,31 @@ if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then
     "x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts"
     "x-pack/test_serverless/functional/test_suites/search/config.ts"
   )
+elif [[ "$SERVERLESS_ENVIRONMENT" == "search.examples" ]]; then
+  SERVERLESS_CONFIGS=(
+    "x-pack/test_serverless/functional/test_suites/search/config.examples.ts"
+  )
 elif [[ "$SERVERLESS_ENVIRONMENT" == "observability" ]]; then
   SERVERLESS_CONFIGS=(
     "x-pack/test_serverless/api_integration/test_suites/observability/config.ts"
     "x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts"
     "x-pack/test_serverless/functional/test_suites/observability/config.ts"
     "x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts"
   )
+elif [[ "$SERVERLESS_ENVIRONMENT" == "observability.examples" ]]; then
+  SERVERLESS_CONFIGS=(
+    "x-pack/test_serverless/functional/test_suites/observability/config.examples.ts"
+  )
 elif [[ "$SERVERLESS_ENVIRONMENT" == "security" ]]; then
   SERVERLESS_CONFIGS=(
     "x-pack/test_serverless/api_integration/test_suites/security/config.ts"
     "x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts"
     "x-pack/test_serverless/functional/test_suites/security/config.ts"
   )
+elif [[ "$SERVERLESS_ENVIRONMENT" == "security.examples" ]]; then
+  SERVERLESS_CONFIGS=(
+    "x-pack/test_serverless/functional/test_suites/security/config.examples.ts"
+  )
 fi
 
 EXIT_CODE=0

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -834,6 +834,13 @@ packages/kbn-yarn-lock-validator @elastic/kibana-operations
 /x-pack/test_serverless/api_integration/test_suites/common/scripts_tests @elastic/kibana-data-discovery
 /x-pack/test_serverless/api_integration/test_suites/common/search_oss @elastic/kibana-data-discovery
 /x-pack/test_serverless/api_integration/test_suites/common/search_xpack @elastic/kibana-data-discovery
+/x-pack/test_serverless/functional/test_suites/common/examples/data_view_field_editor_example @elastic/kibana-data-discovery
+/x-pack/test_serverless/functional/test_suites/common/examples/discover_customization_examples @elastic/kibana-data-discovery
+/x-pack/test_serverless/functional/test_suites/common/examples/field_formats @elastic/kibana-data-discovery
+/x-pack/test_serverless/functional/test_suites/common/examples/partial_results @elastic/kibana-data-discovery
+/x-pack/test_serverless/functional/test_suites/common/examples/search @elastic/kibana-data-discovery
+/x-pack/test_serverless/functional/test_suites/common/examples/search_examples @elastic/kibana-data-discovery
+/x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples @elastic/kibana-data-discovery
 
 # Visualizations
 /src/plugins/visualize/ @elastic/kibana-visualizations

diff --git a/x-pack/test_serverless/functional/config.base.ts b/x-pack/test_serverless/functional/config.base.ts
@@ -27,6 +27,7 @@ export function createTestConfig(options: CreateTestConfigOptions) {
         serverArgs: [
           ...svlSharedConfig.get('kbnTestServer.serverArgs'),
           `--serverless=${options.serverlessProject}`,
+          ...(options.kbnServerArgs ?? []),
         ],
       },
       testFiles: options.testFiles,