[FTR] Move serverless tests depending on feature flag to its own conf…

…ig file (#163929)

## Summary

It seems beneficial to have feature flag tests in a separate test config
file
 - tests are still run on Kibana CI automatically
- tests are not run on MKI projects automatically, but you can deploy
custom project and run tests via feature flags config

All the feature flags within the same project should be places in the
same config to make sure there is no arguments conflict.
When the flag is moved to the yml configuration, we can rely on Kibana
CI and manually triggered deployment to make sure projects are
functioning correctly.

---------

Co-authored-by: Robert Oskamp <[email protected]>

.buildkite/ftr_configs.yml

@@ -81,12 +81,18 @@ disabled:
   # Serverless configs, currently only for manual tests runs, CI integration planned
   - x-pack/test_serverless/api_integration/test_suites/common/config.ts
   - x-pack/test_serverless/api_integration/test_suites/observability/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts
   - x-pack/test_serverless/api_integration/test_suites/search/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts
   - x-pack/test_serverless/api_integration/test_suites/security/config.ts
+  - x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts
   - x-pack/test_serverless/functional/test_suites/common/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
+  - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts
   - x-pack/test_serverless/functional/test_suites/search/config.ts
+  - x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts
   - x-pack/test_serverless/functional/test_suites/security/config.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
 
 defaultQueue: 'n2-4-spot'
 enabled:

.buildkite/scripts/steps/functional/serverless_ftr.sh

@@ -9,17 +9,20 @@ export JOB="kibana-serverless-$SERVERLESS_ENVIRONMENT"
 if [[ "$SERVERLESS_ENVIRONMENT" == "search" ]]; then
   SERVERLESS_CONFIGS=(
     "x-pack/test_serverless/api_integration/test_suites/search/config.ts"
+    "x-pack/test_serverless/api_integration/test_suites/search/config.feature_flags.ts"
     "x-pack/test_serverless/functional/test_suites/search/config.ts"
   )
 elif [[ "$SERVERLESS_ENVIRONMENT" == "observability" ]]; then
   SERVERLESS_CONFIGS=(
     "x-pack/test_serverless/api_integration/test_suites/observability/config.ts"
+    "x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts"
     "x-pack/test_serverless/functional/test_suites/observability/config.ts"
     "x-pack/test_serverless/functional/test_suites/observability/cypress/config_headless.ts"
   )
 elif [[ "$SERVERLESS_ENVIRONMENT" == "security" ]]; then
   SERVERLESS_CONFIGS=(
     "x-pack/test_serverless/api_integration/test_suites/security/config.ts"
+    "x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts"
     "x-pack/test_serverless/functional/test_suites/security/config.ts"
   )
 fi

x-pack/test_serverless/README.md

@@ -97,6 +97,33 @@ tests that should run in a serverless environment have to be added to the
 Tests in this area should be clearly designed for the serverless environment,
 particularly when it comes to timing for API requests and UI interaction.
 
+### Testing with feature flags
+
+**tl;dr:** Tests specific to functionality behind a feature flag need special
+handling and are by default only tested locally / in CI but excluded from regular
+test runs in MKI.
+
+New features might be gated behind a feature flag and can only be enabled
+through a yml configuration entry. By default, these features are not enabled
+so they're not available in a regular serverless MKI project, which would make
+end-to-end tests for such a feature fail. In order to still have tests for
+features behind a feature flag, these tests need to be separated from the
+regular tests.
+
+For every project's `test_suites` directory, there are feature flags specific
+config (`config.feature_flags.ts`) and index (`index.feature_flags.ts`) files
+next to the regular `config.ts` and `index.ts`. These extra files are used to
+cover all feature flag tests of the respective area.
+If you want to add feature flag specific tests:
+- Add your feature flag(s) to the `kbnServerArgs` in the `config.feature_flags.ts` file
+- Load your test file(s) in the `index.feature_flags.ts` file
+
+As mentioned above, these tests are not part of the regular test run against MKI
+projects. If you still want to run feature flag tests against an MKI project,
+this requires a Kibana docker build that has the feature flags enabled by default.
+This docker image can then be used to create a project in serverless QA and the
+feature flags tests can be pointed to the project.
+
 ## Run tests
 Similar to how functional tests are run in `x-pack/test`, you can point the
 functional tests server and test runner to config files in this `x-pack/test_serverless`

x-pack/test_serverless/api_integration/config.base.ts

@@ -25,9 +25,7 @@ export function createTestConfig(options: CreateTestConfigOptions) {
         serverArgs: [
           ...svlSharedConfig.get('kbnTestServer.serverArgs'),
           `--serverless=${options.serverlessProject}`,
-          `--xpack.alerting.enableFrameworkAlerts=true`,
-          '--xpack.observability.unsafe.thresholdRule.enabled=true',
-          '--server.publicBaseUrl=https://localhost:5601',
+          ...(options.kbnServerArgs || []),
         ],
       },
       testFiles: options.testFiles,

x-pack/test_serverless/api_integration/services/alerting_api.ts

@@ -10,6 +10,9 @@ import type {
   SearchResponse,
 } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
+import { MetricThresholdParams } from '@kbn/infra-plugin/common/alerting/metrics';
+import { ThresholdParams } from '@kbn/observability-plugin/common/threshold_rule/types';
+
 import { FtrProviderContext } from '../ftr_provider_context';
 
 export function AlertingApiProvider({ getService }: FtrProviderContext) {
@@ -86,5 +89,56 @@ export function AlertingApiProvider({ getService }: FtrProviderContext) {
         return response;
       });
     },
+
+    async createIndexConnector({ name, indexName }: { name: string; indexName: string }) {
+      const { body } = await supertest
+        .post(`/api/actions/connector`)
+        .set('kbn-xsrf', 'foo')
+        .set('x-elastic-internal-origin', 'foo')
+        .send({
+          name,
+          config: {
+            index: indexName,
+            refresh: true,
+          },
+          connector_type_id: '.index',
+        });
+      return body.id as string;
+    },
+
+    async createRule({
+      name,
+      ruleTypeId,
+      params,
+      actions = [],
+      tags = [],
+      schedule,
+      consumer,
+    }: {
+      ruleTypeId: string;
+      name: string;
+      params: MetricThresholdParams | ThresholdParams;
+      actions?: any[];
+      tags?: any[];
+      schedule?: { interval: string };
+      consumer: string;
+    }) {
+      const { body } = await supertest
+        .post(`/api/alerting/rule`)
+        .set('kbn-xsrf', 'foo')
+        .set('x-elastic-internal-origin', 'foo')
+        .send({
+          params,
+          consumer,
+          schedule: schedule || {
+            interval: '5m',
+          },
+          tags,
+          name,
+          rule_type_id: ruleTypeId,
+          actions,
+        });
+      return body;
+    },
   };
 }

x-pack/test_serverless/api_integration/services/data_view_api.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export function DataViewApiProvider({ getService }: FtrProviderContext) {
+  const supertest = getService('supertest');
+
+  return {
+    async create({ id, name, title }: { id: string; name: string; title: string }) {
+      const { body } = await supertest
+        .post(`/api/content_management/rpc/create`)
+        .set('kbn-xsrf', 'foo')
+        .set('x-elastic-internal-origin', 'foo')
+        .send({
+          contentTypeId: 'index-pattern',
+          data: {
+            fieldAttrs: '{}',
+            title,
+            timeFieldName: '@timestamp',
+            sourceFilters: '[]',
+            fields: '[]',
+            fieldFormatMap: '{}',
+            typeMeta: '{}',
+            runtimeFieldMap: '{}',
+            name,
+          },
+          options: { id },
+          version: 1,
+        });
+      return body;
+    },
+
+    async delete({ id }: { id: string }) {
+      const { body } = await supertest
+        .post(`/api/content_management/rpc/delete`)
+        .set('kbn-xsrf', 'foo')
+        .set('x-elastic-internal-origin', 'foo')
+        .send({
+          contentTypeId: 'index-pattern',
+          id,
+          options: { force: true },
+          version: 1,
+        });
+      return body;
+    },
+  };
+}

x-pack/test_serverless/api_integration/services/index.ts

@@ -12,13 +12,15 @@ import { services as svlSharedServices } from '../../shared/services';
 
 import { SvlCommonApiServiceProvider } from './svl_common_api';
 import { AlertingApiProvider } from './alerting_api';
+import { DataViewApiProvider } from './data_view_api';
 
 export const services = {
   ...xpackApiIntegrationServices,
   ...svlSharedServices,
 
   svlCommonApi: SvlCommonApiServiceProvider,
   alertingApi: AlertingApiProvider,
+  dataViewApi: DataViewApiProvider,
 };
 
 export type InheritedFtrProviderContext = GenericFtrProviderContext<typeof services, {}>;

x-pack/test_serverless/api_integration/test_suites/observability/cases/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Cases', function () {
+    loadTestFile(require.resolve('./get_case'));
+    loadTestFile(require.resolve('./find_cases'));
+    loadTestFile(require.resolve('./post_case'));
+  });
+}

x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createTestConfig } from '../../config.base';
+import { services } from './apm_api_integration/common/services';
+
+/**
+ * Make sure to create a MKI deployment with custom Kibana image, that includes feature flags arguments
+ * This tests most likely will fail on default MKI project
+ */
+export default createTestConfig({
+  serverlessProject: 'oblt',
+  junit: {
+    reportName: 'Serverless Observability Feature Flags API Integration Tests',
+  },
+  suiteTags: { exclude: ['skipSvlOblt'] },
+  services,
+  // add feature flags
+  kbnServerArgs: ['--xpack.observability.unsafe.thresholdRule.enabled=true'],
+  // load tests in the index file
+  testFiles: [require.resolve('./index.feature_flags.ts')],
+});

x-pack/test_serverless/api_integration/test_suites/observability/fleet.ts → x-pack/test_serverless/api_integration/test_suites/observability/fleet/fleet.ts

@@ -6,7 +6,7 @@
  */
 
 import expect from 'expect';
-import { FtrProviderContext } from '../../ftr_provider_context';
+import { FtrProviderContext } from '../../../ftr_provider_context';
 
 export default function ({ getService }: FtrProviderContext) {
   const svlCommonApi = getService('svlCommonApi');