[Defend Workflows] Enable FTRs for serverless using tags (elastic#169946

)

## Summary

This PR enables **most of** our FTR tests in
`x-pack/test/security_solution_endpoint/` to run against both ESS and
Serverless, using the following tags:
- `@ess`
- `@serverless`
- `@brokenInServerless`
- `@skipInServerless`
- `@skipInEss`

The ones that are not enabled are the ones that use different
users/roles.


### Usage
```ts
describe('This is the suite.', function() {
    // a custom function wraps the `this.tags()` function to provide type safety
    targetTags(this, ['@ess', '@serverless', '@skipInServerless']);
})
```
> **Note**
> Only `describe()` blocks can be tagged.

---------

Co-authored-by: Kibana Machine <[email protected]>

.buildkite/ftr_configs.yml

@@ -15,7 +15,7 @@ disabled:
   - x-pack/test/fleet_api_integration/config.base.ts
   - x-pack/test/security_solution_api_integration/config/ess/config.base.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
-
+  - x-pack/test/security_solution_endpoint/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -387,7 +387,9 @@ enabled:
   - x-pack/test/security_functional/expired_session.config.ts
   - x-pack/test/security_solution_endpoint_api_int/config.ts
   - x-pack/test/security_solution_endpoint/endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/integrations.config.ts
+  - x-pack/test/security_solution_endpoint/serverless.integrations.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts

x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts

@@ -14,6 +14,7 @@ import {
   deleteAllDocsFromMetadataCurrentIndex,
   deleteAllDocsFromMetadataUnitedIndex,
 } from '../../../security_solution_endpoint_api_int/apis/data_stream_helper';
+import { targetTags } from '../../target_tags';
 
 export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const pageObjects = getPageObjects(['common', 'endpoint', 'header', 'endpointPageUtils']);
@@ -85,6 +86,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   };
 
   describe('endpoint list', function () {
+    targetTags(this, ['@ess', '@serverless']);
+
     let indexedData: IndexedHostsAndAlertsResponse;
     describe('when initially navigating to page', () => {
       before(async () => {

x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_permissions.ts

@@ -10,13 +10,16 @@ import { IndexedHostsAndAlertsResponse } from '@kbn/security-solution-plugin/com
 import { SecurityRoleName } from '@kbn/security-solution-plugin/common/test';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { createUserAndRole, deleteUserAndRole } from '../../../common/services/security_solution';
+import { targetTags } from '../../target_tags';
 
 export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const PageObjects = getPageObjects(['security', 'endpoint', 'detections', 'hosts']);
   const testSubjects = getService('testSubjects');
   const endpointTestResources = getService('endpointTestResources');
 
-  describe('Endpoint permissions:', () => {
+  describe('Endpoint permissions:', function () {
+    targetTags(this, ['@ess']);
+
     let indexedData: IndexedHostsAndAlertsResponse;
 
     before(async () => {

x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_solution_integrations.ts

@@ -11,6 +11,7 @@ import { TimelineResponse } from '@kbn/security-solution-plugin/common/api/timel
 import { kibanaPackageJson } from '@kbn/repo-info';
 import { type IndexedEndpointRuleAlerts } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/index_endpoint_rule_alerts';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { targetTags } from '../../target_tags';
 
 /**
  * Test suite is meant to cover usages of endpoint functionality or access to endpoint
@@ -24,7 +25,9 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const testSubjects = getService('testSubjects');
   const pageObjects = getPageObjects(['common', 'timeline']);
 
-  describe('App level Endpoint functionality', () => {
+  describe('App level Endpoint functionality', function () {
+    targetTags(this, ['@ess']);
+
     let indexedData: IndexedHostsAndAlertsResponse;
     let indexedAlerts: IndexedEndpointRuleAlerts;
     let endpointAgentId: string;

x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_telemetry.ts

@@ -7,6 +7,7 @@
 
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { targetTags } from '../../target_tags';
 
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
@@ -15,7 +16,9 @@ export default function ({ getService }: FtrProviderContext) {
 
   // The source of the data for these tests have changed and need to be updated
   // There are currently tests in the security_solution application being maintained
-  describe.skip('security solution endpoint telemetry', () => {
+  describe.skip('security solution endpoint telemetry', function () {
+    targetTags(this, ['@ess']);
+
     after(async () => {
       await kibanaServer.savedObjects.cleanStandardList();
     });

x-pack/test/security_solution_endpoint/apps/endpoint/index.ts

@@ -6,19 +6,21 @@
  */
 
 import { getRegistryUrl as getRegistryUrlFromIngest } from '@kbn/fleet-plugin/server';
+import { isServerlessKibanaFlavor } from '@kbn/security-solution-plugin/scripts/endpoint/common/stack_services';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import {
   getRegistryUrlFromTestEnv,
   isRegistryEnabled,
 } from '../../../security_solution_endpoint_api_int/registry';
 
 export default function (providerContext: FtrProviderContext) {
-  const { loadTestFile, getService } = providerContext;
+  const { loadTestFile, getService, getPageObjects } = providerContext;
 
   describe('endpoint', function () {
     const ingestManager = getService('ingestManager');
     const log = getService('log');
     const endpointTestResources = getService('endpointTestResources');
+    const kbnClient = getService('kibanaServer');
 
     if (!isRegistryEnabled()) {
       log.warning('These tests are being run with an external package registry');
@@ -33,6 +35,12 @@ export default function (providerContext: FtrProviderContext) {
 
       log.info('installing/upgrading Endpoint fleet package');
       await endpointTestResources.installOrUpgradeEndpointFleetPackage();
+
+      if (await isServerlessKibanaFlavor(kbnClient)) {
+        log.info('login for serverless environment');
+        const pageObjects = getPageObjects(['svlCommonPage']);
+        await pageObjects.svlCommonPage.login();
+      }
     });
     loadTestFile(require.resolve('./endpoint_list'));
     loadTestFile(require.resolve('./endpoint_telemetry'));

x-pack/test/security_solution_endpoint/apps/endpoint/responder.ts

@@ -10,6 +10,7 @@ import { TimelineResponse } from '@kbn/security-solution-plugin/common/api/timel
 import { type IndexedEndpointRuleAlerts } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/index_endpoint_rule_alerts';
 import { DATE_RANGE_OPTION_TO_TEST_SUBJ_MAP } from '@kbn/security-solution-plugin/common/test';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { targetTags } from '../../target_tags';
 
 export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const pageObjects = getPageObjects([
@@ -82,6 +83,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   };
 
   describe('Response Actions Responder', function () {
+    targetTags(this, ['@ess', '@serverless']);
+
     let indexedData: IndexedHostsAndAlertsResponse;
     let endpointAgentId: string;
 

x-pack/test/security_solution_endpoint/apps/integrations/artifact_entries_list.ts

@@ -26,6 +26,7 @@ import {
   MultipleArtifactActionsType,
 } from './mocks';
 import { PolicyTestResourceInfo } from '../../services/endpoint_policy';
+import { targetTags } from '../../target_tags';
 
 export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const pageObjects = getPageObjects(['common', 'artifactEntriesList']);
@@ -52,6 +53,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   };
 
   describe('For each artifact list under management', function () {
+    targetTags(this, ['@ess', '@serverless']);
+
     this.timeout(60_000 * 5);
     let indexedData: IndexedHostsAndAlertsResponse;
     let policyInfo: PolicyTestResourceInfo;

x-pack/test/security_solution_endpoint/apps/integrations/fleet_integrations.ts

@@ -10,6 +10,7 @@ import {
   deleteMetadataStream,
   deleteAllDocsFromMetadataCurrentIndex,
 } from '../../../security_solution_endpoint_api_int/apis/data_stream_helper';
+import { targetTags } from '../../target_tags';
 
 export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const { fleetIntegrations, trustedApps } = getPageObjects(['trustedApps', 'fleetIntegrations']);
@@ -19,6 +20,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const browser = getService('browser');
 
   describe('When in the Fleet application', function () {
+    targetTags(this, ['@ess', '@serverless']);
+
     before(async () => {
       await esArchiver.load('x-pack/test/functional/es_archives/endpoint/metadata/api_feature', {
         useCreate: true,

x-pack/test/security_solution_endpoint/apps/integrations/index.ts

@@ -6,19 +6,21 @@
  */
 
 import { getRegistryUrl as getRegistryUrlFromIngest } from '@kbn/fleet-plugin/server';
+import { isServerlessKibanaFlavor } from '@kbn/security-solution-plugin/scripts/endpoint/common/stack_services';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import {
   isRegistryEnabled,
   getRegistryUrlFromTestEnv,
 } from '../../../security_solution_endpoint_api_int/registry';
 
 export default function (providerContext: FtrProviderContext) {
-  const { loadTestFile, getService } = providerContext;
+  const { loadTestFile, getService, getPageObjects } = providerContext;
 
   describe('endpoint', function () {
     const ingestManager = getService('ingestManager');
     const log = getService('log');
     const endpointTestResources = getService('endpointTestResources');
+    const kbnClient = getService('kibanaServer');
 
     if (!isRegistryEnabled()) {
       log.warning('These tests are being run with an external package registry');
@@ -33,6 +35,12 @@ export default function (providerContext: FtrProviderContext) {
 
       log.info('installing/upgrading Endpoint fleet package');
       await endpointTestResources.installOrUpgradeEndpointFleetPackage();
+
+      if (await isServerlessKibanaFlavor(kbnClient)) {
+        log.info('login for serverless environment');
+        const pageObjects = getPageObjects(['svlCommonPage']);
+        await pageObjects.svlCommonPage.login();
+      }
     });
     loadTestFile(require.resolve('./policy_list'));
     loadTestFile(require.resolve('./policy_details'));

x-pack/test/security_solution_endpoint/apps/integrations/policy_details.ts

@@ -11,6 +11,7 @@ import { PROTECTION_NOTICE_SUPPORTED_ENDPOINT_VERSION } from '@kbn/security-solu
 import { getPolicySettingsFormTestSubjects } from '@kbn/security-solution-plugin/public/management/pages/policy/view/policy_settings_form/mocks';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { PolicyTestResourceInfo } from '../../services/endpoint_policy';
+import { targetTags } from '../../target_tags';
 
 export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const browser = getService('browser');
@@ -28,6 +29,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const retry = getService('retry');
 
   describe('When on the Endpoint Policy Details Page', function () {
+    targetTags(this, ['@ess', '@serverless']);
+
     let indexedData: IndexedHostsAndAlertsResponse;
     const formTestSubjects = getPolicySettingsFormTestSubjects();
 
@@ -74,12 +77,16 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
         );
       });
 
-      it('should not hide the side navigation', async () => {
-        await testSubjects.scrollIntoView('solutionSideNavItemLink-get_started');
-        // ensure center of button is visible and not hidden by sticky bottom bar
-        await testSubjects.click('solutionSideNavItemLink-administration', 1000, 15);
-        // test cleanup: go back to policy details page
-        await pageObjects.policy.navigateToPolicyDetails(policyInfo.packagePolicy.id);
+      describe('side navigation', function () {
+        targetTags(this, ['@skipInServerless']);
+
+        it('should not hide the side navigation', async function () {
+          await testSubjects.scrollIntoView('solutionSideNavItemLink-get_started');
+          // ensure center of button is visible and not hidden by sticky bottom bar
+          await testSubjects.click('solutionSideNavItemLink-administration', 1000, 15);
+          // test cleanup: go back to policy details page
+          await pageObjects.policy.navigateToPolicyDetails(policyInfo.packagePolicy.id);
+        });
       });
 
       it('Should show/hide advanced section when button is clicked', async () => {
@@ -148,10 +155,11 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
         it('should show a tooltip on hover', async () => {
           await testSubjects.moveMouseTo(cardTestSubj.notifyCustomMessageTooltipIcon);
 
-          expect(
-            await testSubjects.getVisibleText(cardTestSubj.notifyCustomMessageTooltipInfo)
-          ).equal(
-            `Selecting the user notification option will display a notification to the host user when ${protection} is prevented or detected.\nThe user notification can be customized in the text box below. Bracketed tags can be used to dynamically populate the applicable action (such as prevented or detected) and the filename.`
+          await retry.waitFor(
+            'should show a tooltip on hover',
+            async () =>
+              (await testSubjects.getVisibleText(cardTestSubj.notifyCustomMessageTooltipInfo)) ===
+              `Selecting the user notification option will display a notification to the host user when ${protection} is prevented or detected.\nThe user notification can be customized in the text box below. Bracketed tags can be used to dynamically populate the applicable action (such as prevented or detected) and the filename.`
           );
         });
 
@@ -210,7 +218,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
         await pageObjects.policy.confirmAndSave();
 
         await testSubjects.existOrFail('policyDetailsSuccessMessage');
-        await testSubjects.waitForHidden('toastCloseButton');
+        await testSubjects.existOrFail('toastCloseButton');
         await pageObjects.endpoint.navigateToEndpointList();
         await pageObjects.policy.navigateToPolicyDetails(policyInfo.packagePolicy.id);
 

x-pack/test/security_solution_endpoint/apps/integrations/policy_list.ts

@@ -9,6 +9,7 @@ import expect from '@kbn/expect';
 import { IndexedHostsAndAlertsResponse } from '@kbn/security-solution-plugin/common/endpoint/index_data';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { PolicyTestResourceInfo } from '../../services/endpoint_policy';
+import { targetTags } from '../../target_tags';
 
 export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const browser = getService('browser');
@@ -24,7 +25,9 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const policyTestResources = getService('policyTestResources');
   const endpointTestResources = getService('endpointTestResources');
 
-  describe('When on the Endpoint Policy List Page', () => {
+  describe('When on the Endpoint Policy List Page', function () {
+    targetTags(this, ['@ess', '@serverless']);
+
     before(async () => {
       await browser.refresh();
     });

x-pack/test/security_solution_endpoint/apps/integrations/trusted_apps_list.ts

@@ -8,6 +8,7 @@
 import expect from '@kbn/expect';
 import { IndexedHostsAndAlertsResponse } from '@kbn/security-solution-plugin/common/endpoint/index_data';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { targetTags } from '../../target_tags';
 
 export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const pageObjects = getPageObjects(['common', 'trustedApps']);
@@ -16,6 +17,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
   const endpointTestResources = getService('endpointTestResources');
 
   describe('When on the Trusted Apps list', function () {
+    targetTags(this, ['@ess', '@serverless']);
+
     let indexedData: IndexedHostsAndAlertsResponse;
     before(async () => {
       indexedData = await endpointTestResources.loadEndpointData();