[8.10] Upgrading Cypress to 12.17.4

.buildkite/pipelines/pull_request/fleet_cypress.yml

@@ -4,7 +4,8 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 120
+    timeout_in_minutes: 50
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/osquery_cypress.yml

@@ -23,16 +23,3 @@ steps:
     artifact_paths:
       - "target/kibana-osquery/**/*"
 
-  - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-    label: 'Serverless Osquery Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 50
-    parallelism: 6
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-    artifact_paths:
-      - "target/kibana-osquery/**/*"

.buildkite/scripts/steps/functional/fleet_cypress.sh

@@ -3,12 +3,14 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-fleet-cypress
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
 echo "--- Fleet Cypress tests"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-  --config x-pack/test/fleet_cypress/cli_config.ts
+cd x-pack/plugins/fleet
+
+set +e
+yarn cypress:run:reporter; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/profiling_cypress.sh

@@ -13,5 +13,5 @@ echo "--- Profiling Cypress Tests"
 
 cd "$XPACK_DIR"
 
-node plugins/profiling/scripts/test/e2e.js \
+NODE_OPTIONS=--openssl-legacy-provide node plugins/profiling/scripts/test/e2e.js \
   --kibana-install-dir "$KIBANA_BUILD_LOCATION" \

package.json

@@ -1414,7 +1414,7 @@
     "cssnano": "^5.1.12",
     "cssnano-preset-default": "^5.2.12",
     "csstype": "^3.0.2",
-    "cypress": "^12.13.0",
+    "cypress": "^12.17.4",
     "cypress-axe": "^1.4.0",
     "cypress-file-upload": "^5.0.8",
     "cypress-multi-reporters": "^1.6.3",

test/scripts/jenkins_fleet_cypress.sh

@@ -5,10 +5,8 @@ source test/scripts/jenkins_test_setup_xpack.sh
 echo " -> Running fleet cypress tests"
 cd "$XPACK_DIR"
 
-node scripts/functional_tests \
-  --debug --bail \
-  --kibana-install-dir "$KIBANA_INSTALL_DIR" \
-  --config test/fleet_cypress/cli_config.ts
+cd x-pack/plugins/fleet
+yarn --cwd x-pack/plugins/fleet cypress:run
 
 echo ""
 echo ""

x-pack/plugins/apm/scripts/test/e2e.js

@@ -69,7 +69,11 @@ function runTests() {
 
   return childProcess.spawnSync('node', spawnArgs, {
     cwd: e2eDir,
-    env: { ...process.env, CYPRESS_CLI_ARGS: JSON.stringify(cypressCliArgs) },
+    env: {
+      ...process.env,
+      CYPRESS_CLI_ARGS: JSON.stringify(cypressCliArgs),
+      NODE_OPTIONS: '--openssl-legacy-provider',
+    },
     encoding: 'utf8',
     stdio: 'inherit',
   });

x-pack/plugins/fleet/cypress.config.ts

@@ -26,14 +26,16 @@ export default defineCypressConfig({
   viewportWidth: 1440,
   screenshotOnRunFailure: true,
 
-  env: {
-    protocol: 'http',
-    hostname: 'localhost',
-    configport: '5601',
-  },
-
   e2e: {
     baseUrl: 'http://localhost:5601',
+
+    experimentalRunAllSpecs: true,
+    experimentalMemoryManagement: true,
+    numTestsKeptInMemory: 3,
+
+    specPattern: './cypress/e2e/**/*.cy.ts',
+    supportFile: './cypress/support/e2e.ts',
+
     setupNodeEvents(on, config) {
       // eslint-disable-next-line @typescript-eslint/no-var-requires, @kbn/imports/no_boundary_crossing
       return require('./cypress/plugins')(on, config);

x-pack/plugins/fleet/cypress/README.md

@@ -52,8 +52,9 @@ node scripts/build_kibana_platform_plugins
 
 # launch the cypress test runner
 cd x-pack/plugins/fleet
-yarn cypress:run-as-ci
+yarn cypress:run
 ```
+
 #### FTR + Interactive
 
 This is the preferred mode for developing new tests.
@@ -67,17 +68,7 @@ node scripts/build_kibana_platform_plugins
 
 # launch the cypress test runner
 cd x-pack/plugins/fleet
-yarn cypress:open-as-ci
-```
-
-Alternatively, kibana test server can be started separately, to pick up changes in UI (e.g. change in data-test-subj selector)
-
-```
-# launch kibana test server
-node scripts/functional_tests_server --config x-pack/test/fleet_cypress/config.ts
-
-# launch cypress runner
-node scripts/functional_test_runner --config x-pack/test/fleet_cypress/visual_config.ts
+yarn cypress:open
 ```
 
 Note that you can select the browser you want to use on the top right side of the interactive runner.
@@ -108,7 +99,7 @@ Each file inside the screens folder represents a screen in our application.
 
 _Tasks_ are functions that may be reused across tests.
 
-Each file inside the tasks folder represents a screen of our application. 
+Each file inside the tasks folder represents a screen of our application.
 
 ## Test data
 
@@ -141,7 +132,7 @@ Note that the command will create the folder if it does not exist.
 
 ## Development Best Practices
 
-### Clean up the state 
+### Clean up the state
 
 Remember to clean up the state of the test after its execution, typically with the `cleanKibana` function. Be mindful of failure scenarios, as well: if your test fails, will it leave the environment in a recoverable state?
 
@@ -164,7 +155,7 @@ Remember that minimizing the number of times the web page is loaded, we minimize
 
 The `checkA11y({ skipFailures: false });` call uses [axe-core](https://github.com/dequelabs/axe-core) to perform a full page check for accessibility violations.
 
-See [axe-core](https://github.com/dequelabs/axe-core)'s documentation for details on what is checked for. 
+See [axe-core](https://github.com/dequelabs/axe-core)'s documentation for details on what is checked for.
 
 ## Linting
 

x-pack/plugins/fleet/cypress/e2e/a11y/home_page.cy.ts

@@ -33,11 +33,18 @@ import { AGENT_POLICY_NAME_LINK } from '../../screens/integrations';
 import { cleanupAgentPolicies, unenrollAgent } from '../../tasks/cleanup';
 import { setFleetServerHost } from '../../tasks/fleet_server';
 
+import { login } from '../../tasks/login';
+import { request } from '../../tasks/common';
+
 describe('Home page', () => {
   before(() => {
     setFleetServerHost('https://fleetserver:8220');
   });
 
+  beforeEach(() => {
+    login();
+  });
+
   describe('Agents', () => {
     beforeEach(() => {
       navigateTo(FLEET);
@@ -148,7 +155,7 @@ describe('Home page', () => {
 
   describe('Uninstall Tokens', () => {
     before(() => {
-      cy.request({
+      request({
         method: 'POST',
         url: '/api/fleet/agent_policies',
         body: { name: 'Agent policy for A11y test', namespace: 'default', id: 'agent-policy-a11y' },
@@ -160,7 +167,7 @@ describe('Home page', () => {
       cy.getBySel(UNINSTALL_TOKENS_TAB).click();
     });
     after(() => {
-      cy.request({
+      request({
         method: 'POST',
         url: '/api/fleet/agent_policies/delete',
         body: { agentPolicyId: 'agent-policy-a11y' },
@@ -180,6 +187,7 @@ describe('Home page', () => {
 
   describe('Data Streams', () => {
     before(() => {
+      login();
       navigateTo(FLEET);
       cy.getBySel(DATA_STREAMS_TAB, { timeout: 15000 }).should('be.visible');
       cy.getBySel(DATA_STREAMS_TAB).click();

x-pack/plugins/fleet/cypress/e2e/agent_binary_download_source.cy.ts

@@ -11,13 +11,19 @@ import {
   AGENT_BINARY_SOURCES_FLYOUT,
   AGENT_POLICY_FORM,
 } from '../screens/fleet';
-import { cleanupDownloadSources } from '../tasks/cleanup';
+import { cleanupAgentPolicies, cleanupDownloadSources } from '../tasks/cleanup';
 import { FLEET, navigateTo } from '../tasks/navigation';
 import { CONFIRM_MODAL } from '../screens/navigation';
 
+import { request } from '../tasks/common';
+import { login } from '../tasks/login';
+
 describe('Agent binary download source section', () => {
   beforeEach(() => {
     cleanupDownloadSources();
+    cleanupAgentPolicies();
+
+    login();
     navigateTo(FLEET);
   });
 
@@ -72,7 +78,7 @@ describe('Agent binary download source section', () => {
   });
 
   it('the download source is displayed in agent policy settings', () => {
-    cy.request({
+    request({
       method: 'POST',
       url: `api/fleet/agent_download_sources`,
       body: {
@@ -82,7 +88,7 @@ describe('Agent binary download source section', () => {
       },
       headers: { 'kbn-xsrf': 'kibana' },
     });
-    cy.request({
+    request({
       method: 'POST',
       url: '/api/fleet/agent_policies',
       body: {

x-pack/plugins/fleet/cypress/e2e/agent_policy.cy.ts

@@ -4,11 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { TOAST_CLOSE_BTN } from '../screens/navigation';
 import { setupFleetServer } from '../tasks/fleet_server';
 import { AGENT_FLYOUT, AGENT_POLICY_DETAILS_PAGE } from '../screens/fleet';
+import { login } from '../tasks/login';
+
 describe('Edit agent policy', () => {
   beforeEach(() => {
+    login();
+
     cy.intercept('/api/fleet/agent_policies/policy-1', {
       item: {
         id: 'policy-1',
@@ -35,7 +38,6 @@ describe('Edit agent policy', () => {
 
   it('should edit agent policy', () => {
     cy.visit('/app/fleet/policies/policy-1/settings');
-    cy.getBySel(TOAST_CLOSE_BTN).click();
     cy.get('[placeholder="Optional description"').clear().type('desc');
 
     cy.intercept('/api/fleet/agent_policies/policy-1', {

x-pack/plugins/fleet/cypress/e2e/agents/agent_list.cy.ts

@@ -13,6 +13,9 @@ import { deleteFleetServerDocs, deleteAgentDocs, cleanupAgentPolicies } from '..
 import type { CreateAgentPolicyRequest } from '../../../common/types';
 import { setUISettings } from '../../tasks/ui_settings';
 
+import { request } from '../../tasks/common';
+import { login } from '../../tasks/login';
+
 const createAgentDocs = (kibanaVersion: string) => [
   createAgentDoc('agent-1', 'policy-1'), // this agent will have upgrade available
   createAgentDoc('agent-2', 'policy-2', 'error', kibanaVersion),
@@ -63,7 +66,7 @@ const POLICIES: Array<CreateAgentPolicyRequest['body']> = [
 ];
 
 function createAgentPolicy(body: CreateAgentPolicyRequest['body']) {
-  cy.request({
+  request({
     method: 'POST',
     url: '/api/fleet/agent_policies',
     headers: { 'kbn-xsrf': 'xx' },
@@ -99,11 +102,13 @@ describe('View agents list', () => {
     }
   });
   after(() => {
-    deleteFleetServerDocs();
-    deleteAgentDocs();
+    deleteFleetServerDocs(true);
+    deleteAgentDocs(true);
     cleanupAgentPolicies();
   });
   beforeEach(() => {
+    login();
+
     cy.intercept('/api/fleet/agents/setup', {
       isReady: true,
       missing_optional_features: [],
@@ -367,7 +372,6 @@ describe('View agents list', () => {
       cy.get('.euiModalFooter button:enabled').contains('Assign policy').click();
       cy.wait('@getAgents');
       assertTableIsEmpty();
-      cy.pause();
       // Select new policy is filters
       cy.getBySel(FLEET_AGENT_LIST_PAGE.POLICY_FILTER).click();
       cy.get('button').contains('Agent policy 4').click();

x-pack/plugins/fleet/cypress/e2e/enrollment_token.cy.ts

@@ -8,9 +8,12 @@
 import { cleanupAgentPolicies } from '../tasks/cleanup';
 import { ENROLLMENT_TOKENS } from '../screens/fleet';
 
+import { request } from '../tasks/common';
+import { login } from '../tasks/login';
+
 describe('Enrollment token page', () => {
   before(() => {
-    cy.request({
+    request({
       method: 'POST',
       url: '/api/fleet/agent_policies',
       body: {
@@ -28,6 +31,10 @@ describe('Enrollment token page', () => {
     cleanupAgentPolicies();
   });
 
+  beforeEach(() => {
+    login();
+  });
+
   it('Create new Token', () => {
     cy.visit('app/fleet/enrollment-tokens');
     cy.getBySel(ENROLLMENT_TOKENS.CREATE_TOKEN_BUTTON).click();

x-pack/plugins/fleet/cypress/e2e/fleet_agent_flyout.cy.ts

@@ -10,6 +10,9 @@ import { cleanupAgentPolicies, deleteFleetServerDocs, deleteAgentDocs } from '..
 import { createAgentDoc } from '../tasks/agents';
 import { setFleetServerHost } from '../tasks/fleet_server';
 import { FLEET, navigateTo } from '../tasks/navigation';
+import { request } from '../tasks/common';
+
+import { login } from '../tasks/login';
 
 const FLEET_SERVER_POLICY_ID = 'fleet-server-policy';
 
@@ -25,7 +28,7 @@ describe('Fleet add agent flyout', () => {
       cleanUp();
       let policyId: string;
       // Create a Fleet server policy
-      cy.request({
+      request({
         method: 'POST',
         url: '/api/fleet/agent_policies',
         headers: { 'kbn-xsrf': 'xx' },
@@ -59,6 +62,8 @@ describe('Fleet add agent flyout', () => {
         });
         setFleetServerHost();
       });
+
+      login();
     });
 
     afterEach(() => {