Migrate existing Cypress tests to Cypress + Cucumber (#57299)

* chore: move gitignore to the cypress directory

* chore: ignore more test files

* fix: do not check the Loading Message

It seems not relevant to the main purpose of these tests

* chore: use cypres + webpack + cucumber scaffolding

See https://github.com/TheBrainFamily/cypress-cucumber-webpack-typescript-example

* chore: add eslint and prettier for code linting

* feat: convert existing Cypress test into BDD style

* feat: add support for using proper Node version in MacOSX

* chore: use tslint

* chore: use old layout

We are keeping cypress as rootDir to follow project's structure. On the other hand,
having a second cypress directory at the 2nd level is the default structure, as shown
in the examples:
- https://github.com/TheBrainFamily/cypress-cucumber-webpack-typescript-example
- https://github.com/cypress-io/cypress-example-recipes/tree/a240054d7f5626ffcd7bd668dded96d219c4a7eb/examples/preprocessors__typescript-webpack

* chore: remove prelint script meanwhile we fix TS lint

* chore: move test results to a specific directory

* chore: rename variable following old code

* chore: remove non-needed lints, as we are going to use kibana build

* chore: import snapshot function from cypress

* chore: add readFile utils back from a bad removal

* chore: change format of JSON spec file

It was automatically changed by tests

* chore: move CI directory to the proper layout in order for Jenkins to work

* chore: store test-results from proper dir on Jenkins

* chore: store artifacts properly on Jenkins

* Fix type issues

* chore: rename test application to e2e (end-to-end)

We are keeping the build system within the test application, isolating dependencies

* docs: reorganise docs for APM UI e2e tests

* fix: Use proper cypress support file

* chore: use existing NPM script for running cypress on CI

* chore: update paths in CI scripts

* docs: document how the CI runs the tests

* chore: use Node 10 for tests

* chore: Use kibana's Node version for tests

* chore: run yarn install

* docs: update docs

* fix: path was wrong

* docs: fix paths and flags used to load data

* docs: elasticsearch fix flag

* docs: Bootstrap kibana before running it

* docs: remove outdated info

* chore: move background steps to the scenario

This would avoid not reading the background when the number of scenarios grows

Co-authored-by: Dario Gieselaar <[email protected]>
Co-authored-by: Elastic Machine <[email protected]>

.ci/end2end.groovy

@@ -13,7 +13,7 @@ pipeline {
     BASE_DIR = 'src/github.com/elastic/kibana'
     HOME = "${env.WORKSPACE}"
     APM_ITS = 'apm-integration-testing'
-    CYPRESS_DIR = 'x-pack/legacy/plugins/apm/cypress'
+    CYPRESS_DIR = 'x-pack/legacy/plugins/apm/e2e'
     PIPELINE_LOG_LEVEL = 'DEBUG'
   }
   options {
@@ -107,7 +107,7 @@ pipeline {
         dir("${BASE_DIR}"){
           sh '''
             jobs -l
-            docker build --tag cypress ${CYPRESS_DIR}/ci
+            docker build --tag cypress --build-arg NODE_VERSION=$(cat .node-version) ${CYPRESS_DIR}/ci
             docker run --rm -t --user "$(id -u):$(id -g)" \
                     -v `pwd`:/app --network="host" \
                     --name cypress cypress'''
@@ -116,8 +116,8 @@ pipeline {
       post {
         always {
           dir("${BASE_DIR}"){
-            archiveArtifacts(allowEmptyArchive: false, artifacts: "${CYPRESS_DIR}/screenshots/**,${CYPRESS_DIR}/videos/**,${CYPRESS_DIR}/*e2e-tests.xml")
-            junit(allowEmptyResults: true, testResults: "${CYPRESS_DIR}/*e2e-tests.xml")
+            archiveArtifacts(allowEmptyArchive: false, artifacts: "${CYPRESS_DIR}/**/screenshots/**,${CYPRESS_DIR}/**/videos/**,${CYPRESS_DIR}/**/test-results/*e2e-tests.xml")
+            junit(allowEmptyResults: true, testResults: "${CYPRESS_DIR}/**/test-results/*e2e-tests.xml")
           }
           dir("${APM_ITS}"){
             sh 'docker-compose logs > apm-its.log || true'

.eslintignore

@@ -39,7 +39,7 @@ src/legacy/core_plugins/vis_type_vislib/public/vislib/__tests__/lib/fixtures/moc
 /x-pack/legacy/plugins/infra/common/graphql/types.ts
 /x-pack/legacy/plugins/infra/public/graphql/types.ts
 /x-pack/legacy/plugins/infra/server/graphql/types.ts
-/x-pack/legacy/plugins/apm/cypress/**/snapshots.js
+/x-pack/legacy/plugins/apm/e2e/cypress/**/snapshots.js
 /src/legacy/plugin_discovery/plugin_pack/__tests__/fixtures/plugins/broken
 **/graphql/types.ts
 **/*.js.snap

src/cli/cluster/cluster_manager.ts

@@ -264,7 +264,7 @@ export class ClusterManager {
       fromRoot('src/legacy/server/sass/__tmp__'),
       fromRoot('x-pack/legacy/plugins/reporting/.chromium'),
       fromRoot('x-pack/legacy/plugins/siem/cypress'),
-      fromRoot('x-pack/legacy/plugins/apm/cypress'),
+      fromRoot('x-pack/legacy/plugins/apm/e2e/cypress'),
       fromRoot('x-pack/legacy/plugins/apm/scripts'),
       fromRoot('x-pack/legacy/plugins/canvas/canvas_plugin_src'), // prevents server from restarting twice for Canvas plugin changes,
       'plugins/java_languageserver',

src/dev/ci_setup/setup_env.sh

@@ -168,4 +168,4 @@ if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then
   export JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
 fi
 
-export CI_ENV_SETUP=true
+export CI_ENV_SETUP=true

src/dev/run_check_lockfile_symlinks.js

@@ -35,7 +35,7 @@ const IGNORE_FILE_GLOBS = [
   // fixtures aren't used in production, ignore them
   '**/*fixtures*/**/*',
   // cypress isn't used in production, ignore it
-  'x-pack/legacy/plugins/apm/cypress/*',
+  'x-pack/legacy/plugins/apm/e2e/*',
 ];
 
 run(async ({ log }) => {

src/dev/typescript/projects.ts

@@ -30,7 +30,7 @@ export const PROJECTS = [
   new Project(resolve(REPO_ROOT, 'x-pack/legacy/plugins/siem/cypress/tsconfig.json'), {
     name: 'siem/cypress',
   }),
-  new Project(resolve(REPO_ROOT, 'x-pack/legacy/plugins/apm/cypress/tsconfig.json'), {
+  new Project(resolve(REPO_ROOT, 'x-pack/legacy/plugins/apm/e2e/tsconfig.json'), {
     name: 'apm/cypress',
     disableTypeCheck: true,
   }),

x-pack/legacy/plugins/apm/e2e/.gitignore

@@ -0,0 +1,4 @@
+cypress/ingest-data/events.json
+cypress/screenshots/*
+
+cypress/test-results

x-pack/legacy/plugins/apm/e2e/README.md

@@ -0,0 +1,68 @@
+# End-To-End (e2e) Test for APM UI
+
+## Ingest static data into Elasticsearch via APM Server
+
+1. Start Elasticsearch and APM Server, using [apm-integration-testing](https://github.com/elastic/apm-integration-testing):
+
+```shell
+$ git clone https://github.com/elastic/apm-integration-testing.git
+$ cd apm-integration-testing
+./scripts/compose.py start master --no-kibana --no-xpack-secure
+```
+
+2. Download [static data file](https://storage.googleapis.com/apm-ui-e2e-static-data/events.json)
+
+```shell
+$ cd x-pack/legacy/plugins/apm/e2e/cypress/ingest-data
+$ curl https://storage.googleapis.com/apm-ui-e2e-static-data/events.json --output events.json
+```
+
+3. Post to APM Server
+
+```shell
+$ cd x-pack/legacy/plugins/apm/e2e/cypress/ingest-data
+$ node replay.js --server-url http://localhost:8200 --secret-token abcd --events ./events.json
+```
+>This process will take a few minutes to ingest all data
+
+4. Start Kibana
+
+```shell
+$ yarn kbn bootstrap
+$ yarn start --no-base-path --csp.strict=false
+```
+
+> Content Security Policy (CSP) Settings: Your Kibana instance must have the `csp.strict: false`.
+
+## How to run the tests
+
+_Note: Run the following commands from `kibana/x-pack/legacy/plugins/apm/e2e/cypress`._
+
+### Interactive mode
+
+```
+yarn cypress open
+```
+
+### Headless mode
+
+```
+yarn cypress run
+```
+
+## Reproducing CI builds
+
+>This process is very slow compared to the local development described above. Consider that the CI must install and configure the build tools and create a Docker image for the project to run tests in a consistent manner.
+
+The Jenkins CI uses a shell script to prepare Kibana:
+
+```shell
+# Prepare and run Kibana locally
+$ x-pack/legacy/plugins/apm/e2e/ci/prepare-kibana.sh
+# Build Docker image for Kibana
+$ docker build --tag cypress --build-arg NODE_VERSION=$(cat .node-version) x-pack/legacy/plugins/apm/e2e/ci 
+# Run Docker image
+$ docker run --rm -t --user "$(id -u):$(id -g)" \
+    -v `pwd`:/app --network="host" \
+    --name cypress cypress
+```

x-pack/legacy/plugins/apm/cypress/ci/Dockerfile → x-pack/legacy/plugins/apm/e2e/ci/Dockerfile

@@ -1,4 +1,5 @@
-FROM node:12
+ARG NODE_VERSION
+FROM node:$NODE_VERSION
 
 RUN apt-get -qq update \
   && apt-get -y -qq install xvfb \

x-pack/legacy/plugins/apm/cypress/ci/entrypoint.sh → x-pack/legacy/plugins/apm/e2e/ci/entrypoint.sh

@@ -21,9 +21,9 @@ npm config set cache ${HOME}
 #      --exclude=packages/ \
 #      --exclude=built_assets  --exclude=target \
 #      --exclude=data /app ${HOME}/
-#cd ${HOME}/app/x-pack/legacy/plugins/apm/cypress
+#cd ${HOME}/app/x-pack/legacy/plugins/apm/e2e/cypress
 
-cd /app/x-pack/legacy/plugins/apm/cypress
+cd /app/x-pack/legacy/plugins/apm/e2e
 ## Install dependencies for cypress
 CI=true npm install
 yarn install
@@ -33,4 +33,4 @@ npm install wait-on
 ./node_modules/.bin/wait-on ${CYPRESS_BASE_URL}/status && echo 'Kibana is up and running'
 
 # Run cypress
-./node_modules/.bin/cypress run
+npm run cypress:run

x-pack/legacy/plugins/apm/cypress/ci/prepare-kibana.sh → x-pack/legacy/plugins/apm/e2e/ci/prepare-kibana.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-CYPRESS_DIR="x-pack/legacy/plugins/apm/cypress"
+CYPRESS_DIR="x-pack/legacy/plugins/apm/e2e"
 
 echo "1/3 Install dependencies ..."
 # shellcheck disable=SC1091

x-pack/legacy/plugins/apm/e2e/cypress.json

@@ -0,0 +1,19 @@
+{
+  "baseUrl": "http://localhost:5601",
+  "video": false,
+  "trashAssetsBeforeRuns": false,
+  "fileServerFolder": "../",
+  "fixturesFolder": "./cypress/fixtures",
+  "integrationFolder": "./cypress/integration",
+  "pluginsFile": "./cypress/plugins/index.js",
+  "screenshotsFolder": "./cypress/screenshots",
+  "supportFile": "./cypress/support/index.ts",
+  "videosFolder": "./cypress/videos",
+  "useRelativeSnapshots": true,
+  "reporter": "junit",
+  "reporterOptions": {
+    "mochaFile": "./cypress/test-results/[hash]-e2e-tests.xml",
+    "toConsole": false
+  },
+  "testFiles": "**/*.{feature,features}"
+}

x-pack/legacy/plugins/apm/e2e/cypress/integration/apm.feature

@@ -0,0 +1,7 @@
+Feature: APM
+
+  Scenario: Transaction duration charts
+    Given a user browses the APM UI application
+    When the user inspects the opbeans-go service
+    Then should redirect to correct path with correct params
+      And should have correct y-axis ticks

x-pack/legacy/plugins/apm/cypress/integration/snapshots.js → x-pack/legacy/plugins/apm/e2e/cypress/integration/snapshots.js

@@ -8,5 +8,12 @@ module.exports = {
       }
     }
   },
-  "__version": "3.4.1"
+  "__version": "3.8.3",
+  "APM": {
+    "Transaction duration charts": {
+      "1": "3.7 min",
+      "2": "1.8 min",
+      "3": "0.0 min"
+    }
+  }
 }