Merge pull request #7193 from owncloud/remove-vrt-nightwatch

[full-ci] Remove nightwatch-vrt && visual regression tests from CI
owncloud · Jul 5, 2022 · da9cde0 · da9cde0
2 parents e7f34e1 + d828a2e
commit da9cde0
Show file tree

Hide file tree

Showing 12 changed files with 8 additions and 565 deletions.
diff --git a/.drone.star b/.drone.star
@@ -174,7 +174,6 @@ config = {
                 "EXPECTED_FAILURES_FILE": "%s/tests/acceptance/expected-failures-with-oc10-server-oauth2-login.md" % dir["web"],
                 "WEB_UI_CONFIG": "%s/dist/config.json" % dir["web"],
             },
-            "visualTesting": False,
             "screenShots": True,
         },
         # These suites have all or most of their scenarios expected to fail.
@@ -188,7 +187,6 @@ config = {
                 "EXPECTED_FAILURES_FILE": "%s/tests/acceptance/expected-failures-with-oc10-server-oauth2-login.md" % dir["web"],
                 "WEB_UI_CONFIG": "%s/dist/config.json" % dir["web"],
             },
-            "visualTesting": False,
             "screenShots": True,
             "retry": False,
         },
@@ -205,7 +203,6 @@ config = {
                 "EXPECTED_FAILURES_FILE": "%s/tests/acceptance/expected-failures-with-oc10-server-oauth2-login.md" % dir["web"],
                 "WEB_UI_CONFIG": "%s/dist/config.json" % dir["web"],
             },
-            "visualTesting": False,
             "screenShots": True,
             "notificationsAppNeeded": True,
         },
@@ -513,7 +510,6 @@ config = {
                 "EXPECTED_FAILURES_FILE": "%s/tests/acceptance/expected-failures-with-ocis-server-ocis-storage.md" % dir["web"],
             },
             "runningOnOCIS": True,
-            "visualTesting": False,
             "filterTags": "not @skip and not @skipOnOCIS and not @notToImplementOnOCIS",
             "screenShots": True,
         },
@@ -1262,7 +1258,6 @@ def acceptance(ctx):
         "federatedServerVersion": OC10_VERSION,
         "runningOnOCIS": False,
         "screenShots": False,
-        "visualTesting": False,
         "openIdConnect": False,
         "oc10IntegrationAppIncluded": False,
         "skip": False,
@@ -1396,12 +1391,7 @@ def acceptance(ctx):
                         steps += waitForMiddlewareService()
 
                         # run the acceptance tests
-                        steps += runWebuiAcceptanceTests(ctx, suite, alternateSuiteName, params["filterTags"], params["extraEnvironment"], params["visualTesting"], params["screenShots"], params["retry"])
-
-                        # capture the screenshots from visual regression testing (only runs on failure)
-                        if (params["visualTesting"]):
-                            steps += listScreenShots() + uploadVisualDiff() + uploadVisualScreenShots()
-                            steps += buildGithubCommentVisualDiff(ctx, suiteName, params["runningOnOCIS"])
+                        steps += runWebuiAcceptanceTests(ctx, suite, alternateSuiteName, params["filterTags"], params["extraEnvironment"], params["screenShots"], params["retry"])
 
                         # Capture the screenshots from acceptance tests (only runs on failure)
                         if (params["screenShots"]):
@@ -2347,7 +2337,7 @@ def copyFilesForUpload():
         ],
     }]
 
-def runWebuiAcceptanceTests(ctx, suite, alternateSuiteName, filterTags, extraEnvironment, visualTesting, screenShots, retry):
+def runWebuiAcceptanceTests(ctx, suite, alternateSuiteName, filterTags, extraEnvironment, screenShots, retry):
     environment = {}
     if (filterTags != ""):
         environment["TEST_TAGS"] = filterTags
@@ -2363,8 +2353,6 @@ def runWebuiAcceptanceTests(ctx, suite, alternateSuiteName, filterTags, extraEnv
 
     if (ctx.build.event == "cron") or (not retry):
         environment["RERUN_FAILED_WEBUI_SCENARIOS"] = "false"
-    if (visualTesting):
-        environment["VISUAL_TEST"] = "true"
     if (screenShots):
         environment["SCREENSHOTS"] = "true"
     environment["SERVER_HOST"] = "http://web"
@@ -2578,134 +2566,6 @@ def uploadScreenshots():
         },
     }]
 
-def listScreenShots():
-    return [{
-        "name": "list screenshots-visual",
-        "image": OC_CI_NODEJS,
-        "commands": [
-            "ls -laR %s/tests/vrt" % dir["web"],
-        ],
-        "when": {
-            "status": [
-                "failure",
-            ],
-        },
-    }]
-
-def uploadVisualDiff():
-    return [{
-        "name": "upload-diff-screenshots",
-        "image": PLUGINS_S3,
-        "pull": "if-not-exists",
-        "settings": {
-            "bucket": {
-                "from_secret": "cache_public_s3_bucket",
-            },
-            "endpoint": {
-                "from_secret": "cache_public_s3_server",
-            },
-            "path_style": True,
-            "source": "%s/tests/vrt/diff/**/*" % dir["web"],
-            "strip_prefix": "%s/tests/vrt" % dir["web"],
-            "target": "/${DRONE_REPO}/${DRONE_BUILD_NUMBER}/vrt/screenshots",
-        },
-        "environment": {
-            "AWS_ACCESS_KEY_ID": {
-                "from_secret": "cache_public_s3_access_key",
-            },
-            "AWS_SECRET_ACCESS_KEY": {
-                "from_secret": "cache_public_s3_secret_key",
-            },
-        },
-        "when": {
-            "status": [
-                "failure",
-            ],
-            "event": [
-                "pull_request",
-            ],
-        },
-    }]
-
-def uploadVisualScreenShots():
-    return [{
-        "name": "upload-latest-screenshots",
-        "image": PLUGINS_S3,
-        "pull": "if-not-exists",
-        "settings": {
-            "bucket": {
-                "from_secret": "cache_public_s3_bucket",
-            },
-            "endpoint": {
-                "from_secret": "cache_public_s3_server",
-            },
-            "path_style": True,
-            "source": "%s/tests/vrt/latest/**/*" % dir["web"],
-            "strip_prefix": "%s/tests/vrt" % dir["web"],
-            "target": "/${DRONE_REPO}/${DRONE_BUILD_NUMBER}/vrt/screenshots",
-        },
-        "environment": {
-            "AWS_ACCESS_KEY_ID": {
-                "from_secret": "cache_public_s3_access_key",
-            },
-            "AWS_SECRET_ACCESS_KEY": {
-                "from_secret": "cache_public_s3_secret_key",
-            },
-        },
-        "when": {
-            "status": [
-                "failure",
-            ],
-            "event": [
-                "pull_request",
-            ],
-        },
-    }]
-
-def buildGithubCommentVisualDiff(ctx, suite, runningOnOCIS):
-    backend = "ocis" if runningOnOCIS else "oc10"
-    branch = ctx.build.source if ctx.build.event == "pull_request" else "master"
-    return [{
-        "name": "build-github-comment-vrt",
-        "image": OC_UBUNTU,
-        "commands": [
-            "cd %s/tests/vrt" % dir["web"],
-            "if [ ! -d diff ]; then exit 0; fi",
-            "cd diff",
-            "if [ ! -d %s ]; then exit 0; fi" % backend,
-            "cd %s" % backend,
-            "ls -la",
-            'echo "<details><summary>:boom: Visual regression tests failed. Please find the screenshots inside ...</summary>\\n\\n<p>\\n\\n" >> %s/comments.file' % dir["web"],
-            'echo "Diff Image: </br>" >> %s/comments.file' % dir["web"],
-            'for f in *.png; do echo \'!\'"[$f]($CACHE_ENDPOINT/$CACHE_BUCKET/${DRONE_REPO}/${DRONE_BUILD_NUMBER}/vrt/screenshots/diff/%s/$f)" >> %s/comments.file; done' % (backend, dir["web"]),
-            "cd ../../latest",
-            "cd %s" % backend,
-            'echo "Actual Image: </br>" >> %s/comments.file' % dir["web"],
-            'for f in *.png; do echo \'!\'"[$f]($CACHE_ENDPOINT/$CACHE_BUCKET/${DRONE_REPO}/${DRONE_BUILD_NUMBER}/vrt/screenshots/latest/%s/$f)" >> %s/comments.file; done' % (backend, dir["web"]),
-            'echo "Comparing Against: </br>" >> %s/comments.file' % dir["web"],
-            'for f in *.png; do echo \'!\'"[$f](https://raw.githubusercontent.com/owncloud/web/%s/tests/vrt/baseline/%s/$f)" >> %s/comments.file; done' % (branch, backend, dir["web"]),
-            'echo "\n</p></details>" >> %s/comments.file' % dir["web"],
-            "more %s/comments.file" % dir["web"],
-        ],
-        "environment": {
-            "TEST_CONTEXT": suite,
-            "CACHE_ENDPOINT": {
-                "from_secret": "cache_public_s3_server",
-            },
-            "CACHE_BUCKET": {
-                "from_secret": "cache_public_s3_bucket",
-            },
-        },
-        "when": {
-            "status": [
-                "failure",
-            ],
-            "event": [
-                "pull_request",
-            ],
-        },
-    }]
-
 def buildGithubComment(suite):
     return [{
         "name": "build-github-comment",

diff --git a/.gitignore b/.gitignore
@@ -60,10 +60,6 @@ tests/acceptance/.yarn/*
 !tests/acceptance/.yarn/sdks
 !tests/acceptance/.yarn/versions
 
-# visual regression testing artifacts
-/tests/vrt/diff
-/tests/vrt/latest
-
 # acceptance test artifacts
 logfile.txt
 comments.file

diff --git a/docs/testing/acceptance-tests-all.md b/docs/testing/acceptance-tests-all.md
@@ -92,32 +92,6 @@ see [available settings](#available-settings-to-be-set-by-environment-variables)
 - run `yarn test:acceptance:ocis <feature-files-to-test>`
 - If you are a mac user, run `STORAGE_HOME_DATA_SERVER_URL='http://host.docker.internal:9155/data' STORAGE_DATAGATEWAY_PUBLIC_URL='https://host.docker.internal:9200/data' STORAGE_USERS_DATA_SERVER_URL='http://host.docker.internal:9158/data' STORAGE_FRONTEND_PUBLIC_URL='https://host.docker.internal:9200' PROXY_ENABLE_BASIC_AUTH=true PROXY_OIDC_ISSUER='https://host.docker.internal:9200' IDP_INSECURE='true' IDP_IDENTIFIER_REGISTRATION_CONF='./mac-identifier-registration.yml' IDP_ISS='https://host.docker.internal:9200' IDP_TLS='true' yarn test:acceptance:ocis <feature-files-to-test>`
 
-### Visual Regression Testing
-
-The test suite consists of snapshots of UI components which can be compared for visual regression testing when running the acceptance tests. These comparisons are done in the existing scenarios. You can check the existing snapshots of the components in the directory `/tests/vrt/baseline`.
-
-#### Running the visual regression tests
-
-When you run the acceptance tests as usual, all the visual regression comparisons are skipped. To run the acceptance test suite with the visual comparison enabled you need to set the env variable, `VISUAL_TEST` to `true`
-
-eg.
-```
-VISUAL_TEST=true yarn test:acceptance:oc10 <feature-file-to-test>
-```
-
-#### Updating the snapshots
-
-If there is some change in the components, and you want to update the snapshots of the components you can run the tests with `UPDATE_VRT_SCREENSHOTS` set to `true`. When this env variable is set, the testrunner will ignore if the visual comparison fails and updates the baseline images with the latest images if the comparison fails.
-
-eg.
-```
-VISUAL_TEST=true UPDATE_VRT_SCREENSHOTS=true yarn test:acceptance:oc10 <feature-file-to-test>
-```
-
-**note** Visual regression testing may not be completely reliable every time as small changes such as window size and screen resolution may affect the result. For better results it is recommended that you run the tests using the `selenium/standalone-chrome-debug` image of selenium and window size of `1280x1024`
-
-see [available settings](#available-settings-to-be-set-by-environment-variables) for further setup if needed
-
 ## Available settings to be set by environment variables
 
 These values can be set using the environment variables to configure `yarn test:acceptance:oc10` and `yarn test:acceptance:ocis` to match your local test environment.
@@ -138,8 +112,6 @@ These values can be set using the environment variables to configure `yarn test:
 | `OCIS_REVA_DATA_ROOT`       | Data directory of oCIS                                             | /var/tmp/reva |
 | `TESTING_DATA_DIR`       | Testing data directory for new users                                                           | - |
 | `WEB_UI_CONFIG`       | Path for the web config file (usually in the dist folder)                       | - |
-| `VISUAL_TEST`       | Run the visual regression comparison while running the acceptance tests                       | - |
-| `UPDATE_VRT_SCREENSHOTS`       | Update the baseline snapshots with the latest images for visual regression tests                       | - |
 | `MIDDLEWARE_HOST` | Middleware host URL | http://host.docker.internal:3000 |
 
 ## Tips

diff --git a/tests/acceptance/features/webUIFilesList/fileList.feature b/tests/acceptance/features/webUIFilesList/fileList.feature
@@ -12,11 +12,11 @@ Feature: User can view files inside a folder
     And user "Alice" has uploaded file "data.zip" to "data.zip" in the server
     And user "Alice" has logged in using the webUI
 
-  Scenario: Resources are listed
+  Scenario: Resources are listed and files list displays quicklink quick action
     When the user browses to the files page
+    Then quick action "quicklink" should be displayed on the webUI
     Then folder "simple-folder" should be listed on the webUI
     And file "textfile0.txt" should be listed on the webUI
-    And the top bar should match the default baseline
 
   @issue-1910
   Scenario: Empty folders display no resources in the list
@@ -25,11 +25,6 @@ Feature: User can view files inside a folder
     Then there should be no resources listed on the webUI
 
 
-  Scenario: All files list displays quicklink quick action
-    When the user browses to the files page
-    Then quick action "quicklink" should be displayed on the webUI
-    And the sidebar should match the default baseline
-
   Scenario: files are not selected when the user logs in
     When the user browses to the files page
     Then these files should not be selected on the webUI

diff --git a/tests/acceptance/nightwatch.conf.js b/tests/acceptance/nightwatch.conf.js
@@ -31,22 +31,14 @@ const OPENID_LOGIN = RUN_ON_OCIS || process.env.OPENID_LOGIN === 'true'
 const WEB_UI_CONFIG = process.env.WEB_UI_CONFIG || path.join(__dirname, 'dist/config.json')
 const SCREENSHOTS = process.env.SCREENSHOTS === 'true'
 
-const VISUAL_TEST = process.env.VISUAL_TEST === 'true'
-const UPDATE_VRT_SCREENSHOTS = process.env.UPDATE_VRT_SCREENSHOTS === 'true'
-
 const MIDDLEWARE_HOST = withHttp(
   process.env.MIDDLEWARE_HOST ||
     (RUN_ON_OCIS ? 'http://host.docker.internal:3000' : 'http://host.docker.internal:3001')
 )
 
-function generateScreenshotFilePath(nightwatchClient, basePath, imagePath) {
-  return path.join(process.cwd(), basePath, imagePath)
-}
-
 const config = {
   page_objects_path: './pageObjects',
-  custom_commands_path: ['./customCommands', 'node_modules/nightwatch-vrt/commands'],
-  custom_assertions_path: ['node_modules/nightwatch-vrt/assertions'],
+  custom_commands_path: ['./customCommands'],
   test_settings: {
     default: {
       // ocis doesn't have '#' in the url path anymore
@@ -70,19 +62,6 @@ const config = {
         testing_data_dir: TESTING_DATA_DIR,
         ldap_password: LDAP_ADMIN_PASSWORD,
         webUIConfig: WEB_UI_CONFIG,
-        visual_test: VISUAL_TEST,
-        visual_regression_settings: {
-          generate_screenshot_path: generateScreenshotFilePath,
-          latest_screenshots_path: 'tests/vrt/latest',
-          latest_suffix: '',
-          baseline_screenshots_path: 'tests/vrt/baseline',
-          baseline_suffix: '',
-          diff_screenshots_path: 'tests/vrt/diff',
-          diff_suffix: '',
-          threshold: 0.002,
-          prompt: false,
-          always_save_diff_screenshot: UPDATE_VRT_SCREENSHOTS
-        },
         screenshots: SCREENSHOTS,
         middlewareUrl: MIDDLEWARE_HOST
       },
@@ -126,8 +105,4 @@ const config = {
   }
 }
 
-if (VISUAL_TEST) {
-  process.env.SCREEN_RESOLUTION = '1280x1024'
-}
-
 module.exports = config
diff --git a/tests/acceptance/package.json b/tests/acceptance/package.json
@@ -25,7 +25,6 @@
     "lodash": "^4.17.21",
     "nightwatch": "1.7.11",
     "nightwatch-api": "3.0.1",
-    "nightwatch-vrt": "^0.2.10",
     "node-fetch": "^2.6.7",
     "tslib": "^2.2.0",
     "url-search-params-polyfill": "^8.0.0",

diff --git a/tests/acceptance/stepDefinitions/visualContext.js b/tests/acceptance/stepDefinitions/visualContext.js