Reduce Cypress test flakiness (#574) (#576)

* Add retries to first test in each Cypress suite to counter random timeout failures

Signed-off-by: Mohammad Qureshi <[email protected]>

* Update dashboard version in CI to 2.4.0 until branch is updated to newer version

Signed-off-by: Mohammad Qureshi <[email protected]>

* Move Cypress retry to global config

Signed-off-by: Mohammad Qureshi <[email protected]>

* Reduce Cypress retry count to 2

Signed-off-by: Mohammad Qureshi <[email protected]>

* Remove MacOS from Dashboard test matrix for now

Signed-off-by: Mohammad Qureshi <[email protected]>

* Remove include args for macos dashboard test

Signed-off-by: Mohammad Qureshi <[email protected]>

Signed-off-by: Mohammad Qureshi <[email protected]>
(cherry picked from commit fb31fa4)

Co-authored-by: Mohammad Qureshi <[email protected]>

.github/workflows/dashboards-notifications-test-and-build-workflow.yml

@@ -9,7 +9,7 @@ on: [pull_request, push]
 
 env:
   PLUGIN_NAME: notifications-dashboards
-  OPENSEARCH_DASHBOARDS_VERSION: '2.4'
+  OPENSEARCH_DASHBOARDS_VERSION: '2.4.0'
   OPENSEARCH_VERSION: '2.4.0-SNAPSHOT'
 
 jobs:
@@ -25,7 +25,8 @@ jobs:
       # This setting says that all jobs should finish, even if one fails
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        # TODO: macos-latest was removed due to CI flakiness, we can revisit adding it back later
+        os: [ ubuntu-latest, windows-latest ]
         include:
           - os: windows-latest
             working_directory: X:\
@@ -34,8 +35,9 @@ jobs:
           - os: ubuntu-latest
             jest_test_args: --coverage
             cypress_cache_folder: ~/.cache/Cypress
-          - os: macos-latest
-            cypress_cache_folder: ~/Library/Caches/Cypress
+            # TODO: Add back when macos-latest is back in the test matrix
+#          - os: macos-latest
+#            cypress_cache_folder: ~/Library/Caches/Cypress
 
 
     name: Test and Build Dashboards Notifications on ${{ matrix.os }}

dashboards-notifications/cypress.json

@@ -11,6 +11,10 @@
   "requestTimeout": 60000,
   "responseTimeout": 60000,
   "defaultCommandTimeout": 60000,
+  "retries": {
+    "runMode": 2,
+    "openMode": 2
+  },
   "env": {
     "opensearch": "localhost:9200",
     "opensearchDashboards": "localhost:5601",