diff --git a/.github/workflows/dashboards-notifications-test-and-build-workflow.yml b/.github/workflows/dashboards-notifications-test-and-build-workflow.yml
index c04eaf85..3e892740 100644
--- a/.github/workflows/dashboards-notifications-test-and-build-workflow.yml
+++ b/.github/workflows/dashboards-notifications-test-and-build-workflow.yml
@@ -9,7 +9,7 @@ on: [pull_request, push]
 
 env:
   PLUGIN_NAME: notifications-dashboards
-  OPENSEARCH_DASHBOARDS_VERSION: '2.4'
+  OPENSEARCH_DASHBOARDS_VERSION: '2.4.0'
   OPENSEARCH_VERSION: '2.4.0-SNAPSHOT'
 
 jobs:
@@ -25,7 +25,8 @@ jobs:
       # This setting says that all jobs should finish, even if one fails
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        # TODO: macos-latest was removed due to CI flakiness, we can revisit adding it back later
+        os: [ ubuntu-latest, windows-latest ]
         include:
           - os: windows-latest
             working_directory: X:\
@@ -34,8 +35,9 @@ jobs:
           - os: ubuntu-latest
             jest_test_args: --coverage
             cypress_cache_folder: ~/.cache/Cypress
-          - os: macos-latest
-            cypress_cache_folder: ~/Library/Caches/Cypress
+            # TODO: Add back when macos-latest is back in the test matrix
+#          - os: macos-latest
+#            cypress_cache_folder: ~/Library/Caches/Cypress
 
 
     name: Test and Build Dashboards Notifications on ${{ matrix.os }}
diff --git a/dashboards-notifications/cypress.json b/dashboards-notifications/cypress.json
index 4585c272..fa5812d9 100644
--- a/dashboards-notifications/cypress.json
+++ b/dashboards-notifications/cypress.json
@@ -11,6 +11,10 @@
   "requestTimeout": 60000,
   "responseTimeout": 60000,
   "defaultCommandTimeout": 60000,
+  "retries": {
+    "runMode": 2,
+    "openMode": 2
+  },
   "env": {
     "opensearch": "localhost:9200",
     "opensearchDashboards": "localhost:5601",