diff --git a/build.gradle b/build.gradle
index 58ac63aa7..4ed83b401 100644
--- a/build.gradle
+++ b/build.gradle
@@ -39,6 +39,7 @@ plugins {
     id 'nebula.ospackage' version "8.3.0" apply false
     id "com.diffplug.gradle.spotless" version "3.26.1"
     id 'java-library'
+    id 'org.gradle.test-retry' version '1.3.1'
 }
 
 tasks.withType(JavaCompile) {
@@ -148,8 +149,15 @@ def _numNodes = findProperty('numNodes') as Integer ?: 1
 
 def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
 opensearch_tmp_dir.mkdirs()
-
+boolean isCiServer = System.getenv().containsKey("CI")
 test {
+    retry {
+        if (isCiServer) {
+            failOnPassedAfterRetry = false
+            maxRetries = 6
+            maxFailures = 10
+        }
+    }
     include '**/*Tests.class'
     systemProperty 'tests.security.manager', 'false'
 }
@@ -162,6 +170,13 @@ task integTest(type: RestIntegTestTask) {
 tasks.named("check").configure { dependsOn(integTest) }
 
 integTest {
+    retry {
+        if (isCiServer) {
+            failOnPassedAfterRetry = false
+            maxRetries = 6
+            maxFailures = 10
+        }
+    }
     dependsOn "bundlePlugin"
     systemProperty 'tests.security.manager', 'false'
     systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
diff --git a/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java b/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java
index 16c136fe2..16ef77731 100644
--- a/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java
+++ b/src/test/java/org/opensearch/ad/e2e/DetectionResultEvalutationIT.java
@@ -427,6 +427,7 @@ private void waitAllSyncheticDataIngested(int expectedSize, String datasetName,
                 request = new Request("POST", String.format(Locale.ROOT, "/%s/_refresh", datasetName));
                 client.performRequest(request);
             }
+            Thread.sleep(1_000);
         } while (maxWaitCycles-- >= 0);
     }
 
@@ -592,26 +593,21 @@ private void indexTrainData(String datasetName, List<JsonObject> data, int train
                 throw new RuntimeException(e);
             }
         });
-        Thread.sleep(1_000);
+        Thread.sleep(3_000);
     }
 
     public void testRestartHCADDetector() throws Exception {
         // TODO: this test case will run for a much longer time and timeout with security enabled
         if (!isHttps()) {
-            int maxRetries = 3;
-            int i = 0;
-            for (; i < maxRetries; i++) {
-                try {
-                    disableResourceNotFoundFaultTolerence();
-                    verifyRestart("synthetic", 1, 8);
-                    break;
-                } catch (Throwable throwable) {
-                    LOG.info("Retry restart test case", throwable);
-                    cleanUpCluster();
-                    wipeAllODFEIndices();
-                }
+            try {
+                disableResourceNotFoundFaultTolerence();
+                verifyRestart("synthetic", 1, 8);
+            } catch (Throwable throwable) {
+                LOG.info("Retry restart test case", throwable);
+                cleanUpCluster();
+                wipeAllODFEIndices();
+                fail();
             }
-            assertTrue("failed all retries", i < maxRetries);
         }
     }