Abstract waitForInit to minimize duplication and improve test reliability

src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java

@@ -363,13 +363,13 @@ public Map<CType, SecurityDynamicConfiguration<?>> getConfigurationsFromIndex(Co
                 } else {
                     LOGGER.debug("security index exists and was created with ES 7 (new layout)");
                 }
-                retVal.putAll(validate(cl.load(configTypes.toArray(new CType[0]), 5, TimeUnit.SECONDS, acceptInvalid), configTypes.size()));
+                retVal.putAll(validate(cl.load(configTypes.toArray(new CType[0]), 10, TimeUnit.SECONDS, acceptInvalid), configTypes.size()));
 
 
             } else {
                 //wait (and use new layout)
                 LOGGER.debug("security index not exists (yet)");
-                retVal.putAll(validate(cl.load(configTypes.toArray(new CType[0]), 5, TimeUnit.SECONDS, acceptInvalid), configTypes.size()));
+                retVal.putAll(validate(cl.load(configTypes.toArray(new CType[0]), 10, TimeUnit.SECONDS, acceptInvalid), configTypes.size()));
             }
 
         } catch (Exception e) {

src/test/java/org/opensearch/security/RolesInjectorIntegTest.java

@@ -74,18 +74,6 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         }
     }
 
-    //Wait for the security plugin to load roles.
-    private void waitForInit(Client client) throws Exception {
-        try {
-            client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
-        } catch (OpenSearchSecurityException ex) {
-            if(ex.getMessage().contains("OpenSearch Security not initialized")) {
-                Thread.sleep(500);
-                waitForInit(client);
-            }
-        }
-    }
-
     @Test
     public void testRolesInject() throws Exception {
         setup(Settings.EMPTY, new DynamicSecurityConfig().setSecurityRoles("roles.yml"), Settings.EMPTY);

src/test/java/org/opensearch/security/RolesValidationIntegTest.java

@@ -20,7 +20,6 @@
 import org.junit.Test;
 
 import org.opensearch.OpenSearchSecurityException;
-import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.opensearch.action.admin.indices.create.CreateIndexRequest;
 import org.opensearch.action.admin.indices.create.CreateIndexResponse;
 import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest;
@@ -71,18 +70,6 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         }
     }
 
-    //Wait for the security plugin to load roles.
-    private void waitForInit(Client client) throws Exception {
-        try {
-            client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
-        } catch (OpenSearchSecurityException ex) {
-            if(ex.getMessage().contains("OpenSearch Security not initialized")) {
-                Thread.sleep(500);
-                waitForInit(client);
-            }
-        }
-    }
-
     @Test
     public void testRolesValidation() throws Exception {
         setup(Settings.EMPTY, new DynamicSecurityConfig().setSecurityRoles("roles.yml"), Settings.EMPTY);

src/test/java/org/opensearch/security/TransportUserInjectorIntegTest.java

@@ -20,7 +20,6 @@
 import org.junit.Test;
 
 import org.opensearch.OpenSearchSecurityException;
-import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.opensearch.action.admin.indices.create.CreateIndexRequest;
 import org.opensearch.action.admin.indices.create.CreateIndexResponse;
 import org.opensearch.client.Client;
@@ -67,18 +66,6 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         }
     }
 
-    //Wait for the security plugin to load roles.
-    private void waitForInit(Client client) throws Exception {
-        try {
-            client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
-        } catch (OpenSearchSecurityException ex) {
-            if(ex.getMessage().contains("OpenSearch Security not initialized")) {
-                Thread.sleep(500);
-                waitForInit(client);
-            }
-        }
-    }
-
     @Test
     public void testSecurityUserInjection() throws Exception {
         final Settings clusterNodeSettings = Settings.builder()

src/test/java/org/opensearch/security/auditlog/integration/TestAuditlogImpl.java

@@ -74,7 +74,7 @@ public static List<AuditMessage> doThenWaitForMessages(final Runnable action, fi
                 throw new MessagesNotFoundException(expectedCount, messages);
             }
             if (messages.size() != expectedCount) {
-                throw new RuntimeException("Unexpected number of messages, was expecting " + expectedCount + ", recieved " + messages.size());
+                throw new RuntimeException("Unexpected number of messages, was expecting " + expectedCount + ", received " + messages.size());
             }
         } catch (final InterruptedException e) {
             throw new RuntimeException("Unexpected exception", e);
@@ -119,7 +119,7 @@ public List<AuditMessage> getFoundMessages() {
 
         private static String createDetailMessage(final int expectedCount, final List<AuditMessage> foundMessages) {
             return new StringBuilder()
-                .append("Did not recieve all " + expectedCount + " audit messages after a short wait. ")
+                .append("Did not receive all " + expectedCount + " audit messages after a short wait. ")
                 .append("Missing " + (expectedCount - foundMessages.size()) + " messages.")
                 .append("Messages found during this time: \n\n")
                 .append(foundMessages.stream()

src/test/java/org/opensearch/security/ccstest/CrossClusterSearchTests.java

@@ -982,24 +982,6 @@ public void testCcsWithDiffCertsWithNodesDnDynamicallyAdded() throws Exception {
         assertThat(ccs.getBody(), containsString("cross_cluster_two:twitter"));
     }
 
-    //Wait for the security plugin to load roles.
-    private void waitOrThrow(Client client) throws Exception {
-        int failures = 0;
-        while(failures < 5) {
-            try {
-                client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
-                break;
-            } catch (OpenSearchSecurityException ex) {
-                if (ex.getMessage().contains("OpenSearch Security not initialized")) {
-                    Thread.sleep(500);
-                    failures++;
-                } else {
-                    throw ex;
-                }
-            }
-        }
-    }
-
     @Test
     public void testCcsWithRoleInjection() throws Exception {
         setupCcs(new DynamicSecurityConfig().setSecurityRoles("roles.yml"));
@@ -1041,7 +1023,7 @@ public void testCcsWithRoleInjection() throws Exception {
         RolesInjectorIntegTest.RolesInjectorPlugin.injectedRoles = "invalid_user|invalid_role";
         try (Node node = new PluginAwareNode(false, tcSettings, Netty4Plugin.class,
                 OpenSearchSecurityPlugin.class, RolesInjectorIntegTest.RolesInjectorPlugin.class).start()) {
-            waitOrThrow(node.client());
+            waitForInit(node.client());
             Client remoteClient = node.client().getRemoteClusterClient("cross_cluster_two");
             GetRequest getReq = new GetRequest("twitter", "0");
             getReq.realtime(true);
@@ -1061,7 +1043,7 @@ public void testCcsWithRoleInjection() throws Exception {
         RolesInjectorIntegTest.RolesInjectorPlugin.injectedRoles = "valid_user|opendistro_security_all_access";
         try (Node node = new PluginAwareNode(false, tcSettings, Netty4Plugin.class,
                 OpenSearchSecurityPlugin.class, RolesInjectorIntegTest.RolesInjectorPlugin.class).start()) {
-            waitOrThrow(node.client());
+            waitForInit(node.client());
             Client remoteClient = node.client().getRemoteClusterClient("cross_cluster_two");
             GetRequest getReq = new GetRequest("twitter", "0");
             getReq.realtime(true);

src/test/java/org/opensearch/security/dlic/dlsfls/CCReplicationTest.java

@@ -158,20 +158,8 @@ protected void doExecute(Task task, MockReplicationRequest request, ActionListen
 
     //Wait for the security plugin to load roles.
     private void waitOrThrow(Client client, String index) throws Exception {
-        int failures = 0;
-        while(failures < 5) {
-            try {
-                client.execute(MockReplicationAction.INSTANCE, new MockReplicationRequest(index)).actionGet();
-                break;
-            } catch (OpenSearchSecurityException ex) {
-                if (ex.getMessage().contains("OpenSearch Security not initialized")) {
-                    Thread.sleep(500);
-                    failures++;
-                } else {
-                    throw ex;
-                }
-            }
-        }
+        waitForInit(client);
+        client.execute(MockReplicationAction.INSTANCE, new MockReplicationRequest(index)).actionGet();
     }
 
     void populateData(Client tc) {

src/test/java/org/opensearch/security/dlic/rest/api/TenantInfoActionTest.java

@@ -60,6 +60,7 @@ public void testTenantInfoAPIAccess() throws Exception {
     public void testTenantInfoAPIUpdate() throws Exception {
         Settings settings = Settings.builder().put(ConfigConstants.SECURITY_UNSUPPORTED_RESTAPI_ALLOW_SECURITYCONFIG_MODIFICATION, true).build();
         setup(settings);
+
         rh.keystore = "restapi/kirk-keystore.jks";
         rh.sendHTTPClientCredentials = true;
         rh.sendAdminCertificate = true;

src/test/java/org/opensearch/security/test/AbstractSecurityUnitTest.java

@@ -34,6 +34,7 @@
 import java.util.Base64;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.net.ssl.SSLContext;
@@ -57,6 +58,8 @@
 import org.junit.rules.TestName;
 import org.junit.rules.TestWatcher;
 
+import org.opensearch.OpenSearchSecurityException;
+import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest;
 import org.opensearch.action.admin.indices.create.CreateIndexRequest;
 import org.opensearch.action.index.IndexRequest;
@@ -83,7 +86,7 @@
 
 /*
  * There are real thread leaks during test execution, not all threads are 
- * properly waited on or interupted.  While this normally doesn't create test
+ * properly waited on or interrupted.  While this normally doesn't create test
  * failures, retries mitigate this.  Remove this attribute to explore these
  * issues.
  */ 
@@ -164,6 +167,33 @@ protected RestHighLevelClient getRestClient(ClusterInfo info, String keyStoreNam
         }
     }
 
+    /** Wait for the security plugin to load roles. */
+    public void waitForInit(Client client) {
+        int maxRetries = 5;
+        Optional<Exception> retainedException = Optional.empty();
+        for (int i = 0; i < maxRetries; i++) {
+            try {
+                client.admin().cluster().health(new ClusterHealthRequest()).actionGet();
+                retainedException = Optional.empty();
+                return;
+            } catch (OpenSearchSecurityException ex) {
+                if(ex.getMessage().contains("OpenSearch Security not initialized")) {
+                    retainedException = Optional.of(ex);
+                    try {
+                        Thread.sleep(500);
+                    } catch (InterruptedException e) { /* ignored */ }
+                } else {
+                    // plugin is initialized, but another error received.
+                    // Example could be user does not have permissions for cluster:monitor/health
+                    retainedException = Optional.empty();
+                }
+            }
+        }
+        if (retainedException.isPresent()) {
+            throw new RuntimeException(retainedException.get());
+        }
+    }
+
     protected void initialize(ClusterHelper clusterHelper, ClusterInfo clusterInfo, DynamicSecurityConfig securityConfig) throws IOException {
         try (Client tc = clusterHelper.nodeClient()) {
             Assert.assertEquals(clusterInfo.numNodes,