Validate requests are not impacted during security cache invalidation

src/integrationTest/java/org/opensearch/security/http/AsyncTests.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.security.http;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import com.nimbusds.jose.util.StandardCharset;
+
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opensearch.security.IndexOperationsHelper;
+import org.opensearch.security.support.ConfigConstants;
+import org.opensearch.test.framework.AsyncActions;
+import org.opensearch.test.framework.RolesMapping;
+import org.opensearch.test.framework.TestSecurityConfig;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.CompletableFuture;
+
+import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL;
+import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS;
+
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class AsyncTests {
+    private static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").backendRoles("admin");
+
+    @ClassRule
+    public static LocalCluster cluster = new LocalCluster.Builder().singleNode()
+        .authc(AUTHC_HTTPBASIC_INTERNAL)
+        .users(ADMIN_USER)
+        .rolesMapping(new RolesMapping(ALL_ACCESS).backendRoles("admin"))
+        .anonymousAuth(false)
+        .nodeSettings(Map.of(ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED, List.of(ALL_ACCESS.getName())))
+        .build();
+
+    @Test
+    public void testBulkAndCacheInvalidationMixed() throws Exception {
+        String indexName = "test-index";
+        final String invalidateCachePath = "_plugins/_security/api/cache";
+        final String nodesPath = "_nodes";
+        final String bulkPath = "/_bulk";
+        final String document = ("{ \"index\": { \"_index\": \"" + indexName + "\" }}\n{ \"foo\": \"bar\" }\n").repeat(5);
+        final int parallelism = 5;
+        final int totalNumberOfRequests = 30;
+
+        try (final TestRestClient client = cluster.getRestClient(ADMIN_USER)) {
+            IndexOperationsHelper.createIndex(cluster, indexName);
+
+            final CountDownLatch countDownLatch = new CountDownLatch(1);
+
+            List<CompletableFuture<HttpResponse>> allRequests = new ArrayList<CompletableFuture<HttpResponse>>();
+
+            allRequests.addAll(AsyncActions.generate(() -> {
+                countDownLatch.await();
+                return client.delete(invalidateCachePath);
+            }, parallelism, totalNumberOfRequests));
+
+            allRequests.addAll(AsyncActions.generate(() -> {
+                countDownLatch.await();
+                final HttpPost post = new HttpPost(client.getHttpServerUri() + bulkPath);
+                post.setEntity(new ByteArrayEntity(document.getBytes(StandardCharset.UTF_8), ContentType.APPLICATION_JSON));
+                return client.executeRequest(post);
+            }, parallelism, totalNumberOfRequests));
+
+            allRequests.addAll(AsyncActions.generate(() -> {
+                countDownLatch.await();
+                return client.get(nodesPath);
+            }, parallelism, totalNumberOfRequests));
+
+            // Make sure all requests start at the same time
+            countDownLatch.countDown();
+            Collections.shuffle(allRequests);
+
+            AsyncActions.getAll(allRequests, 30, TimeUnit.SECONDS).forEach((response) -> { response.assertStatusCode(HttpStatus.SC_OK); });
+        }
+    }
+}

src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java

@@ -40,7 +40,7 @@ public class BasicAuthTests {
     static final User TEST_USER = new User("test_user").password("s3cret");
 
     public static final String CUSTOM_ATTRIBUTE_NAME = "superhero";
-    static final User SUPER_USER = new User("super-user").password("super-password").attr(CUSTOM_ATTRIBUTE_NAME, true);
+    static final User SUPER_USER = new User("super-user").password("super-password").attr(CUSTOM_ATTRIBUTE_NAME, "true");
     public static final String NOT_EXISTING_USER = "not-existing-user";
     public static final String INVALID_PASSWORD = "secret-password";
 

src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java

@@ -261,7 +261,9 @@ public static class User implements UserCredentialsHolder, ToXContentObject {
         String name;
         private String password;
         List<Role> roles = new ArrayList<>();
-        private Map<String, Object> attributes = new HashMap<>();
+        List<String> backendRoles = new ArrayList<>();
+        String requestedTenant;
+        private Map<String, String> attributes = new HashMap<>();
 
         public User(String name) {
             this.name = name;
@@ -282,7 +284,12 @@ public User roles(Role... roles) {
             return this;
         }
 
-        public User attr(String key, Object value) {
+        public User backendRoles(String... backendRoles) {
+            this.backendRoles.addAll(Arrays.asList(backendRoles));
+            return this;
+        }
+
+        public User attr(String key, String value) {
             this.attributes.put(key, value);
             return this;
         }
@@ -315,6 +322,10 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params
                 xContentBuilder.field("opendistro_security_roles", roleNames);
             }
 
+            if (!backendRoles.isEmpty()) {
+                xContentBuilder.field("backend_roles", backendRoles);
+            }
+
             if (attributes != null && attributes.size() != 0) {
                 xContentBuilder.field("attributes", attributes);
             }