Validate requests are not impacted during security cache invalidation

src/integrationTest/java/org/opensearch/security/http/AsyncTests.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+
+package org.opensearch.security.http;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import com.nimbusds.jose.util.StandardCharset;
+
+import org.apache.hc.client5.http.classic.methods.HttpDelete;
+import org.apache.hc.client5.http.classic.methods.HttpGet;
+import org.apache.hc.client5.http.classic.methods.HttpPost;
+import org.apache.hc.core5.http.ContentType;
+import org.apache.hc.core5.http.HttpStatus;
+import org.apache.hc.core5.http.io.entity.ByteArrayEntity;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.opensearch.security.IndexOperationsHelper;
+import org.opensearch.security.support.ConfigConstants;
+import org.opensearch.test.framework.AsyncActions;
+import org.opensearch.test.framework.RolesMapping;
+import org.opensearch.test.framework.TestSecurityConfig;
+import org.opensearch.test.framework.cluster.LocalCluster;
+import org.opensearch.test.framework.cluster.TestRestClient;
+import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.CompletableFuture;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL;
+import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS;
+import static org.opensearch.test.framework.cluster.TestRestClientConfiguration.getBasicAuthHeader;
+
+@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+public class AsyncTests {
+    private static final TestSecurityConfig.User ADMIN_USER = new TestSecurityConfig.User("admin").backendRoles("admin");
+
+    @ClassRule
+    public static LocalCluster cluster = new LocalCluster.Builder().singleNode()
+        .authc(AUTHC_HTTPBASIC_INTERNAL)
+        .users(ADMIN_USER)
+        .rolesMapping(new RolesMapping(ALL_ACCESS).backendRoles("admin"))
+        .anonymousAuth(false)
+        .nodeSettings(Map.of(ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED, List.of(ALL_ACCESS.getName())))
+        .build();
+
+    @Test
+    public void testBulkAndCacheInvalidationMixed() throws Exception {
+        String indexName = "test-index";
+        final String invalidateCachePath = "/_plugins/_security/api/cache";
+        final String nodesPath = "/_nodes";
+        final String bulkPath = "/_bulk";
+        final String document = ("{ \"index\": { \"_index\": \"" + indexName + "\" }}\n{ \"foo\": \"bar\" }\n").repeat(5);
+        final int parallelism = 5;
+        final int totalNumberOfRequests = 30;
+
+        try (final TestRestClient client = cluster.getRestClient()) {
+            IndexOperationsHelper.createIndex(cluster, indexName);
+
+            final CountDownLatch countDownLatch = new CountDownLatch(1);
+
+            final var cacheInvalidationRequests = AsyncActions.generate(() -> {
+                countDownLatch.await();
+                System.err.println("Generation triggered invalidate cache requests");
+                final HttpDelete delete = new HttpDelete(client.getHttpServerUri() + invalidateCachePath);
+                return client.executeRequest(delete, getBasicAuthHeader(ADMIN_USER.getName(), ADMIN_USER.getPassword()));
+            }, parallelism, totalNumberOfRequests);
+
+            final var bulkRequests = AsyncActions.generate(() -> {
+                countDownLatch.await();
+                System.err.println("Generation triggered bulk requests");
+                final HttpPost post = new HttpPost(client.getHttpServerUri() + bulkPath);
+                post.setEntity(new ByteArrayEntity(document.getBytes(StandardCharset.UTF_8), ContentType.APPLICATION_JSON));
+                return client.executeRequest(post, getBasicAuthHeader(ADMIN_USER.getName(), ADMIN_USER.getPassword()));
+            }, parallelism, totalNumberOfRequests);
+
+            final var nodesRequests = AsyncActions.generate(() -> {
+                countDownLatch.await();
+                System.err.println("Generation triggered node requests");
+                final HttpGet get = new HttpGet(client.getHttpServerUri() + nodesPath);
+                return client.executeRequest(get, getBasicAuthHeader(ADMIN_USER.getName(), ADMIN_USER.getPassword()));
+            }, parallelism, totalNumberOfRequests);
+
+            // Make sure all requests start at the same time
+            countDownLatch.countDown();
+
+            List<CompletableFuture<HttpResponse>> allRequests = new ArrayList<CompletableFuture<HttpResponse>>();
+            allRequests.addAll(cacheInvalidationRequests);
+            allRequests.addAll(bulkRequests);
+            allRequests.addAll(nodesRequests);
+            Collections.shuffle(allRequests);
+            AsyncActions.getAll(allRequests, 150, TimeUnit.SECONDS).forEach((response) -> {
+                if (response.getStatusCode() == HttpStatus.SC_INTERNAL_SERVER_ERROR) {
+                    System.out.println("Response body: " + response.getBody());
+                }
+                assertThat(response.getStatusCode(), equalTo(HttpStatus.SC_OK));
+            });
+        }
+    }
+}

src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java

@@ -40,7 +40,7 @@ public class BasicAuthTests {
     static final User TEST_USER = new User("test_user").password("s3cret");
 
     public static final String CUSTOM_ATTRIBUTE_NAME = "superhero";
-    static final User SUPER_USER = new User("super-user").password("super-password").attr(CUSTOM_ATTRIBUTE_NAME, true);
+    static final User SUPER_USER = new User("super-user").password("super-password").attr(CUSTOM_ATTRIBUTE_NAME, "true");
     public static final String NOT_EXISTING_USER = "not-existing-user";
     public static final String INVALID_PASSWORD = "secret-password";
 

src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java

@@ -29,6 +29,7 @@
 package org.opensearch.test.framework;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.io.UnsupportedEncodingException;
 import java.nio.ByteBuffer;
 import java.security.SecureRandom;
@@ -58,6 +59,9 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.security.securityconf.impl.CType;
 import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
 
 import static org.apache.http.HttpHeaders.AUTHORIZATION;
 import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -252,7 +256,7 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params
         }
     }
 
-    public static class User implements UserCredentialsHolder, ToXContentObject {
+    public static class User implements UserCredentialsHolder, ToXContentObject, Serializable, Writeable {
 
         public final static TestSecurityConfig.User USER_ADMIN = new TestSecurityConfig.User("admin").roles(
             new Role("allaccess").indexPermissions("*").on("*").clusterPermissions("*")
@@ -261,13 +265,27 @@ public static class User implements UserCredentialsHolder, ToXContentObject {
         String name;
         private String password;
         List<Role> roles = new ArrayList<>();
-        private Map<String, Object> attributes = new HashMap<>();
+        List<String> backendRoles = new ArrayList<>();
+        String requestedTenant;
+        private Map<String, String> attributes = new HashMap<>();
 
         public User(String name) {
             this.name = name;
             this.password = "secret";
         }
 
+        public User(final StreamInput in) throws IOException {
+            super();
+            name = in.readString();
+            roles.addAll(in.readList(StreamInput::readString).stream().map(r -> new Role(r)).collect(Collectors.toList()));
+            requestedTenant = in.readString();
+            if (requestedTenant.isEmpty()) {
+                requestedTenant = null;
+            }
+            attributes = in.readMap(StreamInput::readString, StreamInput::readString);
+            backendRoles.addAll(in.readList(StreamInput::readString));
+        }
+
         public User password(String password) {
             this.password = password;
             return this;
@@ -282,7 +300,12 @@ public User roles(Role... roles) {
             return this;
         }
 
-        public User attr(String key, Object value) {
+        public User backendRoles(String... backendRoles) {
+            this.backendRoles.addAll(Arrays.asList(backendRoles));
+            return this;
+        }
+
+        public User attr(String key, String value) {
             this.attributes.put(key, value);
             return this;
         }
@@ -315,13 +338,26 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params
                 xContentBuilder.field("opendistro_security_roles", roleNames);
             }
 
+            if (!backendRoles.isEmpty()) {
+                xContentBuilder.field("backend_roles", backendRoles);
+            }
+
             if (attributes != null && attributes.size() != 0) {
                 xContentBuilder.field("attributes", attributes);
             }
 
             xContentBuilder.endObject();
             return xContentBuilder;
         }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(name);
+            out.writeStringCollection(roles.stream().map(r -> r.getName()).collect(Collectors.toList()));
+            out.writeString(requestedTenant == null ? "" : requestedTenant);
+            out.writeMap(attributes, StreamOutput::writeString, StreamOutput::writeString);
+            out.writeStringCollection(backendRoles == null ? Collections.emptyList() : new ArrayList<String>(backendRoles));
+        }
     }
 
     public static class Role implements ToXContentObject {