Merge branch 'main' into bulk-request-concurrency-fix

opensearch-project · Aug 7, 2023 · c604f93 · c604f93
2 parents 3cac3e4 + 6cc90e6
commit c604f93
Show file tree

Hide file tree

Showing 44 changed files with 202 additions and 74 deletions.
diff --git a/build.gradle b/build.gradle
@@ -506,6 +506,9 @@ dependencies {
     implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.14'
     implementation 'org.apache.commons:commons-collections4:4.4'
 
+    //Password generation
+    implementation 'org.passay:passay:1.6.3'
+
     //JSON path
     implementation 'com.jayway.jsonpath:json-path:2.8.0'
     implementation 'net.minidev:json-smart:2.4.11'

diff --git a/release-notes/opensearch-security.release-notes-1.3.12.0.md b/release-notes/opensearch-security.release-notes-1.3.12.0.md
@@ -0,0 +1,9 @@
+## 2023-08-10 Version 1.3.12.0
+
+Compatible with OpenSearch 1.3.12
+
+### Maintenance
+
+* Bump BouncyCastle from jdk15on to jdk15to18 ([#2901](https://github.com/opensearch-project/security/pull/2901)) [#2931](https://github.com/opensearch-project/security/pull/2931)
+* Update guava to address CVE-2023-2976 ([#3060](https://github.com/opensearch-project/security/pull/3060))
+* Bump the version of kafka and spring-kafka-test (CVE Related) ([#3087](https://github.com/opensearch-project/security/pull/3087))
diff --git a/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java b/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java
@@ -14,17 +14,28 @@
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 import org.apache.hc.core5.http.HttpStatus;
 import org.junit.ClassRule;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.security.auditlog.AuditLog;
+import org.opensearch.test.framework.AuditCompliance;
+import org.opensearch.test.framework.AuditConfiguration;
+import org.opensearch.test.framework.AuditFilters;
 import org.opensearch.test.framework.TestSecurityConfig;
 import org.opensearch.test.framework.TestSecurityConfig.Role;
+import org.opensearch.test.framework.audit.AuditLogsRule;
 import org.opensearch.test.framework.cluster.ClusterManager;
 import org.opensearch.test.framework.cluster.LocalCluster;
 import org.opensearch.test.framework.cluster.TestRestClient;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
+import static org.opensearch.security.auditlog.impl.AuditCategory.GRANTED_PRIVILEGES;
+import static org.opensearch.security.auditlog.impl.AuditCategory.MISSING_PRIVILEGES;
 import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL;
+import static org.opensearch.test.framework.audit.AuditMessagePredicate.auditPredicate;
+import static org.opensearch.test.framework.audit.AuditMessagePredicate.userAuthenticated;
 
 @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE)
@@ -50,12 +61,35 @@ public class WhoAmITests {
     public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS)
         .authc(AUTHC_HTTPBASIC_INTERNAL)
         .users(WHO_AM_I, WHO_AM_I_LEGACY, WHO_AM_I_NO_PERM)
+        .audit(
+            new AuditConfiguration(true).compliance(new AuditCompliance().enabled(true))
+                .filters(new AuditFilters().enabledRest(true).enabledTransport(true).resolveBulkRequests(true))
+        )
         .build();
 
+    @Rule
+    public AuditLogsRule auditLogsRule = new AuditLogsRule();
+
     @Test
-    public void testWhoAmIWithGetPermissions() throws Exception {
+    public void testWhoAmIWithGetPermissions() {
         try (TestRestClient client = cluster.getRestClient(WHO_AM_I)) {
             assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK));
+
+            // audit log, named route
+            auditLogsRule.assertExactly(
+                1,
+                userAuthenticated(WHO_AM_I).withLayer(AuditLog.Origin.REST)
+                    .withRestMethod(RestRequest.Method.GET)
+                    .withRequestPath("/" + WHOAMI_PROTECTED_ENDPOINT)
+                    .withInitiatingUser(WHO_AM_I)
+            );
+            auditLogsRule.assertExactly(
+                1,
+                auditPredicate(GRANTED_PRIVILEGES).withLayer(AuditLog.Origin.REST)
+                    .withRestMethod(RestRequest.Method.GET)
+                    .withRequestPath("/" + WHOAMI_PROTECTED_ENDPOINT)
+                    .withEffectiveUser(WHO_AM_I)
+            );
         }
 
         try (TestRestClient client = cluster.getRestClient(WHO_AM_I)) {
@@ -64,29 +98,60 @@ public void testWhoAmIWithGetPermissions() throws Exception {
     }
 
     @Test
-    public void testWhoAmIWithGetPermissionsLegacy() throws Exception {
+    public void testWhoAmIWithGetPermissionsLegacy() {
         try (TestRestClient client = cluster.getRestClient(WHO_AM_I_LEGACY)) {
             assertThat(client.get(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK));
         }
 
         try (TestRestClient client = cluster.getRestClient(WHO_AM_I_LEGACY)) {
             assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK));
+
+            // audit log, named route
+            auditLogsRule.assertExactly(
+                1,
+                userAuthenticated(WHO_AM_I_LEGACY).withLayer(AuditLog.Origin.REST)
+                    .withRestMethod(RestRequest.Method.GET)
+                    .withRequestPath("/" + WHOAMI_PROTECTED_ENDPOINT)
+                    .withInitiatingUser(WHO_AM_I_LEGACY)
+            );
+            auditLogsRule.assertExactly(
+                1,
+                auditPredicate(GRANTED_PRIVILEGES).withLayer(AuditLog.Origin.REST)
+                    .withRestMethod(RestRequest.Method.GET)
+                    .withRequestPath("/" + WHOAMI_PROTECTED_ENDPOINT)
+                    .withEffectiveUser(WHO_AM_I_LEGACY)
+            );
         }
     }
 
     @Test
-    public void testWhoAmIWithoutGetPermissions() throws Exception {
+    public void testWhoAmIWithoutGetPermissions() {
         try (TestRestClient client = cluster.getRestClient(WHO_AM_I_NO_PERM)) {
             assertThat(client.get(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK));
         }
 
         try (TestRestClient client = cluster.getRestClient(WHO_AM_I_NO_PERM)) {
             assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_UNAUTHORIZED));
+
+            // audit log, named route
+            auditLogsRule.assertExactly(
+                1,
+                userAuthenticated(WHO_AM_I_NO_PERM).withLayer(AuditLog.Origin.REST)
+                    .withRestMethod(RestRequest.Method.GET)
+                    .withRequestPath("/" + WHOAMI_PROTECTED_ENDPOINT)
+            );
+            auditLogsRule.assertExactly(
+                1,
+                auditPredicate(MISSING_PRIVILEGES).withLayer(AuditLog.Origin.REST)
+                    .withRestMethod(RestRequest.Method.GET)
+                    .withRequestPath("/" + WHOAMI_PROTECTED_ENDPOINT)
+                    .withEffectiveUser(WHO_AM_I_NO_PERM)
+            );
         }
     }
 
     @Test
-    public void testWhoAmIPost() throws Exception {
+    public void testWhoAmIPost() {
         try (TestRestClient client = cluster.getRestClient(WHO_AM_I)) {
             assertThat(client.post(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK));
         }

diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java
@@ -52,7 +52,6 @@
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.client.Client;
-import org.opensearch.common.Strings;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.xcontent.ToXContentObject;
@@ -680,7 +679,7 @@ private static String configToJson(CType configType, Map<String, ? extends ToXCo
 
         builder.endObject();
 
-        return Strings.toString(builder);
+        return builder.toString();
     }
 
     private void writeSingleEntryConfigToIndex(Client client, CType configType, ToXContentObject config) {
@@ -701,7 +700,7 @@ private void writeSingleEntryConfigToIndex(Client client, CType configType, Stri
 
             builder.endObject();
 
-            String json = Strings.toString(builder);
+            String json = builder.toString();
 
             log.info("Writing security plugin configuration into index " + configType + ":\n" + json);
 

diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java
@@ -57,10 +57,10 @@
 import org.opensearch.client.AdminClient;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.health.ClusterHealthStatus;
-import org.opensearch.common.Strings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.Strings;
 import org.opensearch.http.BindHttpException;
 import org.opensearch.node.PluginAwareNode;
 import org.opensearch.plugins.Plugin;

diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java
@@ -69,8 +69,8 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import org.opensearch.common.Strings;
 import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.Strings;
 import org.opensearch.core.xcontent.ToXContentObject;
 import org.opensearch.security.DefaultObjectMapper;
 

diff --git a/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java b/src/main/java/org/opensearch/security/OpenSearchSecurityPlugin.java
@@ -187,7 +187,7 @@
 import org.opensearch.transport.TransportRequest;
 import org.opensearch.transport.TransportRequestHandler;
 import org.opensearch.transport.TransportRequestOptions;
-import org.opensearch.transport.TransportResponse;
+import org.opensearch.core.transport.TransportResponse;
 import org.opensearch.transport.TransportResponseHandler;
 import org.opensearch.transport.TransportService;
 import org.opensearch.watcher.ResourceWatcherService;

diff --git a/src/main/java/org/opensearch/security/action/whoami/WhoAmIResponse.java b/src/main/java/org/opensearch/security/action/whoami/WhoAmIResponse.java
@@ -29,10 +29,10 @@
 import java.io.IOException;
 
 import org.opensearch.action.ActionResponse;
-import org.opensearch.common.Strings;
+import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
 
@@ -105,6 +105,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
     @Override
     public String toString() {
-        return Strings.toString(XContentType.JSON, this, true, true);
+        return Strings.toString(MediaTypeRegistry.JSON, this, true, true);
     }
 }
diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java b/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java
@@ -42,11 +42,10 @@
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.Strings;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.common.xcontent.XContentType;
@@ -496,7 +495,7 @@ public void logDocumentRead(String index, String id, ShardId shardId, Map<String
                         builder.field("field_names", fieldNameValues.keySet());
                         builder.endObject();
                         builder.close();
-                        msg.addUnescapedJsonToRequestBody(Strings.toString(builder));
+                        msg.addUnescapedJsonToRequestBody(builder.toString());
                     } catch (IOException e) {
                         log.error(e.toString());
                     }
@@ -739,7 +738,7 @@ public Map run() {
             builder.endObject();
             builder.endObject();
             builder.close();
-            msg.addUnescapedJsonToRequestBody(Strings.toString(builder));
+            msg.addUnescapedJsonToRequestBody(builder.toString());
         } catch (Exception e) {
             log.error("Unable to build message", e);
         }

diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java b/src/main/java/org/opensearch/security/auditlog/impl/AuditMessage.java
@@ -36,7 +36,7 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.common.xcontent.XContentType;
@@ -466,15 +466,15 @@ public String getDocId() {
     @Override
     public String toString() {
         try {
-            return org.opensearch.common.Strings.toString(JsonXContent.contentBuilder().map(getAsMap()));
+            return JsonXContent.contentBuilder().map(getAsMap()).toString();
         } catch (final IOException e) {
             throw ExceptionsHelper.convertToOpenSearchException(e);
         }
     }
 
     public String toPrettyString() {
         try {
-            return org.opensearch.common.Strings.toString(JsonXContent.contentBuilder().prettyPrint().map(getAsMap()));
+            return JsonXContent.contentBuilder().prettyPrint().map(getAsMap()).toString();
         } catch (final IOException e) {
             throw ExceptionsHelper.convertToOpenSearchException(e);
         }

diff --git a/src/main/java/org/opensearch/security/auditlog/impl/RequestResolver.java b/src/main/java/org/opensearch/security/auditlog/impl/RequestResolver.java
@@ -41,11 +41,10 @@
 import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.Strings;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.common.xcontent.XContentType;
@@ -314,7 +313,7 @@ private static AuditMessage resolveInner(
                         builder.field("transient_settings", Utils.convertJsonToxToStructuredMap(persistentSettings));
                     }
                     builder.endObject();
-                    msg.addUnescapedJsonToRequestBody(builder == null ? null : Strings.toString(builder));
+                    msg.addUnescapedJsonToRequestBody(builder == null ? null : builder.toString());
                 } catch (IOException e) {
                     log.error(e.toString());
                 } finally {

diff --git a/src/main/java/org/opensearch/security/auth/BackendRegistry.java b/src/main/java/org/opensearch/security/auth/BackendRegistry.java
@@ -49,7 +49,7 @@
 
 import org.opensearch.OpenSearchSecurityException;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.rest.BytesRestResponse;
 import org.opensearch.rest.RestChannel;

diff --git a/src/main/java/org/opensearch/security/auth/UserInjector.java b/src/main/java/org/opensearch/security/auth/UserInjector.java
@@ -37,7 +37,7 @@
 import org.apache.logging.log4j.Logger;
 
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.security.auditlog.AuditLog;
 import org.opensearch.security.http.XFFResolver;

diff --git a/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java b/src/main/java/org/opensearch/security/configuration/ConfigurationRepository.java
@@ -59,11 +59,11 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.Strings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.common.util.concurrent.ThreadContext.StoredContext;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.security.auditlog.AuditLog;
@@ -464,7 +464,7 @@ public Map<CType, SecurityDynamicConfiguration<?>> getConfigurationsFromIndex(
         if (logComplianceEvent && auditLog.getComplianceConfig().isEnabled()) {
             CType configurationType = configTypes.iterator().next();
             Map<String, String> fields = new HashMap<String, String>();
-            fields.put(configurationType.toLCString(), Strings.toString(XContentType.JSON, retVal.get(configurationType)));
+            fields.put(configurationType.toLCString(), Strings.toString(MediaTypeRegistry.JSON, retVal.get(configurationType)));
             auditLog.logDocumentRead(this.securityIndex, configurationType.toLCString(), null, fields);
         }
 

diff --git a/src/main/java/org/opensearch/security/configuration/DlsFlsValveImpl.java b/src/main/java/org/opensearch/security/configuration/DlsFlsValveImpl.java
@@ -48,10 +48,10 @@
 import org.opensearch.client.Client;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.Strings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.concurrent.ThreadContext;
-import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.index.query.ParsedQuery;
 import org.opensearch.core.rest.RestStatus;
@@ -230,10 +230,10 @@ public boolean invoke(
                     StringBuilder sb = new StringBuilder();
 
                     if (searchRequest.source() != null) {
-                        sb.append(Strings.toString(XContentType.JSON, searchRequest.source()) + System.lineSeparator());
+                        sb.append(Strings.toString(MediaTypeRegistry.JSON, searchRequest.source()) + System.lineSeparator());
                     }
 
-                    sb.append(Strings.toString(XContentType.JSON, af) + System.lineSeparator());
+                    sb.append(Strings.toString(MediaTypeRegistry.JSON, af) + System.lineSeparator());
 
                     LogManager.getLogger("debuglogger").error(sb.toString());
 
@@ -245,7 +245,9 @@ public boolean invoke(
                     LogManager.getLogger("debuglogger")
                         .error(
                             "Shard requestcache enabled for "
-                                + (searchRequest.source() == null ? "<NULL>" : Strings.toString(XContentType.JSON, searchRequest.source()))
+                                + (searchRequest.source() == null
+                                    ? "<NULL>"
+                                    : Strings.toString(MediaTypeRegistry.JSON, searchRequest.source()))
                         );
                 }
 

diff --git a/src/main/java/org/opensearch/security/configuration/SecurityIndexSearcherWrapper.java b/src/main/java/org/opensearch/security/configuration/SecurityIndexSearcherWrapper.java
@@ -36,7 +36,7 @@
 
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.transport.TransportAddress;
+import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.index.Index;
 import org.opensearch.index.IndexService;