Adds a test to check similarity between audit logs generated from REST layer and TRANSPORT layer

src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java

@@ -19,6 +19,7 @@
 import org.junit.runner.RunWith;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.security.auditlog.AuditLog;
+import org.opensearch.security.auditlog.impl.AuditMessage;
 import org.opensearch.test.framework.AuditCompliance;
 import org.opensearch.test.framework.AuditConfiguration;
 import org.opensearch.test.framework.AuditFilters;
@@ -29,12 +30,20 @@
 import org.opensearch.test.framework.cluster.LocalCluster;
 import org.opensearch.test.framework.cluster.TestRestClient;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
 import static org.opensearch.security.auditlog.impl.AuditCategory.GRANTED_PRIVILEGES;
 import static org.opensearch.security.auditlog.impl.AuditCategory.MISSING_PRIVILEGES;
 import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL;
 import static org.opensearch.test.framework.audit.AuditMessagePredicate.auditPredicate;
+import static org.opensearch.test.framework.audit.AuditMessagePredicate.grantedPrivilege;
 import static org.opensearch.test.framework.audit.AuditMessagePredicate.userAuthenticated;
 
 @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
@@ -44,6 +53,10 @@ public class WhoAmITests {
         new Role("who_am_i_role").clusterPermissions("security:whoamiprotected")
     );
 
+    protected final static TestSecurityConfig.User AUDIT_LOG_VERIFIER = new TestSecurityConfig.User("audit_log_verifier").roles(
+        new Role("audit_log_verifier_role").clusterPermissions("*").indexPermissions("*").on("*")
+    );
+
     protected final static TestSecurityConfig.User WHO_AM_I_LEGACY = new TestSecurityConfig.User("who_am_i_user_legacy").roles(
         new Role("who_am_i_role_legacy").clusterPermissions("cluster:admin/opendistro_security/whoamiprotected")
     );
@@ -60,7 +73,7 @@ public class WhoAmITests {
     @ClassRule
     public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS)
         .authc(AUTHC_HTTPBASIC_INTERNAL)
-        .users(WHO_AM_I, WHO_AM_I_LEGACY, WHO_AM_I_NO_PERM)
+        .users(WHO_AM_I, WHO_AM_I_LEGACY, WHO_AM_I_NO_PERM, AUDIT_LOG_VERIFIER)
         .audit(
             new AuditConfiguration(true).compliance(new AuditCompliance().enabled(true))
                 .filters(new AuditFilters().enabledRest(true).enabledTransport(true).resolveBulkRequests(true))
@@ -169,4 +182,64 @@ public void testWhoAmIPost() {
         }
 
     }
+
+    @Test
+    public void testAuditLogSimilarityWithTransportLayer() {
+        try (TestRestClient client = cluster.getRestClient(AUDIT_LOG_VERIFIER)) {
+            assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK));
+
+            auditLogsRule.assertExactly(
+                1,
+                auditPredicate(GRANTED_PRIVILEGES).withLayer(AuditLog.Origin.REST)
+                    .withRestMethod(RestRequest.Method.GET)
+                    .withRequestPath("/" + WHOAMI_PROTECTED_ENDPOINT)
+                    .withEffectiveUser(AUDIT_LOG_VERIFIER)
+            );
+
+            assertThat(client.get("_cat/indices").getStatusCode(), equalTo(HttpStatus.SC_OK));
+
+            auditLogsRule.assertExactly(2, grantedPrivilege(AUDIT_LOG_VERIFIER, "GetSettingsRequest"));
+
+            List<AuditMessage> grantedPrivilegesMessages = auditLogsRule.getCurrentTestAuditMessages()
+                .stream()
+                .filter(msg -> msg.getCategory().equals(GRANTED_PRIVILEGES))
+                .collect(Collectors.toList());
+            verifyAuditLogSimilarity(grantedPrivilegesMessages);
+        }
+    }
+
+    private void verifyAuditLogSimilarity(List<AuditMessage> currentTestAuditMessages) {
+        List<AuditMessage> restSet = new ArrayList<>();
+        List<AuditMessage> transportSet = new ArrayList<>();
+
+        // It is okay to loop through all even though we end up using only 2, as the total number of messages should be around 8
+        for (AuditMessage auditMessage : currentTestAuditMessages) {
+            if ("REST".equals(auditMessage.getAsMap().get(AuditMessage.REQUEST_LAYER).toString())) {
+                restSet.add(auditMessage);
+            } else if ("TRANSPORT".equals(auditMessage.getAsMap().get(AuditMessage.REQUEST_LAYER).toString())) {
+                transportSet.add(auditMessage);
+            }
+        }
+        // We pass 1 message from each layer to check for similarity
+        checkForStructuralSimilarity(restSet.get(0), transportSet.get(0));
+    }
+
+    private void checkForStructuralSimilarity(AuditMessage restAuditMessage, AuditMessage transportAuditMessage) {
+
+        Set<String> restAuditSet = restAuditMessage.getAsMap().keySet();
+        Set<String> transportAuditSet = transportAuditMessage.getAsMap().keySet();
+
+        // Added a magic number here and below, because there are always 15 or more items in each message generated via Audit logs
+        assertThat(restAuditSet.size(), greaterThan(14));
+        assertThat(transportAuditSet.size(), greaterThan(14));
+
+        restAuditSet.removeAll(transportAuditMessage.getAsMap().keySet());
+        transportAuditSet.removeAll(restAuditMessage.getAsMap().keySet());
+
+        // We compare two sets and see there were more than 10 items with same keys indicating these logs are similar
+        // There are a few headers that are generated different for REST vs TRANSPORT layer audit logs, but that is expected
+        // The end goal of this test is to ensure similarity, not equality.
+        assertThat(restAuditSet.size(), lessThan(5));
+        assertThat(transportAuditSet.size(), lessThan(5));
+    }
 }

src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java

@@ -40,6 +40,10 @@ public class AuditLogsRule implements TestRule {
 
     private List<AuditMessage> currentTestAuditMessages;
 
+    public List<AuditMessage> getCurrentTestAuditMessages() {
+        return currentTestAuditMessages;
+    }
+
     public void waitForAuditLogs() {
         try {
             TimeUnit.SECONDS.sleep(3);