diff --git a/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java b/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java index 651ddc2303..cbb5ec11f0 100644 --- a/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java +++ b/src/integrationTest/java/org/opensearch/security/SearchOperationTest.java @@ -13,6 +13,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.google.common.base.Stopwatch; @@ -23,7 +25,6 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -97,6 +98,7 @@ import org.opensearch.index.reindex.ReindexRequest; import org.opensearch.repositories.RepositoryMissingException; import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.security.auditlog.AuditLog; import org.opensearch.test.framework.AuditCompliance; import org.opensearch.test.framework.AuditConfiguration; import org.opensearch.test.framework.AuditFilters; @@ -359,7 +361,7 @@ public class SearchOperationTest { .on(INDICES_ON_WHICH_USER_CAN_PERFORM_INDEX_OPERATIONS_PREFIX.concat("*")) ); - private static User USER_ALLOWED_TO_CREATE_INDEX = new User("user-allowed-to-create-index").roles( + private static final User USER_ALLOWED_TO_CREATE_INDEX = new User("user-allowed-to-create-index").roles( new Role("create-index-role").indexPermissions("indices:admin/create").on("*") ); @@ -462,7 +464,7 @@ public void cleanData() throws ExecutionException, InterruptedException { if (indicesExistsResponse.isExists()) { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexToBeDeleted); indices.delete(deleteIndexRequest).actionGet(); - Awaitility.await().ignoreExceptions().until(() -> indices.exists(indicesExistsRequest).get().isExists() == false); + Awaitility.await().ignoreExceptions().until(() -> !indices.exists(indicesExistsRequest).get().isExists()); } } @@ -976,12 +978,11 @@ public void shouldIndexDocumentInBulkRequest_positive() throws IOException { } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertAtLeast(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER));// sometimes 4 or 6 - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest"));// sometimes 2 or 4 + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldIndexDocumentInBulkRequest_partiallyPositive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1004,10 +1005,10 @@ public void shouldIndexDocumentInBulkRequest_partiallyPositive() throws IOExcept } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest").withIndex(SONG_INDEX_NAME)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1036,7 +1037,6 @@ public void shouldIndexDocumentInBulkRequest_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest").withIndex(SONG_INDEX_NAME)); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldUpdateDocumentsInBulk_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1058,13 +1058,10 @@ public void shouldUpdateDocumentsInBulk_positive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldUpdateDocumentsInBulk_partiallyPositive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1087,10 +1084,10 @@ public void shouldUpdateDocumentsInBulk_partiallyPositive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest").withIndex(SONG_INDEX_NAME)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1120,6 +1117,10 @@ public void shouldUpdateDocumentsInBulk_negative() throws IOException { @Test public void shouldDeleteDocumentInBulk_positive() throws IOException { + // create index + Settings sourceIndexSettings = Settings.builder().put("index.number_of_replicas", 2).put("index.number_of_shards", 2).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, sourceIndexSettings); + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(IMMEDIATE); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("one").source(SONGS[0].asMap())); @@ -1144,14 +1145,16 @@ public void shouldDeleteDocumentInBulk_positive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactly(2, auditPredicate(null).withLayer(AuditLog.Origin.TRANSPORT)); + auditLogsRule.assertAtLeastTransportMessages(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); + auditLogsRule.assertAtLeastTransportMessages(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); } - @Ignore("Audit log verification is shown to be flaky in this test") @Test public void shouldDeleteDocumentInBulk_partiallyPositive() throws IOException { + Settings indexSettings = Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", 1).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(IMMEDIATE); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("one").source(SONGS[0].asMap())); @@ -1176,10 +1179,8 @@ public void shouldDeleteDocumentInBulk_partiallyPositive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "BulkShardRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1208,7 +1209,6 @@ public void shouldDeleteDocumentInBulk_negative() throws IOException { } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1227,14 +1227,13 @@ public void shouldReindexDocuments_positive() throws IOException { auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "ReindexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "SearchRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(REINDEXING_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(REINDEXING_USER, "PutMappingRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "CreateIndexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(REINDEXING_USER, "SearchScrollRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(REINDEXING_USER)); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(REINDEXING_USER)); auditLogsRule.assertExactlyOne(missingPrivilege(REINDEXING_USER, "ClearScrollRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(REINDEXING_USER, "PutMappingRequest")); } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_negativeSource() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1249,7 +1248,6 @@ public void shouldReindexDocuments_negativeSource() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(REINDEXING_USER, "SearchRequest")); } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_negativeDestination() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1268,7 +1266,6 @@ public void shouldReindexDocuments_negativeDestination() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(REINDEXING_USER, "ClearScrollRequest")); } - @Ignore("Seems like reindixing isn't completing before the test proceeds") @Test public void shouldReindexDocuments_negativeSourceAndDestination() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(REINDEXING_USER)) { @@ -1341,7 +1338,6 @@ public void shouldDeleteDocument_negative() throws IOException { } } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldCreateAlias_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1355,11 +1351,10 @@ public void shouldCreateAlias_positive() throws IOException { assertThat(internalClient, clusterContainsDocument(TEMPORARY_ALIAS_NAME, ID_S1)); } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_READ_USER).withRestRequest(POST, "/_aliases")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); - auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); + auditLogsRule.assertExactlyOne(auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldCreateAlias_negative() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1377,7 +1372,6 @@ public void shouldCreateAlias_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldDeleteAlias_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1394,8 +1388,8 @@ public void shouldDeleteAlias_positive() throws IOException { assertThat(internalClient, not(clusterContainsDocument(TEMPORARY_ALIAS_NAME, ID_S1))); } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_READ_USER).withRestRequest(POST, "/_aliases")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); - auditLogsRule.assertExactly(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); + auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); + auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_READ_USER)); } @Test @@ -1415,7 +1409,6 @@ public void shouldDeleteAlias_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "IndicesAliasesRequest")); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldCreateIndexTemplate_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1439,12 +1432,12 @@ public void shouldCreateIndexTemplate_positive() throws IOException { } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_template/musical-index-template")); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/song-transcription-jazz/_doc/0001")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "IndexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(8, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1477,9 +1470,9 @@ public void shouldDeleteTemplate_positive() throws IOException { } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_template/musical-index-template")); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(DELETE, "/_template/musical-index-template")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "DeleteIndexTemplateRequest")); - auditLogsRule.assertExactly(4, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "DeleteIndexTemplateRequest")); + auditLogsRule.assertExactly(2, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); } @Test @@ -1497,7 +1490,6 @@ public void shouldDeleteTemplate_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "DeleteIndexTemplateRequest")); } - @Ignore("Create alias / delete alias isn't resolving in a timely manner for this test") @Test public void shouldUpdateTemplate_positive() throws IOException { try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { @@ -1525,12 +1517,12 @@ public void shouldUpdateTemplate_positive() throws IOException { } auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_template/musical-index-template")); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/song-transcription-jazz/_doc/000one")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); + auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutIndexTemplateRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "IndexRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(10, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); + auditLogsRule.assertExactly(3, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -1620,7 +1612,7 @@ public void shouldCreateSnapshotRepository_positive() throws IOException { assertThat(internalClient, clusterContainsSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME)); } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_snapshot/test-snapshot-repository")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); } @Test @@ -1656,8 +1648,8 @@ public void shouldDeleteSnapshotRepository_positive() throws IOException { auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_WRITE_USER).withRestRequest(DELETE, "/_snapshot/test-snapshot-repository") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "DeleteRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "DeleteRepositoryRequest")); } @Test @@ -1674,9 +1666,10 @@ public void shouldDeleteSnapshotRepository_negative() throws IOException { auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "DeleteRepositoryRequest")); } - @Test // Bug which can be reproduced with the below test: https://github.com/opensearch-project/security/issues/2169 + @Test public void shouldCreateSnapshot_positive() throws IOException { final String snapshotName = "snapshot-positive-test"; + long snapshotGetCount; try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); steps.createSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME, cluster.getSnapshotDirPath(), "fs"); @@ -1685,20 +1678,21 @@ public void shouldCreateSnapshot_positive() throws IOException { assertThat(response, notNullValue()); assertThat(response.status(), equalTo(RestStatus.ACCEPTED)); - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); assertThat(internalClient, clusterContainSuccessSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName)); } auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_snapshot/test-snapshot-repository")); auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_WRITE_USER).withRestRequest(PUT, "/_snapshot/test-snapshot-repository/snapshot-positive-test") ); - auditLogsRule.assertAtLeast( - 1, - userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/snapshot-positive-test") + auditLogsRule.assertExactly( + snapshotGetCount, + userAuthenticated(LIMITED_WRITE_USER).withEffectiveUser(LIMITED_WRITE_USER) + .withRestRequest(GET, "/_snapshot/test-snapshot-repository/snapshot-positive-test") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertAtLeast(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); } @Test @@ -1723,12 +1717,13 @@ public void shouldCreateSnapshot_negative() throws IOException { @Test public void shouldDeleteSnapshot_positive() throws IOException { String snapshotName = "delete-snapshot-positive"; + long snapshotGetCount; try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); restHighLevelClient.snapshot(); steps.createSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME, cluster.getSnapshotDirPath(), "fs"); steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, SONG_INDEX_NAME); - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); var response = steps.deleteSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); @@ -1742,24 +1737,25 @@ public void shouldDeleteSnapshot_positive() throws IOException { auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_WRITE_USER).withRestRequest(DELETE, "/_snapshot/test-snapshot-repository/delete-snapshot-positive") ); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/delete-snapshot-positive") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "DeleteSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "DeleteSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); } @Test public void shouldDeleteSnapshot_negative() throws IOException { String snapshotName = "delete-snapshot-negative"; + long snapshotGetCount; try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); steps.createSnapshotRepository(TEST_SNAPSHOT_REPOSITORY_NAME, cluster.getSnapshotDirPath(), "fs"); steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, SONG_INDEX_NAME); - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); } try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); @@ -1774,23 +1770,27 @@ public void shouldDeleteSnapshot_negative() throws IOException { auditLogsRule.assertExactlyOne( userAuthenticated(LIMITED_READ_USER).withRestRequest(DELETE, "/_snapshot/test-snapshot-repository/delete-snapshot-negative") ); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/delete-snapshot-negative") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); - auditLogsRule.assertExactly(1, missingPrivilege(LIMITED_READ_USER, "DeleteSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_READ_USER, "DeleteSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); } - @Ignore("Audit log entries verifcation isn't always consistant") @Test public void shouldRestoreSnapshot_positive() throws IOException { final String snapshotName = "restore-snapshot-positive"; + long snapshotGetCount; + AtomicInteger restoredCount = new AtomicInteger(); try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); // 1. create some documents + Settings indexSettings = Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", 1).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); + BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("Eins").source(SONGS[0].asMap())); bulkRequest.add(new IndexRequest(WRITE_SONG_INDEX_NAME).id("Zwei").source(SONGS[1].asMap())); @@ -1804,7 +1804,7 @@ public void shouldRestoreSnapshot_positive() throws IOException { steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, WRITE_SONG_INDEX_NAME); // 4. wait till snapshot is ready - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); // 5. introduce some changes bulkRequest = new BulkRequest(); @@ -1824,8 +1824,12 @@ public void shouldRestoreSnapshot_positive() throws IOException { CountRequest countRequest = new CountRequest(RESTORED_SONG_INDEX_NAME); Awaitility.await() .ignoreExceptions() + .pollInterval(100, TimeUnit.MILLISECONDS) .alias("Index contains proper number of documents restored from snapshot.") - .until(() -> restHighLevelClient.count(countRequest, DEFAULT).getCount() == 2); + .until(() -> { + restoredCount.incrementAndGet(); + return restHighLevelClient.count(countRequest, DEFAULT).getCount() == 2; + }); // 8. verify that document are present in restored index assertThat( @@ -1849,30 +1853,33 @@ public void shouldRestoreSnapshot_positive() throws IOException { "/_snapshot/test-snapshot-repository/restore-snapshot-positive/_restore" ) ); + auditLogsRule.assertExactly( + restoredCount.get(), + userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/restored_write_song_index/_count") + ); auditLogsRule.assertExactly(2, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(GET, "/_snapshot/test-snapshot-repository/restore-snapshot-positive") ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - - // Audit events generated in step 7 above - auditLogsRule.assertAtLeast(1, userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/restored_write_song_index/_count")); - auditLogsRule.assertAtLeast(1, grantedPrivilege(LIMITED_WRITE_USER, "SearchRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertExactly(restoredCount.get(), grantedPrivilege(LIMITED_WRITE_USER, "SearchRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test public void shouldRestoreSnapshot_failureForbiddenIndex() throws IOException { final String snapshotName = "restore-snapshot-negative-forbidden-index"; String restoreToIndex = "forbidden_index"; + long snapshotGetCount; + Settings indexSettings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { + SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); // 1. create some documents BulkRequest bulkRequest = new BulkRequest(); @@ -1888,7 +1895,7 @@ public void shouldRestoreSnapshot_failureForbiddenIndex() throws IOException { steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, WRITE_SONG_INDEX_NAME); // 4. wait till snapshot is ready - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); // 5. restore the snapshot assertThatThrownBy( @@ -1914,27 +1921,28 @@ public void shouldRestoreSnapshot_failureForbiddenIndex() throws IOException { ) ); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest( GET, "/_snapshot/test-snapshot-repository/restore-snapshot-negative-forbidden-index" ) ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); - auditLogsRule.assertExactlyOne(missingPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertAtLeastTransportMessages(1, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyScanAll(1, missingPrivilege(LIMITED_WRITE_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test public void shouldRestoreSnapshot_failureOperationForbidden() throws IOException { String snapshotName = "restore-snapshot-negative-forbidden-operation"; + long snapshotGetCount; + Settings indexSettings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).build(); + IndexOperationsHelper.createIndex(cluster, WRITE_SONG_INDEX_NAME, indexSettings); try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_WRITE_USER)) { SnapshotSteps steps = new SnapshotSteps(restHighLevelClient); // 1. create some documents @@ -1951,7 +1959,7 @@ public void shouldRestoreSnapshot_failureOperationForbidden() throws IOException steps.createSnapshot(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName, WRITE_SONG_INDEX_NAME); // 4. wait till snapshot is ready - steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); + snapshotGetCount = steps.waitForSnapshotCreation(TEST_SNAPSHOT_REPOSITORY_NAME, snapshotName); } // 5. restore the snapshot try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_READ_USER)) { @@ -1979,21 +1987,19 @@ public void shouldRestoreSnapshot_failureOperationForbidden() throws IOException ) ); auditLogsRule.assertExactlyOne(userAuthenticated(LIMITED_WRITE_USER).withRestRequest(POST, "/_bulk")); - auditLogsRule.assertAtLeast( - 1, + auditLogsRule.assertExactly( + snapshotGetCount, userAuthenticated(LIMITED_WRITE_USER).withRestRequest( GET, "/_snapshot/test-snapshot-repository/restore-snapshot-negative-forbidden-operation" ) ); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "PutRepositoryRequest")); + auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "CreateSnapshotRequest")); auditLogsRule.assertExactlyOne(grantedPrivilege(LIMITED_WRITE_USER, "BulkRequest")); - auditLogsRule.assertExactly(2, grantedPrivilege(LIMITED_WRITE_USER, "CreateIndexRequest")); - auditLogsRule.assertExactly(4, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); - auditLogsRule.assertExactly(1, missingPrivilege(LIMITED_READ_USER, "RestoreSnapshotRequest")); - auditLogsRule.assertAtLeast(2, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); - auditLogsRule.assertExactly(6, auditPredicate(INDEX_EVENT).withEffectiveUser(LIMITED_WRITE_USER)); + auditLogsRule.assertExactlyScanAll(1, missingPrivilege(LIMITED_READ_USER, "RestoreSnapshotRequest")); + auditLogsRule.assertExactly(snapshotGetCount, grantedPrivilege(LIMITED_WRITE_USER, "GetSnapshotsRequest")); + auditLogsRule.assertAtLeastTransportMessages(2, grantedPrivilege(LIMITED_WRITE_USER, "PutMappingRequest")); } @Test @@ -2131,11 +2137,11 @@ public void shouldDeleteIndexByAliasRequest_positive() throws IOException { userAuthenticated(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES).withRestRequest(POST, "/_aliases") ); auditLogsRule.assertExactly( - 2, + 1, grantedPrivilege(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES, "IndicesAliasesRequest") ); auditLogsRule.assertExactly( - 2, + 1, auditPredicate(INDEX_EVENT).withEffectiveUser(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES) ); } @@ -2722,11 +2728,11 @@ public void shouldCreateIndexWithAlias_positive() throws IOException { ) ); auditLogsRule.assertExactly( - 2, + 1, grantedPrivilege(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES, "CreateIndexRequest") ); auditLogsRule.assertExactly( - 2, + 1, auditPredicate(INDEX_EVENT).withEffectiveUser(USER_ALLOWED_TO_PERFORM_INDEX_OPERATIONS_ON_SELECTED_INDICES) ); } diff --git a/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java b/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java index 28aa6abd43..a03891ecca 100644 --- a/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java +++ b/src/integrationTest/java/org/opensearch/security/SnapshotSteps.java @@ -11,6 +11,8 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.awaitility.Awaitility; @@ -58,13 +60,21 @@ public CreateSnapshotResponse createSnapshot(String repositoryName, String snaps return snapshotClient.create(createSnapshotRequest, DEFAULT); } - public void waitForSnapshotCreation(String repositoryName, String snapshotName) { + public int waitForSnapshotCreation(String repositoryName, String snapshotName) { + AtomicInteger count = new AtomicInteger(); GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest(repositoryName, new String[] { snapshotName }); - Awaitility.await().alias("wait for snapshot creation").ignoreExceptions().until(() -> { - GetSnapshotsResponse snapshotsResponse = snapshotClient.get(getSnapshotsRequest, DEFAULT); - SnapshotInfo snapshotInfo = snapshotsResponse.getSnapshots().get(0); - return SnapshotState.SUCCESS.equals(snapshotInfo.state()); - }); + Awaitility.await() + .pollDelay(250, TimeUnit.MILLISECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .alias("wait for snapshot creation") + .ignoreExceptions() + .until(() -> { + count.incrementAndGet(); + GetSnapshotsResponse snapshotsResponse = snapshotClient.get(getSnapshotsRequest, DEFAULT); + SnapshotInfo snapshotInfo = snapshotsResponse.getSnapshots().get(0); + return SnapshotState.SUCCESS.equals(snapshotInfo.state()); + }); + return count.get(); } // CS-SUPPRESS-SINGLE: RegexpSingleline It is not possible to use phrase "cluster manager" instead of master here diff --git a/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java b/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java index badb938b3a..c41b5f4cda 100644 --- a/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java +++ b/src/integrationTest/java/org/opensearch/security/rest/WhoAmITests.java @@ -11,21 +11,48 @@ package org.opensearch.security.rest; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import org.apache.http.HttpStatus; import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; +import org.opensearch.security.auditlog.impl.AuditMessage; +import org.opensearch.test.framework.AuditCompliance; +import org.opensearch.test.framework.AuditConfiguration; +import org.opensearch.test.framework.AuditFilters; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.audit.AuditLogsRule; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import joptsimple.internal.Strings; + import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; +import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.security.auditlog.impl.AuditCategory.GRANTED_PRIVILEGES; +import static org.opensearch.security.auditlog.impl.AuditCategory.MISSING_PRIVILEGES; import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.audit.AuditMessagePredicate.grantedPrivilege; +import static org.opensearch.test.framework.audit.AuditMessagePredicate.privilegePredicateRESTLayer; +import static org.opensearch.test.framework.audit.AuditMessagePredicate.userAuthenticatedPredicate; +import static org.junit.Assert.assertTrue; @RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) @@ -34,6 +61,10 @@ public class WhoAmITests { new Role("who_am_i_role").clusterPermissions("security:whoamiprotected") ); + protected final static TestSecurityConfig.User AUDIT_LOG_VERIFIER = new TestSecurityConfig.User("audit_log_verifier").roles( + new Role("audit_log_verifier_role").clusterPermissions("*").indexPermissions("*").on("*") + ); + protected final static TestSecurityConfig.User WHO_AM_I_LEGACY = new TestSecurityConfig.User("who_am_i_user_legacy").roles( new Role("who_am_i_role_legacy").clusterPermissions("cluster:admin/opendistro_security/whoamiprotected") ); @@ -44,50 +75,71 @@ public class WhoAmITests { protected final static TestSecurityConfig.User WHO_AM_I_UNREGISTERED = new TestSecurityConfig.User("who_am_i_user_no_perm"); + protected final String expectedAuthorizedBody = "{\"dn\":null,\"is_admin\":false,\"is_node_certificate_request\":false}"; + protected final String expectedUnuauthorizedBody = + "no permissions for [security:whoamiprotected] and User [name=who_am_i_user_no_perm, backend_roles=[], requestedTenant=null]"; + public static final String WHOAMI_ENDPOINT = "_plugins/_security/whoami"; public static final String WHOAMI_PROTECTED_ENDPOINT = "_plugins/_security/whoamiprotected"; @ClassRule public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.THREE_CLUSTER_MANAGERS) .authc(AUTHC_HTTPBASIC_INTERNAL) - .users(WHO_AM_I, WHO_AM_I_LEGACY, WHO_AM_I_NO_PERM) + .users(WHO_AM_I, WHO_AM_I_LEGACY, WHO_AM_I_NO_PERM, AUDIT_LOG_VERIFIER) + .audit( + new AuditConfiguration(true).compliance(new AuditCompliance().enabled(true)) + .filters(new AuditFilters().enabledRest(true).enabledTransport(true).resolveBulkRequests(true)) + ) .build(); + @Rule + public AuditLogsRule auditLogsRule = new AuditLogsRule(); + @Test - public void testWhoAmIWithGetPermissions() throws Exception { - try (TestRestClient client = cluster.getRestClient(WHO_AM_I)) { - assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); - } + public void testWhoAmIWithGetPermissions() { try (TestRestClient client = cluster.getRestClient(WHO_AM_I)) { - assertThat(client.get(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); + assertResponse(client.get(WHOAMI_PROTECTED_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); + + // audit log, named route + auditLogsRule.assertExactlyOne(userAuthenticatedPredicate(WHO_AM_I, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + auditLogsRule.assertExactlyOne(privilegePredicateRESTLayer(GRANTED_PRIVILEGES, WHO_AM_I, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + + assertResponse(client.get(WHOAMI_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); } } @Test - public void testWhoAmIWithGetPermissionsLegacy() throws Exception { + public void testWhoAmIWithGetPermissionsLegacy() { try (TestRestClient client = cluster.getRestClient(WHO_AM_I_LEGACY)) { - assertThat(client.get(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); - } + assertResponse(client.get(WHOAMI_PROTECTED_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); - try (TestRestClient client = cluster.getRestClient(WHO_AM_I_LEGACY)) { - assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); + // audit log, named route + auditLogsRule.assertExactlyOne(userAuthenticatedPredicate(WHO_AM_I_LEGACY, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + auditLogsRule.assertExactlyOne( + privilegePredicateRESTLayer(GRANTED_PRIVILEGES, WHO_AM_I_LEGACY, GET, "/" + WHOAMI_PROTECTED_ENDPOINT) + ); + + assertResponse(client.get(WHOAMI_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); } } @Test - public void testWhoAmIWithoutGetPermissions() throws Exception { + public void testWhoAmIWithoutGetPermissions() { try (TestRestClient client = cluster.getRestClient(WHO_AM_I_NO_PERM)) { - assertThat(client.get(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); - } + assertResponse(client.get(WHOAMI_PROTECTED_ENDPOINT), HttpStatus.SC_UNAUTHORIZED, expectedUnuauthorizedBody); + // audit log, named route + auditLogsRule.assertExactlyOne(userAuthenticatedPredicate(WHO_AM_I_NO_PERM, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + auditLogsRule.assertExactlyOne( + privilegePredicateRESTLayer(MISSING_PRIVILEGES, WHO_AM_I_NO_PERM, GET, "/" + WHOAMI_PROTECTED_ENDPOINT) + ); - try (TestRestClient client = cluster.getRestClient(WHO_AM_I_NO_PERM)) { - assertThat(client.get(WHOAMI_PROTECTED_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_UNAUTHORIZED)); + assertResponse(client.get(WHOAMI_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); } } @Test - public void testWhoAmIPost() throws Exception { + public void testWhoAmIPost() { try (TestRestClient client = cluster.getRestClient(WHO_AM_I)) { assertThat(client.post(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); } @@ -104,5 +156,173 @@ public void testWhoAmIPost() throws Exception { assertThat(client.post(WHOAMI_ENDPOINT).getStatusCode(), equalTo(HttpStatus.SC_OK)); } + // No audit logs generated because `/whoami` is passthrough at Transport Layer, and POST route is not a NamedRoute + auditLogsRule.assertAuditLogsCount(0, 0); + } + + @Test + public void testAuditLogSimilarityWithTransportLayer() { + try (TestRestClient client = cluster.getRestClient(AUDIT_LOG_VERIFIER)) { + assertResponse(client.get(WHOAMI_PROTECTED_ENDPOINT), HttpStatus.SC_OK, expectedAuthorizedBody); + auditLogsRule.assertExactlyOne(userAuthenticatedPredicate(AUDIT_LOG_VERIFIER, GET, "/" + WHOAMI_PROTECTED_ENDPOINT)); + auditLogsRule.assertExactlyOne( + privilegePredicateRESTLayer(GRANTED_PRIVILEGES, AUDIT_LOG_VERIFIER, GET, "/" + WHOAMI_PROTECTED_ENDPOINT) + ); + + assertThat(client.get("_cat/indices").getStatusCode(), equalTo(HttpStatus.SC_OK)); + + // transport layer audit messages + auditLogsRule.assertExactly(1, grantedPrivilege(AUDIT_LOG_VERIFIER, "GetSettingsRequest")); + + List grantedPrivilegesMessages = auditLogsRule.getCurrentTestAuditMessages() + .stream() + .filter(msg -> msg.getCategory().equals(GRANTED_PRIVILEGES)) + .collect(Collectors.toList()); + + verifyAuditLogSimilarity(grantedPrivilegesMessages); + } + } + + private void assertResponse(TestRestClient.HttpResponse response, int expectedStatus, String expectedBody) { + assertThat(response.getStatusCode(), equalTo(expectedStatus)); + assertThat(response.getBody(), equalTo(expectedBody)); + } + + private void verifyAuditLogSimilarity(List currentTestAuditMessages) { + List restSet = new ArrayList<>(); + List transportSet = new ArrayList<>(); + + // It is okay to loop through all even though we end up using only 2, as the total number of messages should be around 8 + for (AuditMessage auditMessage : currentTestAuditMessages) { + if ("REST".equals(auditMessage.getAsMap().get(AuditMessage.REQUEST_LAYER).toString())) { + restSet.add(auditMessage); + } else if ("TRANSPORT".equals(auditMessage.getAsMap().get(AuditMessage.REQUEST_LAYER).toString())) { + transportSet.add(auditMessage); + } + } + // We pass 1 message from each layer to check for similarity + checkForStructuralSimilarity(restSet.get(0), transportSet.get(0)); + } + + /** + * Checks for structural similarity between audit message generated at Rest layer vs transport layer + * Example REST audit message for GRANTED_PRIVILEGES: + * { + * "audit_cluster_name":"local_cluster_1", + * "audit_node_name":"data_0", + * "audit_rest_request_method":"GET", + * "audit_category":"GRANTED_PRIVILEGES", + * "audit_request_origin":"REST", + * "audit_node_id":"Dez5cwAAQAC6cdmK_____w", + * "audit_request_layer":"REST", + * "audit_rest_request_path":"/_plugins/_security/whoamiprotected", + * "@timestamp":"2023-08-16T17:35:53.531+00:00", + * "audit_format_version":4, + * "audit_request_remote_address":"127.0.0.1", + * "audit_node_host_address":"127.0.0.1", + * "audit_rest_request_headers":{ + * "Connection":[ + * "keep-alive" + * ], + * "User-Agent":[ + * "Apache-HttpClient/5.2.1 (Java/19.0.1)" + * ], + * "content-length":[ + * "0" + * ], + * "Host":[ + * "127.0.0.1:47210" + * ], + * "Accept-Encoding":[ + * "gzip, x-gzip, deflate" + * ] + * }, + * "audit_request_effective_user":"audit_log_verifier", + * "audit_node_host_name":"127.0.0.1" + * } + * + * + * Example Transport audit message for GRANTED_PRIVILEGES: + * { + * "audit_cluster_name":"local_cluster_1", + * "audit_transport_headers":{ + * "_system_index_access_allowed":"false" + * }, + * "audit_node_name":"data_0", + * "audit_trace_task_id":"Dez5cwAAQAC6cdmK_____w:87", + * "audit_transport_request_type":"GetSettingsRequest", + * "audit_category":"GRANTED_PRIVILEGES", + * "audit_request_origin":"REST", + * "audit_node_id":"Dez5cwAAQAC6cdmK_____w", + * "audit_request_layer":"TRANSPORT", + * "@timestamp":"2023-08-16T17:35:53.621+00:00", + * "audit_format_version":4, + * "audit_request_remote_address":"127.0.0.1", + * "audit_request_privilege":"indices:monitor/settings/get", + * "audit_node_host_address":"127.0.0.1", + * "audit_request_effective_user":"audit_log_verifier", + * "audit_node_host_name":"127.0.0.1" + * } + * + * + * @param restAuditMessage audit message generated at REST layer + * @param transportAuditMessage audit message generated at Transport layer + */ + private void checkForStructuralSimilarity(AuditMessage restAuditMessage, AuditMessage transportAuditMessage) { + + Map restMsgFields = restAuditMessage.getAsMap(); + Map transportMsgFields = transportAuditMessage.getAsMap(); + + Set restAuditSet = restMsgFields.keySet(); + Set transportAuditSet = transportMsgFields.keySet(); + + // Added a magic number here and below, because there are always 15 or more items in each message generated via Audit logs + assertThat(restAuditSet.size(), greaterThan(14)); + assertThat(transportAuditSet.size(), greaterThan(14)); + + // check for values of common fields + Set commonFields = new HashSet<>(restAuditSet); + commonFields.retainAll(transportAuditSet); + + assertCommonFields(commonFields, restMsgFields, transportMsgFields); + + // check for values of uncommon fields + restAuditSet.removeAll(transportAuditMessage.getAsMap().keySet()); + transportAuditSet.removeAll(restAuditMessage.getAsMap().keySet()); + + // We compare two sets and see there were more than 10 items with same keys indicating these logs are similar + // There are a few headers that are generated different for REST vs TRANSPORT layer audit logs, but that is expected + // The end goal of this test is to ensure similarity, not equality. + assertThat(restAuditSet.size(), lessThan(5)); + assertThat(transportAuditSet.size(), lessThan(5)); + + assertThat(restMsgFields.get("audit_rest_request_path"), equalTo("/_plugins/_security/whoamiprotected")); + assertThat(restMsgFields.get("audit_rest_request_method").toString(), equalTo("GET")); + assertThat(restMsgFields.get("audit_rest_request_headers").toString().contains("Connection"), equalTo(true)); + + assertThat(transportMsgFields.get("audit_transport_request_type"), equalTo("GetSettingsRequest")); + assertThat(transportMsgFields.get("audit_request_privilege"), equalTo("indices:monitor/settings/get")); + assertThat(Strings.isNullOrEmpty(transportMsgFields.get("audit_trace_task_id").toString()), equalTo(false)); + } + + private void assertCommonFields(Set commonFields, Map restMsgFields, Map transportMsgFields) { + for (String key : commonFields) { + if (key.equals("@timestamp")) { + String restTimeStamp = restMsgFields.get(key).toString(); + String transportTimeStamp = transportMsgFields.get(key).toString(); + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + LocalDateTime restDateTime = LocalDateTime.parse(restTimeStamp, formatter); + LocalDateTime transportDateTime = LocalDateTime.parse(transportTimeStamp, formatter); + + // assert that these log messages are generated within 10 seconds of each other + assertTrue(Duration.between(restDateTime, transportDateTime).getSeconds() < 10); + } else if (key.equals("audit_request_layer")) { + assertThat(restMsgFields.get(key).toString(), equalTo("REST")); + assertThat(transportMsgFields.get(key).toString(), equalTo("TRANSPORT")); + } else { + assertThat(restMsgFields.get(key), equalTo(transportMsgFields.get(key))); + } + } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java index 3d13d731eb..3f9a0ae466 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java +++ b/src/integrationTest/java/org/opensearch/test/framework/audit/AuditLogsRule.java @@ -39,6 +39,7 @@ public class AuditLogsRule implements TestRule { private static final Logger log = LogManager.getLogger(AuditLogsRule.class); private List currentTestAuditMessages; + private List currentTransportTestAuditMessages; public List getCurrentTestAuditMessages() { return currentTestAuditMessages; @@ -56,6 +57,7 @@ public void waitForAuditLogs() { private void afterWaitingForAuditLogs() { if (log.isDebugEnabled()) { log.debug("Audit records captured during test:\n{}", auditMessagesToString(currentTestAuditMessages)); + log.debug("Audit transport records captured during test:\n{}", auditMessagesToString(currentTransportTestAuditMessages)); } } @@ -63,6 +65,13 @@ public void assertExactlyOne(Predicate predicate) { assertExactly(1, predicate); } + public void assertExactlyScanAll(long expectedNumberOfAuditMessages, Predicate predicate) { + List auditMessages = new ArrayList<>(currentTestAuditMessages); + auditMessages.addAll(currentTransportTestAuditMessages); + assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate), auditMessages); + + } + public void assertAuditLogsCount(int from, int to) { int actualCount = currentTestAuditMessages.size(); String message = "Expected audit log count is between " + from + " and " + to + " but was " + actualCount; @@ -70,10 +79,10 @@ public void assertAuditLogsCount(int from, int to) { } public void assertExactly(long expectedNumberOfAuditMessages, Predicate predicate) { - assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate)); + assertExactly(exactNumberOfAuditsFulfillPredicate(expectedNumberOfAuditMessages, predicate), currentTestAuditMessages); } - private void assertExactly(Matcher> matcher) { + private void assertExactly(Matcher> matcher, List currentTestAuditMessages) { // pollDelay - initial delay before first evaluation Awaitility.await("Await for audit logs") .atMost(3, TimeUnit.SECONDS) @@ -82,7 +91,11 @@ private void assertExactly(Matcher> matcher) { } public void assertAtLeast(long minCount, Predicate predicate) { - assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate)); + assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate), currentTestAuditMessages); + } + + public void assertAtLeastTransportMessages(long minCount, Predicate predicate) { + assertExactly(atLeastCertainNumberOfAuditsFulfillPredicate(minCount, predicate), currentTransportTestAuditMessages); } private static String auditMessagesToString(List audits) { @@ -122,16 +135,35 @@ private void whenTimeoutOccurs(String methodName) { private void afterTest() { TestRuleAuditLogSink.unregisterListener(); this.currentTestAuditMessages = null; + this.currentTransportTestAuditMessages = null; } private void beforeTest(String methodName) { log.info("Start collecting audit logs before test {}", methodName); this.currentTestAuditMessages = synchronizedList(new ArrayList<>()); + this.currentTransportTestAuditMessages = synchronizedList(new ArrayList<>()); TestRuleAuditLogSink.registerListener(this); } public void onAuditMessage(AuditMessage auditMessage) { - currentTestAuditMessages.add(auditMessage); - log.debug("New audit message received '{}', total number of audit messages '{}'.", auditMessage, currentTestAuditMessages.size()); + if (auditMessage.getAsMap().keySet().contains("audit_transport_headers")) { + if (log.isDebugEnabled()) { + log.debug( + "New transport audit message received '{}', total number of transport audit messages '{}'.", + auditMessage, + currentTransportTestAuditMessages.size() + ); + } + currentTransportTestAuditMessages.add(auditMessage); + } else { + if (log.isDebugEnabled()) { + log.debug( + "New audit message received '{}', total number of audit messages '{}'.", + auditMessage, + currentTestAuditMessages.size() + ); + } + currentTestAuditMessages.add(auditMessage); + } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index 0bf50c7a4d..551a231830 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -74,6 +74,15 @@ public enum ClusterManager { new NodeSettings(NodeRole.CLUSTER_MANAGER), new NodeSettings(NodeRole.DATA), new NodeSettings(NodeRole.DATA) + ), + + THREE_CLUSTER_MANAGERS_COORDINATOR( + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.CLUSTER_MANAGER), + new NodeSettings(NodeRole.DATA), + new NodeSettings(NodeRole.DATA), + new NodeSettings() ); private List nodeSettings = new LinkedList<>();