diff --git a/src/integrationTest/java/org/opensearch/security/CrossClusterSearchTests.java b/src/integrationTest/java/org/opensearch/security/CrossClusterSearchTests.java new file mode 100644 index 0000000000..5f812b921b --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/CrossClusterSearchTests.java @@ -0,0 +1,435 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.security; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.action.search.SearchRequest; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.client.Client; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.TestSecurityConfig.User; +import org.opensearch.test.framework.certificate.TestCertificates; +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.SearchRequestFactory; +import org.opensearch.test.framework.cluster.TestRestClient; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.opensearch.client.RequestOptions.DEFAULT; +import static org.opensearch.rest.RestStatus.FORBIDDEN; +import static org.opensearch.security.Song.ARTIST_FIRST; +import static org.opensearch.security.Song.FIELD_ARTIST; +import static org.opensearch.security.Song.FIELD_GENRE; +import static org.opensearch.security.Song.FIELD_LYRICS; +import static org.opensearch.security.Song.FIELD_STARS; +import static org.opensearch.security.Song.FIELD_TITLE; +import static org.opensearch.security.Song.GENRE_JAZZ; +import static org.opensearch.security.Song.GENRE_ROCK; +import static org.opensearch.security.Song.QUERY_TITLE_MAGNUM_OPUS; +import static org.opensearch.security.Song.SONGS; +import static org.opensearch.security.Song.TITLE_MAGNUM_OPUS; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.Role.ALL_ACCESS; +import static org.opensearch.test.framework.cluster.SearchRequestFactory.queryStringQueryRequest; +import static org.opensearch.test.framework.matcher.ExceptionMatcherAssert.assertThatThrownBy; +import static org.opensearch.test.framework.matcher.OpenSearchExceptionMatchers.statusException; +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.isSuccessfulSearchResponse; +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.numberOfTotalHitsIsEqualTo; +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.searchHitContainsFieldWithValue; +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.searchHitDoesNotContainField; +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.searchHitsContainDocumentWithId; +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.searchHitsContainDocumentsInAnyOrder; + +/** +* This is a parameterized test so that one test class is used to test security plugin behaviour when ccsMinimizeRoundtrips +* option is enabled or disabled. Method {@link #parameters()} is a source of parameters values. +*/ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class CrossClusterSearchTests { + + private static final String SONG_INDEX_NAME = "song_lyrics"; + + private static final String PROHIBITED_SONG_INDEX_NAME = "prohibited_song_lyrics"; + + public static final String REMOTE_CLUSTER_NAME = "ccsRemote"; + public static final String REMOTE_SONG_INDEX = REMOTE_CLUSTER_NAME + ":" + SONG_INDEX_NAME; + + public static final String SONG_ID_1R = "remote-00001"; + public static final String SONG_ID_2L = "local-00002"; + public static final String SONG_ID_3R = "remote-00003"; + public static final String SONG_ID_4L = "local-00004"; + public static final String SONG_ID_5R = "remote-00005"; + public static final String SONG_ID_6R = "remote-00006"; + + private static final Role LIMITED_ROLE = new Role("limited_role") + .indexPermissions("indices:data/read/search", "indices:admin/shards/search_shards") + .on(SONG_INDEX_NAME, "user-${user.name}-${attr.internal.type}"); + + private static final Role DLS_ROLE_ROCK = new Role("dls_role_rock") + .indexPermissions("indices:data/read/search", "indices:data/read/get", "indices:admin/shards/search_shards") + .dls(String.format("{\"match\":{\"%s\":\"%s\"}}", FIELD_GENRE, GENRE_ROCK)) + .on(SONG_INDEX_NAME); + + private static final Role DLS_ROLE_JAZZ = new Role("dls_role_jazz") + .indexPermissions("indices:data/read/search", "indices:data/read/get", "indices:admin/shards/search_shards") + .dls(String.format("{\"match\":{\"%s\":\"%s\"}}", FIELD_GENRE, GENRE_JAZZ)) + .on(SONG_INDEX_NAME); + + private static final Role FLS_EXCLUDE_LYRICS_ROLE = new Role("fls_exclude_lyrics_role") + .indexPermissions("indices:data/read/search", "indices:data/read/get", "indices:admin/shards/search_shards") + .fls("~" + FIELD_LYRICS) + .on(SONG_INDEX_NAME); + + private static final Role FLS_INCLUDE_TITLE_ROLE = new Role("fls_include_title_role") + .indexPermissions("indices:data/read/search", "indices:data/read/get", "indices:admin/shards/search_shards") + .fls(FIELD_TITLE) + .on(SONG_INDEX_NAME); + + public static final String TYPE_ATTRIBUTE = "type"; + + private static final User ADMIN_USER = new User("admin").roles(ALL_ACCESS).attr(TYPE_ATTRIBUTE, "administrative"); + private static final User LIMITED_USER = new User("limited_user").attr(TYPE_ATTRIBUTE, "personal"); + + private static final User FLS_INCLUDE_TITLE_USER = new User("fls_include_title_user"); + + private static final User FLS_EXCLUDE_LYRICS_USER = new User("fls_exclude_lyrics_user"); + + private static final User DLS_USER_ROCK = new User("dls-user-rock"); + + private static final User DLS_USER_JAZZ = new User("dls-user-jazz"); + + public static final String LIMITED_USER_INDEX_NAME = "user-" + LIMITED_USER.getName() + "-" + LIMITED_USER.getAttribute(TYPE_ATTRIBUTE); + public static final String ADMIN_USER_INDEX_NAME = "user-" + ADMIN_USER.getName() + "-" + ADMIN_USER.getAttribute(TYPE_ATTRIBUTE); + + private static final TestCertificates TEST_CERTIFICATES = new TestCertificates(); + + private final boolean ccsMinimizeRoundtrips; + + public static final String PLUGINS_SECURITY_RESTAPI_ROLES_ENABLED = "plugins.security.restapi.roles_enabled"; + @ClassRule + public static final LocalCluster remoteCluster = new LocalCluster.Builder().certificates(TEST_CERTIFICATES) + .clusterManager(ClusterManager.SINGLENODE).anonymousAuth(false).clusterName(REMOTE_CLUSTER_NAME) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .roles(LIMITED_ROLE, DLS_ROLE_ROCK, DLS_ROLE_JAZZ, FLS_EXCLUDE_LYRICS_ROLE, FLS_INCLUDE_TITLE_ROLE).users(ADMIN_USER) + .build(); + + @ClassRule + public static final LocalCluster cluster = new LocalCluster.Builder().certificates(TEST_CERTIFICATES) + .clusterManager(ClusterManager.SINGLE_REMOTE_CLIENT).anonymousAuth(false).clusterName("ccsLocal") + .nodeSettings(Map.of(PLUGINS_SECURITY_RESTAPI_ROLES_ENABLED, List.of("user_" + ADMIN_USER.getName() +"__" + ALL_ACCESS.getName()))) + .remote(REMOTE_CLUSTER_NAME, remoteCluster) + .roles(LIMITED_ROLE, DLS_ROLE_ROCK, DLS_ROLE_JAZZ, FLS_EXCLUDE_LYRICS_ROLE, FLS_INCLUDE_TITLE_ROLE).authc(AUTHC_HTTPBASIC_INTERNAL) + .users(ADMIN_USER, LIMITED_USER, DLS_USER_ROCK, DLS_USER_JAZZ, FLS_INCLUDE_TITLE_USER, FLS_EXCLUDE_LYRICS_USER) + .build(); + + @ParametersFactory(shuffle = false) + public static Iterable parameters() { + return List.of(new Object[] { true }, new Object[] { false }); + } + + public CrossClusterSearchTests(Boolean ccsMinimizeRoundtrips) { + this.ccsMinimizeRoundtrips = ccsMinimizeRoundtrips; + } + + @BeforeClass + public static void createTestData() { + try(Client client = remoteCluster.getInternalNodeClient()){ + client.prepareIndex(SONG_INDEX_NAME).setId(SONG_ID_1R).setRefreshPolicy(IMMEDIATE).setSource(SONGS[0]).get(); + client.prepareIndex(SONG_INDEX_NAME).setId(SONG_ID_6R).setRefreshPolicy(IMMEDIATE).setSource(SONGS[5]).get(); + client.prepareIndex(PROHIBITED_SONG_INDEX_NAME).setId(SONG_ID_3R).setRefreshPolicy(IMMEDIATE).setSource(SONGS[1]).get(); + client.prepareIndex(LIMITED_USER_INDEX_NAME).setId(SONG_ID_5R).setRefreshPolicy(IMMEDIATE).setSource(SONGS[4]).get(); + } + try(Client client = cluster.getInternalNodeClient()){ + client.prepareIndex(SONG_INDEX_NAME).setId(SONG_ID_2L).setRefreshPolicy(IMMEDIATE).setSource(SONGS[2]).get(); + client.prepareIndex(PROHIBITED_SONG_INDEX_NAME).setId(SONG_ID_4L).setRefreshPolicy(IMMEDIATE).setSource(SONGS[3]).get(); + } + try(TestRestClient client = cluster.getRestClient(ADMIN_USER)) { + client.assignRoleToUser(LIMITED_USER.getName(), LIMITED_ROLE.getName()).assertStatusCode(200); + client.assignRoleToUser(DLS_USER_ROCK.getName(), DLS_ROLE_ROCK.getName()).assertStatusCode(200); + client.assignRoleToUser(DLS_USER_JAZZ.getName(), DLS_ROLE_JAZZ.getName()).assertStatusCode(200); + client.assignRoleToUser(FLS_INCLUDE_TITLE_USER.getName(), FLS_INCLUDE_TITLE_ROLE.getName()).assertStatusCode(200); + client.assignRoleToUser(FLS_EXCLUDE_LYRICS_USER.getName(), FLS_EXCLUDE_LYRICS_ROLE.getName()).assertStatusCode(200); + } + } + + @Test + public void shouldFindDocumentOnRemoteCluster_positive() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_SONG_INDEX); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(2)); + assertThat(response, searchHitsContainDocumentWithId(0, SONG_INDEX_NAME, SONG_ID_1R)); + assertThat(response, searchHitsContainDocumentWithId(1, SONG_INDEX_NAME, SONG_ID_6R)); + } + } + + private SearchRequest searchAll(String indexName) { + SearchRequest searchRequest = SearchRequestFactory.searchAll(indexName); + searchRequest.setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + return searchRequest; + } + + @Test + public void shouldFindDocumentOnRemoteCluster_negative() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_CLUSTER_NAME + ":" + PROHIBITED_SONG_INDEX_NAME); + + assertThatThrownBy(() -> restHighLevelClient.search(searchRequest, DEFAULT), statusException(FORBIDDEN)); + } + } + + @Test + public void shouldSearchForDocumentOnRemoteClustersWhenStarIsUsedAsClusterName_positive() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll("*" + ":" + SONG_INDEX_NAME); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + //only remote documents are found + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(2)); + assertThat(response, searchHitsContainDocumentWithId(0, SONG_INDEX_NAME, SONG_ID_1R)); + assertThat(response, searchHitsContainDocumentWithId(1, SONG_INDEX_NAME, SONG_ID_6R)); + } + } + + @Test + public void shouldSearchForDocumentOnRemoteClustersWhenStarIsUsedAsClusterName_negative() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll("*" + ":" + PROHIBITED_SONG_INDEX_NAME); + + assertThatThrownBy(() -> restHighLevelClient.search(searchRequest, DEFAULT), statusException(FORBIDDEN)); + } + } + + @Test + public void shouldSearchForDocumentOnBothClustersWhenIndexOnBothClusterArePointedOut_positive() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = SearchRequestFactory.searchAll(REMOTE_SONG_INDEX, SONG_INDEX_NAME); + searchRequest.setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(3)); + assertThat(response, searchHitsContainDocumentsInAnyOrder( + Pair.of(SONG_INDEX_NAME, SONG_ID_1R), + Pair.of(SONG_INDEX_NAME, SONG_ID_2L), + Pair.of(SONG_INDEX_NAME, SONG_ID_6R)) + ); + } + } + + @Test + public void shouldSearchForDocumentOnBothClustersWhenIndexOnBothClusterArePointedOut_negativeLackOfLocalAccess() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + var searchRequest = SearchRequestFactory.searchAll(REMOTE_SONG_INDEX, PROHIBITED_SONG_INDEX_NAME); + searchRequest.setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + + assertThatThrownBy(() -> restHighLevelClient.search(searchRequest, DEFAULT), statusException(FORBIDDEN)); + } + } + + @Test + public void shouldSearchForDocumentOnBothClustersWhenIndexOnBothClusterArePointedOut_negativeLackOfRemoteAccess() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + String remoteIndex = REMOTE_CLUSTER_NAME + ":" + PROHIBITED_SONG_INDEX_NAME; + SearchRequest searchRequest = SearchRequestFactory.searchAll(remoteIndex, SONG_INDEX_NAME); + searchRequest.setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + + assertThatThrownBy(() -> restHighLevelClient.search(searchRequest, DEFAULT), statusException(FORBIDDEN)); + } + } + + @Test + public void shouldSearchViaAllAliasOnRemoteCluster_positive() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(ADMIN_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_CLUSTER_NAME + ":_all"); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(4)); + assertThat(response, searchHitsContainDocumentsInAnyOrder( + Pair.of(SONG_INDEX_NAME, SONG_ID_1R), + Pair.of(SONG_INDEX_NAME, SONG_ID_6R), + Pair.of(PROHIBITED_SONG_INDEX_NAME, SONG_ID_3R), + Pair.of(LIMITED_USER_INDEX_NAME, SONG_ID_5R)) + ); + } + } + + @Test + public void shouldSearchViaAllAliasOnRemoteCluster_negative() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_CLUSTER_NAME + ":_all"); + + assertThatThrownBy(() -> restHighLevelClient.search(searchRequest, DEFAULT), statusException(FORBIDDEN)); + } + } + + @Test + public void shouldSearchAllIndexOnRemoteClusterWhenStarIsUsedAsIndexName_positive() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(ADMIN_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_CLUSTER_NAME + ":*"); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(4)); + assertThat(response, searchHitsContainDocumentsInAnyOrder( + Pair.of(SONG_INDEX_NAME, SONG_ID_1R), + Pair.of(SONG_INDEX_NAME, SONG_ID_6R), + Pair.of(PROHIBITED_SONG_INDEX_NAME, SONG_ID_3R), + Pair.of(LIMITED_USER_INDEX_NAME, SONG_ID_5R)) + ); + } + } + + @Test + public void shouldSearchAllIndexOnRemoteClusterWhenStarIsUsedAsIndexName_negative() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_CLUSTER_NAME + ":*"); + + assertThatThrownBy(() -> restHighLevelClient.search(searchRequest, DEFAULT), statusException(FORBIDDEN)); + } + } + + @Test + public void shouldResolveUserNameExpressionInRoleIndexPattern_positive() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_CLUSTER_NAME + ":" + LIMITED_USER_INDEX_NAME); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + assertThat(response, numberOfTotalHitsIsEqualTo(1)); + } + } + + @Test + public void shouldResolveUserNameExpressionInRoleIndexPattern_negative() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_CLUSTER_NAME + ":" + ADMIN_USER_INDEX_NAME); + + assertThatThrownBy(() -> restHighLevelClient.search(searchRequest, DEFAULT), statusException(FORBIDDEN)); + } + } + + @Test + public void shouldSearchInIndexWithPrefix_positive() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_CLUSTER_NAME + ":song*"); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(2)); + assertThat(response, searchHitsContainDocumentsInAnyOrder( + Pair.of(SONG_INDEX_NAME, SONG_ID_1R), + Pair.of(SONG_INDEX_NAME, SONG_ID_6R) + )); + } + } + + @Test + public void shouldSearchInIndexWithPrefix_negative() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(LIMITED_USER)) { + SearchRequest searchRequest = searchAll(REMOTE_CLUSTER_NAME + ":prohibited*"); + + assertThatThrownBy(() -> restHighLevelClient.search(searchRequest, DEFAULT), statusException(FORBIDDEN)); + } + } + + @Test + public void shouldEvaluateDocumentLevelSecurityRulesOnRemoteClusterOnSearchRequest_caseRock() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(DLS_USER_ROCK)) { + SearchRequest searchRequest = searchAll(REMOTE_SONG_INDEX); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + //searching for all documents, so is it important that result contain only one document with id SONG_ID_1 + // and document with SONG_ID_6 is excluded from result set by DLS + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(1)); + assertThat(response, searchHitsContainDocumentWithId(0, SONG_INDEX_NAME, SONG_ID_1R)); + } + } + + @Test + public void shouldEvaluateDocumentLevelSecurityRulesOnRemoteClusterOnSearchRequest_caseJazz() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(DLS_USER_JAZZ)) { + SearchRequest searchRequest = searchAll(REMOTE_SONG_INDEX); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + //searching for all documents, so is it important that result contain only one document with id SONG_ID_6 + // and document with SONG_ID_1 is excluded from result set by DLS + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(1)); + assertThat(response, searchHitsContainDocumentWithId(0, SONG_INDEX_NAME, SONG_ID_6R)); + } + } + + @Test + public void shouldHaveAccessOnlyToSpecificField() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(FLS_INCLUDE_TITLE_USER)) { + SearchRequest searchRequest = queryStringQueryRequest(REMOTE_SONG_INDEX, QUERY_TITLE_MAGNUM_OPUS); + searchRequest.setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(1)); + //document should contain only title field + assertThat(response, searchHitContainsFieldWithValue(0, FIELD_TITLE, TITLE_MAGNUM_OPUS)); + assertThat(response, searchHitDoesNotContainField(0, FIELD_ARTIST)); + assertThat(response, searchHitDoesNotContainField(0, FIELD_LYRICS)); + assertThat(response, searchHitDoesNotContainField(0, FIELD_STARS)); + assertThat(response, searchHitDoesNotContainField(0, FIELD_GENRE)); + } + } + + @Test + public void shouldLackAccessToSpecificField() throws IOException { + try (RestHighLevelClient restHighLevelClient = cluster.getRestHighLevelClient(FLS_EXCLUDE_LYRICS_USER)) { + SearchRequest searchRequest = queryStringQueryRequest(REMOTE_SONG_INDEX, QUERY_TITLE_MAGNUM_OPUS); + searchRequest.setCcsMinimizeRoundtrips(ccsMinimizeRoundtrips); + + SearchResponse response = restHighLevelClient.search(searchRequest, DEFAULT); + + assertThat(response, isSuccessfulSearchResponse()); + assertThat(response, numberOfTotalHitsIsEqualTo(1)); + //document should not contain lyrics field + assertThat(response, searchHitDoesNotContainField(0, FIELD_LYRICS)); + + assertThat(response, searchHitContainsFieldWithValue(0, FIELD_TITLE, TITLE_MAGNUM_OPUS)); + assertThat(response, searchHitContainsFieldWithValue(0, FIELD_ARTIST, ARTIST_FIRST)); + assertThat(response, searchHitContainsFieldWithValue(0, FIELD_STARS, 1)); + assertThat(response, searchHitContainsFieldWithValue(0, FIELD_GENRE, GENRE_ROCK)); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/Song.java b/src/integrationTest/java/org/opensearch/security/Song.java index e71a258169..b5ca1895b5 100644 --- a/src/integrationTest/java/org/opensearch/security/Song.java +++ b/src/integrationTest/java/org/opensearch/security/Song.java @@ -9,14 +9,17 @@ */ package org.opensearch.security; +import java.util.Map; +import java.util.Objects; -class Song { +public class Song { static final String FIELD_TITLE = "title"; static final String FIELD_ARTIST = "artist"; static final String FIELD_LYRICS = "lyrics"; - static final String FIELD_STARS = "stars"; + + static final String FIELD_GENRE = "genre"; static final String ARTIST_FIRST = "First artist"; static final String ARTIST_STRING = "String"; static final String ARTIST_TWINS = "Twins"; @@ -26,19 +29,57 @@ class Song { static final String ARTIST_NO = "No!"; static final String TITLE_POISON = "Poison"; + public static final String ARTIST_YES = "yes"; + + public static final String TITLE_AFFIRMATIVE = "Affirmative"; + + public static final String ARTIST_UNKNOWN = "unknown"; + public static final String TITLE_CONFIDENTIAL = "confidential"; + public static final String LYRICS_1 = "Very deep subject"; public static final String LYRICS_2 = "Once upon a time"; public static final String LYRICS_3 = "giant nonsense"; public static final String LYRICS_4 = "Much too much"; + public static final String LYRICS_5 = "Little to little"; + public static final String LYRICS_6 = "confidential secret classified"; + + static final String GENRE_ROCK = "rock"; + static final String GENRE_JAZZ = "jazz"; + static final String GENRE_BLUES = "blues"; static final String QUERY_TITLE_NEXT_SONG = FIELD_TITLE + ":" + "\"" + TITLE_NEXT_SONG + "\""; static final String QUERY_TITLE_POISON = FIELD_TITLE + ":" + TITLE_POISON; static final String QUERY_TITLE_MAGNUM_OPUS = FIELD_TITLE + ":" + TITLE_MAGNUM_OPUS; - static final Object[][] SONGS = { - {FIELD_ARTIST, ARTIST_FIRST, FIELD_TITLE, TITLE_MAGNUM_OPUS ,FIELD_LYRICS, LYRICS_1, FIELD_STARS, 1}, - {FIELD_ARTIST, ARTIST_STRING, FIELD_TITLE, TITLE_SONG_1_PLUS_1, FIELD_LYRICS, LYRICS_2, FIELD_STARS, 2}, - {FIELD_ARTIST, ARTIST_TWINS, FIELD_TITLE, TITLE_NEXT_SONG, FIELD_LYRICS, LYRICS_3, FIELD_STARS, 3}, - {FIELD_ARTIST, ARTIST_NO, FIELD_TITLE, TITLE_POISON, FIELD_LYRICS, LYRICS_4, FIELD_STARS, 4} + static final Map[] SONGS = { + new Song(ARTIST_FIRST, TITLE_MAGNUM_OPUS ,LYRICS_1, 1, GENRE_ROCK).asMap(), + new Song(ARTIST_STRING, TITLE_SONG_1_PLUS_1, LYRICS_2, 2, GENRE_BLUES).asMap(), + new Song(ARTIST_TWINS, TITLE_NEXT_SONG, LYRICS_3, 3, GENRE_JAZZ).asMap(), + new Song(ARTIST_NO, TITLE_POISON, LYRICS_4, 4, GENRE_ROCK).asMap(), + new Song(ARTIST_YES, TITLE_AFFIRMATIVE,LYRICS_5, 5, GENRE_BLUES).asMap(), + new Song(ARTIST_UNKNOWN, TITLE_CONFIDENTIAL, LYRICS_6, 6, GENRE_JAZZ).asMap() }; + + private final String artist; + private final String title; + private final String lyrics; + private final Integer stars; + + private final String genre; + + public Song(String artist, String title, String lyrics, Integer stars, String genre) { + this.artist = Objects.requireNonNull(artist, "Artist is required"); + this.title = Objects.requireNonNull(title, "Title is required"); + this.lyrics = Objects.requireNonNull(lyrics, "Lyrics is required"); + this.stars = Objects.requireNonNull(stars, "Stars field is required"); + this.genre = Objects.requireNonNull(genre, "Genre field is required"); + } + + public Map asMap() { + return Map.of(FIELD_ARTIST, artist, + FIELD_TITLE, title, + FIELD_LYRICS, lyrics, + FIELD_STARS, stars, + FIELD_GENRE, genre); + } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index b0eea6c336..6ae9d93d34 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -196,6 +196,10 @@ public Set getRoleNames() { return roles.stream().map(Role::getName).collect(Collectors.toSet()); } + public Object getAttribute(String attributeName) { + return attributes.get(attributeName); + } + @Override public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { xContentBuilder.startObject(); diff --git a/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java index df9bcfa41d..c770def8a8 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java +++ b/src/integrationTest/java/org/opensearch/test/framework/certificate/TestCertificates.java @@ -36,6 +36,9 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import static org.opensearch.test.framework.certificate.PublicKeyUsage.CLIENT_AUTH; import static org.opensearch.test.framework.certificate.PublicKeyUsage.CRL_SIGN; import static org.opensearch.test.framework.certificate.PublicKeyUsage.DIGITAL_SIGNATURE; @@ -50,6 +53,8 @@ */ public class TestCertificates { + private static final Logger log = LogManager.getLogger(TestCertificates.class); + public static final Integer MAX_NUMBER_OF_NODE_CERTIFICATES = 3; private static final String CA_SUBJECT = "DC=com,DC=example,O=Example Com Inc.,OU=Example Com Inc. Root CA,CN=Example Com Inc. Root CA"; @@ -68,6 +73,7 @@ public TestCertificates() { .mapToObj(this::createNodeCertificate) .collect(Collectors.toList()); this.adminCertificate = createAdminCertificate(); + log.info("Test certificates successfully generated"); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java index 760820a3b6..35c5229bfb 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/ClusterManager.java @@ -31,9 +31,11 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.LinkedList; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import org.opensearch.index.reindex.ReindexModulePlugin; @@ -50,17 +52,18 @@ import static org.opensearch.test.framework.cluster.NodeType.DATA; public enum ClusterManager { - //3 nodes (1m, 2d) - DEFAULT(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)), + DEFAULT(new NodeSettings(NodeRole.CLUSTER_MANAGER), new NodeSettings(NodeRole.DATA), new NodeSettings(NodeRole.DATA)), //1 node (1md) - SINGLENODE(new NodeSettings(true, true)), + SINGLENODE(new NodeSettings(NodeRole.CLUSTER_MANAGER, NodeRole.DATA)), + + SINGLE_REMOTE_CLIENT(new NodeSettings(NodeRole.CLUSTER_MANAGER, NodeRole.DATA, NodeRole.REMOTE_CLUSTER_CLIENT)), //4 node (1m, 2d, 1c) - CLIENTNODE(new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true), new NodeSettings(false, false)), + CLIENTNODE(new NodeSettings(NodeRole.CLUSTER_MANAGER), new NodeSettings(NodeRole.DATA), new NodeSettings(NodeRole.DATA), new NodeSettings()), - THREE_CLUSTER_MANAGERS(new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(true, false), new NodeSettings(false, true), new NodeSettings(false, true)); + THREE_CLUSTER_MANAGERS(new NodeSettings(NodeRole.CLUSTER_MANAGER), new NodeSettings(NodeRole.CLUSTER_MANAGER), new NodeSettings(NodeRole.CLUSTER_MANAGER), new NodeSettings(NodeRole.DATA), new NodeSettings(NodeRole.DATA)); private List nodeSettings = new LinkedList<>(); @@ -73,11 +76,11 @@ public List getNodeSettings() { } public List getClusterManagerNodeSettings() { - return unmodifiableList(nodeSettings.stream().filter(a -> a.clusterManagerNode).collect(Collectors.toList())); + return unmodifiableList(nodeSettings.stream().filter(a -> a.containRole(NodeRole.CLUSTER_MANAGER)).collect(Collectors.toList())); } public List getNonClusterManagerNodeSettings() { - return unmodifiableList(nodeSettings.stream().filter(a -> !a.clusterManagerNode).collect(Collectors.toList())); + return unmodifiableList(nodeSettings.stream().filter(a -> !a.containRole(NodeRole.CLUSTER_MANAGER)).collect(Collectors.toList())); } public int getNodes() { @@ -85,39 +88,48 @@ public int getNodes() { } public int getClusterManagerNodes() { - return (int) nodeSettings.stream().filter(a -> a.clusterManagerNode).count(); + return (int) nodeSettings.stream().filter(a -> a.containRole(NodeRole.CLUSTER_MANAGER)).count(); } public int getDataNodes() { - return (int) nodeSettings.stream().filter(a -> a.dataNode).count(); + return (int) nodeSettings.stream().filter(a -> a.containRole(NodeRole.DATA)).count(); } public int getClientNodes() { - return (int) nodeSettings.stream().filter(a -> !a.clusterManagerNode && !a.dataNode).count(); + return (int) nodeSettings.stream().filter(a -> a.isClientNode()).count(); } + public static class NodeSettings { private final static List> DEFAULT_PLUGINS = List.of(Netty4ModulePlugin.class, OpenSearchSecurityPlugin.class, MatrixAggregationModulePlugin.class, ParentJoinModulePlugin.class, PercolatorModulePlugin.class, ReindexModulePlugin.class); - public final boolean clusterManagerNode; - public final boolean dataNode; + + private final Set roles; public final List> plugins; - public NodeSettings(boolean clusterManagerNode, boolean dataNode) { - this(clusterManagerNode, dataNode, Collections.emptyList()); + public NodeSettings(NodeRole...roles) { + this(roles.length == 0 ? Collections.emptySet() : EnumSet.copyOf(Arrays.asList(roles)), Collections.emptyList()); } - public NodeSettings(boolean clusterManagerNode, boolean dataNode, List> additionalPlugins) { + public NodeSettings(Set roles, List> additionalPlugins) { super(); - this.clusterManagerNode = clusterManagerNode; - this.dataNode = dataNode; + this.roles = Objects.requireNonNull(roles, "Node roles set must not be null"); this.plugins = mergePlugins(additionalPlugins, DEFAULT_PLUGINS); } + + public boolean containRole(NodeRole nodeRole) { + return roles.contains(nodeRole); + } + + public boolean isClientNode() { + return (roles.contains(NodeRole.DATA) == false) && (roles.contains(NodeRole.CLUSTER_MANAGER)); + } + NodeType recognizeNodeType() { - if (clusterManagerNode) { + if (roles.contains(NodeRole.CLUSTER_MANAGER)) { return CLUSTER_MANAGER; - } else if (dataNode) { + } else if (roles.contains(NodeRole.DATA)) { return DATA; } else { return CLIENT; diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index f6f4610121..334e408f0d 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -121,8 +121,11 @@ public void before() throws Throwable { for (Map.Entry entry : remotes.entrySet()) { @SuppressWarnings("resource") InetSocketAddress transportAddress = entry.getValue().localOpenSearchCluster.clusterManagerNode().getTransportAddress(); + String key = "cluster.remote." + entry.getKey() + ".seeds"; + String value = transportAddress.getHostString() + ":" + transportAddress.getPort(); + log.info("Remote cluster '{}' added to configuration with the following seed '{}'", key, value); nodeOverride = Settings.builder().put(nodeOverride) - .putList("cluster.remote." + entry.getKey() + ".seeds", transportAddress.getHostString() + ":" + transportAddress.getPort()) + .putList(key, value) .build(); } @@ -250,7 +253,6 @@ public static class Builder { private boolean loadConfigurationIntoIndex = true; public Builder() { - this.testCertificates = new TestCertificates(); } public Builder dependsOn(Object object) { @@ -367,9 +369,16 @@ public Builder loadConfigurationIntoIndex(boolean loadConfigurationIntoIndex) { this.loadConfigurationIntoIndex = loadConfigurationIntoIndex; return this; } + public Builder certificates(TestCertificates certificates) { + this.testCertificates = certificates; + return this; + } public LocalCluster build() { try { + if(testCertificates == null){ + testCertificates = new TestCertificates(); + } clusterName += "_" + num.incrementAndGet(); Settings settings = nodeOverrideSettingsBuilder.build(); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java index 66f4f095cb..9a8602fb2e 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java @@ -370,7 +370,7 @@ boolean hasAssignedType(NodeType type) { CompletableFuture start() { CompletableFuture completableFuture = new CompletableFuture<>(); Class[] mergedPlugins = nodeSettings.pluginsWithAddition(additionalPlugins); - this.node = new PluginAwareNode(nodeSettings.clusterManagerNode, getOpenSearchSettings(), mergedPlugins); + this.node = new PluginAwareNode(nodeSettings.containRole(NodeRole.CLUSTER_MANAGER), getOpenSearchSettings(), mergedPlugins); new Thread(new Runnable() { @@ -495,12 +495,15 @@ private Settings getMinimalOpenSearchSettings() { private List createNodeRolesSettings() { final ImmutableList.Builder nodeRolesBuilder = ImmutableList.builder(); - if (nodeSettings.dataNode) { + if (nodeSettings.containRole(NodeRole.DATA)) { nodeRolesBuilder.add("data"); } - if (nodeSettings.clusterManagerNode) { + if (nodeSettings.containRole(NodeRole.CLUSTER_MANAGER)) { nodeRolesBuilder.add("cluster_manager"); } + if(nodeSettings.containRole(NodeRole.REMOTE_CLUSTER_CLIENT)) { + nodeRolesBuilder.add("remote_cluster_client"); + } return nodeRolesBuilder.build(); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeRole.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeRole.java new file mode 100644 index 0000000000..7dc77ef37e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/NodeRole.java @@ -0,0 +1,14 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.cluster; + +enum NodeRole { + DATA, CLUSTER_MANAGER, REMOTE_CLUSTER_CLIENT +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java index 5c1d25911a..40f0d63c19 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java @@ -53,6 +53,14 @@ public static SearchRequest searchRequestWithScroll(String indexName, int pageSi return searchRequest; } + public static SearchRequest searchAll(String...indexNames) { + SearchRequest searchRequest = new SearchRequest(indexNames); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchRequest.source(searchSourceBuilder); + return searchRequest; + } + public static SearchScrollRequest getSearchScrollRequest(SearchResponse searchResponse) { SearchScrollRequest scrollRequest = new SearchScrollRequest(searchResponse.getScrollId()); scrollRequest.scroll(new TimeValue(1, MINUTES)); diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index b4b2ca0583..aac11008d6 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -37,6 +37,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import javax.net.ssl.SSLContext; @@ -167,6 +168,13 @@ public HttpResponse patch(String path, String body) { return executeRequest(uriRequest, CONTENT_TYPE_JSON); } + public HttpResponse assignRoleToUser(String username, String roleName) { + Objects.requireNonNull(roleName, "Role name is required"); + Objects.requireNonNull(username, "User name is required"); + String body = String.format("[{\"op\":\"add\",\"path\":\"/opendistro_security_roles\",\"value\":[\"%s\"]}]", roleName); + return patch("_plugins/_security/api/internalusers/" + username, body); + } + public HttpResponse executeRequest(HttpUriRequest uriRequest, Header... requestSpecificHeaders) { try(CloseableHttpClient httpClient = getHTTPClient()) { diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentFieldValueMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentFieldValueMatcher.java index 54b29949c1..72418fe6e0 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentFieldValueMatcher.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentFieldValueMatcher.java @@ -31,6 +31,10 @@ public GetResponseDocumentFieldValueMatcher(String fieldName, Object fieldValue) @Override protected boolean matchesSafely(GetResponse response, Description mismatchDescription) { Map source = response.getSource(); + if(source == null) { + mismatchDescription.appendText("Source is not available in search results"); + return false; + } if(source.containsKey(fieldName) == false) { mismatchDescription.appendText("Document does not contain field ").appendValue(fieldName); return false; diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentIdMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentIdMatcher.java index 64a64f93dd..85519a7261 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentIdMatcher.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/GetResponseDocumentIdMatcher.java @@ -36,6 +36,10 @@ protected boolean matchesSafely(GetResponse response, Description mismatchDescri mismatchDescription.appendText("Document contain incorrect id which is ").appendValue(response.getId()); return false; } + if(response.isExists() == false) { + mismatchDescription.appendText("Document does not exist or is inaccessible"); + return false; + } return true; } diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java index 2a8e866901..aa9cfe6864 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitContainsFieldWithValueMatcher.java @@ -49,7 +49,7 @@ class SearchHitContainsFieldWithValueMatcher extends TypeSafeDiagnosingMatche mismatchDescription.appendText("Source document is null, is fetch source option set to true?"); return false; } - if(!source.containsKey(fieldName)) { + if(source.containsKey(fieldName) == false) { mismatchDescription.appendText("Document does not contain field ").appendValue(fieldName); return false; } diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitDoesNotContainFieldMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitDoesNotContainFieldMatcher.java new file mode 100644 index 0000000000..106f2ee943 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitDoesNotContainFieldMatcher.java @@ -0,0 +1,63 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Map; + +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHit; + +import static java.util.Objects.requireNonNull; +import static org.opensearch.test.framework.matcher.SearchResponseMatchers.readTotalHits; + +class SearchHitDoesNotContainFieldMatcher extends TypeSafeDiagnosingMatcher { + + private final int hitIndex; + + private final String fieldName; + + public SearchHitDoesNotContainFieldMatcher(int hitIndex, String fieldName) { + this.hitIndex = hitIndex; + this.fieldName = requireNonNull(fieldName, "Field name is required."); + } + + @Override + protected boolean matchesSafely(SearchResponse searchResponse, Description mismatchDescription) { + Long numberOfHits = readTotalHits(searchResponse); + if(numberOfHits == null) { + mismatchDescription.appendText("Total number of hits is unknown."); + return false; + } + if(hitIndex >= numberOfHits) { + mismatchDescription.appendText("Search result contain only ").appendValue(numberOfHits).appendText(" hits"); + return false; + } + SearchHit searchHit = searchResponse.getHits().getAt(hitIndex); + Map source = searchHit.getSourceAsMap(); + if(source == null){ + mismatchDescription.appendText("Source document is null, is fetch source option set to true?"); + return false; + } + if(source.containsKey(fieldName)) { + mismatchDescription.appendText(" document contains field ").appendValue(fieldName); + return false; + } + return true; + } + + @Override + public void describeTo(Description description) { + description.appendText("search hit with index ").appendValue(hitIndex).appendText(" does not contain field ") + .appendValue(fieldName); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitsContainDocumentsInAnyOrderMatcher.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitsContainDocumentsInAnyOrderMatcher.java new file mode 100644 index 0000000000..78fd20557e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchHitsContainDocumentsInAnyOrderMatcher.java @@ -0,0 +1,74 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +*/ +package org.opensearch.test.framework.matcher; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.tuple.Pair; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.search.SearchHit; +import org.opensearch.search.SearchHits; + +import static java.util.Objects.requireNonNull; + +class SearchHitsContainDocumentsInAnyOrderMatcher extends TypeSafeDiagnosingMatcher { + + /** + * Pair contain index name and document id + */ + private final List> documentIds; + + /** + * + * @param documentIds Pair contain index name and document id + */ + public SearchHitsContainDocumentsInAnyOrderMatcher(List> documentIds) { + this.documentIds = requireNonNull(documentIds, "Document ids are required."); + } + + @Override + protected boolean matchesSafely(SearchResponse response, Description mismatchDescription) { + SearchHits hits = response.getHits(); + if(hits == null) { + mismatchDescription.appendText("Search response does not contains hits (null)."); + return false; + } + SearchHit[] hitsArray = hits.getHits(); + if(hitsArray == null) { + mismatchDescription.appendText("Search hits array is null"); + return false; + } + Set> actualDocumentIds = Arrays.stream(hitsArray) + .map(result -> Pair.of(result.getIndex(), result.getId())) + .collect(Collectors.toSet()); + for(Pair desiredDocumentId : documentIds) { + if(actualDocumentIds.contains(desiredDocumentId) == false) { + mismatchDescription.appendText("search result does not contain document with id ") + .appendValue(desiredDocumentId.getKey()).appendText("/").appendValue(desiredDocumentId.getValue()); + return false; + } + } + return true; + } + + @Override + public void describeTo(Description description) { + String documentIdsString = documentIds.stream() + .map(pair -> pair.getKey() + "/" + pair.getValue()) + .collect(Collectors.joining(", ")); + description.appendText("Search response should contains following documents ").appendValue(documentIdsString); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseMatchers.java index efd7da8cf9..acb669a85e 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/SearchResponseMatchers.java @@ -9,8 +9,11 @@ */ package org.opensearch.test.framework.matcher; +import java.util.Arrays; +import java.util.List; import java.util.Optional; +import org.apache.commons.lang3.tuple.Pair; import org.hamcrest.Matcher; import org.opensearch.action.search.SearchResponse; @@ -37,6 +40,11 @@ public static Matcher searchHitContainsFieldWithValue(int hi return new SearchHitContainsFieldWithValueMatcher<>(hitIndex, fieldName, expectedValue); } + public static Matcher searchHitDoesNotContainField(int hitIndex, String fieldName) { + return new SearchHitDoesNotContainFieldMatcher(hitIndex, fieldName); + } + + public static Matcher searchHitsContainDocumentWithId(int hitIndex, String indexName, String documentId) { return new SearchHitsContainDocumentWithIdMatcher(hitIndex, indexName, documentId); } @@ -53,6 +61,20 @@ public static Matcher containAggregationWithNameAndType(String e return new ContainsAggregationWithNameAndTypeMatcher(expectedAggregationName, expectedAggregationType); } + /** + * Matcher checks if search result contains all expected documents + * + * @param documentIds Pair contain index name and document id + * @return matcher + */ + public static Matcher searchHitsContainDocumentsInAnyOrder(List> documentIds) { + return new SearchHitsContainDocumentsInAnyOrderMatcher(documentIds); + } + + public static Matcher searchHitsContainDocumentsInAnyOrder(Pair...documentIds) { + return new SearchHitsContainDocumentsInAnyOrderMatcher(Arrays.asList(documentIds)); + } + static Long readTotalHits(SearchResponse searchResponse) { return Optional.ofNullable(searchResponse) .map(SearchResponse::getHits)