Wide range of integration tests.

build.gradle

@@ -430,6 +430,9 @@ dependencies {
     integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
     integrationTestImplementation "org.bouncycastle:bcpkix-jdk15on:${versions.bouncycastle}"
     integrationTestImplementation "org.bouncycastle:bcutil-jdk15on:${versions.bouncycastle}"
+    integrationTestImplementation('org.awaitility:awaitility:4.2.0') {
+        exclude(group: 'org.hamcrest', module: 'hamcrest')
+    }
 }
 
 jar {

src/integrationTest/java/org/opensearch/security/SnapshotSteps.java

@@ -0,0 +1,87 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.security;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.awaitility.Awaitility;
+
+import org.opensearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
+import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
+import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
+import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
+import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
+import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
+import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.client.SnapshotClient;
+import org.opensearch.snapshots.SnapshotInfo;
+import org.opensearch.snapshots.SnapshotState;
+
+import static java.util.Objects.requireNonNull;
+import static org.opensearch.client.RequestOptions.DEFAULT;
+
+class SnapshotSteps {
+
+	private final SnapshotClient snapshotClient;
+
+	public SnapshotSteps(RestHighLevelClient restHighLevelClient) {
+		this.snapshotClient = requireNonNull(restHighLevelClient, "Rest high level client is required.").snapshot();
+	}
+
+	// CS-SUPPRESS-SINGLE: RegexpSingleline It is not possible to use phrase "cluster manager" instead of master here
+	public org.opensearch.action.support.master.AcknowledgedResponse createSnapshotRepository(String repositoryName, String snapshotDirPath, String type)
+	//CS-ENFORCE-SINGLE
+		throws IOException {
+		PutRepositoryRequest createRepositoryRequest = new PutRepositoryRequest().name(repositoryName).type(type)
+			.settings(Map.of("location", snapshotDirPath));
+		return snapshotClient.createRepository(createRepositoryRequest, DEFAULT);
+	}
+
+	public CreateSnapshotResponse createSnapshot(String repositoryName, String snapshotName, String...indices) throws IOException {
+		CreateSnapshotRequest createSnapshotRequest = new CreateSnapshotRequest(repositoryName, snapshotName)
+			.indices(indices);
+		return snapshotClient.create(createSnapshotRequest, DEFAULT);
+	}
+
+	public void waitForSnapshotCreation(String repositoryName, String snapshotName) {
+		GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest(repositoryName, new String[] { snapshotName });
+		Awaitility.await().until(() -> {
+			GetSnapshotsResponse snapshotsResponse = snapshotClient.get(getSnapshotsRequest, DEFAULT);
+			SnapshotInfo snapshotInfo = snapshotsResponse.getSnapshots().get(0);
+			return SnapshotState.SUCCESS.equals(snapshotInfo.state());
+		});
+	}
+
+	//CS-SUPPRESS-SINGLE: RegexpSingleline It is not possible to use phrase "cluster manager" instead of master here
+	public org.opensearch.action.support.master.AcknowledgedResponse deleteSnapshotRepository(String repositoryName) throws IOException {
+	//CS-ENFORCE-SINGLE
+		DeleteRepositoryRequest request = new DeleteRepositoryRequest(repositoryName);
+		return snapshotClient.deleteRepository(request, DEFAULT);
+	}
+
+	//CS-SUPPRESS-SINGLE: RegexpSingleline: It is not possible to use phrase "cluster manager" instead of master here
+	public org.opensearch.action.support.master.AcknowledgedResponse deleteSnapshot(String repositoryName, String snapshotName) throws IOException {
+	//CS-ENFORCE-SINGLE
+		return snapshotClient.delete(new DeleteSnapshotRequest(repositoryName, snapshotName), DEFAULT);
+	}
+
+	public RestoreSnapshotResponse restoreSnapshot(
+		String repositoryName, String snapshotName, String renamePattern,
+		String renameReplacement) throws IOException {
+		RestoreSnapshotRequest restoreSnapshotRequest = new RestoreSnapshotRequest(repositoryName, snapshotName)
+			.renamePattern(renamePattern)
+			.renameReplacement(renameReplacement);
+		return snapshotClient.restore(restoreSnapshotRequest, DEFAULT);
+	}
+}

src/integrationTest/java/org/opensearch/security/Song.java

@@ -0,0 +1,44 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.security;
+
+
+class Song {
+
+	static final String FIELD_TITLE = "title";
+	static final String FIELD_ARTIST = "artist";
+	static final String FIELD_LYRICS = "lyrics";
+
+	static final String FIELD_STARS = "stars";
+	static final String ARTIST_FIRST = "First artist";
+	static final String ARTIST_STRING = "String";
+	static final String ARTIST_TWINS = "Twins";
+	static final String TITLE_MAGNUM_OPUS = "Magnum Opus";
+	static final String TITLE_SONG_1_PLUS_1 = "Song 1+1";
+	static final String TITLE_NEXT_SONG = "Next song";
+	static final String ARTIST_NO = "No!";
+	static final String TITLE_POISON = "Poison";
+
+	public static final String LYRICS_1 = "Very deep subject";
+	public static final String LYRICS_2 = "Once upon a time";
+	public static final String LYRICS_3 = "giant nonsense";
+	public static final String LYRICS_4 = "Much too much";
+
+	static final String QUERY_TITLE_NEXT_SONG = FIELD_TITLE + ":" + "\"" + TITLE_NEXT_SONG + "\"";
+	static final String QUERY_TITLE_POISON = FIELD_TITLE + ":" + TITLE_POISON;
+	static final String QUERY_TITLE_MAGNUM_OPUS = FIELD_TITLE + ":" + TITLE_MAGNUM_OPUS;
+
+	static final Object[][] SONGS = {
+		{FIELD_ARTIST, ARTIST_FIRST, FIELD_TITLE, TITLE_MAGNUM_OPUS ,FIELD_LYRICS, LYRICS_1, FIELD_STARS, 1},
+		{FIELD_ARTIST, ARTIST_STRING, FIELD_TITLE, TITLE_SONG_1_PLUS_1, FIELD_LYRICS, LYRICS_2, FIELD_STARS, 2},
+		{FIELD_ARTIST, ARTIST_TWINS, FIELD_TITLE, TITLE_NEXT_SONG, FIELD_LYRICS, LYRICS_3, FIELD_STARS, 3},
+		{FIELD_ARTIST, ARTIST_NO, FIELD_TITLE, TITLE_POISON, FIELD_LYRICS, LYRICS_4, FIELD_STARS, 4}
+	};
+}

src/integrationTest/java/org/opensearch/security/http/BasicAuthTests.java

@@ -46,7 +46,7 @@ public class BasicAuthTests {
 	public static final String INVALID_PASSWORD = "secret-password";
 
 	public static final AuthcDomain AUTHC_DOMAIN = new AuthcDomain("basic", 0)
-		.challengingAuthenticator("basic").backend("internal");
+		.httpAuthenticatorWithChallenge("basic").backend("internal");
 
 	@ClassRule
 	public static final LocalCluster cluster = new LocalCluster.Builder()
@@ -95,7 +95,7 @@ public void shouldRespondWith200WhenCredentialsAreCorrect() {
 	}
 
 	@Test
-	public void browserShouldRequestUserForCredentials() {
+	public void testBrowserShouldRequestForCredentials() {
 		try (TestRestClient client = cluster.getRestClient()) {
 
 			HttpResponse response = client.getAuthInfo();
@@ -122,7 +122,7 @@ public void testUserShouldNotHaveAssignedCustomAttributes() {
 	}
 
 	@Test
-	public void userShouldHaveAssignedCustomAttributes() {
+	public void testUserShouldHaveAssignedCustomAttributes() {
 		try (TestRestClient client = cluster.getRestClient(SUPER_USER)) {
 
 			HttpResponse response = client.getAuthInfo();

src/integrationTest/java/org/opensearch/security/http/BasicAuthWithoutChallengeTests.java

@@ -45,7 +45,7 @@ public void browserShouldNotRequestUserForCredentials() {
 	}
 
 	private void assertThatBrowserDoesNotAskUserForCredentials(HttpResponse response) {
-		String reason = "Browser ask user for credentials what is not expected";
+		String reason = "Browser asked user for credentials which is not expected";
 		assertThat(reason, response.containHeader(HttpHeaders.WWW_AUTHENTICATE), equalTo(false));
 	}
 }

src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java

@@ -340,7 +340,7 @@ public static class AuthcDomain implements ToXContentObject {
 		private static String PUBLIC_KEY = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqZbjLUAWc+DZTkinQAdvy1GFjPHPnxheU89hSiWoDD3NOW76H3u3T7cCDdOah2msdxSlBmCBH6wik8qLYkcV8owWukQg3PQmbEhrdPaKo0QCgomWs4nLgtmEYqcZ+QQldd82MdTlQ1QmoQmI9Uxqs1SuaKZASp3Gy19y8su5CV+FZ6BruUw9HELK055sAwl3X7j5ouabXGbcib2goBF3P52LkvbJLuWr5HDZEOeSkwIeqSeMojASM96K5SdotD+HwEyjaTjzRPL2Aa1BEQFWOQ6CFJLyLH7ZStDuPM1mJU1VxIVfMbZrhsUBjAnIhRynmWxML7YlNqkP9j6jyOIYQIDAQAB";
 
 		public final static AuthcDomain AUTHC_HTTPBASIC_INTERNAL = new TestSecurityConfig.AuthcDomain("basic", 0)
-				.challengingAuthenticator("basic").backend("internal");
+				.httpAuthenticatorWithChallenge("basic").backend("internal");
 
 		public final static AuthcDomain AUTHC_HTTPBASIC_INTERNAL_WITHOUT_CHALLENGE = new TestSecurityConfig.AuthcDomain("basic", 0)
 			.httpAuthenticator("basic").backend("internal");
@@ -380,7 +380,7 @@ public AuthcDomain jwtHttpAuthenticator(String headerName, String signingKey) {
 			return this;
 		}
 
-		public AuthcDomain challengingAuthenticator(String type) {
+		public AuthcDomain httpAuthenticatorWithChallenge(String type) {
 			this.httpAuthenticator = new HttpAuthenticator(type).challenge(true);
 			return this;
 		}

src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java

@@ -98,6 +98,10 @@ private LocalCluster(String clusterName, TestSecurityConfig testSgConfig, Settin
 		this.testIndices = testIndices;
 	}
 
+	public String getSnapshotDirPath() {
+		return localOpenSearchCluster.getSnapshotDirPath();
+	}
+
 	@Override
 	public void before() throws Throwable {
 		if (localOpenSearchCluster == null) {

src/integrationTest/java/org/opensearch/test/framework/cluster/LocalOpenSearchCluster.java

@@ -102,6 +102,8 @@ public class LocalOpenSearchCluster {
 	private boolean started;
 	private Random random = new Random();
 
+	private File snapshotDir;
+
 	public LocalOpenSearchCluster(String clusterName, ClusterManager clusterManager, NodeSettingsSupplier nodeSettingsSupplier,
 						List<Class<? extends Plugin>> additionalPlugins, TestCertificates testCertificates) {
 		this.clusterName = clusterName;
@@ -110,12 +112,23 @@ public LocalOpenSearchCluster(String clusterName, ClusterManager clusterManager,
 		this.additionalPlugins = additionalPlugins;
 		this.testCertificates = testCertificates;
 		try {
-			this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName).toFile();
+			createClusterDirectory(clusterName);
 		} catch (IOException e) {
 			throw new IllegalStateException(e);
 		}        
 	}
 
+	public String getSnapshotDirPath() {
+		return snapshotDir.getAbsolutePath();
+	}
+
+	private void createClusterDirectory(String clusterName) throws IOException {
+		this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName).toFile();
+		log.debug("Cluster home directory '{}'.", clusterHomeDir.getAbsolutePath());
+		this.snapshotDir = new File(this.clusterHomeDir, "snapshots");
+		this.snapshotDir.mkdir();
+	}
+
 	private List<Node> getNodesByType(NodeType nodeType) {
 		return nodes.stream()
 			.filter(currentNode -> currentNode.hasAssignedType(nodeType))
@@ -230,8 +243,7 @@ private void retry() throws Exception {
 		this.nodes.clear();
 		this.seedHosts = null;
 		this.initialClusterManagerHosts = null;
-		this.clusterHomeDir = Files.createTempDirectory("local_cluster_" + clusterName + "_retry_" + retry).toFile();
-
+		createClusterDirectory("local_cluster_" + clusterName + "_retry_" + retry);
 		start();
 	}
 
@@ -458,13 +470,15 @@ public InetSocketAddress getTransportAddress() {
 		}
 
 		private Settings getOpenSearchSettings() {
-			Settings settings = getMinimalOpenSearchSettings();
+			Settings settings = Settings.builder()
+				.put(getMinimalOpenSearchSettings())
+				.putList("path.repo", List.of(getSnapshotDirPath()))
+				.build();
 
 			if (nodeSettingsSupplier != null) {
 				// TODO node number
 				return Settings.builder().put(settings).put(nodeSettingsSupplier.get(0)).build();
 			}
-
 			return settings;
 		}
 

src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java

@@ -44,8 +44,18 @@
 import javax.net.ssl.TrustManagerFactory;
 
 import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 
+import org.opensearch.client.RestClient;
+import org.opensearch.client.RestClientBuilder;
+import org.opensearch.client.RestHighLevelClient;
 import org.opensearch.security.support.PemKeyReader;
 import org.opensearch.test.framework.certificate.TestCertificates;
 
@@ -82,6 +92,25 @@ default TestRestClient getRestClient(UserCredentialsHolder user, Header... heade
 		return getRestClient(user.getName(), user.getPassword(), headers);
 	}
 
+	default RestHighLevelClient getRestHighLevelClient(UserCredentialsHolder user) {
+		InetSocketAddress httpAddress = getHttpAddress();
+		CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+		credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user.getName(), user.getPassword()));
+
+		RestClientBuilder.HttpClientConfigCallback configCallback = httpClientBuilder -> {
+			httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setSSLStrategy(
+				new SSLIOSessionStrategy(getSSLContext(), null, null, NoopHostnameVerifier.INSTANCE));
+
+			return httpClientBuilder;
+		};
+
+		RestClientBuilder builder = RestClient.builder(new HttpHost(httpAddress.getHostString(), httpAddress.getPort(), "https"))
+			.setHttpClientConfigCallback(configCallback);
+
+
+		return new RestHighLevelClient(builder);
+	}
+
 	/**
 	* Returns a REST client that sends requests with basic authentication for the specified user name and password. Optionally,
 	* additional HTTP headers can be specified which will be sent with each request.

src/integrationTest/java/org/opensearch/test/framework/cluster/SearchRequestFactory.java

@@ -0,0 +1,79 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*
+*/
+package org.opensearch.test.framework.cluster;
+
+import org.opensearch.action.search.SearchRequest;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.action.search.SearchScrollRequest;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.search.aggregations.AggregationBuilders;
+import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.sort.FieldSortBuilder;
+import org.opensearch.search.sort.SortOrder;
+
+import static java.util.concurrent.TimeUnit.MINUTES;
+
+public final class SearchRequestFactory {
+
+	private SearchRequestFactory() {
+
+	}
+	public static SearchRequest queryStringQueryRequest(String indexName, String queryString) {
+		SearchRequest searchRequest = new SearchRequest(indexName);
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.query(QueryBuilders.queryStringQuery(queryString));
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+
+	public static SearchRequest queryStringQueryRequest(String queryString) {
+		SearchRequest searchRequest = new SearchRequest();
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.query(QueryBuilders.queryStringQuery(queryString));
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+
+	public static SearchRequest searchRequestWithScroll(String indexName, int pageSize) {
+		SearchRequest searchRequest = new SearchRequest(indexName);
+		searchRequest.scroll(new TimeValue(1, MINUTES));
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.query(QueryBuilders.matchAllQuery());
+		searchSourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC));
+		searchSourceBuilder.size(pageSize);
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+
+	public static SearchScrollRequest getSearchScrollRequest(SearchResponse searchResponse) {
+		SearchScrollRequest scrollRequest = new SearchScrollRequest(searchResponse.getScrollId());
+		scrollRequest.scroll(new TimeValue(1, MINUTES));
+		return scrollRequest;
+	}
+
+	public static SearchRequest averageAggregationRequest(String indexName, String aggregationName, String fieldName) {
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.aggregation(AggregationBuilders.avg(aggregationName).field(fieldName));
+		searchSourceBuilder.size(0);
+		SearchRequest searchRequest = new SearchRequest(indexName);
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+
+	public static SearchRequest statsAggregationRequest(String indexName, String aggregationName, String fieldName) {
+		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+		searchSourceBuilder.aggregation(AggregationBuilders.stats(aggregationName).field(fieldName));
+		searchSourceBuilder.size(0);
+		SearchRequest searchRequest = new SearchRequest(indexName);
+		searchRequest.source(searchSourceBuilder);
+		return searchRequest;
+	}
+}