Moved auth interceptors to common module

Signed-off-by: Vamsi Manohar <[email protected]>

build.gradle

@@ -76,6 +76,7 @@ repositories {
     mavenLocal()
     maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
     mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available
+    maven { url 'https://jitpack.io' }
 }
 
 allprojects {
@@ -98,8 +99,8 @@ subprojects {
         mavenLocal()
         maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
         mavenCentral()
-        // todo. remove this when lucene 9.4.0 is released
         maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" }
+        maven { url 'https://jitpack.io' }
     }
 }
 

common/build.gradle

@@ -36,9 +36,17 @@ dependencies {
     api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
     api group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1'
     api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
+    api group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3'
+    implementation 'com.github.babbel:okhttp-aws-signer:1.0.2'
+    api group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.1'
+    api group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.1'
 
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.9.1'
     testImplementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre'
     testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1'
+    testImplementation('org.junit.jupiter:junit-jupiter:5.6.2')
+    testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4'
+    testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4'
+    testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: '4.9.3'
 }

prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptor.java → common/src/main/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptor.java

@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.prometheus.authinterceptors;
+package org.opensearch.sql.common.authinterceptors;
 
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;

prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptor.java → common/src/main/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptor.java

@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.prometheus.authinterceptors;
+package org.opensearch.sql.common.authinterceptors;
 
 import java.io.IOException;
 import lombok.NonNull;

common/src/main/java/org/opensearch/sql/common/setting/Settings.java

@@ -39,8 +39,13 @@ public enum Key {
      */
     QUERY_MEMORY_LIMIT("plugins.query.memory_limit"),
     QUERY_SIZE_LIMIT("plugins.query.size_limit"),
+    ENCYRPTION_MASTER_KEY("plugins.query.datasources.encryption.masterkey"),
+    DATASOURCES_URI_ALLOWHOSTS("plugins.query.datasources.uri.allowhosts"),
+
     METRICS_ROLLING_WINDOW("plugins.query.metrics.rolling_window"),
-    METRICS_ROLLING_INTERVAL("plugins.query.metrics.rolling_interval");
+    METRICS_ROLLING_INTERVAL("plugins.query.metrics.rolling_interval"),
+
+    CLUSTER_NAME("cluster.name");
 
     @Getter
     private final String keyValue;

prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptorTest.java → common/src/test/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptorTest.java

@@ -5,7 +5,7 @@
  *
  */
 
-package org.opensearch.sql.prometheus.authinterceptors;
+package org.opensearch.sql.common.authinterceptors;
 
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -15,7 +15,6 @@
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
-import com.amazonaws.auth.STSSessionCredentialsProvider;
 import lombok.SneakyThrows;
 import okhttp3.Interceptor;
 import okhttp3.Request;
@@ -25,7 +24,9 @@
 import org.mockito.ArgumentCaptor;
 import org.mockito.Captor;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.opensearch.sql.common.authinterceptors.AwsSigningInterceptor;
 
 @ExtendWith(MockitoExtension.class)
 public class AwsSigningInterceptorTest {
@@ -54,15 +55,15 @@ void testConstructors() {
   @Test
   @SneakyThrows
   void testIntercept() {
-    when(chain.request()).thenReturn(new Request.Builder()
+    Mockito.when(chain.request()).thenReturn(new Request.Builder()
         .url("http://localhost:9090")
         .build());
     AwsSigningInterceptor awsSigningInterceptor
         = new AwsSigningInterceptor(
             getStaticAWSCredentialsProvider("testAccessKey", "testSecretKey"),
             "us-east-1", "aps");
     awsSigningInterceptor.intercept(chain);
-    verify(chain).proceed(requestArgumentCaptor.capture());
+    Mockito.verify(chain).proceed(requestArgumentCaptor.capture());
     Request request = requestArgumentCaptor.getValue();
     Assertions.assertNotNull(request.headers("Authorization"));
     Assertions.assertNotNull(request.headers("x-amz-date"));
@@ -73,16 +74,16 @@ void testIntercept() {
   @Test
   @SneakyThrows
   void testSTSCredentialsProviderInterceptor() {
-    when(chain.request()).thenReturn(new Request.Builder()
+    Mockito.when(chain.request()).thenReturn(new Request.Builder()
         .url("http://localhost:9090")
         .build());
-    when(stsAssumeRoleSessionCredentialsProvider.getCredentials())
+    Mockito.when(stsAssumeRoleSessionCredentialsProvider.getCredentials())
         .thenReturn(getAWSSessionCredentials());
     AwsSigningInterceptor awsSigningInterceptor
         = new AwsSigningInterceptor(stsAssumeRoleSessionCredentialsProvider,
             "us-east-1", "aps");
     awsSigningInterceptor.intercept(chain);
-    verify(chain).proceed(requestArgumentCaptor.capture());
+    Mockito.verify(chain).proceed(requestArgumentCaptor.capture());
     Request request = requestArgumentCaptor.getValue();
     Assertions.assertNotNull(request.headers("Authorization"));
     Assertions.assertNotNull(request.headers("x-amz-date"));

prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptorTest.java → common/src/test/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptorTest.java

@@ -5,10 +5,7 @@
  *
  */
 
-package org.opensearch.sql.prometheus.authinterceptors;
-
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
+package org.opensearch.sql.common.authinterceptors;
 
 import java.util.Collections;
 import lombok.SneakyThrows;
@@ -21,7 +18,7 @@
 import org.mockito.ArgumentCaptor;
 import org.mockito.Captor;
 import org.mockito.Mock;
-import org.mockito.internal.matchers.Null;
+import org.mockito.Mockito;
 import org.mockito.junit.jupiter.MockitoExtension;
 
 @ExtendWith(MockitoExtension.class)
@@ -45,13 +42,13 @@ void testConstructors() {
   @Test
   @SneakyThrows
   void testIntercept() {
-    when(chain.request()).thenReturn(new Request.Builder()
+    Mockito.when(chain.request()).thenReturn(new Request.Builder()
             .url("http://localhost:9090")
             .build());
     BasicAuthenticationInterceptor basicAuthenticationInterceptor
         = new BasicAuthenticationInterceptor("testAdmin", "testPassword");
     basicAuthenticationInterceptor.intercept(chain);
-    verify(chain).proceed(requestArgumentCaptor.capture());
+    Mockito.verify(chain).proceed(requestArgumentCaptor.capture());
     Request request = requestArgumentCaptor.getValue();
     Assertions.assertEquals(
         Collections.singletonList(Credentials.basic("testAdmin", "testPassword")),

docs/user/ppl/admin/datasources.rst

@@ -136,6 +136,17 @@ Master Key config for encrypting credential information
     # Print the master key
     print("Generated master key:", master_key)
 
+Datasource Allow Hosts Config
+========================================================
+* In the OpenSearch configuration file (opensearch.yml), the parameter "plugins.query.datasources.uri.allowhosts" can be utilized to control the permitted hosts within the datasource URI configuration.
+* By default, the value is set to `.*`, which allows any domain to be accepted.
+* For instance, if you set the value to `dummy.*.com`, the following URIs are some examples that would be allowed in the datasource configuration:
+   - https://dummy.prometheus.com:9080
+   - http://dummy.prometheus.com
+
+Note: The mentioned URIs are just examples to illustrate the concept.
+
+
 Using a datasource in PPL command
 ====================================
 Datasource is referred in source command as show in the code block below.

integ-test/src/test/java/org/opensearch/sql/datasource/DataSourceAPIsIT.java

@@ -22,6 +22,7 @@
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
+import org.opensearch.action.update.UpdateRequest;
 import org.opensearch.client.Request;
 import org.opensearch.client.RequestOptions;
 import org.opensearch.client.Response;
@@ -89,7 +90,7 @@ public void updateDataSourceAPITest() {
     //update datasource
     DataSourceMetadata updateDSM =
         new DataSourceMetadata("update_prometheus", DataSourceType.PROMETHEUS,
-            ImmutableList.of(), ImmutableMap.of("prometheus.uri", "https://randomtest:9090"));
+            ImmutableList.of(), ImmutableMap.of("prometheus.uri", "https://randomtest.com:9090"));
     Request updateRequest = getUpdateDataSourceRequest(updateDSM);
     Response updateResponse = client().performRequest(updateRequest);
     Assert.assertEquals(200, updateResponse.getStatusLine().getStatusCode());
@@ -99,6 +100,22 @@ public void updateDataSourceAPITest() {
     //Datasource is not immediately updated. so introducing a sleep of 2s.
     Thread.sleep(2000);
 
+    //update datasource with invalid URI
+    updateDSM =
+        new DataSourceMetadata("update_prometheus", DataSourceType.PROMETHEUS,
+            ImmutableList.of(), ImmutableMap.of("prometheus.uri", "https://randomtest:9090"));
+    final Request illFormedUpdateRequest
+        = getUpdateDataSourceRequest(updateDSM);
+    ResponseException updateResponseException
+        = Assert.assertThrows(ResponseException.class, () -> client().performRequest(illFormedUpdateRequest));
+    Assert.assertEquals(400, updateResponseException.getResponse().getStatusLine().getStatusCode());
+    updateResponseString = getResponseBody(updateResponseException.getResponse());
+    JsonObject errorMessage = new Gson().fromJson(updateResponseString, JsonObject.class);
+    Assert.assertEquals("Invalid hostname in the uri: https://randomtest:9090",
+        errorMessage.get("error").getAsJsonObject().get("details").getAsString());
+
+    Thread.sleep(2000);
+
     //get datasource to validate the modification.
     //get datasource
     Request getRequest = getFetchDataSourceRequest("update_prometheus");
@@ -107,7 +124,7 @@ public void updateDataSourceAPITest() {
     String getResponseString = getResponseBody(getResponse);
     DataSourceMetadata dataSourceMetadata =
         new Gson().fromJson(getResponseString, DataSourceMetadata.class);
-    Assert.assertEquals("https://randomtest:9090",
+    Assert.assertEquals("https://randomtest.com:9090",
         dataSourceMetadata.getProperties().get("prometheus.uri"));
   }
 

legacy/src/test/java/org/opensearch/sql/legacy/executor/AsyncRestExecutorTest.java

@@ -25,6 +25,7 @@
 import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnitRunner;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.ClusterName;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.rest.RestChannel;
 import org.opensearch.sql.legacy.esdomain.LocalClusterState;
@@ -62,6 +63,7 @@ public class AsyncRestExecutorTest {
     public void setUp() {
         when(client.threadPool()).thenReturn(mock(ThreadPool.class));
         when(action.getSqlRequest()).thenReturn(SqlRequest.NULL);
+        when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT);
 
         OpenSearchSettings settings = spy(new OpenSearchSettings(clusterSettings));
         doReturn(emptyList()).when(settings).getSettings();

legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java

@@ -26,6 +26,7 @@
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterStateListener;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.ClusterSettings;
@@ -181,6 +182,7 @@ public void getMappingFromCache() throws IOException {
 
     @Test
     public void getDefaultValueForQuerySlowLog() {
+        when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT);
         OpenSearchSettings settings = new OpenSearchSettings(clusterSettings);
         assertEquals(Integer.valueOf(2), settings.getSettingValue(Settings.Key.SQL_SLOWLOG));
     }

legacy/src/test/java/org/opensearch/sql/legacy/unittest/planner/QueryPlannerTest.java

@@ -38,6 +38,7 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.SearchScrollRequestBuilder;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.ClusterName;
 import org.opensearch.common.bytes.BytesArray;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
@@ -105,8 +106,9 @@ public static void initLogger() {
     @Before
     public void init() {
         MockitoAnnotations.initMocks(this);
-
+        when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT);
         OpenSearchSettings settings = spy(new OpenSearchSettings(clusterSettings));
+
         // Force return empty list to avoid ClusterSettings be invoked which is a final class and hard to mock.
         // In this case, default value in Setting will be returned all the time.
         doReturn(emptyList()).when(settings).getSettings();