Merge branch 'opensearch-project:2.x' into 2.x

YANG-DB · Oct 9, 2023 · d044caa · d044caa
2 parents aeccc83 + a877ea8
commit d044caa
Show file tree

Hide file tree

Showing 117 changed files with 9,166 additions and 224 deletions.
diff --git a/build.gradle b/build.gradle
@@ -6,7 +6,7 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "2.10.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.11.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')

diff --git a/common/build.gradle b/common/build.gradle
@@ -38,8 +38,8 @@ dependencies {
     api group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
     api group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3'
     implementation 'com.github.babbel:okhttp-aws-signer:1.0.2'
-    api group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.1'
-    api group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.1'
+    api group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.545'
+    api group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.545'
     implementation "com.github.seancfoley:ipaddress:5.4.0"
 
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'

diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java
@@ -35,7 +35,7 @@ public enum Key {
 
     METRICS_ROLLING_WINDOW("plugins.query.metrics.rolling_window"),
     METRICS_ROLLING_INTERVAL("plugins.query.metrics.rolling_interval"),
-
+    SPARK_EXECUTION_ENGINE_CONFIG("plugins.query.executionengine.spark.config"),
     CLUSTER_NAME("cluster.name");
 
     @Getter private final String keyValue;

diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java b/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java
@@ -39,6 +39,15 @@ public interface DataSourceService {
    */
   DataSourceMetadata getDataSourceMetadata(String name);
 
+  /**
+   * Returns dataSourceMetadata object with specific name. The returned objects contain all the
+   * metadata information without any filtering.
+   *
+   * @param name name of the {@link DataSource}.
+   * @return set of {@link DataSourceMetadata}.
+   */
+  DataSourceMetadata getRawDataSourceMetadata(String name);
+
   /**
    * Register {@link DataSource} defined by {@link DataSourceMetadata}.
    *

diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java
@@ -11,26 +11,29 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableMap;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
-import lombok.NoArgsConstructor;
 import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
 import org.opensearch.sql.datasource.DataSourceService;
 
 @Getter
 @Setter
 @AllArgsConstructor
-@NoArgsConstructor
 @EqualsAndHashCode
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class DataSourceMetadata {
 
   @JsonProperty private String name;
 
+  @JsonProperty private String description;
+
   @JsonProperty
   @JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES)
   private DataSourceType connector;
@@ -39,6 +42,28 @@ public class DataSourceMetadata {
 
   @JsonProperty private Map<String, String> properties;
 
+  @JsonProperty private String resultIndex;
+
+  public DataSourceMetadata(
+      String name,
+      DataSourceType connector,
+      List<String> allowedRoles,
+      Map<String, String> properties,
+      String resultIndex) {
+    this.name = name;
+    this.connector = connector;
+    this.description = StringUtils.EMPTY;
+    this.properties = properties;
+    this.allowedRoles = allowedRoles;
+    this.resultIndex = resultIndex;
+  }
+
+  public DataSourceMetadata() {
+    this.description = StringUtils.EMPTY;
+    this.allowedRoles = new ArrayList<>();
+    this.properties = new HashMap<>();
+  }
+
   /**
    * Default OpenSearch {@link DataSourceMetadata}. Which is used to register default OpenSearch
    * {@link DataSource} to {@link DataSourceService}.
@@ -48,6 +73,7 @@ public static DataSourceMetadata defaultOpenSearchDataSourceMetadata() {
         DEFAULT_DATASOURCE_NAME,
         DataSourceType.OPENSEARCH,
         Collections.emptyList(),
-        ImmutableMap.of());
+        ImmutableMap.of(),
+        null);
   }
 }
diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java
@@ -8,7 +8,9 @@
 public enum DataSourceType {
   PROMETHEUS("prometheus"),
   OPENSEARCH("opensearch"),
-  SPARK("spark");
+  SPARK("spark"),
+  S3GLUE("s3glue");
+
   private String text;
 
   DataSourceType(String text) {

diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java
@@ -45,7 +45,7 @@ public String explain() {
   @Override
   public void open() {
     List<ExprValue> exprValues = new ArrayList<>();
-    Set<DataSourceMetadata> dataSourceMetadataSet = dataSourceService.getDataSourceMetadata(true);
+    Set<DataSourceMetadata> dataSourceMetadataSet = dataSourceService.getDataSourceMetadata(false);
     for (DataSourceMetadata dataSourceMetadata : dataSourceMetadataSet) {
       exprValues.add(
           new ExprTupleValue(

diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java
@@ -199,7 +199,8 @@ public Set<DataSourceMetadata> getDataSourceMetadata(boolean isDefaultDataSource
                       ds.getName(),
                       ds.getConnectorType(),
                       Collections.emptyList(),
-                      ImmutableMap.of()))
+                      ImmutableMap.of(),
+                      null))
           .collect(Collectors.toSet());
     }
 
@@ -208,6 +209,11 @@ public DataSourceMetadata getDataSourceMetadata(String name) {
       return null;
     }
 
+    @Override
+    public DataSourceMetadata getRawDataSourceMetadata(String name) {
+      return null;
+    }
+
     @Override
     public void createDataSource(DataSourceMetadata metadata) {
       throw new UnsupportedOperationException("unsupported operation");

diff --git a/...src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java b/...src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java
@@ -64,9 +64,10 @@ void testIterator() {
                         dataSource.getName(),
                         dataSource.getConnectorType(),
                         Collections.emptyList(),
-                        ImmutableMap.of()))
+                        ImmutableMap.of(),
+                        null))
             .collect(Collectors.toSet());
-    when(dataSourceService.getDataSourceMetadata(true)).thenReturn(dataSourceMetadata);
+    when(dataSourceService.getDataSourceMetadata(false)).thenReturn(dataSourceMetadata);
 
     assertFalse(dataSourceTableScan.hasNext());
     dataSourceTableScan.open();

diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java
@@ -12,6 +12,7 @@
 import java.util.Map;
 
 public enum AuthenticationType {
+  NOAUTH("noauth"),
   BASICAUTH("basicauth"),
   AWSSIGV4AUTH("awssigv4");
 

diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactory.java b/datasources/src/main/java/org/opensearch/sql/datasources/glue/GlueDataSourceFactory.java
@@ -0,0 +1,79 @@
+package org.opensearch.sql.datasources.glue;
+
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.util.Map;
+import java.util.Set;
+import lombok.RequiredArgsConstructor;
+import org.opensearch.sql.common.setting.Settings;
+import org.opensearch.sql.datasource.model.DataSource;
+import org.opensearch.sql.datasource.model.DataSourceMetadata;
+import org.opensearch.sql.datasource.model.DataSourceType;
+import org.opensearch.sql.datasources.auth.AuthenticationType;
+import org.opensearch.sql.datasources.utils.DatasourceValidationUtils;
+import org.opensearch.sql.storage.DataSourceFactory;
+
+@RequiredArgsConstructor
+public class GlueDataSourceFactory implements DataSourceFactory {
+
+  private final Settings pluginSettings;
+
+  // Glue configuration properties
+  public static final String GLUE_AUTH_TYPE = "glue.auth.type";
+  public static final String GLUE_ROLE_ARN = "glue.auth.role_arn";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_URI = "glue.indexstore.opensearch.uri";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_AUTH = "glue.indexstore.opensearch.auth";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_AUTH_USERNAME =
+      "glue.indexstore.opensearch.auth.username";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_AUTH_PASSWORD =
+      "glue.indexstore.opensearch.auth.password";
+  public static final String GLUE_INDEX_STORE_OPENSEARCH_REGION =
+      "glue.indexstore.opensearch.region";
+
+  @Override
+  public DataSourceType getDataSourceType() {
+    return DataSourceType.S3GLUE;
+  }
+
+  @Override
+  public DataSource createDataSource(DataSourceMetadata metadata) {
+    try {
+      validateGlueDataSourceConfiguration(metadata.getProperties());
+      return new DataSource(
+          metadata.getName(),
+          metadata.getConnector(),
+          (dataSourceSchemaName, tableName) -> {
+            throw new UnsupportedOperationException("Glue storage engine is not supported.");
+          });
+    } catch (URISyntaxException | UnknownHostException e) {
+      throw new IllegalArgumentException("Invalid flint host in properties.");
+    }
+  }
+
+  private void validateGlueDataSourceConfiguration(Map<String, String> dataSourceMetadataConfig)
+      throws URISyntaxException, UnknownHostException {
+
+    DatasourceValidationUtils.validateLengthAndRequiredFields(
+        dataSourceMetadataConfig,
+        Set.of(
+            GLUE_AUTH_TYPE,
+            GLUE_ROLE_ARN,
+            GLUE_INDEX_STORE_OPENSEARCH_URI,
+            GLUE_INDEX_STORE_OPENSEARCH_AUTH));
+    AuthenticationType authenticationType =
+        AuthenticationType.get(dataSourceMetadataConfig.get(GLUE_INDEX_STORE_OPENSEARCH_AUTH));
+    if (AuthenticationType.BASICAUTH.equals(authenticationType)) {
+      DatasourceValidationUtils.validateLengthAndRequiredFields(
+          dataSourceMetadataConfig,
+          Set.of(
+              GLUE_INDEX_STORE_OPENSEARCH_AUTH_USERNAME,
+              GLUE_INDEX_STORE_OPENSEARCH_AUTH_PASSWORD));
+    } else if (AuthenticationType.AWSSIGV4AUTH.equals(authenticationType)) {
+      DatasourceValidationUtils.validateLengthAndRequiredFields(
+          dataSourceMetadataConfig, Set.of(GLUE_INDEX_STORE_OPENSEARCH_REGION));
+    }
+    DatasourceValidationUtils.validateHost(
+        dataSourceMetadataConfig.get(GLUE_INDEX_STORE_OPENSEARCH_URI),
+        pluginSettings.getSettingValue(Settings.Key.DATASOURCES_URI_HOSTS_DENY_LIST));
+  }
+}
diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java
@@ -88,8 +88,7 @@ public List<Route> routes() {
         new Route(GET, BASE_DATASOURCE_ACTION_URL),
 
         /*
-         * GET datasources
-         * Request URL: GET
+         * PUT datasources
          * Request body:
          * Ref
          * [org.opensearch.sql.plugin.transport.datasource.model.UpdateDataSourceActionRequest]
@@ -100,8 +99,7 @@ public List<Route> routes() {
         new Route(PUT, BASE_DATASOURCE_ACTION_URL),
 
         /*
-         * GET datasources
-         * Request URL: GET
+         * DELETE datasources
          * Request body: Ref
          * [org.opensearch.sql.plugin.transport.datasource.model.DeleteDataSourceActionRequest]
          * Response body: Ref

diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java
@@ -64,29 +64,17 @@ public Set<DataSourceMetadata> getDataSourceMetadata(boolean isDefaultDataSource
   }
 
   @Override
-  public DataSourceMetadata getDataSourceMetadata(String datasourceName) {
-    Optional<DataSourceMetadata> dataSourceMetadataOptional =
-        getDataSourceMetadataFromName(datasourceName);
-    if (dataSourceMetadataOptional.isEmpty()) {
-      throw new IllegalArgumentException(
-          "DataSource with name: " + datasourceName + " doesn't exist.");
-    }
-    removeAuthInfo(dataSourceMetadataOptional.get());
-    return dataSourceMetadataOptional.get();
+  public DataSourceMetadata getDataSourceMetadata(String dataSourceName) {
+    DataSourceMetadata dataSourceMetadata = getRawDataSourceMetadata(dataSourceName);
+    removeAuthInfo(dataSourceMetadata);
+    return dataSourceMetadata;
   }
 
   @Override
   public DataSource getDataSource(String dataSourceName) {
-    Optional<DataSourceMetadata> dataSourceMetadataOptional =
-        getDataSourceMetadataFromName(dataSourceName);
-    if (dataSourceMetadataOptional.isEmpty()) {
-      throw new DataSourceNotFoundException(
-          String.format("DataSource with name %s doesn't exist.", dataSourceName));
-    } else {
-      DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get();
-      this.dataSourceUserAuthorizationHelper.authorizeDataSource(dataSourceMetadata);
-      return dataSourceLoaderCache.getOrLoadDataSource(dataSourceMetadata);
-    }
+    DataSourceMetadata dataSourceMetadata = getRawDataSourceMetadata(dataSourceName);
+    this.dataSourceUserAuthorizationHelper.authorizeDataSource(dataSourceMetadata);
+    return dataSourceLoaderCache.getOrLoadDataSource(dataSourceMetadata);
   }
 
   @Override
@@ -146,11 +134,20 @@ private void validateDataSourceMetaData(DataSourceMetadata metadata) {
             + " Properties are required parameters.");
   }
 
-  private Optional<DataSourceMetadata> getDataSourceMetadataFromName(String dataSourceName) {
+  @Override
+  public DataSourceMetadata getRawDataSourceMetadata(String dataSourceName) {
     if (dataSourceName.equals(DEFAULT_DATASOURCE_NAME)) {
-      return Optional.of(DataSourceMetadata.defaultOpenSearchDataSourceMetadata());
+      return DataSourceMetadata.defaultOpenSearchDataSourceMetadata();
+
     } else {
-      return this.dataSourceMetadataStorage.getDataSourceMetadata(dataSourceName);
+      Optional<DataSourceMetadata> dataSourceMetadataOptional =
+          this.dataSourceMetadataStorage.getDataSourceMetadata(dataSourceName);
+      if (dataSourceMetadataOptional.isEmpty()) {
+        throw new DataSourceNotFoundException(
+            String.format("DataSource with name %s doesn't exist.", dataSourceName));
+      } else {
+        return dataSourceMetadataOptional.get();
+      }
     }
   }
 

diff --git a/...main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java b/...main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java
@@ -43,7 +43,6 @@
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.sql.datasource.model.DataSourceMetadata;
-import org.opensearch.sql.datasources.auth.AuthenticationType;
 import org.opensearch.sql.datasources.encryptor.Encryptor;
 import org.opensearch.sql.datasources.exceptions.DataSourceNotFoundException;
 import org.opensearch.sql.datasources.service.DataSourceMetadataStorage;
@@ -93,7 +92,9 @@ public Optional<DataSourceMetadata> getDataSourceMetadata(String datasourceName)
       createDataSourcesIndex();
       return Optional.empty();
     }
-    return searchInDataSourcesIndex(QueryBuilders.termQuery("name", datasourceName)).stream()
+    // todo, in case docId == datasourceName, could read doc directly.
+    return searchInDataSourcesIndex(QueryBuilders.termQuery("name.keyword", datasourceName))
+        .stream()
         .findFirst()
         .map(x -> this.encryptDecryptAuthenticationData(x, false));
   }
@@ -252,26 +253,13 @@ private List<DataSourceMetadata> searchInDataSourcesIndex(QueryBuilder query) {
     }
   }
 
-  @SuppressWarnings("missingswitchdefault")
+  // Encrypt and Decrypt irrespective of auth type.If properties name ends in username, password,
+  // secret_key and access_key.
   private DataSourceMetadata encryptDecryptAuthenticationData(
       DataSourceMetadata dataSourceMetadata, Boolean isEncryption) {
     Map<String, String> propertiesMap = dataSourceMetadata.getProperties();
-    Optional<AuthenticationType> authTypeOptional =
-        propertiesMap.keySet().stream()
-            .filter(s -> s.endsWith("auth.type"))
-            .findFirst()
-            .map(propertiesMap::get)
-            .map(AuthenticationType::get);
-    if (authTypeOptional.isPresent()) {
-      switch (authTypeOptional.get()) {
-        case BASICAUTH:
-          handleBasicAuthPropertiesEncryptionDecryption(propertiesMap, isEncryption);
-          break;
-        case AWSSIGV4AUTH:
-          handleSigV4PropertiesEncryptionDecryption(propertiesMap, isEncryption);
-          break;
-      }
-    }
+    handleBasicAuthPropertiesEncryptionDecryption(propertiesMap, isEncryption);
+    handleSigV4PropertiesEncryptionDecryption(propertiesMap, isEncryption);
     return dataSourceMetadata;
   }