[main]Changes in DataSourceService and DataSourceMetadataStorage inte…

…rface (#1408) * Added DataSourceMetadataStorage interface Signed-off-by: Vamsi Manohar <[email protected]> * Added new interface methods in DataSourceService and DataSourceMetadataStorage Signed-off-by: Vamsi Manohar <[email protected]> --------- Signed-off-by: Vamsi Manohar <[email protected]>
opensearch-project · Mar 8, 2023 · 98fb969 · 98fb969
1 parent 0ccb20a
commit 98fb969
Show file tree

Hide file tree

Showing 13 changed files with 221 additions and 53 deletions.
diff --git a/.gitignore b/.gitignore
@@ -46,3 +46,4 @@ gen
 /.prom.pid.lock
 
 .java-version
+.worktrees
diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java
@@ -64,7 +64,7 @@
 import org.opensearch.sql.data.model.ExprMissingValue;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.datasource.DataSourceService;
-import org.opensearch.sql.datasource.model.DataSource;
+import org.opensearch.sql.datasource.model.DataSourceMetadata;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.DSL;
 import org.opensearch.sql.expression.Expression;
@@ -134,9 +134,9 @@ public LogicalPlan analyze(UnresolvedPlan unresolved, AnalysisContext context) {
   @Override
   public LogicalPlan visitRelation(Relation node, AnalysisContext context) {
     QualifiedName qualifiedName = node.getTableQualifiedName();
-    Set<String> allowedDataSourceNames = dataSourceService.getDataSources()
+    Set<String> allowedDataSourceNames = dataSourceService.getDataSourceMetadataSet()
         .stream()
-        .map(DataSource::getName)
+        .map(DataSourceMetadata::getName)
         .collect(Collectors.toSet());
     DataSourceSchemaIdentifierNameResolver dataSourceSchemaIdentifierNameResolver
         = new DataSourceSchemaIdentifierNameResolver(qualifiedName.getParts(),
@@ -182,9 +182,9 @@ public LogicalPlan visitRelationSubquery(RelationSubquery node, AnalysisContext
   @Override
   public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext context) {
     QualifiedName qualifiedName = node.getFunctionName();
-    Set<String> allowedDataSourceNames = dataSourceService.getDataSources()
+    Set<String> allowedDataSourceNames = dataSourceService.getDataSourceMetadataSet()
         .stream()
-        .map(DataSource::getName)
+        .map(DataSourceMetadata::getName)
         .collect(Collectors.toSet());
     DataSourceSchemaIdentifierNameResolver dataSourceSchemaIdentifierNameResolver
         = new DataSourceSchemaIdentifierNameResolver(qualifiedName.getParts(),

diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceMetadataStorage.java b/core/src/main/java/org/opensearch/sql/datasource/DataSourceMetadataStorage.java
@@ -0,0 +1,64 @@
+/*
+ *
+ *  * Copyright OpenSearch Contributors
+ *  * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.opensearch.sql.datasource;
+
+import java.util.List;
+import java.util.Optional;
+import javax.xml.crypto.Data;
+import org.opensearch.sql.datasource.model.DataSource;
+import org.opensearch.sql.datasource.model.DataSourceMetadata;
+
+/**
+ * Interface for DataSourceMetadata Storage
+ * which will be only used by DataSourceService for Storage.
+ */
+public interface DataSourceMetadataStorage {
+
+  /**
+   * Returns all dataSource Metadata objects. The returned objects won't contain
+   * any of the credential info.
+   *
+   * @return list of {@link DataSourceMetadata}.
+   */
+  List<DataSourceMetadata> getDataSourceMetadata();
+
+
+  /**
+   * Gets {@link DataSourceMetadata} corresponding to the
+   * datasourceName from underlying storage.
+   *
+   * @param datasourceName name of the {@link DataSource}.
+   */
+  Optional<DataSourceMetadata> getDataSourceMetadata(String datasourceName);
+
+
+  /**
+   * Stores {@link DataSourceMetadata} in underlying storage.
+   *
+   * @param dataSourceMetadata {@link DataSourceMetadata}.
+   */
+  void createDataSourceMetadata(DataSourceMetadata dataSourceMetadata);
+
+
+  /**
+   * Updates {@link DataSourceMetadata} in underlying storage.
+   *
+   * @param dataSourceMetadata {@link DataSourceMetadata}.
+   */
+  void updateDataSourceMetadata(DataSourceMetadata dataSourceMetadata);
+
+
+  /**
+   * Deletes {@link DataSourceMetadata} corresponding to the
+   * datasourceName from underlying storage.
+   *
+   * @param datasourceName name of the {@link DataSource}.
+   */
+  void deleteDataSourceMetadata(String datasourceName);
+
+}
diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java b/core/src/main/java/org/opensearch/sql/datasource/DataSourceService.java
@@ -15,26 +15,49 @@
 public interface DataSourceService {
 
   /**
-   * Returns all DataSource objects.
+   * Returns {@link DataSource} corresponding to the DataSource name.
    *
-   * @return set of {@link DataSource}.
+   * @param dataSourceName Name of the {@link DataSource}.
+   * @return {@link DataSource}.
    */
-  Set<DataSource> getDataSources();
+  DataSource getDataSource(String dataSourceName);
+
 
   /**
-   * Returns {@link DataSource} with corresponding to the DataSource name.
+   * Returns all dataSource Metadata objects. The returned objects won't contain
+   * any of the credential info.
    *
-   * @param dataSourceName Name of the {@link DataSource}.
-   * @return {@link DataSource}.
+   * @return set of {@link DataSourceMetadata}.
    */
-  DataSource getDataSource(String dataSourceName);
+  Set<DataSourceMetadata> getDataSourceMetadataSet();
 
   /**
    * Register {@link DataSource} defined by {@link DataSourceMetadata}.
    *
    * @param metadatas list of {@link DataSourceMetadata}.
    */
-  void addDataSource(DataSourceMetadata... metadatas);
+  void createDataSource(DataSourceMetadata... metadatas);
+
+  /**
+   * Updates {@link DataSource} corresponding to dataSourceMetadata.
+   *
+   * @param dataSourceMetadata {@link DataSourceMetadata}.
+   */
+  void updateDataSource(DataSourceMetadata dataSourceMetadata);
+
+
+  /**
+   * Deletes {@link DataSource} corresponding to the DataSource name.
+   *
+   * @param dataSourceName name of the {@link DataSource}.
+   */
+  void deleteDataSource(String dataSourceName);
+
+  /**
+   * This method is to bootstrap
+   * datasources during the startup of the plugin.
+   */
+  void bootstrapDataSources();
 
   /**
    * remove all the registered {@link DataSource}.

diff --git a/core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java b/core/src/main/java/org/opensearch/sql/datasource/DataSourceServiceImpl.java
@@ -7,6 +7,8 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -45,8 +47,12 @@ public DataSourceServiceImpl(Set<DataSourceFactory> dataSourceFactories) {
   }
 
   @Override
-  public Set<DataSource> getDataSources() {
-    return Set.copyOf(dataSourceMap.values());
+  public Set<DataSourceMetadata> getDataSourceMetadataSet() {
+    return dataSourceMap.values().stream()
+        .map(dataSource
+            -> new DataSourceMetadata(dataSource.getName(),
+            dataSource.getConnectorType(), ImmutableMap.of()))
+        .collect(Collectors.toSet());
   }
 
   @Override
@@ -59,7 +65,7 @@ public DataSource getDataSource(String dataSourceName) {
   }
 
   @Override
-  public void addDataSource(DataSourceMetadata... metadatas) {
+  public void createDataSource(DataSourceMetadata... metadatas) {
     for (DataSourceMetadata metadata : metadatas) {
       validateDataSourceMetaData(metadata);
       dataSourceMap.put(
@@ -68,6 +74,21 @@ public void addDataSource(DataSourceMetadata... metadatas) {
     }
   }
 
+  @Override
+  public void updateDataSource(DataSourceMetadata dataSourceMetadata) {
+    throw new UnsupportedOperationException("will be supported in future");
+  }
+
+  @Override
+  public void deleteDataSource(String dataSourceName) {
+    throw new UnsupportedOperationException("will be supported in future");
+  }
+
+  @Override
+  public void bootstrapDataSources() {
+    throw new UnsupportedOperationException("will be supported in future");
+  }
+
   @Override
   public void clear() {
     dataSourceMap.clear();

diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceMetadata.java
@@ -13,14 +13,19 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.ImmutableMap;
 import java.util.Map;
+import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 import org.opensearch.sql.datasource.DataSourceService;
 
 @JsonIgnoreProperties(ignoreUnknown = true)
 @Getter
 @Setter
+@AllArgsConstructor
+@NoArgsConstructor
 @EqualsAndHashCode
 public class DataSourceMetadata {
 
@@ -39,10 +44,7 @@ public class DataSourceMetadata {
    * {@link DataSource} to {@link DataSourceService}.
    */
   public static DataSourceMetadata defaultOpenSearchDataSourceMetadata() {
-    DataSourceMetadata dataSourceMetadata = new DataSourceMetadata();
-    dataSourceMetadata.setName(DEFAULT_DATASOURCE_NAME);
-    dataSourceMetadata.setConnector(DataSourceType.OPENSEARCH);
-    dataSourceMetadata.setProperties(ImmutableMap.of());
-    return dataSourceMetadata;
+    return new DataSourceMetadata(DEFAULT_DATASOURCE_NAME,
+        DataSourceType.OPENSEARCH, ImmutableMap.of());
   }
 }
diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java
@@ -18,7 +18,7 @@
 import org.opensearch.sql.data.model.ExprValue;
 import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.datasource.DataSourceService;
-import org.opensearch.sql.datasource.model.DataSource;
+import org.opensearch.sql.datasource.model.DataSourceMetadata;
 import org.opensearch.sql.storage.TableScanOperator;
 
 /**
@@ -47,14 +47,15 @@ public String explain() {
   @Override
   public void open() {
     List<ExprValue> exprValues = new ArrayList<>();
-    Set<DataSource> dataSources = dataSourceService.getDataSources();
-    for (DataSource dataSource : dataSources) {
+    Set<DataSourceMetadata> dataSourceMetadataSet
+        = dataSourceService.getDataSourceMetadataSet();
+    for (DataSourceMetadata dataSourceMetadata : dataSourceMetadataSet) {
       exprValues.add(
           new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of(
               "DATASOURCE_NAME",
-              ExprValueUtils.stringValue(dataSource.getName()),
+              ExprValueUtils.stringValue(dataSourceMetadata.getName()),
               "CONNECTOR_TYPE",
-              ExprValueUtils.stringValue(dataSource.getConnectorType().name())))));
+              ExprValueUtils.stringValue(dataSourceMetadata.getConnector().name())))));
     }
     iterator = exprValues.iterator();
   }

diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java
@@ -10,6 +10,7 @@
 import static org.opensearch.sql.data.type.ExprCoreType.LONG;
 import static org.opensearch.sql.data.type.ExprCoreType.STRING;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import java.util.List;
 import java.util.Map;
@@ -164,8 +165,9 @@ private class DefaultDataSourceService implements DataSourceService {
 
 
     @Override
-    public Set<DataSource> getDataSources() {
-      return ImmutableSet.of(dataSource);
+    public Set<DataSourceMetadata> getDataSourceMetadataSet() {
+      return ImmutableSet.of(new DataSourceMetadata(dataSource.getName(),
+          dataSource.getConnectorType(), ImmutableMap.of()));
     }
 
     @Override
@@ -174,10 +176,25 @@ public DataSource getDataSource(String dataSourceName) {
     }
 
     @Override
-    public void addDataSource(DataSourceMetadata... metadatas) {
+    public void createDataSource(DataSourceMetadata... metadatas) {
       throw new UnsupportedOperationException();
     }
 
+    @Override
+    public void updateDataSource(DataSourceMetadata dataSourceMetadata) {
+
+    }
+
+    @Override
+    public void deleteDataSource(String dataSourceName) {
+
+    }
+
+    @Override
+    public void bootstrapDataSources() {
+
+    }
+
     @Override
     public void clear() {
       throw new UnsupportedOperationException();
Original file line number	Diff line number	Diff line change
Expand Up		@@ -46,3 +46,4 @@ gen
		/.prom.pid.lock

		.java-version
		.worktrees