Add IT test for Glue metastore creation (#137)

Co-authored-by: Marco Scalzo <[email protected]>
agile-lab-dev · Dec 22, 2023 · 7ccfa42 · 7ccfa42
1 parent 7a35adc
commit 7ccfa42
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 26 deletions.
diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java
@@ -3,8 +3,12 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import com.github.mrpowers.spark.fast.tests.DatasetComparer;
+import io.whitefox.api.client.model.CreateMetastore;
+import io.whitefox.api.client.model.Metastore;
 import io.whitefox.api.models.MrFoxDeltaTableSchema;
+import io.whitefox.api.utils.SparkUtil;
 import io.whitefox.api.utils.StorageManagerInitializer;
+import io.whitefox.api.utils.TablePath;
 import java.util.List;
 import org.apache.spark.sql.SparkSession;
 import org.apache.spark.sql.types.DataType;
@@ -17,20 +21,18 @@
 import scala.collection.GenMap;
 
 @Tag("clientSparkTest")
-public class ITDeltaSharingClient implements DatasetComparer {
+public class ITDeltaSharingClient implements DatasetComparer, SparkUtil {
 
-  private final String tablePath = String.format(
-      "%s#%s.%s.%s",
-      getClass().getClassLoader().getResource("MrFoxProfile.json"),
-      "s3share",
-      "s3schema",
-      "s3Table1");
+  private final StorageManagerInitializer storageManagerInitializer;
+  private final String deltaTablePath;
+  private final SparkSession spark;
 
-  private final SparkSession spark = SparkSession.builder()
-      .appName("delta sharing client test")
-      .config("spark.driver.host", "localhost")
-      .master("local[1, 4]")
-      .getOrCreate();
+  public ITDeltaSharingClient() {
+    this.storageManagerInitializer = new StorageManagerInitializer();
+    this.deltaTablePath =
+        TablePath.getDeltaTablePath(getClass().getClassLoader().getResource("MrFoxProfile.json"));
+    this.spark = newSparkSession();
+  }
 
   @BeforeAll
   static void initStorageManager() {
@@ -39,7 +41,8 @@ static void initStorageManager() {
 
   @Test
   void showS3Table1withQueryTableApi() {
-    var ds = spark.read().format("deltaSharing").load(tablePath);
+    storageManagerInitializer.createS3DeltaTable();
+    var ds = spark.read().format("deltaSharing").load(deltaTablePath);
     var expectedSchema = new StructType(new StructField[] {
       new StructField("id", DataType.fromDDL("long"), true, new Metadata(GenMap.empty()))
     });
@@ -58,4 +61,11 @@ void showS3Table1withQueryTableApi() {
     assertEquals(5, ds.count());
     assertSmallDatasetEquality(ds, expectedData, true, false, false, 500);
   }
+
+  @Test
+  void createGlueMetastore() {
+    Metastore metastore = storageManagerInitializer.createGlueMetastore();
+    assertEquals(metastore.getName(), "MrFoxMetastore");
+    assertEquals(metastore.getType(), CreateMetastore.TypeEnum.GLUE.getValue());
+  }
 }
diff --git a/client-spark/src/test/java/io/whitefox/api/utils/SparkUtil.java b/client-spark/src/test/java/io/whitefox/api/utils/SparkUtil.java
@@ -0,0 +1,14 @@
+package io.whitefox.api.utils;
+
+import org.apache.spark.sql.SparkSession;
+
+public interface SparkUtil {
+
+  default SparkSession newSparkSession() {
+    return SparkSession.builder()
+        .appName("delta sharing client test")
+        .config("spark.driver.host", "localhost")
+        .master("local[1, 4]")
+        .getOrCreate();
+  }
+}
diff --git a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java
@@ -4,10 +4,12 @@
 import io.whitefox.api.client.model.*;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 public class StorageManagerInitializer {
   private final S3TestConfig s3TestConfig;
   private final StorageV1Api storageV1Api;
+  private final MetastoreV1Api metastoreV1Api;
   private final ProviderV1Api providerV1Api;
   private final TableV1Api tableV1Api;
   private final ShareV1Api shareV1Api;
@@ -21,33 +23,48 @@ public StorageManagerInitializer() {
     this.tableV1Api = new TableV1Api(apiClient);
     this.shareV1Api = new ShareV1Api(apiClient);
     this.schemaV1Api = new SchemaV1Api(apiClient);
+    this.metastoreV1Api = new MetastoreV1Api(apiClient);
   }
 
   public void initStorageManager() {
     storageV1Api.createStorage(createStorageRequest(s3TestConfig));
-    providerV1Api.addProvider(addProviderRequest());
-    tableV1Api.createTableInProvider(addProviderRequest().getName(), createTableRequest());
     shareV1Api.createShare(createShareRequest());
-    schemaV1Api.createSchema(createShareRequest().getName(), createSchemaRequest());
+  }
+
+  public void createS3DeltaTable() {
+    var providerRequest = addProviderRequest(Optional.empty(), TableFormat.delta);
+    providerV1Api.addProvider(providerRequest);
+    var createTableRequest = createDeltaTableRequest();
+    tableV1Api.createTableInProvider(providerRequest.getName(), createTableRequest);
+    var shareRequest = createShareRequest();
+    var schemaRequest = createSchemaRequest(TableFormat.delta);
+    schemaV1Api.createSchema(shareRequest.getName(), schemaRequest);
     schemaV1Api.addTableToSchema(
-        createShareRequest().getName(), createSchemaRequest(), addTableToSchemaRequest());
+        shareRequest.getName(),
+        schemaRequest,
+        addTableToSchemaRequest(providerRequest.getName(), createTableRequest.getName()));
+  }
+
+  public Metastore createGlueMetastore() {
+    var metastoreRequest = createMetastoreRequest(s3TestConfig, CreateMetastore.TypeEnum.GLUE);
+    return metastoreV1Api.createMetastore(metastoreRequest);
   }
 
-  private String createSchemaRequest() {
-    return "s3schema";
+  private String createSchemaRequest(TableFormat tableFormat) {
+    return format("s3schema", tableFormat);
   }
 
-  private AddTableToSchemaRequest addTableToSchemaRequest() {
+  private AddTableToSchemaRequest addTableToSchemaRequest(String providerName, String tableName) {
     return new AddTableToSchemaRequest()
-        .name("s3Table1")
-        .reference(new TableReference().providerName("MrFoxProvider").name("s3Table1"));
+        .name(tableName)
+        .reference(new TableReference().providerName(providerName).name(tableName));
   }
 
   private CreateShareInput createShareRequest() {
     return new CreateShareInput().name("s3share").recipients(List.of("Mr.Fox")).schemas(List.of());
   }
 
-  private CreateTableInput createTableRequest() {
+  private CreateTableInput createDeltaTableRequest() {
     return new CreateTableInput()
         .name("s3Table1")
         .skipValidation(true)
@@ -56,11 +73,12 @@ private CreateTableInput createTableRequest() {
             "location", "s3a://whitefox-s3-test-bucket/delta/samples/delta-table"));
   }
 
-  private ProviderInput addProviderRequest() {
+  private ProviderInput addProviderRequest(
+      Optional<String> metastoreName, TableFormat tableFormat) {
     return new ProviderInput()
-        .name("MrFoxProvider")
+        .name(format("MrFoxProvider", tableFormat))
         .storageName("MrFoxStorage")
-        .metastoreName(null);
+        .metastoreName(metastoreName.orElse(null));
   }
 
   private CreateStorage createStorageRequest(S3TestConfig s3TestConfig) {
@@ -74,4 +92,22 @@ private CreateStorage createStorageRequest(S3TestConfig s3TestConfig) {
                 .awsSecretAccessKey(s3TestConfig.getSecretKey()))))
         .skipValidation(true);
   }
+
+  private CreateMetastore createMetastoreRequest(
+      S3TestConfig s3TestConfig, CreateMetastore.TypeEnum type) {
+    return new CreateMetastore()
+        .name("MrFoxMetastore")
+        .type(type)
+        .skipValidation(true)
+        .properties(new MetastoreProperties(new GlueProperties()
+            .catalogId("catalogId") // TODO
+            .credentials(new SimpleAwsCredentials()
+                .region(s3TestConfig.getRegion())
+                .awsAccessKeyId(s3TestConfig.getAccessKey())
+                .awsSecretAccessKey(s3TestConfig.getSecretKey()))));
+  }
+
+  private String format(String value, TableFormat tableFormat) {
+    return String.format("%s%s", value, tableFormat.name());
+  }
 }
diff --git a/client-spark/src/test/java/io/whitefox/api/utils/TableFormat.java b/client-spark/src/test/java/io/whitefox/api/utils/TableFormat.java
@@ -0,0 +1,6 @@
+package io.whitefox.api.utils;
+
+public enum TableFormat {
+  delta,
+  iceberg
+}
diff --git a/client-spark/src/test/java/io/whitefox/api/utils/TablePath.java b/client-spark/src/test/java/io/whitefox/api/utils/TablePath.java
@@ -0,0 +1,10 @@
+package io.whitefox.api.utils;
+
+import java.net.URL;
+
+public class TablePath {
+
+  public static String getDeltaTablePath(URL resource) {
+    return String.format("%s#%s.%s.%s", resource, "s3share", "s3schemadelta", "s3Table1");
+  }
+}