diff --git a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java index 4124c87c0..e5c1d18df 100644 --- a/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java +++ b/client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java @@ -3,8 +3,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.github.mrpowers.spark.fast.tests.DatasetComparer; +import io.whitefox.api.client.model.CreateMetastore; +import io.whitefox.api.client.model.Metastore; import io.whitefox.api.models.MrFoxDeltaTableSchema; +import io.whitefox.api.utils.SparkUtil; import io.whitefox.api.utils.StorageManagerInitializer; +import io.whitefox.api.utils.TablePath; import java.util.List; import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataType; @@ -17,20 +21,18 @@ import scala.collection.GenMap; @Tag("clientSparkTest") -public class ITDeltaSharingClient implements DatasetComparer { +public class ITDeltaSharingClient implements DatasetComparer, SparkUtil { - private final String tablePath = String.format( - "%s#%s.%s.%s", - getClass().getClassLoader().getResource("MrFoxProfile.json"), - "s3share", - "s3schema", - "s3Table1"); + private final StorageManagerInitializer storageManagerInitializer; + private final String deltaTablePath; + private final SparkSession spark; - private final SparkSession spark = SparkSession.builder() - .appName("delta sharing client test") - .config("spark.driver.host", "localhost") - .master("local[1, 4]") - .getOrCreate(); + public ITDeltaSharingClient() { + this.storageManagerInitializer = new StorageManagerInitializer(); + this.deltaTablePath = + TablePath.getDeltaTablePath(getClass().getClassLoader().getResource("MrFoxProfile.json")); + this.spark = newSparkSession(); + } @BeforeAll static void initStorageManager() { @@ -39,7 +41,8 @@ static void initStorageManager() { @Test void showS3Table1withQueryTableApi() { - var ds = spark.read().format("deltaSharing").load(tablePath); + storageManagerInitializer.createS3DeltaTable(); + var ds = spark.read().format("deltaSharing").load(deltaTablePath); var expectedSchema = new StructType(new StructField[] { new StructField("id", DataType.fromDDL("long"), true, new Metadata(GenMap.empty())) }); @@ -58,4 +61,11 @@ void showS3Table1withQueryTableApi() { assertEquals(5, ds.count()); assertSmallDatasetEquality(ds, expectedData, true, false, false, 500); } + + @Test + void createGlueMetastore() { + Metastore metastore = storageManagerInitializer.createGlueMetastore(); + assertEquals(metastore.getName(), "MrFoxMetastore"); + assertEquals(metastore.getType(), CreateMetastore.TypeEnum.GLUE.getValue()); + } } diff --git a/client-spark/src/test/java/io/whitefox/api/utils/SparkUtil.java b/client-spark/src/test/java/io/whitefox/api/utils/SparkUtil.java new file mode 100644 index 000000000..9536ce11f --- /dev/null +++ b/client-spark/src/test/java/io/whitefox/api/utils/SparkUtil.java @@ -0,0 +1,14 @@ +package io.whitefox.api.utils; + +import org.apache.spark.sql.SparkSession; + +public interface SparkUtil { + + default SparkSession newSparkSession() { + return SparkSession.builder() + .appName("delta sharing client test") + .config("spark.driver.host", "localhost") + .master("local[1, 4]") + .getOrCreate(); + } +} diff --git a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java index fd6639943..47d3f3603 100644 --- a/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java +++ b/client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java @@ -4,10 +4,12 @@ import io.whitefox.api.client.model.*; import java.util.List; import java.util.Map; +import java.util.Optional; public class StorageManagerInitializer { private final S3TestConfig s3TestConfig; private final StorageV1Api storageV1Api; + private final MetastoreV1Api metastoreV1Api; private final ProviderV1Api providerV1Api; private final TableV1Api tableV1Api; private final ShareV1Api shareV1Api; @@ -21,33 +23,48 @@ public StorageManagerInitializer() { this.tableV1Api = new TableV1Api(apiClient); this.shareV1Api = new ShareV1Api(apiClient); this.schemaV1Api = new SchemaV1Api(apiClient); + this.metastoreV1Api = new MetastoreV1Api(apiClient); } public void initStorageManager() { storageV1Api.createStorage(createStorageRequest(s3TestConfig)); - providerV1Api.addProvider(addProviderRequest()); - tableV1Api.createTableInProvider(addProviderRequest().getName(), createTableRequest()); shareV1Api.createShare(createShareRequest()); - schemaV1Api.createSchema(createShareRequest().getName(), createSchemaRequest()); + } + + public void createS3DeltaTable() { + var providerRequest = addProviderRequest(Optional.empty(), TableFormat.delta); + providerV1Api.addProvider(providerRequest); + var createTableRequest = createDeltaTableRequest(); + tableV1Api.createTableInProvider(providerRequest.getName(), createTableRequest); + var shareRequest = createShareRequest(); + var schemaRequest = createSchemaRequest(TableFormat.delta); + schemaV1Api.createSchema(shareRequest.getName(), schemaRequest); schemaV1Api.addTableToSchema( - createShareRequest().getName(), createSchemaRequest(), addTableToSchemaRequest()); + shareRequest.getName(), + schemaRequest, + addTableToSchemaRequest(providerRequest.getName(), createTableRequest.getName())); + } + + public Metastore createGlueMetastore() { + var metastoreRequest = createMetastoreRequest(s3TestConfig, CreateMetastore.TypeEnum.GLUE); + return metastoreV1Api.createMetastore(metastoreRequest); } - private String createSchemaRequest() { - return "s3schema"; + private String createSchemaRequest(TableFormat tableFormat) { + return format("s3schema", tableFormat); } - private AddTableToSchemaRequest addTableToSchemaRequest() { + private AddTableToSchemaRequest addTableToSchemaRequest(String providerName, String tableName) { return new AddTableToSchemaRequest() - .name("s3Table1") - .reference(new TableReference().providerName("MrFoxProvider").name("s3Table1")); + .name(tableName) + .reference(new TableReference().providerName(providerName).name(tableName)); } private CreateShareInput createShareRequest() { return new CreateShareInput().name("s3share").recipients(List.of("Mr.Fox")).schemas(List.of()); } - private CreateTableInput createTableRequest() { + private CreateTableInput createDeltaTableRequest() { return new CreateTableInput() .name("s3Table1") .skipValidation(true) @@ -56,11 +73,12 @@ private CreateTableInput createTableRequest() { "location", "s3a://whitefox-s3-test-bucket/delta/samples/delta-table")); } - private ProviderInput addProviderRequest() { + private ProviderInput addProviderRequest( + Optional metastoreName, TableFormat tableFormat) { return new ProviderInput() - .name("MrFoxProvider") + .name(format("MrFoxProvider", tableFormat)) .storageName("MrFoxStorage") - .metastoreName(null); + .metastoreName(metastoreName.orElse(null)); } private CreateStorage createStorageRequest(S3TestConfig s3TestConfig) { @@ -74,4 +92,22 @@ private CreateStorage createStorageRequest(S3TestConfig s3TestConfig) { .awsSecretAccessKey(s3TestConfig.getSecretKey())))) .skipValidation(true); } + + private CreateMetastore createMetastoreRequest( + S3TestConfig s3TestConfig, CreateMetastore.TypeEnum type) { + return new CreateMetastore() + .name("MrFoxMetastore") + .type(type) + .skipValidation(true) + .properties(new MetastoreProperties(new GlueProperties() + .catalogId("catalogId") // TODO + .credentials(new SimpleAwsCredentials() + .region(s3TestConfig.getRegion()) + .awsAccessKeyId(s3TestConfig.getAccessKey()) + .awsSecretAccessKey(s3TestConfig.getSecretKey())))); + } + + private String format(String value, TableFormat tableFormat) { + return String.format("%s%s", value, tableFormat.name()); + } } diff --git a/client-spark/src/test/java/io/whitefox/api/utils/TableFormat.java b/client-spark/src/test/java/io/whitefox/api/utils/TableFormat.java new file mode 100644 index 000000000..3e4239712 --- /dev/null +++ b/client-spark/src/test/java/io/whitefox/api/utils/TableFormat.java @@ -0,0 +1,6 @@ +package io.whitefox.api.utils; + +public enum TableFormat { + delta, + iceberg +} diff --git a/client-spark/src/test/java/io/whitefox/api/utils/TablePath.java b/client-spark/src/test/java/io/whitefox/api/utils/TablePath.java new file mode 100644 index 000000000..63a910545 --- /dev/null +++ b/client-spark/src/test/java/io/whitefox/api/utils/TablePath.java @@ -0,0 +1,10 @@ +package io.whitefox.api.utils; + +import java.net.URL; + +public class TablePath { + + public static String getDeltaTablePath(URL resource) { + return String.format("%s#%s.%s.%s", resource, "s3share", "s3schemadelta", "s3Table1"); + } +}