Make adjustments for #2397

apache · Mar 14, 2024 · d8ed020 · d8ed020
1 parent 6330c29
commit d8ed020
Show file tree

Hide file tree

Showing 5 changed files with 65 additions and 40 deletions.
diff --git a/catalogs/catalog-jdbc-doris/build.gradle.kts b/catalogs/catalog-jdbc-doris/build.gradle.kts
@@ -15,30 +15,39 @@ dependencies {
   implementation(project(":catalogs:catalog-jdbc-common"))
   implementation(project(":common"))
   implementation(project(":core"))
+
   implementation(libs.bundles.log4j)
   implementation(libs.commons.collections4)
   implementation(libs.commons.lang3)
   implementation(libs.guava)
   implementation(libs.jsqlparser)
+  implementation(libs.slf4j.api)
+
+  testImplementation(project(":catalogs:catalog-jdbc-common", "testArtifacts"))
+  testImplementation(project(":clients:client-java"))
+  testImplementation(project(":integration-test-common", "testArtifacts"))
+  testImplementation(project(":server"))
+  testImplementation(project(":server-common"))
 
-  testImplementation(project(":catalogs:catalog-jdbc-common"))
-  testImplementation(libs.mysql.driver)
-  testImplementation(libs.guava)
   testImplementation(libs.commons.lang3)
+  testImplementation(libs.guava)
+  testImplementation(libs.mysql.driver)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
+  testImplementation(libs.testcontainers)
+  testImplementation(libs.testcontainers.mysql)
 
   testRuntimeOnly(libs.junit.jupiter.engine)
 }
 
 tasks {
-  val copyDepends by registering(Copy::class) {
+  val runtimeJars by registering(Copy::class) {
     from(configurations.runtimeClasspath)
-    into("build/libs_all")
+    into("build/libs")
   }
   val copyCatalogLibs by registering(Copy::class) {
-    dependsOn(copyDepends, "build")
-    from("build/libs_all", "build/libs")
+    dependsOn("jar", "runtimeJars")
+    from("build/libs")
     into("$rootDir/distribution/package/catalogs/jdbc-doris/libs")
   }
 
@@ -57,3 +66,26 @@ tasks {
     dependsOn(copyCatalogLibs, copyCatalogConfig)
   }
 }
+
+tasks.test {
+  val skipUTs = project.hasProperty("skipTests")
+  if (skipUTs) {
+    // Only run integration tests
+    include("**/integration/**")
+  }
+
+  val skipITs = project.hasProperty("skipITs")
+  if (skipITs) {
+    // Exclude integration tests
+    exclude("**/integration/**")
+  } else {
+    dependsOn(tasks.jar)
+
+    doFirst {
+      environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "datastrato/gravitino-ci-doris:0.1.0")
+    }
+
+    val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
+    init(this)
+  }
+}
diff --git a/...st/catalog/jdbc/doris/CatalogDorisIT.java → ...oris/integration/test/CatalogDorisIT.java b/...st/catalog/jdbc/doris/CatalogDorisIT.java → ...oris/integration/test/CatalogDorisIT.java
@@ -2,9 +2,8 @@
  * Copyright 2024 Datastrato Pvt Ltd.
  * This software is licensed under the Apache License version 2.
  */
-package com.datastrato.gravitino.integration.test.catalog.jdbc.doris;
+package com.datastrato.gravitino.catalog.doris.integration.test;
 
-import static com.datastrato.gravitino.integration.test.catalog.jdbc.TestJdbcAbstractIT.assertColumn;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import com.datastrato.gravitino.Catalog;
@@ -15,12 +14,12 @@
 import com.datastrato.gravitino.dto.rel.ColumnDTO;
 import com.datastrato.gravitino.exceptions.NoSuchSchemaException;
 import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException;
-import com.datastrato.gravitino.integration.test.catalog.jdbc.utils.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.container.ContainerSuite;
 import com.datastrato.gravitino.integration.test.container.DorisContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
+import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.rel.Schema;
 import com.datastrato.gravitino.rel.SupportsSchemas;
 import com.datastrato.gravitino.rel.Table;
@@ -43,7 +42,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
-import org.apache.commons.lang.math.RandomUtils;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
@@ -84,12 +82,10 @@ public class CatalogDorisIT extends AbstractIT {
 
   protected Catalog catalog;
 
-  protected final String TEST_DB_NAME = RandomUtils.nextInt(10000) + "_test_db";
-
   @BeforeAll
   public void startup() throws IOException {
 
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
+    if (!ITUtils.EMBEDDED_TEST_MODE.equals(AbstractIT.testMode)) {
       String gravitinoHome = System.getenv("GRAVITINO_HOME");
       Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-doris/libs");
       JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString());
@@ -105,7 +101,7 @@ public void startup() throws IOException {
   @AfterAll
   public void stop() {
     clearTableAndSchema();
-    client.dropMetalake(NameIdentifier.of(metalakeName));
+    AbstractIT.client.dropMetalake(NameIdentifier.of(metalakeName));
   }
 
   @AfterEach
@@ -114,6 +110,16 @@ private void resetSchema() {
     createSchema();
   }
 
+  private static void waitForDorisOperation() {
+    // TODO: use a better way to wait for the operation to complete
+    // see: https://doris.apache.org/docs/1.2/advanced/alter-table/schema-change/
+    try {
+      Thread.sleep(2000);
+    } catch (InterruptedException e) {
+      // do nothing
+    }
+  }
+
   private void clearTableAndSchema() {
     NameIdentifier[] nameIdentifiers =
         catalog.asTableCatalog().listTables(Namespace.of(metalakeName, catalogName, schemaName));
@@ -124,12 +130,14 @@ private void clearTableAndSchema() {
   }
 
   private void createMetalake() {
-    GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes();
+    GravitinoMetaLake[] gravitinoMetaLakes = AbstractIT.client.listMetalakes();
     Assertions.assertEquals(0, gravitinoMetaLakes.length);
 
     GravitinoMetaLake createdMetalake =
-        client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());
-    GravitinoMetaLake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
+        AbstractIT.client.createMetalake(
+            NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());
+    GravitinoMetaLake loadMetalake =
+        AbstractIT.client.loadMetalake(NameIdentifier.of(metalakeName));
     Assertions.assertEquals(createdMetalake, loadMetalake);
 
     metalake = loadMetalake;
@@ -327,7 +335,7 @@ void testCreateAndLoadDorisTable() {
     Assertions.assertEquals(createdTable.columns().length, columns.length);
 
     for (int i = 0; i < columns.length; i++) {
-      assertColumn(columns[i], createdTable.columns()[i]);
+      AbstractIT.assertColumn(columns[i], createdTable.columns()[i]);
     }
 
     // test load table
@@ -341,7 +349,7 @@ void testCreateAndLoadDorisTable() {
     }
     Assertions.assertEquals(loadTable.columns().length, columns.length);
     for (int i = 0; i < columns.length; i++) {
-      assertColumn(columns[i], loadTable.columns()[i]);
+      AbstractIT.assertColumn(columns[i], loadTable.columns()[i]);
     }
   }
 
@@ -374,11 +382,7 @@ void testDorisIndex() {
         TableChange.addIndex(
             Index.IndexType.PRIMARY_KEY, "k1_index", new String[][] {{DORIS_COL_NAME1}}));
 
-    try {
-      Thread.sleep(1000);
-    } catch (InterruptedException e) {
-      // do nothing
-    }
+    waitForDorisOperation();
     Table table =
         tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName));
     Index[] indexes = table.index();
@@ -391,11 +395,7 @@ void testDorisIndex() {
         TableChange.addIndex(
             Index.IndexType.PRIMARY_KEY, "k2_index", new String[][] {{DORIS_COL_NAME2}}));
 
-    try {
-      Thread.sleep(1000);
-    } catch (InterruptedException e) {
-      // do nothing
-    }
+    waitForDorisOperation();
 
     table =
         tableCatalog.loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName));

diff --git a/...og/jdbc/doris/DorisTableOperationsIT.java → ...egration/test/DorisTableOperationsIT.java b/...og/jdbc/doris/DorisTableOperationsIT.java → ...egration/test/DorisTableOperationsIT.java
@@ -2,7 +2,7 @@
  * Copyright 2023 Datastrato Pvt Ltd.
  * This software is licensed under the Apache License version 2.
  */
-package com.datastrato.gravitino.integration.test.catalog.jdbc.doris;
+package com.datastrato.gravitino.catalog.doris.integration.test;
 
 import com.datastrato.gravitino.catalog.doris.converter.DorisColumnDefaultValueConverter;
 import com.datastrato.gravitino.catalog.doris.converter.DorisExceptionConverter;
@@ -13,10 +13,10 @@
 import com.datastrato.gravitino.catalog.jdbc.JdbcTable;
 import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig;
 import com.datastrato.gravitino.catalog.jdbc.converter.JdbcExceptionConverter;
+import com.datastrato.gravitino.catalog.jdbc.integration.test.TestJdbcAbstractIT;
 import com.datastrato.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations;
 import com.datastrato.gravitino.catalog.jdbc.operation.JdbcTableOperations;
 import com.datastrato.gravitino.catalog.jdbc.utils.DataSourceUtils;
-import com.datastrato.gravitino.integration.test.catalog.jdbc.TestJdbcAbstractIT;
 import com.datastrato.gravitino.integration.test.container.ContainerSuite;
 import com.datastrato.gravitino.integration.test.container.DorisContainer;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;

diff --git a/.../test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java b/.../test/java/com/datastrato/gravitino/catalog/doris/operation/TestDorisTableOperations.java
diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts
@@ -140,6 +140,7 @@ tasks.test {
   } else {
     dependsOn(":trino-connector:jar")
     dependsOn(":catalogs:catalog-lakehouse-iceberg:jar", ":catalogs:catalog-lakehouse-iceberg:runtimeJars")
+    dependsOn(":catalogs:catalog-jdbc-doris:jar", ":catalogs:catalog-jdbc-doris:runtimeJars")
     dependsOn(":catalogs:catalog-jdbc-mysql:jar", ":catalogs:catalog-jdbc-mysql:runtimeJars")
     dependsOn(":catalogs:catalog-jdbc-postgresql:jar", ":catalogs:catalog-jdbc-postgresql:runtimeJars")
     dependsOn(":catalogs:catalog-hadoop:jar", ":catalogs:catalog-hadoop:runtimeJars")