[#3524] feat(trino-connector): Load catalog by Trino dynamic catalog feature

build.gradle.kts

@@ -218,10 +218,6 @@ nexusPublishing {
   packageGroup.set("com.datastrato.gravitino")
 }
 
-dependencies {
-  testImplementation(libs.testng)
-}
-
 subprojects {
   // Gravitino Python client project didn't need to apply the java plugin
   if (project.name == "client-python") {
@@ -411,13 +407,8 @@ subprojects {
     reports.html.outputLocation.set(file("${rootProject.projectDir}/build/reports/"))
     val skipTests = project.hasProperty("skipTests")
     if (!skipTests) {
-      if (project.name == "trino-connector") {
-        useTestNG()
-        maxHeapSize = "2G"
-      } else {
-        useJUnitPlatform()
-      }
-
+      jvmArgs = listOf("-Xmx2G")
+      useJUnitPlatform()
       jvmArgs(project.property("extraJvmArgs") as List<*>)
       finalizedBy(tasks.getByName("jacocoTestReport"))
     }

catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisDriverIT.java

@@ -10,7 +10,5 @@
 public class CatalogDorisDriverIT extends CatalogDorisIT {
   public CatalogDorisDriverIT() {
     super();
-    mysqlDriverDownloadUrl =
-        "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.11/mysql-connector-java-8.0.11.jar";
   }
 }

catalogs/catalog-jdbc-doris/src/test/java/com/datastrato/gravitino/catalog/doris/integration/test/CatalogDorisIT.java

@@ -19,7 +19,6 @@
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Table;
 import com.datastrato.gravitino.rel.TableCatalog;
@@ -34,8 +33,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Map;
@@ -57,8 +54,6 @@
 public class CatalogDorisIT extends AbstractIT {
 
   private static final String provider = "jdbc-doris";
-  private static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
 
   private static final String DRIVER_CLASS_NAME = "com.mysql.jdbc.Driver";
 
@@ -87,17 +82,8 @@ public class CatalogDorisIT extends AbstractIT {
 
   protected Catalog catalog;
 
-  protected String mysqlDriverDownloadUrl = DOWNLOAD_JDBC_DRIVER_URL;
-
   @BeforeAll
   public void startup() throws IOException {
-
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(AbstractIT.testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-doris/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(mysqlDriverDownloadUrl, tmpPath.toString());
-    }
-
     containerSuite.startDorisContainer();
 
     createMetalake();

catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java

@@ -18,17 +18,13 @@
 import com.datastrato.gravitino.integration.test.container.MySQLContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
-import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Table;
 import com.datastrato.gravitino.rel.TableChange;
 import com.datastrato.gravitino.rel.types.Types;
 import com.google.common.collect.Maps;
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.Map;
@@ -44,8 +40,6 @@ public class AuditCatalogMysqlIT extends AbstractIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   public static final String metalakeName = GravitinoITUtils.genRandomName("audit_mysql_metalake");
   private static final String expectUser = System.getProperty("user.name");
-  public static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
   protected static TestDatabaseName TEST_DB_NAME;
   private static final String provider = "jdbc-mysql";
 
@@ -60,12 +54,6 @@ public static void startIntegrationTest() throws Exception {
     registerCustomConfigs(configs);
     AbstractIT.startIntegrationTest();
 
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-mysql/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString());
-    }
-
     containerSuite.startMySQLContainer(TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT);
     MYSQL_CONTAINER = containerSuite.getMySQLContainer();
     TEST_DB_NAME = TestDatabaseName.MYSQL_AUDIT_CATALOG_MYSQL_IT;

catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlDriverIT.java

@@ -11,7 +11,5 @@
 public class CatalogMysqlDriverIT extends CatalogMysqlIT {
   public CatalogMysqlDriverIT() {
     super();
-    mysqlDriverDownloadUrl =
-        "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.11/mysql-connector-java-8.0.11.jar";
   }
 }

catalogs/catalog-jdbc-mysql/src/test/java/com/datastrato/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java

@@ -25,7 +25,6 @@
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Column.ColumnImpl;
@@ -47,8 +46,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.Collections;
@@ -72,8 +69,6 @@
 public class CatalogMysqlIT extends AbstractIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   private static final String provider = "jdbc-mysql";
-  public static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
 
   public String metalakeName = GravitinoITUtils.genRandomName("mysql_it_metalake");
   public String catalogName = GravitinoITUtils.genRandomName("mysql_it_catalog");
@@ -104,21 +99,12 @@ public class CatalogMysqlIT extends AbstractIT {
 
   protected String mysqlImageName = defaultMysqlImageName;
 
-  protected String mysqlDriverDownloadUrl = DOWNLOAD_JDBC_DRIVER_URL;
-
   boolean SupportColumnDefaultValueExpression() {
     return true;
   }
 
   @BeforeAll
   public void startup() throws IOException, SQLException {
-
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-mysql/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(mysqlDriverDownloadUrl, tmpPath.toString());
-    }
-
     TEST_DB_NAME = TestDatabaseName.MYSQL_CATALOG_MYSQL_IT;
 
     if (mysqlImageName.equals("mysql:5.7")) {

catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java

@@ -24,7 +24,6 @@
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
 import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
 import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.Table;
@@ -46,8 +45,6 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.Arrays;
@@ -72,8 +69,6 @@
 public class CatalogPostgreSqlIT extends AbstractIT {
   private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
   public static final PGImageName DEFAULT_POSTGRES_IMAGE = PGImageName.VERSION_13;
-  public static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar";
 
   public String metalakeName = GravitinoITUtils.genRandomName("postgresql_it_metalake");
   public String catalogName = GravitinoITUtils.genRandomName("postgresql_it_catalog");
@@ -101,12 +96,6 @@ public class CatalogPostgreSqlIT extends AbstractIT {
 
   @BeforeAll
   public void startup() throws IOException, SQLException {
-
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      String gravitinoHome = System.getenv("GRAVITINO_HOME");
-      Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-postgresql/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString());
-    }
     containerSuite.startPostgreSQLContainer(TEST_DB_NAME, postgreImageName);
     POSTGRESQL_CONTAINER = containerSuite.getPostgreSQLContainer(postgreImageName);
 

catalogs/catalog-jdbc-postgresql/src/test/java/com/datastrato/gravitino/catalog/postgresql/integration/test/TestMultipleJDBCLoad.java

@@ -13,17 +13,13 @@
 import com.datastrato.gravitino.integration.test.container.MySQLContainer;
 import com.datastrato.gravitino.integration.test.container.PostgreSQLContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
-import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.types.Types;
 import com.datastrato.gravitino.utils.RandomNameUtils;
 import com.google.common.collect.Maps;
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.Map;
@@ -41,38 +37,8 @@ public class TestMultipleJDBCLoad extends AbstractIT {
   private static MySQLContainer mySQLContainer;
   private static PostgreSQLContainer postgreSQLContainer;
 
-  private static final String DOWNLOAD_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
-
   @BeforeAll
   public static void startup() throws IOException {
-    String gravitinoHome = System.getenv("GRAVITINO_HOME");
-
-    // Deploy mode
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      Path icebergLibsPath = Paths.get(gravitinoHome, "/catalogs/lakehouse-iceberg/libs");
-      Path pgDirPath = Paths.get(gravitinoHome, "/catalogs/jdbc-postgresql/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(
-          CatalogPostgreSqlIT.DOWNLOAD_JDBC_DRIVER_URL,
-          pgDirPath.toString(),
-          icebergLibsPath.toString());
-
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_JDBC_DRIVER_URL, pgDirPath.toString(), icebergLibsPath.toString());
-    } else {
-      // embedded mode
-      Path icebergLibsPath =
-          Paths.get(gravitinoHome, "/catalogs/catalog-lakehouse-iceberg/build/libs");
-      Path pgDirPath = Paths.get(gravitinoHome, "/catalogs/catalog-jdbc-postgresql/build/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(
-          CatalogPostgreSqlIT.DOWNLOAD_JDBC_DRIVER_URL,
-          icebergLibsPath.toString(),
-          pgDirPath.toString());
-
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_JDBC_DRIVER_URL, pgDirPath.toString(), icebergLibsPath.toString());
-    }
-
     containerSuite.startMySQLContainer(TEST_DB_NAME);
     mySQLContainer = containerSuite.getMySQLContainer();
     containerSuite.startPostgreSQLContainer(TEST_DB_NAME);

catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java

@@ -15,17 +15,13 @@
 import com.datastrato.gravitino.integration.test.container.MySQLContainer;
 import com.datastrato.gravitino.integration.test.container.PostgreSQLContainer;
 import com.datastrato.gravitino.integration.test.util.AbstractIT;
-import com.datastrato.gravitino.integration.test.util.ITUtils;
-import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
 import com.datastrato.gravitino.integration.test.util.TestDatabaseName;
 import com.datastrato.gravitino.rel.Column;
 import com.datastrato.gravitino.rel.types.Types;
 import com.datastrato.gravitino.utils.RandomNameUtils;
 import com.google.common.collect.Maps;
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.util.Collections;
 import java.util.Map;
@@ -42,34 +38,10 @@ public class TestMultipleJDBCLoad extends AbstractIT {
   private static MySQLContainer mySQLContainer;
   private static PostgreSQLContainer postgreSQLContainer;
 
-  private static final String DOWNLOAD_MYSQL_JDBC_DRIVER_URL =
-      "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar";
   public static final String DEFAULT_POSTGRES_IMAGE = "postgres:13";
-  public static final String DOWNLOAD_PG_JDBC_DRIVER_URL =
-      "https://jdbc.postgresql.org/download/postgresql-42.7.0.jar";
 
   @BeforeAll
   public static void startup() throws IOException {
-    String gravitinoHome = System.getenv("GRAVITINO_HOME");
-
-    // Deploy mode
-    if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) {
-      Path icebergLibsPath = Paths.get(gravitinoHome, "/catalogs/lakehouse-iceberg/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString());
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_PG_JDBC_DRIVER_URL, icebergLibsPath.toString());
-    } else {
-      // embedded mode
-      Path icebergLibsPath =
-          Paths.get(gravitinoHome, "/catalogs/catalog-lakehouse-iceberg/build/libs");
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_MYSQL_JDBC_DRIVER_URL, icebergLibsPath.toString());
-
-      JdbcDriverDownloader.downloadJdbcDriver(
-          DOWNLOAD_PG_JDBC_DRIVER_URL, icebergLibsPath.toString());
-    }
-
     containerSuite.startMySQLContainer(TEST_DB_NAME);
     mySQLContainer = containerSuite.getMySQLContainer();
     containerSuite.startPostgreSQLContainer(TEST_DB_NAME);

gradle/libs.versions.toml

@@ -13,8 +13,9 @@ log4j = "2.22.0"
 jetty = "9.4.51.v20230217"
 jersey = "2.41"
 mockito = "4.11.0"
-airlift-units = "1.8"
+airlift-json = "237"
 airlift-log = "231"
+airlift-units = "1.8"
 hive2 = "2.3.9"
 hadoop2 = "2.10.2"
 hadoop3 = "3.1.0"
@@ -29,19 +30,17 @@ caffeine = "2.9.3"
 rocksdbjni = "7.10.2"
 iceberg = '1.3.1' # used for Gravitino Iceberg catalog and Iceberg REST service
 iceberg4spark = "1.4.1" # used for compile spark connector
-trino = '426'
-spark33 = "3.3.4" 
+spark33 = "3.3.4"
 spark34 = "3.4.3"
 spark35 = "3.5.1"
 kyuubi4spark33 = "1.7.4"
 kyuubi4spark34 = "1.8.2"
 kyuubi4spark35 = "1.9.0"
+trino = '435'
 scala-collection-compat = "2.7.0"
 scala-java-compat = "1.0.2"
 sqlite-jdbc = "3.42.0.0"
-testng = "7.5.1"
 testcontainers = "1.19.0"
-trino-jdbc = "426"
 jwt = "0.11.1"
 jline = "3.21.0"
 okhttp3 = "4.11.0"
@@ -118,6 +117,7 @@ hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common", version.
 hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client", version.ref = "hadoop3"}
 hadoop3-mapreduce-client-core = { group = "org.apache.hadoop", name = "hadoop-mapreduce-client-core", version.ref = "hadoop3"}
 hadoop3-minicluster = { group = "org.apache.hadoop", name = "hadoop-minicluster", version.ref = "hadoop-minikdc"}
+airlift-json = { group = "io.airlift", name = "json", version.ref = "airlift-json"}
 airlift-units = { group = "io.airlift", name = "units", version.ref = "airlift-units"}
 airlift-log = { group = "io.airlift", name = "log", version.ref = "airlift-log"}
 httpclient5 = { group = "org.apache.httpcomponents.client5", name = "httpclient5", version.ref = "httpclient5" }
@@ -138,13 +138,12 @@ trino-memory= { group = "io.trino", name = "trino-memory", version.ref = "trino"
 trino-cli= { group = "io.trino", name = "trino-cli", version.ref = "trino" }
 trino-client= { group = "io.trino", name = "trino-client", version.ref = "trino" }
 sqlite-jdbc = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite-jdbc" }
-testng = { group = "org.testng", name = "testng", version.ref = "testng" }
 commons-dbcp2 = { group = "org.apache.commons", name = "commons-dbcp2", version.ref = "commons-dbcp2" }
 testcontainers = { group = "org.testcontainers", name = "testcontainers", version.ref = "testcontainers" }
 testcontainers-mysql = { group = "org.testcontainers", name = "mysql", version.ref = "testcontainers" }
 testcontainers-postgresql = { group = "org.testcontainers", name = "postgresql", version.ref = "testcontainers" }
 testcontainers-junit-jupiter = { group = "org.testcontainers", name = "junit-jupiter", version.ref = "testcontainers" }
-trino-jdbc = { group = "io.trino", name = "trino-jdbc", version.ref = "trino-jdbc" }
+trino-jdbc = { group = "io.trino", name = "trino-jdbc", version.ref = "trino" }
 jwt-api = { group = "io.jsonwebtoken", name = "jjwt-api", version.ref = "jwt"}
 jwt-impl = { group = "io.jsonwebtoken", name = "jjwt-impl", version.ref = "jwt"}
 jwt-gson = { group = "io.jsonwebtoken", name = "jjwt-gson", version.ref = "jwt"}