support v3.x

settings.gradle.kts

@@ -28,7 +28,13 @@ include(
   "clients:client-python"
 )
 include("trino-connector")
-include("spark-connector:spark-connector-common", "spark-connector:spark3.3", "spark-connector:spark3.4", "spark-connector:spark3.5", "spark-connector:spark-connector-runtime")
+include("spark-connector:spark-common", "spark-connector:spark33", "spark-connector:spark33-runtime", "spark-connector:spark34", "spark-connector:spark34-runtime", "spark-connector:spark35", "spark-connector:spark35-runtime")
+project(":spark-connector:spark33").projectDir = file("spark-connector/v3.3/spark")
+project(":spark-connector:spark33-runtime").projectDir = file("spark-connector/v3.3/spark-runtime")
+project(":spark-connector:spark34").projectDir = file("spark-connector/v3.4/spark")
+project(":spark-connector:spark34-runtime").projectDir = file("spark-connector/v3.4/spark-runtime")
+project(":spark-connector:spark35").projectDir = file("spark-connector/v3.5/spark")
+project(":spark-connector:spark35-runtime").projectDir = file("spark-connector/v3.5/spark-runtime")
 include("flink-connector")
 include("web")
 include("docs")

spark-connector/spark-connector-common/src/main/java/com/datastrato/gravitino/spark/connector/SparkTableChangeConverter.java → spark-connector/spark-common/src/main/java/com/datastrato/gravitino/spark/connector/SparkTableChangeConverter.java

@@ -17,10 +17,17 @@ public SparkTableChangeConverter(SparkTypeConverter sparkTypeConverter) {
   public com.datastrato.gravitino.rel.TableChange toGravitinoTableChange(TableChange change) {
     if (change instanceof TableChange.SetProperty) {
       TableChange.SetProperty setProperty = (TableChange.SetProperty) change;
-      return com.datastrato.gravitino.rel.TableChange.setProperty(
-          setProperty.property(), setProperty.value());
+      if (ConnectorConstants.COMMENT.equals(setProperty.property())) {
+        return com.datastrato.gravitino.rel.TableChange.updateComment(setProperty.value());
+      } else {
+        return com.datastrato.gravitino.rel.TableChange.setProperty(
+            setProperty.property(), setProperty.value());
+      }
     } else if (change instanceof TableChange.RemoveProperty) {
       TableChange.RemoveProperty removeProperty = (TableChange.RemoveProperty) change;
+      Preconditions.checkArgument(
+          ConnectorConstants.COMMENT.equals(removeProperty.property()) == false,
+          "Gravitino doesn't support remove table comment yet");
       return com.datastrato.gravitino.rel.TableChange.removeProperty(removeProperty.property());
     } else if (change instanceof TableChange.AddColumn) {
       TableChange.AddColumn addColumn = (TableChange.AddColumn) change;

spark-connector/spark-connector-common/src/test/java/com/datastrato/gravitino/spark/connector/catalog/TestTransformTableChange.java → spark-connector/spark-common/src/test/java/com/datastrato/gravitino/spark/connector/catalog/TestTransformTableChange.java

@@ -47,15 +47,15 @@ void testTransformRemoveProperty() {
   void testTransformUpdateComment() {
     TableChange sparkSetProperty = TableChange.setProperty(ConnectorConstants.COMMENT, "a");
     com.datastrato.gravitino.rel.TableChange tableChange =
-        BaseCatalog.transformTableChange(sparkSetProperty);
+        sparkTableChangeConverter.toGravitinoTableChange(sparkSetProperty);
     Assertions.assertTrue(
         tableChange instanceof com.datastrato.gravitino.rel.TableChange.UpdateComment);
     Assertions.assertEquals("a", ((UpdateComment) tableChange).getNewComment());
 
     TableChange sparkRemoveProperty = TableChange.removeProperty(ConnectorConstants.COMMENT);
     Assertions.assertThrowsExactly(
         IllegalArgumentException.class,
-        () -> BaseCatalog.transformTableChange(sparkRemoveProperty));
+        () -> sparkTableChangeConverter.toGravitinoTableChange(sparkRemoveProperty));
   }
 
   @Test

spark-connector/v3.3/build.gradle.kts

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+
+tasks.all {
+    enabled = false
+}

spark-connector/spark-connector-runtime/build.gradle.kts → spark-connector/v3.3/spark-runtime/build.gradle.kts

@@ -11,31 +11,20 @@ plugins {
 }
 
 val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
-val sparkVersion: String = project.properties["sparkVersion"] as? String ?: extra["defaultSparkVersion"].toString()
-val baseName = "${rootProject.name}-spark-connector-runtime-${sparkVersion}_$scalaVersion"
+val sparkVersion: String = libs.versions.spark33.get()
+val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
+val baseName = "${rootProject.name}-spark-connector-runtime-${sparkMajorVersion}_$scalaVersion"
 
 dependencies {
   implementation(project(":clients:client-java-runtime", configuration = "shadow"))
-  when (sparkVersion) {
+  when (sparkMajorVersion) {
     "3.3" -> {
       val kyuubiVersion: String = libs.versions.kyuubi4spark33.get()
       println("Applying Spark 3.3 dependencies")
-      implementation(project(":spark-connector:spark3.3"))
+      implementation(project(":spark-connector:spark33"))
       implementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
     }
-    "3.4" -> {
-      val kyuubiVersion: String = libs.versions.kyuubi4spark34.get()
-      println("Applying Spark 3.4 dependencies")
-      implementation(project(":spark-connector:spark3.4"))
-      implementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
-    }
-    "3.5" -> {
-      val kyuubiVersion: String = libs.versions.kyuubi4spark35.get()
-      println("Applying Spark 3.5 dependencies")
-      implementation(project(":spark-connector:spark3.5"))
-      implementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
-    }
-    else -> throw IllegalArgumentException("Unsupported Spark version: $sparkVersion")
+    else -> throw IllegalArgumentException("Unsupported Spark version: $sparkMajorVersion")
   }
 }
 

spark-connector/spark3.3/build.gradle.kts → spark-connector/v3.3/spark/build.gradle.kts

@@ -22,7 +22,7 @@ val scalaJava8CompatVersion: String = libs.versions.scala.java.compat.get()
 val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()
 
 dependencies {
-  implementation(project(":spark-connector:spark-connector-common"))
+  implementation(project(":spark-connector:spark-common"))
   compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
   compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
     exclude("com.fasterxml.jackson")
@@ -48,7 +48,7 @@ dependencies {
   testImplementation(project(":server-common")) {
     exclude("org.apache.logging.log4j")
   }
-  testImplementation(project(":spark-connector:spark-connector-common", "testArtifacts")) {
+  testImplementation(project(":spark-connector:spark-common", "testArtifacts")) {
     exclude("com.fasterxml.jackson")
   }
 

spark-connector/v3.4/build.gradle.kts

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+
+tasks.all {
+    enabled = false
+}

spark-connector/v3.4/spark-runtime/build.gradle.kts

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+  `maven-publish`
+  id("java")
+  alias(libs.plugins.shadow)
+}
+
+val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
+val sparkVersion: String = libs.versions.spark34.get()
+val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
+val baseName = "${rootProject.name}-spark-connector-runtime-${sparkMajorVersion}_$scalaVersion"
+
+dependencies {
+  implementation(project(":clients:client-java-runtime", configuration = "shadow"))
+  when (sparkMajorVersion) {
+    "3.4" -> {
+      val kyuubiVersion: String = libs.versions.kyuubi4spark34.get()
+      println("Applying Spark 3.4 dependencies")
+      implementation(project(":spark-connector:spark34"))
+      implementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
+    }
+    else -> throw IllegalArgumentException("Unsupported Spark version: $sparkMajorVersion")
+  }
+}
+
+tasks.withType<ShadowJar>(ShadowJar::class.java) {
+  isZip64 = true
+  configurations = listOf(project.configurations.runtimeClasspath.get())
+  archiveFileName.set("$baseName-$version.jar")
+  archiveClassifier.set("")
+
+  // Relocate dependencies to avoid conflicts
+  relocate("com.google", "com.datastrato.gravitino.shaded.com.google")
+  relocate("google", "com.datastrato.gravitino.shaded.google")
+  relocate("org.apache.hc", "com.datastrato.gravitino.shaded.org.apache.hc")
+}
+
+tasks.jar {
+  dependsOn(tasks.named("shadowJar"))
+  archiveClassifier.set("empty")
+}

spark-connector/spark3.4/build.gradle.kts → spark-connector/v3.4/spark/build.gradle.kts

@@ -22,7 +22,7 @@ val scalaJava8CompatVersion: String = libs.versions.scala.java.compat.get()
 val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()
 
 dependencies {
-  implementation(project(":spark-connector:spark-connector-common"))
+  implementation(project(":spark-connector:spark-common"))
   compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
   compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
     exclude("com.fasterxml.jackson")
@@ -49,7 +49,7 @@ dependencies {
   testImplementation(project(":server-common")) {
     exclude("org.apache.logging.log4j")
   }
-  testImplementation(project(":spark-connector:spark-connector-common", "testArtifacts")) {
+  testImplementation(project(":spark-connector:spark-common", "testArtifacts")) {
     exclude("com.fasterxml.jackson")
   }
 

spark-connector/v3.5/build.gradle.kts

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+
+tasks.all {
+    enabled = false
+}

spark-connector/v3.5/spark-runtime/build.gradle.kts

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+  `maven-publish`
+  id("java")
+  alias(libs.plugins.shadow)
+}
+
+val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
+val sparkVersion: String = libs.versions.spark35.get()
+val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
+val baseName = "${rootProject.name}-spark-connector-runtime-${sparkMajorVersion}_$scalaVersion"
+
+dependencies {
+  implementation(project(":clients:client-java-runtime", configuration = "shadow"))
+  when (sparkMajorVersion) {
+    "3.5" -> {
+      val kyuubiVersion: String = libs.versions.kyuubi4spark35.get()
+      println("Applying Spark 3.5 dependencies")
+      implementation(project(":spark-connector:spark35"))
+      implementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
+    }
+    else -> throw IllegalArgumentException("Unsupported Spark version: $sparkMajorVersion")
+  }
+}
+
+tasks.withType<ShadowJar>(ShadowJar::class.java) {
+  isZip64 = true
+  configurations = listOf(project.configurations.runtimeClasspath.get())
+  archiveFileName.set("$baseName-$version.jar")
+  archiveClassifier.set("")
+
+  // Relocate dependencies to avoid conflicts
+  relocate("com.google", "com.datastrato.gravitino.shaded.com.google")
+  relocate("google", "com.datastrato.gravitino.shaded.google")
+  relocate("org.apache.hc", "com.datastrato.gravitino.shaded.org.apache.hc")
+}
+
+tasks.jar {
+  dependsOn(tasks.named("shadowJar"))
+  archiveClassifier.set("empty")
+}

spark-connector/spark3.5/build.gradle.kts → spark-connector/v3.5/spark/build.gradle.kts

@@ -6,6 +6,9 @@ plugins {
   `maven-publish`
   id("java")
   id("idea")
+  id("com.dorongold.task-tree") version "2.1.1"
+  //  id("org.barfuin.gradle.taskinfo") version "2.1.0"
+
   alias(libs.plugins.shadow)
 }
 
@@ -22,8 +25,8 @@ val scalaJava8CompatVersion: String = libs.versions.scala.java.compat.get()
 val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()
 
 dependencies {
-  implementation(project(":spark-connector:spark3.4"))
-  implementation(project(":spark-connector:spark-connector-common"))
+  implementation(project(":spark-connector:spark34"))
+  implementation(project(":spark-connector:spark-common"))
   compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
   compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
     exclude("com.fasterxml.jackson")
@@ -50,7 +53,7 @@ dependencies {
   testImplementation(project(":server-common")) {
     exclude("org.apache.logging.log4j")
   }
-  testImplementation(project(":spark-connector:spark-connector-common", "testArtifacts")) {
+  testImplementation(project(":spark-connector:spark-common", "testArtifacts")) {
     exclude("com.fasterxml.jackson")
   }
 

spark-connector/spark3.5/src/test/java/com/datastrato/gravitino/spark/connector/version/TestCatalogNameAdaptor.java → spark-connector/v3.5/spark/src/test/java/com/datastrato/gravitino/spark/connector/version/TestCatalogNameAdaptor.java

@@ -11,7 +11,7 @@
 
 public class TestCatalogNameAdaptor {
   @Test
-  void testSpark34() {
+  void testSpark35() {
     String hiveCatalogName = CatalogNameAdaptor.getCatalogName("hive");
     Assertions.assertEquals(GravitinoHiveCatalogSpark35.class.getName(), hiveCatalogName);