Skip to content

Commit

Permalink
support v3.x
Browse files Browse the repository at this point in the history
  • Loading branch information
FANNG1 committed May 27, 2024
1 parent 4d31464 commit f39ba94
Show file tree
Hide file tree
Showing 72 changed files with 146 additions and 8 deletions.
2 changes: 1 addition & 1 deletion settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ include(
"clients:client-python"
)
include("trino-connector")
include("spark-connector:spark-connector-common", "spark-connector:spark3.3", "spark-connector:spark3.4", "spark-connector:spark3.5", "spark-connector:spark-connector-runtime")
include("spark-connector:common", "spark-connector:v3.3:spark", "spark-connector:v3.3:spark-runtime", "spark-connector:v3.4:spark", "spark-connector:v3.4:spark-runtime", "spark-connector:v3.5:spark", "spark-connector:v3.5:spark-runtime")
include("flink-connector")
include("web")
include("docs")
Expand Down
File renamed without changes.
46 changes: 46 additions & 0 deletions spark-connector/v3.3/spark-runtime/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright 2024 Datastrato Pvt Ltd.
* This software is licensed under the Apache License version 2.
*/
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

plugins {
`maven-publish`
id("java")
alias(libs.plugins.shadow)
}

val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
val sparkVersion: String = libs.versions.spark33.get()
val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
val baseName = "${rootProject.name}-spark-connector-runtime-${sparkMajorVersion}_$scalaVersion"

dependencies {
implementation(project(":clients:client-java-runtime", configuration = "shadow"))
when (sparkMajorVersion) {
"3.3" -> {
val kyuubiVersion: String = libs.versions.kyuubi4spark33.get()
println("Applying Spark 3.3 dependencies")
implementation(project(":spark-connector:v3.3:spark"))
implementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
}
else -> throw IllegalArgumentException("Unsupported Spark version: $sparkMajorVersion")
}
}

tasks.withType<ShadowJar>(ShadowJar::class.java) {
isZip64 = true
configurations = listOf(project.configurations.runtimeClasspath.get())
archiveFileName.set("$baseName-$version.jar")
archiveClassifier.set("")

// Relocate dependencies to avoid conflicts
relocate("com.google", "com.datastrato.gravitino.shaded.com.google")
relocate("google", "com.datastrato.gravitino.shaded.google")
relocate("org.apache.hc", "com.datastrato.gravitino.shaded.org.apache.hc")
}

tasks.jar {
dependsOn(tasks.named("shadowJar"))
archiveClassifier.set("empty")
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ val scalaJava8CompatVersion: String = libs.versions.scala.java.compat.get()
val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()

dependencies {
implementation(project(":spark-connector:spark-connector-common"))
implementation(project(":spark-connector:common"))
compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
exclude("com.fasterxml.jackson")
Expand All @@ -48,7 +48,7 @@ dependencies {
testImplementation(project(":server-common")) {
exclude("org.apache.logging.log4j")
}
testImplementation(project(":spark-connector:spark-connector-common", "testArtifacts")) {
testImplementation(project(":spark-connector:common", "testArtifacts")) {
exclude("com.fasterxml.jackson")
}

Expand Down
46 changes: 46 additions & 0 deletions spark-connector/v3.4/spark-runtime/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright 2024 Datastrato Pvt Ltd.
* This software is licensed under the Apache License version 2.
*/
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

plugins {
`maven-publish`
id("java")
alias(libs.plugins.shadow)
}

val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
val sparkVersion: String = libs.versions.spark34.get()
val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
val baseName = "${rootProject.name}-spark-connector-runtime-${sparkMajorVersion}_$scalaVersion"

dependencies {
implementation(project(":clients:client-java-runtime", configuration = "shadow"))
when (sparkMajorVersion) {
"3.4" -> {
val kyuubiVersion: String = libs.versions.kyuubi4spark34.get()
println("Applying Spark 3.4 dependencies")
implementation(project(":spark-connector:v3.4:spark"))
implementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
}
else -> throw IllegalArgumentException("Unsupported Spark version: $sparkMajorVersion")
}
}

tasks.withType<ShadowJar>(ShadowJar::class.java) {
isZip64 = true
configurations = listOf(project.configurations.runtimeClasspath.get())
archiveFileName.set("$baseName-$version.jar")
archiveClassifier.set("")

// Relocate dependencies to avoid conflicts
relocate("com.google", "com.datastrato.gravitino.shaded.com.google")
relocate("google", "com.datastrato.gravitino.shaded.google")
relocate("org.apache.hc", "com.datastrato.gravitino.shaded.org.apache.hc")
}

tasks.jar {
dependsOn(tasks.named("shadowJar"))
archiveClassifier.set("empty")
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ val scalaJava8CompatVersion: String = libs.versions.scala.java.compat.get()
val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()

dependencies {
implementation(project(":spark-connector:spark-connector-common"))
implementation(project(":spark-connector:common"))
compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
exclude("com.fasterxml.jackson")
Expand All @@ -49,7 +49,7 @@ dependencies {
testImplementation(project(":server-common")) {
exclude("org.apache.logging.log4j")
}
testImplementation(project(":spark-connector:spark-connector-common", "testArtifacts")) {
testImplementation(project(":spark-connector:common", "testArtifacts")) {
exclude("com.fasterxml.jackson")
}

Expand Down
46 changes: 46 additions & 0 deletions spark-connector/v3.5/spark-runtime/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright 2024 Datastrato Pvt Ltd.
* This software is licensed under the Apache License version 2.
*/
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

plugins {
`maven-publish`
id("java")
alias(libs.plugins.shadow)
}

val scalaVersion: String = project.properties["scalaVersion"] as? String ?: extra["defaultScalaVersion"].toString()
val sparkVersion: String = libs.versions.spark35.get()
val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
val baseName = "${rootProject.name}-spark-connector-runtime-${sparkMajorVersion}_$scalaVersion"

dependencies {
implementation(project(":clients:client-java-runtime", configuration = "shadow"))
when (sparkMajorVersion) {
"3.5" -> {
val kyuubiVersion: String = libs.versions.kyuubi4spark35.get()
println("Applying Spark 3.5 dependencies")
implementation(project(":spark-connector:v3.5:spark"))
implementation("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
}
else -> throw IllegalArgumentException("Unsupported Spark version: $sparkMajorVersion")
}
}

tasks.withType<ShadowJar>(ShadowJar::class.java) {
isZip64 = true
configurations = listOf(project.configurations.runtimeClasspath.get())
archiveFileName.set("$baseName-$version.jar")
archiveClassifier.set("")

// Relocate dependencies to avoid conflicts
relocate("com.google", "com.datastrato.gravitino.shaded.com.google")
relocate("google", "com.datastrato.gravitino.shaded.google")
relocate("org.apache.hc", "com.datastrato.gravitino.shaded.org.apache.hc")
}

tasks.jar {
dependsOn(tasks.named("shadowJar"))
archiveClassifier.set("empty")
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ val scalaJava8CompatVersion: String = libs.versions.scala.java.compat.get()
val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get()

dependencies {
implementation(project(":spark-connector:spark3.4"))
implementation(project(":spark-connector:spark-connector-common"))
implementation(project(":spark-connector:v3.4:spark"))
implementation(project(":spark-connector:common"))
compileOnly("org.apache.kyuubi:kyuubi-spark-connector-hive_$scalaVersion:$kyuubiVersion")
compileOnly("org.apache.spark:spark-catalyst_$scalaVersion:$sparkVersion") {
exclude("com.fasterxml.jackson")
Expand All @@ -50,7 +50,7 @@ dependencies {
testImplementation(project(":server-common")) {
exclude("org.apache.logging.log4j")
}
testImplementation(project(":spark-connector:spark-connector-common", "testArtifacts")) {
testImplementation(project(":spark-connector:common", "testArtifacts")) {
exclude("com.fasterxml.jackson")
}

Expand Down

0 comments on commit f39ba94

Please sign in to comment.