From e1cf4971620708fe22a89e2924e0e1c20a2d15a1 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Fri, 11 Aug 2023 17:17:43 +0000 Subject: [PATCH 1/3] Create a two-shim jar by default on Databricks Fixes #9008 Signed-off-by: Gera Shegalov --- jenkins/databricks/build.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh index 15f8777ab44..7f25c97763e 100755 --- a/jenkins/databricks/build.sh +++ b/jenkins/databricks/build.sh @@ -93,6 +93,9 @@ initialize() SKIP_DEP_INSTALL=${SKIP_DEP_INSTALL:-'0'} # export 'M2DIR' so that shims can get the correct Spark dependency info export M2DIR=/home/ubuntu/.m2/repository + # whether to build a two-shim jar with the lowest supported upstream Spark version + WITH_DEFAULT_UPSTREAM_SHIM=${WITH_DEFAULT_UPSTREAM_SHIM:-1} + # Print a banner of the build configurations. printf '+ %*s +\n' 100 '' | tr ' ' = @@ -108,6 +111,7 @@ initialize() echo "Rapids build jar : ${RAPIDS_BUILT_JAR}" echo "Build Version : ${BUILDVER}" echo "Skip Dependencies : ${SKIP_DEP_INSTALL}" + echo "Include Default Spark Shim : ${WITH_DEFAULT_UPSTREAM_SHIM}" printf '+ %*s +\n' 100 '' | tr ' ' = } @@ -143,5 +147,10 @@ fi # Build the RAPIDS plugin by running package command for databricks mvn -B -Ddatabricks -Dbuildver=$BUILDVER clean package -DskipTests $MVN_OPT +if [[ "$WITH_DEFAULT_UPSTREAM_SHIM" != "0" ]]; then + echo "Building the default Spark shim and creatign a two-shim dist jar" + mvn -B package -pl dist -am -Dincluded_buildvers=311,$BUILDVER -DskipTests -Dskip $MVN_OPT +fi + cd /home/ubuntu tar -zcf spark-rapids-built.tgz spark-rapids From d6c521fe08e289f28cbd612a30ddbabd02552d52 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Fri, 11 Aug 2023 10:42:35 -0700 Subject: [PATCH 2/3] Update jenkins/databricks/build.sh Co-authored-by: Thomas Graves --- jenkins/databricks/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh index 7f25c97763e..66a8353d141 100755 --- a/jenkins/databricks/build.sh +++ b/jenkins/databricks/build.sh @@ -148,7 +148,7 @@ fi mvn -B -Ddatabricks -Dbuildver=$BUILDVER clean package -DskipTests $MVN_OPT if [[ "$WITH_DEFAULT_UPSTREAM_SHIM" != "0" ]]; then - echo "Building the default Spark shim and creatign a two-shim dist jar" + echo "Building the default Spark shim and creating a two-shim dist jar" mvn -B package -pl dist -am -Dincluded_buildvers=311,$BUILDVER -DskipTests -Dskip $MVN_OPT fi From 3805fd1226945dd36a07f1af2b38b8b4c36b4bb9 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Fri, 11 Aug 2023 17:45:36 +0000 Subject: [PATCH 3/3] remove hardcoding for 311 --- jenkins/databricks/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh index 7f25c97763e..8e97c4a0359 100755 --- a/jenkins/databricks/build.sh +++ b/jenkins/databricks/build.sh @@ -145,11 +145,12 @@ if [[ "$WITH_BLOOP" == "1" ]]; then fi # Build the RAPIDS plugin by running package command for databricks -mvn -B -Ddatabricks -Dbuildver=$BUILDVER clean package -DskipTests $MVN_OPT +$MVN_CMD -B -Ddatabricks -Dbuildver=$BUILDVER clean package -DskipTests $MVN_OPT if [[ "$WITH_DEFAULT_UPSTREAM_SHIM" != "0" ]]; then echo "Building the default Spark shim and creatign a two-shim dist jar" - mvn -B package -pl dist -am -Dincluded_buildvers=311,$BUILDVER -DskipTests -Dskip $MVN_OPT + UPSTREAM_BUILDVER=$($MVN_CMD help:evaluate -q -pl dist -Dexpression=buildver -DforceStdout) + $MVN_CMD -B package -pl dist -am -Dincluded_buildvers=$UPSTREAM_BUILDVER,$BUILDVER -DskipTests -Dskip $MVN_OPT fi cd /home/ubuntu