diff --git a/jenkins/Jenkinsfile-blossom.premerge b/jenkins/Jenkinsfile-blossom.premerge index dfc8b90cf0c..d9ef63aaa84 100644 --- a/jenkins/Jenkinsfile-blossom.premerge +++ b/jenkins/Jenkinsfile-blossom.premerge @@ -574,7 +574,7 @@ void databricksBuild() { withCredentials([file(credentialsId: 'SPARK_DATABRICKS_PRIVKEY', variable: 'DATABRICKS_PRIVKEY')]) { def BUILD_PARAMS = " -w $DATABRICKS_HOST -t $DATABRICKS_TOKEN -c $CLUSTER_ID -z ./spark-rapids-ci.tgz" + " -p $DATABRICKS_PRIVKEY -l ./jenkins/databricks/build.sh -d /home/ubuntu/build.sh" + - " -b $BUILD_PROFILES -v $BASE_SPARK_VERSION -i $BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS" + " -v $BASE_SPARK_VERSION -i $BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS" sh "python3 ./jenkins/databricks/run-build.py $BUILD_PARAMS" } } diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh index db426897075..1aa51bc7756 100755 --- a/jenkins/databricks/build.sh +++ b/jenkins/databricks/build.sh @@ -20,8 +20,10 @@ set -ex SPARKSRCTGZ=$1 # version of Apache Spark we are building against BASE_SPARK_VERSION=$2 -MVN_OPT=$3 -BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS=$4 +BASE_SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS=$3 + +# Move MVN_OPT to last, as it is empty in most cases +MVN_OPT=$4 MVN_OPT=${MVN_OPT:-''} BASE_SPARK_VERSION=${BASE_SPARK_VERSION:-'3.1.2'} BUILDVER=$(echo ${BASE_SPARK_VERSION} | sed 's/\.//g')db @@ -442,7 +444,7 @@ mvn -B install:install-file \ -Dversion=$SPARK_VERSION_TO_INSTALL_DATABRICKS_JARS \ -Dpackaging=jar -mvn -B -Ddatabricks -Dbuildver=$BUILDVER clean package -DskipTests $MVN_OPT +mvn -B -Ddatabricks -Dbuildver=$BUILDVER clean package -DskipTests cd /home/ubuntu tar -zcf spark-rapids-built.tgz spark-rapids diff --git a/jenkins/databricks/params.py b/jenkins/databricks/params.py index e86973ddf39..5e03a66090d 100644 --- a/jenkins/databricks/params.py +++ b/jenkins/databricks/params.py @@ -26,7 +26,7 @@ base_spark_pom_version = '3.1.1' base_spark_version_to_install_databricks_jars = base_spark_pom_version clusterid = '' -build_profiles = 'databricks,!snapshot-shims' +build_profiles = '' jar_path = '' # `spark_conf` can take comma seperated multiple spark configurations, e.g., spark.foo=1,spark.bar=2,...' spark_conf = '' diff --git a/jenkins/databricks/run-build.py b/jenkins/databricks/run-build.py index 042b872e34a..ca456f3f0cb 100644 --- a/jenkins/databricks/run-build.py +++ b/jenkins/databricks/run-build.py @@ -38,7 +38,7 @@ def main(): print("rsync command: %s" % rsync_command) subprocess.check_call(rsync_command, shell = True) - ssh_command = "bash -c 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@%s -p 2200 -i %s %s %s %s %s %s 2>&1 | tee buildout; if [ `echo ${PIPESTATUS[0]}` -ne 0 ]; then false; else true; fi'" % (master_addr, params.private_key_file, params.script_dest, params.tgz_dest, params.base_spark_pom_version, params.build_profiles, params.base_spark_version_to_install_databricks_jars) + ssh_command = "bash -c 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@%s -p 2200 -i %s %s %s %s %s %s 2>&1 | tee buildout; if [ `echo ${PIPESTATUS[0]}` -ne 0 ]; then false; else true; fi'" % (master_addr, params.private_key_file, params.script_dest, params.tgz_dest, params.base_spark_pom_version, params.base_spark_version_to_install_databricks_jars, params.build_profiles) print("ssh command: %s" % ssh_command) subprocess.check_call(ssh_command, shell = True)