Use dist/pom file as source of truth for spark versions

jenkins/common.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright (c) 2020-2022, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function get_spark_shim_versions() {
+    PROFILE_OPT=$1
+    SPARK_SHIM_VERSIONS_STR=$(mvn -B help:evaluate -q -pl dist $PROFILE_OPT -Dexpression=included_buildvers -DforceStdout)
+    SPARK_SHIM_VERSIONS_STR=$(echo $SPARK_SHIM_VERSIONS_STR)
+    PRE_IFS=$IFS
+    IFS=", " <<< $SPARK_SHIM_VERSIONS_STR read -r -a SPARK_SHIM_VERSIONS
+    IFS=$PRE_IFS
+}

jenkins/spark-premerge-build.sh

@@ -23,7 +23,7 @@ if [[ $# -eq 1 ]]; then
     BUILD_TYPE=$1
 
 elif [[ $# -gt 1 ]]; then
-    echo "ERROR: too many parameters are provided"
+    >&2 echo "ERROR: too many parameters are provided"
     exit 1
 fi
 
@@ -37,27 +37,42 @@ mvn_verify() {
     # file size check for pull request. The size of a committed file should be less than 1.5MiB
     pre-commit run check-added-large-files --from-ref $BASE_REF --to-ref HEAD
 
+    # Get Spark shim versions
+    . $(dirname "$0")/common.sh
+    # snapshots + noSnapshots
+    get_spark_shim_versions -Psnapshots
+    SPARK_SHIM_VERSIONS_ALL=("${SPARK_SHIM_VERSIONS[@]}")
+    # noSnapshots only
+    get_spark_shim_versions -PnoSnapshots
+    SPARK_SHIM_VERSIONS_NOSNAPSHOTS=("${SPARK_SHIM_VERSIONS[@]}")
+
+    # build and run unit tests on one 3.1.X version (base version covers this), one 3.2.X and one 3.3.X version
+    SPARK_SHIM_VERSIONS_TEST=($(for version in ${SPARK_SHIM_VERSIONS_ALL[@]}; do [[ $version != 31* ]] && echo $version; done | sort -n -k1.1,1.2 -u -s))
+
     # build the Spark 2.x explain jar
     env -u SPARK_HOME $MVN_CMD -B $MVN_URM_MIRROR -Dbuildver=24X clean install -DskipTests
 
     MVN_INSTALL_CMD="env -u SPARK_HOME $MVN_CMD -U -B $MVN_URM_MIRROR clean install $MVN_BUILD_ARGS -DskipTests -pl aggregator -am"
-    # build all the versions but only run unit tests on one 3.1.X version (base version covers this), one 3.2.X and one 3.3.X version.
     # All others shims test should be covered in nightly pipelines
-    $MVN_INSTALL_CMD -DskipTests -Dbuildver=321cdh
-    $MVN_INSTALL_CMD -DskipTests -Dbuildver=312
-    $MVN_INSTALL_CMD -DskipTests -Dbuildver=313
-    [[ $BUILD_MAINTENANCE_VERSION_SNAPSHOTS == "true" ]] && $MVN_INSTALL_CMD -Dbuildver=314
-
-    $MVN_INSTALL_CMD -DskipTests -Dbuildver=320
+    for version in "${SPARK_SHIM_VERSIONS_ALL[@]}"
+    do
+        echo "Spark version: $version"
+        # build and run unit test
+        if [[ "${SPARK_SHIM_VERSIONS_TEST[@]}" =~ "$version" ]]; then
+            env -u SPARK_HOME $MVN_CMD -U -B $MVN_URM_MIRROR -Dbuildver=$version clean install $MVN_BUILD_ARGS \
+              -Dpytest.TEST_TAGS='' -pl '!tools'
+        # build only for nosnapshot versions
+        elif [[ "${SPARK_SHIM_VERSIONS_NOSNAPSHOTS[@]}" =~ "$version" ]]; then
+            $MVN_INSTALL_CMD -DskipTests -Dbuildver=$version
+        # build only for snapshot versions
+        elif [[ $BUILD_MAINTENANCE_VERSION_SNAPSHOTS == "true" ]]; then
+            $MVN_INSTALL_CMD -Dbuildver=$version
+        fi
+    done
     # enable UTF-8 for regular expression tests
     env -u SPARK_HOME LC_ALL="en_US.UTF-8" $MVN_CMD $MVN_URM_MIRROR -Dbuildver=320 test $MVN_BUILD_ARGS \
       -Dpytest.TEST_TAGS='' -pl '!tools' \
       -DwildcardSuites=com.nvidia.spark.rapids.ConditionalsSuite,com.nvidia.spark.rapids.RegularExpressionSuite,com.nvidia.spark.rapids.RegularExpressionTranspilerSuite
-    $MVN_INSTALL_CMD -DskipTests -Dbuildver=321
-    $MVN_INSTALL_CMD -DskipTests -Dbuildver=322
-    env -u SPARK_HOME $MVN_CMD -U -B $MVN_URM_MIRROR -Dbuildver=330 clean install $MVN_BUILD_ARGS \
-      -Dpytest.TEST_TAGS='' -pl '!tools'
-    [[ $BUILD_MAINTENANCE_VERSION_SNAPSHOTS == "true" ]] && $MVN_INSTALL_CMD -DskipTests -Dbuildver=331
     # TODO: move it to BUILD_MAINTENANCE_VERSION_SNAPSHOTS when we resolve all spark340 build issues
     [[ $BUILD_FEATURE_VERSION_SNAPSHOTS == "true" ]] && $MVN_INSTALL_CMD -DskipTests -Dbuildver=340
 

jenkins/version-def.sh

@@ -17,6 +17,18 @@
 
 set -e
 
+# PHASE_TYPE: CICD phase at which the script is called, to specify Spark shim versions.
+# regular: noSnapshots + snapshots
+# pre-release: noSnapshots only
+PHASE_TYPE=regular
+
+if [[ $# -eq 1 ]]; then
+    PHASE_TYPE=$1
+elif [[ $# -gt 1 ]]; then
+    >&2 echo "ERROR: too many parameters are provided"
+    exit 1
+fi
+
 # Split abc=123 from $OVERWRITE_PARAMS
 # $OVERWRITE_PARAMS patten 'abc=123;def=456;'
 PRE_IFS=$IFS
@@ -48,9 +60,17 @@ SPARK_REPO=${SPARK_REPO:-"$URM_URL"}
 echo "CUDF_VER: $CUDF_VER, CUDA_CLASSIFIER: $CUDA_CLASSIFIER, PROJECT_VER: $PROJECT_VER \
     SPARK_VER: $SPARK_VER, SCALA_BINARY_VER: $SCALA_BINARY_VER"
 
+case $PHASE_TYPE in
 
-SPARK_SHIM_VERSIONS_STR=${SPARK_SHIM_VERSIONS_STR:-"311 321cdh 312 313 314 320 321 322 330 331"}
+    pre-release)
+        PROFILE_OPT="-Psnapshots" 
+        ;;
 
-IFS=" " <<< $SPARK_SHIM_VERSIONS_STR read -r -a SPARK_SHIM_VERSIONS
+    *)
+        PROFILE_OPT="-PnoSnapshots" 
+        ;;
+esac
 
-SPARK_BASE_SHIM_VERSION=${SPARK_SHIM_VERSIONS[0]}
+. $(dirname "$0")/common.sh
+get_spark_shim_versions $PROFILE_OPT
+SPARK_BASE_SHIM_VERSION=${SPARK_SHIM_VERSIONS[0]}