remove common and seperate ut versions to two parts

Signed-off-by: YanxuanLiu <[email protected]>
NVIDIA · Sep 14, 2022 · f63e2e7 · f63e2e7
1 parent aa8a015
commit f63e2e7
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 38 deletions.
diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml
@@ -42,8 +42,6 @@ jobs:
         id: noSnapshotVersionsStep
         run: |
           set -x
-          # SCRIPT_PATH is to avoid losing script path for wrapper calling common.sh in version-def.sh
-          export SCRIPT_PATH=$(pwd -P)/jenkins
           . jenkins/version-def.sh
           svArrBody=$(printf ",{\"spark-version\":\"%s\"}" "${TAIL_NOSNAPSHOTS_VERSIONS[@]}")
           svArrBody=${svArrBody:1}

diff --git a/dist/pom.xml b/dist/pom.xml
@@ -63,10 +63,12 @@
           base shim version (311 currently) should be covered in default mvn verify command of premerge script, 
           so base shim version is removed from the premergeUT list
         -->
-        <premergeUT.buildvers>
-            320,
+        <premergeUT1.buildvers>
+            320
+        </premergeUT1.buildvers>
+        <premergeUT2.buildvers>
             330
-        </premergeUT.buildvers>
+        </premergeUT2.buildvers>
         <dist.jar.name>${project.build.directory}/${project.build.finalName}-${cuda.version}.jar</dist.jar.name>
     </properties>
     <profiles>
@@ -88,10 +90,18 @@
             </properties>
         </profile>
         <profile>
-            <id>premergeUT</id>
+            <id>premergeUT1</id>
+            <properties>
+                <included_buildvers>
+                    ${premergeUT1.buildvers}
+                </included_buildvers>
+            </properties>
+        </profile>
+        <profile>
+            <id>premergeUT2</id>
             <properties>
                 <included_buildvers>
-                    ${premergeUT.buildvers}
+                    ${premergeUT2.buildvers}
                 </included_buildvers>
             </properties>
         </profile>

diff --git a/jenkins/common.sh b/jenkins/common.sh
diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh
@@ -45,7 +45,8 @@ mvn_verify() {
     do
         echo "Spark version: $version"
         # build and run unit tests on one specific version for each sub-version (e.g. 320, 330) except base version
-        if [[ "${SPARK_SHIM_VERSIONS_TEST[@]}" =~ "$version" ]]; then
+        # separate the versions to two ci stages (mvn_verify, ci_2) for balancing the duration
+        if [[ "${SPARK_SHIM_VERSIONS_PREMERGE_UT_1[@]}" =~ "$version" ]]; then
             env -u SPARK_HOME $MVN_CMD -U -B $MVN_URM_MIRROR -Dbuildver=$version clean install $MVN_BUILD_ARGS \
               -Dpytest.TEST_TAGS='' -pl '!tools'
         # build only for nosnapshot versions
@@ -138,8 +139,11 @@ ci_2() {
 
     # put some mvn tests here to balance durations of parallel stages
     echo "Run mvn package..."
-    env -u SPARK_HOME $MVN_CMD -U -B $MVN_URM_MIRROR -Dbuildver=320 clean package $MVN_BUILD_ARGS \
-      -Dpytest.TEST_TAGS='' -pl '!tools'
+    for version in "${SPARK_SHIM_VERSIONS_PREMERGE_UT_2[@]}"
+    do
+        env -u SPARK_HOME $MVN_CMD -U -B $MVN_URM_MIRROR -Dbuildver=$version clean package $MVN_BUILD_ARGS \
+          -Dpytest.TEST_TAGS='' -pl '!tools'
+    done
 }
 
 

diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh
@@ -61,9 +61,13 @@ echo "CUDF_VER: $CUDF_VER, CUDA_CLASSIFIER: $CUDA_CLASSIFIER, PROJECT_VER: $PROJ
     SPARK_VER: $SPARK_VER, SCALA_BINARY_VER: $SCALA_BINARY_VER"
 
 # Spark shim versions
-# SCRIPT_PATH is to avoid losing script path in wrapper calling. It should be exported if wrapper call existed.
-SCRIPT_PATH=${SCRIPT_PATH:-$(pwd -P)/jenkins}
-. $SCRIPT_PATH/common.sh
+# get Spark shim versions from pom
+function set_env_var_SPARK_SHIM_VERSIONS_ARR() {
+    PROFILE_OPT=$1
+    SPARK_SHIM_VERSIONS_STR=$(mvn -B help:evaluate -q -pl dist $PROFILE_OPT -Dexpression=included_buildvers -DforceStdout)
+    SPARK_SHIM_VERSIONS_STR=$(echo $SPARK_SHIM_VERSIONS_STR)
+    IFS=", " <<< $SPARK_SHIM_VERSIONS_STR read -r -a SPARK_SHIM_VERSIONS_ARR
+}
 # Psnapshots: snapshots + noSnapshots
 set_env_var_SPARK_SHIM_VERSIONS_ARR -Psnapshots
 SPARK_SHIM_VERSIONS_SNAPSHOTS=("${SPARK_SHIM_VERSIONS_ARR[@]}")
@@ -85,7 +89,10 @@ SPARK_BASE_SHIM_VERSION=${SPARK_SHIM_VERSIONS[0]}
 # tail noSnapshots
 TAIL_NOSNAPSHOTS_VERSIONS=("${SPARK_SHIM_VERSIONS_NOSNAPSHOTS[@]:1}")
 # build and run unit tests on one specific version for each sub-version (e.g. 320, 330)
-set_env_var_SPARK_SHIM_VERSIONS_ARR -PpremergeUT
-SPARK_SHIM_VERSIONS_TEST=("${SPARK_SHIM_VERSIONS_ARR[@]}")
+# separate the versions to two parts (premergeUT1, premergeUT2) for balancing the duration
+set_env_var_SPARK_SHIM_VERSIONS_ARR -PpremergeUT1
+SPARK_SHIM_VERSIONS_PREMERGE_UT_1=("${SPARK_SHIM_VERSIONS_ARR[@]}")
+set_env_var_SPARK_SHIM_VERSIONS_ARR -PpremergeUT2
+SPARK_SHIM_VERSIONS_PREMERGE_UT_2=("${SPARK_SHIM_VERSIONS_ARR[@]}")
 
 echo "SPARK_BASE_SHIM_VERSION: $SPARK_BASE_SHIM_VERSION"