Revert "Temporarily lower parallelism for NVIDIA#7092"

This reverts commit c67927b.
pxLi · Nov 18, 2022 · 7d8efa2 · 7d8efa2
1 parent c67927b
commit 7d8efa2
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 9 deletions.
diff --git a/jenkins/databricks/test.sh b/jenkins/databricks/test.sh
@@ -98,9 +98,7 @@ export PYSP_TEST_spark_eventLog_enabled=true
 mkdir -p /tmp/spark-events
 
 ## limit parallelism to avoid OOM kill
-# Temporarily lower the parallelism for https://github.com/NVIDIA/spark-rapids/issues/7092
-#export TEST_PARALLEL=4
-export TEST_PARALLEL=2
+export TEST_PARALLEL=4
 if [ -d "$LOCAL_JAR_PATH" ]; then
     if [[ $TEST_MODE == "DEFAULT" ]]; then
         ## Run tests with jars in the LOCAL_JAR_PATH dir downloading from the dependency repo

diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh
@@ -139,9 +139,7 @@ ci_2() {
     $MVN_CMD -U -B $MVN_URM_MIRROR clean package $MVN_BUILD_ARGS -DskipTests=true
     export TEST_TAGS="not premerge_ci_1"
     export TEST_TYPE="pre-commit"
-    # Temporarily lower the parallelism for https://github.com/NVIDIA/spark-rapids/issues/7092
-    #export TEST_PARALLEL=5
-    export TEST_PARALLEL=2
+    export TEST_PARALLEL=5
     ./integration_tests/run_pyspark_from_build.sh
     # enable avro test separately
     INCLUDE_SPARK_AVRO_JAR=true TEST='avro_test.py' ./integration_tests/run_pyspark_from_build.sh

diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh
@@ -164,9 +164,7 @@ if [[ $PARALLEL_TEST == "true" ]]; then
   fi
   # parallelism > 6 could slow down the whole process, so we have a limitation for it
   # this is based on our CI gpu types, so we do not put it into the run_pyspark_from_build.sh
-  # Temporarily lower the parallelism for https://github.com/NVIDIA/spark-rapids/issues/7092
-  #[[ ${PARALLELISM} -gt 6 ]] && PARALLELISM=6
-  [[ ${PARALLELISM} -gt 2 ]] && PARALLELISM=2
+  [[ ${PARALLELISM} -gt 6 ]] && PARALLELISM=6
   MEMORY_FRACTION=$(python -c "print(1/($PARALLELISM + 0.1))")
 
   export TEST_PARALLEL=${PARALLELISM}