add INCLUDE_SPARK_AVRO_JAR

NVIDIA · Mar 22, 2022 · 0be4e9a · 0be4e9a
1 parent 02cd1c8
commit 0be4e9a
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 4 deletions.
diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
@@ -57,11 +57,14 @@ else
     #
     # `./run_pyspark_from_build.sh -k xxx ` runs all xxx tests with spark-avro.jar in the classpath
     #
-    # `CI_EXCLUDE_AVRO=true ./run_pyspark_from_build.sh -k not avro_test.py` run all tests excluding
-    #                               those in avro_test.py
-    if [[ "${CI_EXCLUDE_AVRO}" != "" ]];
+    # `CI_EXCLUDE_AVRO=true ./run_pyspark_from_build.sh` run all tests (except the marker skipif())
+    #                                           without spark-avro.jar
+    if [[ $( echo ${INCLUDE_SPARK_AVRO_JAR} | tr [:upper:] [:lower:] ) == "true" ]];
     then
-       AVRO_JARS=""
+        export INCLUDE_SPARK_AVRO_JAR=true
+    else
+        export INCLUDE_SPARK_AVRO_JAR=false
+        AVRO_JARS=""
     fi
 
     # Only 3 jars: cudf.jar dist.jar integration-test.jar

diff --git a/integration_tests/src/main/python/avro_test.py b/integration_tests/src/main/python/avro_test.py
@@ -11,6 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
+
 from spark_session import with_cpu_session
 import pytest
 
@@ -19,6 +21,9 @@
 from marks import *
 from pyspark.sql.types import *
 
+if os.environ.get('INCLUDE_SPARK_AVRO_JAR', 'false') == 'false':
+    pytestmark = pytest.mark.skip(reason=str("INCLUDE_SPARK_AVRO_JAR is disabled"))
+
 support_gens = numeric_gens + [string_gen, boolean_gen]
 
 _enable_all_types_conf = {

diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh
@@ -118,6 +118,7 @@ ci_2() {
     TEST_PARALLEL=5 TEST='struct_test or time_window_test' ./integration_tests/run_pyspark_from_build.sh
     TEST='not conditionals_test and not window_function_test and not struct_test and not time_window_test' \
       ./integration_tests/run_pyspark_from_build.sh
+    INCLUDE_SPARK_AVRO_JAR=true TEST='avro_test.py' ./integration_tests/run_pyspark_from_build.sh
 }