From 321f9b98a72ba3465f8d72c42c62ee9c1fd25d10 Mon Sep 17 00:00:00 2001 From: Jim Brennan Date: Fri, 24 Jun 2022 11:22:22 -0500 Subject: [PATCH] Enable testing zstd for spark releases 3.2.0 and later (#5898) Signed-off-by: Jim Brennan --- integration_tests/src/main/python/parquet_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration_tests/src/main/python/parquet_test.py b/integration_tests/src/main/python/parquet_test.py index 6bd839f789b..b5fc42e77cd 100644 --- a/integration_tests/src/main/python/parquet_test.py +++ b/integration_tests/src/main/python/parquet_test.py @@ -145,6 +145,9 @@ def test_parquet_read_round_trip_binary_as_string(std_input_path, read_func, rea conf=all_confs) parquet_compress_options = ['none', 'uncompressed', 'snappy', 'gzip'] +# zstd is available in spark 3.2.0 and later. +if not is_before_spark_320(): + parquet_compress_options.append('zstd') # The following need extra jars 'lzo', 'lz4', 'brotli', 'zstd' # https://github.com/NVIDIA/spark-rapids/issues/143