Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BooleanType test shouldn't xfail #639

Merged
merged 10 commits into from
Sep 4, 2020
Merged

BooleanType test shouldn't xfail #639

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
64714a5
BooleanType test shouldn't xfail
razajafri Sep 2, 2020
3bef9fb
Revert "BooleanType test shouldn't xfail"
razajafri Sep 3, 2020
557df87
xfail only for 3.0.0
razajafri Sep 3, 2020
6472e92
addressed review comments
razajafri Sep 3, 2020
a9d9a04
removed the version method
razajafri Sep 3, 2020
06a0c0a
Revert "removed the version method"
razajafri Sep 3, 2020
e36515e
Revert "addressed review comments"
razajafri Sep 3, 2020
a3a9726
added version checks
razajafri Sep 3, 2020
9e2e1e2
missed changing it in cache_test.py
razajafri Sep 3, 2020
5f991e9
removing unnecessary lambda
razajafri Sep 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 16 additions & 11 deletions integration_tests/src/main/python/cache_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from data_gen import *
from datetime import date
import pyspark.sql.functions as f
from spark_session import with_cpu_session, with_gpu_session
from spark_session import with_cpu_session, with_gpu_session, with_spark_session
from join_test import create_df
from generate_expr_test import four_op_df
from marks import incompat, allow_non_gpu, ignore_order
Expand Down Expand Up @@ -61,8 +61,9 @@ def test_passing_gpuExpr_as_Expr():
@pytest.mark.parametrize('join_type', ['Left', 'Right', 'Inner', 'LeftSemi', 'LeftAnti'], ids=idfn)
@ignore_order
def test_cache_join(data_gen, join_type):
if data_gen.data_type == BooleanType():
revans2 marked this conversation as resolved.
Show resolved Hide resolved
pytest.xfail("https://github.com/NVIDIA/spark-rapids/issues/350")
if with_spark_session(lambda spark : spark.sparkContext.version == "3.0.0") \
and data_gen.data_type == BooleanType():
pytest.xfail("https://issues.apache.org/jira/browse/SPARK-32672")

def do_join(spark):
left, right = create_df(spark, data_gen, 500, 500)
Expand All @@ -81,8 +82,9 @@ def do_join(spark):
@ignore_order
def test_cached_join_filter(data_gen, join_type):
data, filter = data_gen
if data.data_type == BooleanType():
pytest.xfail("https://github.com/NVIDIA/spark-rapids/issues/350")
if with_spark_session(lambda spark : spark.sparkContext.version == "3.0.0") \
and data.data_type == BooleanType():
pytest.xfail("https://issues.apache.org/jira/browse/SPARK-32672")

def do_join(spark):
left, right = create_df(spark, data, 500, 500)
Expand All @@ -96,8 +98,9 @@ def do_join(spark):
@pytest.mark.parametrize('join_type', ['Left', 'Right', 'Inner', 'LeftSemi', 'LeftAnti'], ids=idfn)
@ignore_order
def test_cache_broadcast_hash_join(data_gen, join_type):
if data_gen.data_type == BooleanType():
pytest.xfail("https://github.com/NVIDIA/spark-rapids/issues/350")
if with_spark_session(lambda spark : spark.sparkContext.version == "3.0.0") \
and data_gen.data_type == BooleanType():
pytest.xfail("https://issues.apache.org/jira/browse/SPARK-32672")

def do_join(spark):
left, right = create_df(spark, data_gen, 500, 500)
Expand All @@ -116,8 +119,9 @@ def do_join(spark):
@pytest.mark.parametrize('join_type', ['Left', 'Right', 'Inner', 'LeftSemi', 'LeftAnti'], ids=idfn)
@ignore_order
def test_cache_shuffled_hash_join(data_gen, join_type):
if data_gen.data_type == BooleanType():
pytest.xfail("https://github.com/NVIDIA/spark-rapids/issues/350")
if with_spark_session(lambda spark : spark.sparkContext.version == "3.0.0") \
and data_gen.data_type == BooleanType():
pytest.xfail("https://issues.apache.org/jira/browse/SPARK-32672")

def do_join(spark):
left, right = create_df(spark, data_gen, 50, 500)
Expand Down Expand Up @@ -151,8 +155,9 @@ def do_join(spark):
@pytest.mark.parametrize('data_gen', all_gen_restricting_dates, ids=idfn)
@allow_non_gpu('InMemoryTableScanExec', 'DataWritingCommandExec')
def test_cache_posexplode_makearray(spark_tmp_path, data_gen):
if data_gen.data_type == BooleanType():
pytest.xfail("https://github.com/NVIDIA/spark-rapids/issues/350")
if with_spark_session(lambda spark : spark.sparkContext.version == "3.0.0") \
and data_gen.data_type == BooleanType():
pytest.xfail("https://issues.apache.org/jira/browse/SPARK-32672")
data_path_cpu = spark_tmp_path + '/PARQUET_DATA_CPU'
data_path_gpu = spark_tmp_path + '/PARQUET_DATA_GPU'
def write_posExplode(data_path):
Expand Down
2 changes: 2 additions & 0 deletions integration_tests/src/main/python/spark_init_internal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,5 @@ def get_spark_i_know_what_i_am_doing():
"""
return _spark

def spark_version():
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like this, but there are a number of other places in the code that are doing a

with_spark_session(spark: spark.sparkContext.version ...

It would probably be good to update all of them to be consistent or do it the same way here too.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have reverted the change to be more like the rest to be consistent

return _spark.version