diff --git a/docs/supported_ops.md b/docs/supported_ops.md index 802ee5d0a72..60d25631583 100644 --- a/docs/supported_ops.md +++ b/docs/supported_ops.md @@ -7890,8 +7890,8 @@ are limited. NS -PS
MAP only supports keys and values that are of STRING type;
unsupported child types BOOLEAN, BYTE, SHORT, LONG, FLOAT, DOUBLE, DATE, TIMESTAMP, DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT
-PS
unsupported child types BOOLEAN, BYTE, SHORT, LONG, FLOAT, DOUBLE, DATE, TIMESTAMP, DECIMAL, NULL, BINARY, CALENDAR, MAP, UDT
+PS
MAP only supports keys and values that are of STRING type;
unsupported child types BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, DATE, TIMESTAMP, DECIMAL, NULL, BINARY, CALENDAR, ARRAY, MAP, STRUCT, UDT
+NS diff --git a/integration_tests/src/main/python/json_test.py b/integration_tests/src/main/python/json_test.py index db9e0dfc76f..7c8b2499130 100644 --- a/integration_tests/src/main/python/json_test.py +++ b/integration_tests/src/main/python/json_test.py @@ -380,6 +380,7 @@ def test_from_json_map_fallback(): 'JsonToStructs', conf={"spark.rapids.sql.expression.JsonToStructs": True}) +@pytest.mark.xfail(reason='https://github.com/NVIDIA/spark-rapids/issues/8558') @pytest.mark.parametrize('schema', ['struct', 'struct', 'struct', @@ -393,6 +394,7 @@ def test_from_json_struct(schema): .select(f.from_json('a', schema)), conf={"spark.rapids.sql.expression.JsonToStructs": True}) +@pytest.mark.xfail(reason='https://github.com/NVIDIA/spark-rapids/issues/8558') @pytest.mark.parametrize('schema', ['struct', 'struct>', 'struct>']) @@ -404,6 +406,7 @@ def test_from_json_struct_of_struct(schema): .select(f.from_json('a', schema)), conf={"spark.rapids.sql.expression.JsonToStructs": True}) +@pytest.mark.xfail(reason='https://github.com/NVIDIA/spark-rapids/issues/8558') @pytest.mark.parametrize('schema', ['struct', 'struct>>', 'struct>>']) @@ -416,6 +419,7 @@ def test_from_json_struct_of_list(schema): .select(f.from_json('a', schema)), conf={"spark.rapids.sql.expression.JsonToStructs": True}) +@pytest.mark.xfail(reason='https://github.com/NVIDIA/spark-rapids/issues/8558') @pytest.mark.parametrize('schema', ['struct', 'struct']) def test_from_json_struct_all_empty_string_input(schema): json_string_gen = StringGen('') diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala index 7480ab870aa..baa629a57be 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOverrides.scala @@ -3374,19 +3374,16 @@ object GpuOverrides extends Logging { "Returns a struct value with the given `jsonStr` and `schema`", ExprChecks.projectOnly( TypeSig.MAP.nested(TypeSig.STRING).withPsNote(TypeEnum.MAP, - "MAP only supports keys and values that are of STRING type") + - TypeSig.STRUCT.nested(TypeSig.STRUCT + TypeSig.ARRAY + TypeSig.STRING + TypeSig.INT), + "MAP only supports keys and values that are of STRING type"), (TypeSig.STRUCT + TypeSig.MAP + TypeSig.ARRAY).nested(TypeSig.all), Seq(ParamCheck("jsonStr", TypeSig.STRING, TypeSig.STRING))), (a, conf, p, r) => new UnaryExprMeta[JsonToStructs](a, conf, p, r) { override def tagExprForGpu(): Unit = a.schema match { case MapType(_: StringType, _: StringType, _) => () - case MapType(kt, vt, _) => { - willNotWorkOnGpu("JsonToStructs only supports MapType for " + - s"input MapType schema, but received MapType<$kt, $vt>") - } - case _ => () + case _ => + willNotWorkOnGpu("from_json on GPU only supports MapType " + + "input schema") } GpuJsonScan.tagJsonToStructsSupport(a.options, this) diff --git a/tools/generated_files/supportedExprs.csv b/tools/generated_files/supportedExprs.csv index 32f90ff83db..08fea6d4731 100644 --- a/tools/generated_files/supportedExprs.csv +++ b/tools/generated_files/supportedExprs.csv @@ -265,7 +265,7 @@ IsNotNull,S,`isnotnull`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,N IsNull,S,`isnull`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS IsNull,S,`isnull`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA JsonToStructs,NS,`from_json`,This is disabled by default because parsing JSON from a column has a large number of issues and should be considered beta quality right now.,project,jsonStr,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -JsonToStructs,NS,`from_json`,This is disabled by default because parsing JSON from a column has a large number of issues and should be considered beta quality right now.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,PS,PS,NA +JsonToStructs,NS,`from_json`,This is disabled by default because parsing JSON from a column has a large number of issues and should be considered beta quality right now.,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NS,PS,NS,NA JsonTuple,S,`json_tuple`,None,project,json,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA JsonTuple,S,`json_tuple`,None,project,field,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA JsonTuple,S,`json_tuple`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA