minor change

apache · Jan 9, 2025 · cc68e23 · cc68e23
1 parent edf103d
commit cc68e23
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 0 deletions.
diff --git a/...ends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/...ends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -58,6 +58,8 @@ import org.apache.commons.lang3.ClassUtils
 
 import javax.ws.rs.core.UriBuilder
 
+import java.util.Locale
+
 class VeloxSparkPlanExecApi extends SparkPlanExecApi {
 
   /** Transform GetArrayItem to Substrait. */
@@ -715,6 +717,23 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
       throw new GlutenNotSupportException(
         "'from_json' with 'spark.sql.caseSensitive = true' is not supported in Velox")
     }
+
+    val hasCaseInsensitiveDuplicateKey = expr.schema match {
+      case s: StructType =>
+        !s.filter(
+          f =>
+            !s.names
+              .filter(
+                n => n != f.name && n.toLowerCase(Locale.ROOT) == f.name.toLowerCase(Locale.ROOT))
+              .isEmpty)
+          .isEmpty
+      case other =>
+        false
+    }
+    if (hasCaseInsensitiveDuplicateKey) {
+      throw new GlutenNotSupportException(
+        "'from_json' with case-insensitive duplicate keys is not supported in Velox")
+    }
     val hasCorruptRecord = expr.schema match {
       case s: StructType =>
         !s.filter(_.name == SQLConf.get.getConf(SQLConf.COLUMN_NAME_OF_CORRUPT_RECORD)).isEmpty

diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala
@@ -320,4 +320,27 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
         }
     }
   }
+
+  test("from_json function duplicate key") {
+    withTempPath {
+      path =>
+        Seq[(String)](
+          ("""{"id":1,"Id":2}"""),
+          ("""{"id":3,"Id":4}""")
+        )
+          .toDF("txt")
+          .write
+          .parquet(path.getCanonicalPath)
+
+        spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("tbl")
+
+        runQueryAndCompare("select txt, from_json(txt, 'id INT, Id INT') from tbl") {
+          checkSparkOperatorMatch[ProjectExec]
+        }
+
+        runQueryAndCompare("select txt, from_json(txt, 'id INT') from tbl") {
+          checkSparkOperatorMatch[ProjectExecTransformer]
+        }
+    }
+  }
 }