Copy function isSearchableType from Spark because signature changed in (

NVIDIA#632) 3.0.1 Signed-off-by: Thomas Graves <[email protected]> Co-authored-by: Thomas Graves <[email protected]>
pxLi · Sep 1, 2020 · e1cee47 · e1cee47
1 parent 81c8bf0
commit e1cee47
Showing 1 changed file with 21 additions and 9 deletions.
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/OrcFilters.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/OrcFilters.scala
@@ -121,6 +121,18 @@ object OrcFilters extends OrcFiltersBase {
     }
   }
 
+  /**
+   * Return true if this is a searchable type in ORC.
+   * Both CharType and VarcharType are cleaned at AstBuilder.
+   *
+   * Copied from Spark because scope got changed.
+   */
+  private def isSearchableTypeLocal(dataType: DataType) = dataType match {
+    case BinaryType => false
+    case _: AtomicType => true
+    case _ => false
+  }
+
   /**
    * Build a SearchArgument for a leaf predicate and return the builder so far.
    *
@@ -144,45 +156,45 @@ object OrcFilters extends OrcFiltersBase {
     // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters
     // in order to distinguish predicate pushdown for nested columns.
     expression match {
-      case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+      case EqualTo(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) =>
         val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().equals(quotedName, getType(attribute), castedValue).end())
 
-      case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+      case EqualNullSafe(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) =>
         val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().nullSafeEquals(quotedName, getType(attribute), castedValue).end())
 
-      case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+      case LessThan(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) =>
         val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().lessThan(quotedName, getType(attribute), castedValue).end())
 
-      case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+      case LessThanOrEqual(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) =>
         val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startAnd().lessThanEquals(quotedName, getType(attribute), castedValue).end())
 
-      case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+      case GreaterThan(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) =>
         val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startNot().lessThanEquals(quotedName, getType(attribute), castedValue).end())
 
-      case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) =>
+      case GreaterThanOrEqual(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) =>
         val quotedName = quoteIfNeeded(attribute)
         val castedValue = castLiteralValue(value, dataTypeMap(attribute))
         Some(builder.startNot().lessThan(quotedName, getType(attribute), castedValue).end())
 
-      case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
+      case IsNull(attribute) if isSearchableTypeLocal(dataTypeMap(attribute)) =>
         val quotedName = quoteIfNeeded(attribute)
         Some(builder.startAnd().isNull(quotedName, getType(attribute)).end())
 
-      case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) =>
+      case IsNotNull(attribute) if isSearchableTypeLocal(dataTypeMap(attribute)) =>
         val quotedName = quoteIfNeeded(attribute)
         Some(builder.startNot().isNull(quotedName, getType(attribute)).end())
 
-      case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) =>
+      case In(attribute, values) if isSearchableTypeLocal(dataTypeMap(attribute)) =>
         val quotedName = quoteIfNeeded(attribute)
         val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(attribute)))
         Some(builder.startAnd().in(quotedName, getType(attribute),