diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/OrcFilters.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/OrcFilters.scala index 24eb1620f8c..5f81776d28b 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/OrcFilters.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/OrcFilters.scala @@ -121,6 +121,18 @@ object OrcFilters extends OrcFiltersBase { } } + /** + * Return true if this is a searchable type in ORC. + * Both CharType and VarcharType are cleaned at AstBuilder. + * + * Copied from Spark because scope got changed. + */ + private def isSearchableTypeLocal(dataType: DataType) = dataType match { + case BinaryType => false + case _: AtomicType => true + case _ => false + } + /** * Build a SearchArgument for a leaf predicate and return the builder so far. * @@ -144,45 +156,45 @@ object OrcFilters extends OrcFiltersBase { // Since ORC 1.5.0 (ORC-323), we need to quote for column names with `.` characters // in order to distinguish predicate pushdown for nested columns. expression match { - case EqualTo(attribute, value) if isSearchableType(dataTypeMap(attribute)) => + case EqualTo(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) => val quotedName = quoteIfNeeded(attribute) val castedValue = castLiteralValue(value, dataTypeMap(attribute)) Some(builder.startAnd().equals(quotedName, getType(attribute), castedValue).end()) - case EqualNullSafe(attribute, value) if isSearchableType(dataTypeMap(attribute)) => + case EqualNullSafe(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) => val quotedName = quoteIfNeeded(attribute) val castedValue = castLiteralValue(value, dataTypeMap(attribute)) Some(builder.startAnd().nullSafeEquals(quotedName, getType(attribute), castedValue).end()) - case LessThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) => + case LessThan(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) => val quotedName = quoteIfNeeded(attribute) val castedValue = castLiteralValue(value, dataTypeMap(attribute)) Some(builder.startAnd().lessThan(quotedName, getType(attribute), castedValue).end()) - case LessThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) => + case LessThanOrEqual(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) => val quotedName = quoteIfNeeded(attribute) val castedValue = castLiteralValue(value, dataTypeMap(attribute)) Some(builder.startAnd().lessThanEquals(quotedName, getType(attribute), castedValue).end()) - case GreaterThan(attribute, value) if isSearchableType(dataTypeMap(attribute)) => + case GreaterThan(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) => val quotedName = quoteIfNeeded(attribute) val castedValue = castLiteralValue(value, dataTypeMap(attribute)) Some(builder.startNot().lessThanEquals(quotedName, getType(attribute), castedValue).end()) - case GreaterThanOrEqual(attribute, value) if isSearchableType(dataTypeMap(attribute)) => + case GreaterThanOrEqual(attribute, value) if isSearchableTypeLocal(dataTypeMap(attribute)) => val quotedName = quoteIfNeeded(attribute) val castedValue = castLiteralValue(value, dataTypeMap(attribute)) Some(builder.startNot().lessThan(quotedName, getType(attribute), castedValue).end()) - case IsNull(attribute) if isSearchableType(dataTypeMap(attribute)) => + case IsNull(attribute) if isSearchableTypeLocal(dataTypeMap(attribute)) => val quotedName = quoteIfNeeded(attribute) Some(builder.startAnd().isNull(quotedName, getType(attribute)).end()) - case IsNotNull(attribute) if isSearchableType(dataTypeMap(attribute)) => + case IsNotNull(attribute) if isSearchableTypeLocal(dataTypeMap(attribute)) => val quotedName = quoteIfNeeded(attribute) Some(builder.startNot().isNull(quotedName, getType(attribute)).end()) - case In(attribute, values) if isSearchableType(dataTypeMap(attribute)) => + case In(attribute, values) if isSearchableTypeLocal(dataTypeMap(attribute)) => val quotedName = quoteIfNeeded(attribute) val castedValues = values.map(v => castLiteralValue(v, dataTypeMap(attribute))) Some(builder.startAnd().in(quotedName, getType(attribute),