From 2a27481b63a1967456ac3d3d54f0314c5b5e0f7b Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Thu, 4 Mar 2021 14:17:30 -0600 Subject: [PATCH] Disable coalesce batch spilling to avoid cudf contiguous_split bug Signed-off-by: Jason Lowe --- .../nvidia/spark/rapids/GpuCoalesceBatches.scala | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala index 22ce1e30c59..5de0f4f6884 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala @@ -608,13 +608,15 @@ case class GpuCoalesceBatches(child: SparkPlan, goal: CoalesceGoal) // cache in local vars to avoid serializing the plan val outputSchema = schema val decompressMemoryTarget = maxDecompressBatchMemory - val cannotSpill = child.schema.fields.exists { f => - f.dataType match { - case MapType(_, _, _) | ArrayType(_, _) | StructType(_) => true - case _ => false - } - } - + // disabling spillable batches due to + // https://github.com/rapidsai/cudf/issues/7514 + val cannotSpill = true +// val cannotSpill = child.schema.fields.exists { f => +// f.dataType match { +// case MapType(_, _, _) | ArrayType(_, _) | StructType(_) => true +// case _ => false +// } +// } val batches = child.executeColumnar() batches.mapPartitions { iter => if (outputSchema.isEmpty) {