From 1cf741b078987953016d2d514d5158c0e56ba0fc Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Mon, 24 May 2021 13:40:13 -0500 Subject: [PATCH] Add temporary logging for Dataproc round robin fallback issue (#2490) Signed-off-by: Jason Lowe --- .../spark/sql/rapids/execution/GpuShuffleExchangeExec.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuShuffleExchangeExec.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuShuffleExchangeExec.scala index c91de1a859e..8b09405a8cd 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuShuffleExchangeExec.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/execution/GpuShuffleExchangeExec.scala @@ -57,12 +57,17 @@ class GpuShuffleMeta( shuffle.outputPartitioning match { case _: RoundRobinPartitioning if shuffle.sqlContext.conf.sortBeforeRepartition => val orderableTypes = GpuOverrides.pluginSupportedOrderableSig + System.err.println(s"Checking round-robin sort for ${shuffle.output.map(_.dataType)}") shuffle.output.map(_.dataType) .filterNot(orderableTypes.isSupportedByPlugin(_, conf.decimalTypeEnabled)) .foreach { dataType => + System.err.println(s"Disallowing $dataType for round-robin partitioning") willNotWorkOnGpu(s"round-robin partitioning cannot sort $dataType to run " + s"this on the GPU set ${SQLConf.SORT_BEFORE_REPARTITION.key} to false") } + case _: RoundRobinPartitioning => + System.err.println(s"Allowing ${shuffle.output.map(_.dataType)} for round-robin " + + s"sort-before=${shuffle.sqlContext.conf.sortBeforeRepartition}") case _ => } }