From b3d99a48b7026a49ccacadb681a1bf0d719408d9 Mon Sep 17 00:00:00 2001 From: Navin Kumar Date: Wed, 12 Jan 2022 15:10:16 -0800 Subject: [PATCH] This fixes #1866 Make this more consistent with implementation of FileFormatWriter change implemented in SPARK-30227 by adding finallyBlock with close() call on dataWriter Signed-off-by: Navin Kumar --- .../scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala index cf6edd7ea20..55d1cca9ea2 100644 --- a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala +++ b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/GpuFileFormatWriter.scala @@ -310,6 +310,8 @@ object GpuFileFormatWriter extends Logging { // If there is an error, abort the task dataWriter.abort() logError(s"Job $jobId aborted.") + }, finallyBlock = { + dataWriter.close() }) } catch { case e: FetchFailedException =>