From 499a45bd2bea4b256eb9b0dbae4720cbedd8a295 Mon Sep 17 00:00:00 2001 From: MithunR Date: Thu, 30 May 2024 14:13:52 -0700 Subject: [PATCH] [Spark 4.0] Account for `CommandUtils.uncacheTableOrView` signature change. (#10863) * Account for `CommandUtils.uncacheTableOrView` signature change. Fixes #10710. This commit accounts for the changes in the signature of `CommandUtils.uncacheTableOrView` in Apache Spark 4.0. (See [SPARK-47191](https://github.com/apache/spark/pull/45289).) Signed-off-by: MithunR * Removed unnecessary base class. --------- Signed-off-by: MithunR --- .../hive/rapids/shims/CommandUtilsShim.scala | 57 +++++++++++++++++++ .../rapids/shims/GpuInsertIntoHiveTable.scala | 2 +- .../rapids/shims/GpuInsertIntoHiveTable.scala | 2 +- .../hive/rapids/shims/CommandUtilsShim.scala | 33 +++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala create mode 100644 sql-plugin/src/main/spark400/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala new file mode 100644 index 00000000000..1e1ac57aa60 --- /dev/null +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "311"} +{"spark": "312"} +{"spark": "313"} +{"spark": "320"} +{"spark": "321"} +{"spark": "321cdh"} +{"spark": "322"} +{"spark": "323"} +{"spark": "324"} +{"spark": "330"} +{"spark": "330cdh"} +{"spark": "330db"} +{"spark": "331"} +{"spark": "332"} +{"spark": "332cdh"} +{"spark": "332db"} +{"spark": "333"} +{"spark": "334"} +{"spark": "340"} +{"spark": "341"} +{"spark": "341db"} +{"spark": "342"} +{"spark": "343"} +{"spark": "350"} +{"spark": "351"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.hive.rapids.shims + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.execution.command.CommandUtils + +object CommandUtilsShim { + + // Shim for CommandUtils.uncacheTableOrView, whose signature changed in Apache Spark 4.0. + def uncacheTableOrView(sparkSession: SparkSession, tableId: TableIdentifier): Unit = { + CommandUtils.uncacheTableOrView(sparkSession, tableId.quotedString) + } + +} \ No newline at end of file diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala index 92fb72801c8..2af89bf1170 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/hive/rapids/shims/GpuInsertIntoHiveTable.scala @@ -137,7 +137,7 @@ case class GpuInsertIntoHiveTable( } // un-cache this table. - CommandUtils.uncacheTableOrView(sparkSession, table.identifier.quotedString) + CommandUtilsShim.uncacheTableOrView(sparkSession, table.identifier) sparkSession.sessionState.catalog.refreshTable(table.identifier) CommandUtils.updateTableStats(sparkSession, table) diff --git a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala index 9105ab50e1e..28b8033389a 100644 --- a/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala +++ b/sql-plugin/src/main/spark332db/scala/com/nvidia/spark/rapids/shims/GpuInsertIntoHiveTable.scala @@ -127,7 +127,7 @@ case class GpuInsertIntoHiveTable( } // un-cache this table. - CommandUtils.uncacheTableOrView(sparkSession, table.identifier.quotedString) + CommandUtilsShim.uncacheTableOrView(sparkSession, table.identifier) sparkSession.sessionState.catalog.refreshTable(table.identifier) CommandUtils.updateTableStats(sparkSession, table) diff --git a/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala new file mode 100644 index 00000000000..f5858e4cfd6 --- /dev/null +++ b/sql-plugin/src/main/spark400/scala/org/apache/spark/sql/hive/rapids/shims/CommandUtilsShim.scala @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*** spark-rapids-shim-json-lines +{"spark": "400"} +spark-rapids-shim-json-lines ***/ +package org.apache.spark.sql.hive.rapids.shims + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.execution.command.CommandUtils + +object CommandUtilsShim { + + // Shim for CommandUtils.uncacheTableOrView, whose signature changed in Apache Spark 4.0. + def uncacheTableOrView(sparkSession: SparkSession, tableId: TableIdentifier): Unit = { + CommandUtils.uncacheTableOrView(sparkSession, tableId) + } + +} \ No newline at end of file