From f6296a7217ac7baaa4ea5c04390dce1322f23e09 Mon Sep 17 00:00:00 2001 From: sychen Date: Thu, 23 Jun 2022 20:48:44 +0800 Subject: [PATCH 1/3] fallback --- .../parser/HoodieSpark3_2ExtendedSqlParser.scala | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala b/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala index 59ef8dfe0969..2b8931ace3b8 100644 --- a/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala +++ b/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/parser/HoodieSpark3_2ExtendedSqlParser.scala @@ -32,6 +32,8 @@ import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} import org.apache.spark.sql.types._ import org.apache.spark.sql.{AnalysisException, SparkSession} +import scala.util.control.NonFatal + class HoodieSpark3_2ExtendedSqlParser(session: SparkSession, delegate: ParserInterface) extends ParserInterface with Logging { @@ -39,9 +41,14 @@ class HoodieSpark3_2ExtendedSqlParser(session: SparkSession, delegate: ParserInt private lazy val builder = new HoodieSpark3_2ExtendedSqlAstBuilder(conf, delegate) override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) { parser => - builder.visit(parser.singleStatement()) match { - case plan: LogicalPlan => plan - case _=> delegate.parsePlan(sqlText) + try { + builder.visit(parser.singleStatement()) match { + case plan: LogicalPlan => plan + case _=> delegate.parsePlan(sqlText) + } + } catch { + case NonFatal(_) => + delegate.parsePlan(sqlText) } } From b2364bb320cdec679a6d44eca506d7dce34be766 Mon Sep 17 00:00:00 2001 From: sychen Date: Thu, 23 Jun 2022 22:11:00 +0800 Subject: [PATCH 2/3] add UT --- .../apache/spark/sql/hudi/TestTimeTravelTable.scala | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala index ce0f17c3f569..7449c8d2764a 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala @@ -238,4 +238,15 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase { } } } + + test("Test Unsupported syntax can be parsed") { + if (HoodieSparkUtils.gteqSpark3_2) { + checkAnswer("select 1 distribute by 1")(Seq(1)) + withTempDir { dir => + val path = dir.toURI.getPath + spark.sql(s"insert overwrite local directory '$path' using parquet select 1") + spark.sql(s"insert overwrite local directory '$path' stored as orc select 1") + } + } + } } From c1bc478a28f8f6ad54a1a086e857b9c89353f845 Mon Sep 17 00:00:00 2001 From: sychen Date: Thu, 23 Jun 2022 23:15:24 +0800 Subject: [PATCH 3/3] disable hive --- .../scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala index 7449c8d2764a..4a871a82dd2e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestTimeTravelTable.scala @@ -245,7 +245,8 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase { withTempDir { dir => val path = dir.toURI.getPath spark.sql(s"insert overwrite local directory '$path' using parquet select 1") - spark.sql(s"insert overwrite local directory '$path' stored as orc select 1") + // Requires enable hive support, so didn't test it + // spark.sql(s"insert overwrite local directory '$path' stored as orc select 1") } } }