From dad16d1d712576b3b92389ab6ab045dc16bdafbf Mon Sep 17 00:00:00 2001 From: Alexey Kudinkin Date: Fri, 3 Jun 2022 20:08:40 -0700 Subject: [PATCH] Tidying up --- .../spark/sql/hudi/analysis/HoodieSpark3Analysis.scala | 3 ++- .../apache/spark/sql/hudi/catalog/HoodieCatalog.scala | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala b/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala index 6553fe96d61be..e351174587245 100644 --- a/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala +++ b/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSpark3Analysis.scala @@ -45,6 +45,8 @@ import scala.collection.JavaConverters.mapAsJavaMapConverter * Since Hudi relations don't currently implement DS V2 Read API, we have to fallback to V1 here. * Such fallback will have considerable performance impact, therefore it's only performed in cases * where V2 API have to be used. Currently only such use-case is using of Schema Evolution feature + * + * Check out HUDI-4178 for more details */ class HoodieDataSourceV2ToV1Fallback(sparkSession: SparkSession) extends Rule[LogicalPlan] with ProvidesHoodieConfig { @@ -129,7 +131,6 @@ class HoodieSpark3Analysis(sparkSession: SparkSession) extends Rule[LogicalPlan] /** * Rule for resolve hoodie's extended syntax or rewrite some logical plan. - * @param sparkSession */ case class HoodieSpark3ResolveReferences(sparkSession: SparkSession) extends Rule[LogicalPlan] with SparkAdapterSupport with ProvidesHoodieConfig { diff --git a/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala b/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala index 4272219ca0447..f122e8f8d31c6 100644 --- a/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala +++ b/hudi-spark-datasource/hudi-spark3/src/main/scala/org/apache/spark/sql/hudi/catalog/HoodieCatalog.scala @@ -115,7 +115,14 @@ class HoodieCatalog extends DelegatingCatalogExtension val schemaEvolutionEnabled: Boolean = spark.sessionState.conf.getConfString(DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.key, DataSourceReadOptions.SCHEMA_EVOLUTION_ENABLED.defaultValue.toString).toBoolean - // TODO elaborate + + // NOTE: PLEASE READ CAREFULLY + // + // Since Hudi relations don't currently implement DS V2 Read API, we by default fallback to V1 here. + // Such fallback will have considerable performance impact, therefore it's only performed in cases + // where V2 API have to be used. Currently only such use-case is using of Schema Evolution feature + // + // Check out HUDI-4178 for more details if (schemaEvolutionEnabled) { v2Table } else {