Skip to content

Commit

Permalink
[HUDI-4296]Fix the bug that TestHoodieSparkSqlWriter.testSchemaEvolut…
Browse files Browse the repository at this point in the history
…ionForTableType is flaky
  • Loading branch information
xiarixiaoyao committed Jun 25, 2022
1 parent c86edfc commit 3c419b0
Showing 1 changed file with 4 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,9 @@ class BaseFileOnlyRelation(sqlContext: SQLContext,
DataSource.apply(
sparkSession = sparkSession,
paths = extraReadPaths,
userSpecifiedSchema = userSchema,
// Here we should specify the schema to the latest commit schema since
// the table schema evolution.
userSpecifiedSchema = userSchema.orElse(Some(tableStructSchema)),
className = formatClassName,
// Since we're reading the table as just collection of files we have to make sure
// we only read the latest version of every Hudi's file-group, which might be compacted, clustered, etc.
Expand All @@ -175,8 +177,7 @@ class BaseFileOnlyRelation(sqlContext: SQLContext,
// We rely on [[HoodieROTablePathFilter]], to do proper filtering to assure that
options = optParams ++ Map(
"mapreduce.input.pathFilter.class" -> classOf[HoodieROTablePathFilter].getName
),
partitionColumns = partitionColumns
)
)
.resolveRelation()
.asInstanceOf[HadoopFsRelation]
Expand Down

0 comments on commit 3c419b0

Please sign in to comment.