Skip to content

Commit

Permalink
revise the MIT tests to ensure column matches
Browse files Browse the repository at this point in the history
  • Loading branch information
@Davis-Zhang-Onehouse
Davis-Zhang-Onehouse committed Feb 7, 2025
1 parent e531fff commit 3f42e47
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 54 deletions.
8 changes: 4 additions & 4 deletions hudi-spark-datasource/hudi-spark/src/test/resources/sql-statements.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,10 @@ select id, name, price, cast(dt as string) from h0_p;
# CREATE TABLE

create table h1 (
id bigint,
id int,
name string,
price double,
ts bigint
ts int
) using hudi
options (
type = '${tableType}',
Expand All @@ -79,10 +79,10 @@ location '${tmpDir}/h1';
+----------+

create table h1_p (
id bigint,
id int,
name string,
price double,
ts bigint,
ts int,
dt string
) using hudi
partitioned by (dt)
Expand Down
2 changes: 1 addition & 1 deletion ...-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
|create table ${targetTable} (
| `id` string,
| `name` string,
| `dt` bigint,
| `dt` int,
| `day` STRING,
| `hour` INT
|) using hudi
Expand Down
34 changes: 17 additions & 17 deletions ...tasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
| id int,
| name string,
| price double,
| ts long
| ts int
|) using hudi
| location '${tmp.getCanonicalPath}'
| tblproperties (
Expand All @@ -63,7 +63,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
StructField("id", IntegerType, nullable = true),
StructField("name", StringType, nullable = true),
StructField("price", DoubleType, nullable = true),
StructField("ts", LongType, nullable = true))
StructField("ts", IntegerType, nullable = true))
// First merge with a extra input field 'flag' (insert a new record)
spark.sql(
s"""
Expand Down Expand Up @@ -155,7 +155,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
| name string,
| data int,
| country string,
| ts bigint
| ts int
|) using hudi
|tblproperties (
| type = 'cow',
Expand All @@ -169,7 +169,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
s"""
|merge into $targetTable as target
|using (
|select 1 as id, 'lb' as name, 6 as data, 'shu' as country, 1646643193 as ts
|select 1 as id, 'lb' as name, 6 as data, 'shu' as country, 43193 as ts
|) source
|on source.id = target.id
|when matched then
Expand All @@ -181,7 +181,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
s"""
|merge into $targetTable as target
|using (
|select 1 as id, 'lb' as name, 5 as data, 'shu' as country, 1646643196 as ts
|select 1 as id, 'lb' as name, 5 as data, 'shu' as country, 43196 as ts
|) source
|on source.id = target.id
|when matched and source.data > target.data then
Expand All @@ -193,7 +193,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
|""".stripMargin)

checkAnswer(s"select id, name, data, country, ts from $targetTable")(
Seq(1, "lb", 5, "shu", 1646643196L)
Seq(1, "lb", 5, "shu", 43196)
)
}
}
Expand Down Expand Up @@ -447,7 +447,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
| id int,
| name string,
| price double,
| ts long,
| ts int,
| dt string
| ) using hudi
| tblproperties (
Expand All @@ -462,15 +462,15 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
StructField("id", IntegerType, nullable = true),
StructField("name", StringType, nullable = true),
StructField("price", DoubleType, nullable = true),
StructField("ts", LongType, nullable = true),
StructField("ts", IntegerType, nullable = true),
StructField("dt", StringType, nullable = true))

// Insert data
spark.sql(
s"""
| merge into $tableName as t0
| using (
| select 1 as id, 'a1' as name, 10 as price, 1000L as ts, '2021-03-21' as dt
| select 1 as id, 'a1' as name, 10 as price, 1000 as ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0.id
| when not matched and s0.id % 2 = 1 then insert *
Expand All @@ -485,7 +485,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
s"""
| merge into $tableName as t0
| using (
| select 1 as _id, 'a1' as name, 12 as _price, 1001L as _ts, '2021-03-21' as dt
| select 1 as _id, 'a1' as name, 12 as _price, 1001 as _ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0._id
| when matched and s0._id % 2 = 0 then update set
Expand All @@ -501,7 +501,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
s"""
| merge into $tableName as t0
| using (
| select 1 as _id, 'a1' as name, 12 as _price, 1001L as _ts, '2021-03-21' as dt
| select 1 as _id, 'a1' as name, 12 as _price, 1001 as _ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0._id
| when matched and s0._id % 2 = 1 then update set
Expand All @@ -517,7 +517,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
s"""
| merge into $tableName as t0
| using (
| select 2 as id, 'a2' as name, 10 as price, 1000L as ts, '2021-03-21' as dt
| select 2 as id, 'a2' as name, 10 as price, 1000 as ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0.id
| when not matched and s0.id % 2 = 0 then insert *
Expand All @@ -532,7 +532,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
s"""
| merge into $tableName t0
| using (
| select 2 as s_id, 'a2' as s_name, 15 as s_price, 1001L as s_ts, '2021-03-21' as dt
| select 2 as s_id, 'a2' as s_name, 15 as s_price, 1001 as s_ts, '2021-03-21' as dt
| ) s0
| on t0.id = s0.s_id
| when matched and s_ts = 1001
Expand All @@ -552,7 +552,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
s"""
| merge into $tableName t0
| using (
| select 1 as s_id, 'a2' as s_name, 15 as s_price, 1001L as s_ts, '2021-03-21' as dt
| select 1 as s_id, 'a2' as s_name, 15 as s_price, 1001 as s_ts, '2021-03-21' as dt
| ) s0
| on t0.id = s0.s_id + 1
| when matched and s_ts = 1001 then delete
Expand All @@ -563,7 +563,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
s"""
| merge into $tableName t0
| using (
| select 2 as s_id, 'a2' as s_name, 15 as s_price, 1001L as ts, '2021-03-21' as dt
| select 2 as s_id, 'a2' as s_name, 15 as s_price, 1001 as ts, '2021-03-21' as dt
| ) s0
| on t0.id = s0.s_id
| when matched and s0.ts = 1001 then delete
Expand Down Expand Up @@ -1131,7 +1131,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
StructField("id", IntegerType, nullable = true),
StructField("name", StringType, nullable = true),
StructField("price", DoubleType, nullable = true),
StructField("ts", LongType, nullable = true))
StructField("ts", IntegerType, nullable = true))

spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)")
spark.sql(s"insert into $tableName values(2, 'a2', 10, 1000)")
Expand Down Expand Up @@ -1497,7 +1497,7 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
spark.sql(
s"""
| create table $tableName (
| id bigint,
| id int,
| name string,
| price double,
| ts bigint,
Expand Down
16 changes: 8 additions & 8 deletions ...asource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable2.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
| id int,
| name string,
| price double,
| ts long,
| ts int,
| dt string
| ) using hudi
| tblproperties (
Expand All @@ -57,7 +57,7 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
s"""
| merge into $tableName as t0
| using (
| select 1 as id, 'a1' as name, 10 as price, 1000L as ts, '2021-03-21' as dt
| select 1 as id, 'a1' as name, 10 as price, 1000 as ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0.id
| when not matched and s0.id % 2 = 1 then insert *
Expand All @@ -72,7 +72,7 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
s"""
| merge into $tableName as t0
| using (
| select 2 as id, 'a2' as name, 10 as price, 1000L as ts, '2021-03-21' as dt
| select 2 as id, 'a2' as name, 10 as price, 1000 as ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0.id
| when not matched and s0.id % 2 = 1 then insert *
Expand All @@ -87,7 +87,7 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
s"""
| merge into $tableName as t0
| using (
| select 1 as id, 'a1' as name, 11 as price, 1000L as ts, '2021-03-21' as dt
| select 1 as id, 'a1' as name, 11 as price, 1000 as ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0.id
| when matched and s0.id % 2 = 0 then update set *
Expand All @@ -104,7 +104,7 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
s"""
| merge into $tableName as t0
| using (
| select 1 as id, 'a1' as name, 11 as price, 1000L as ts, '2021-03-21' as dt
| select 1 as id, 'a1' as name, 11 as price, 1000 as ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0.id
| when matched and s0.id % 2 = 1 then update set id = s0.id, name = s0.name,
Expand All @@ -121,7 +121,7 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
s"""
| merge into $tableName as t0
| using (
| select 1 as id, 'a1' as name, 11 as price, 1000L as ts, '2021-03-21' as dt
| select 1 as id, 'a1' as name, 11 as price, 1000 as ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0.id
| when matched and s0.id % 2 = 0 then update set id = s0.id, name = s0.name,
Expand All @@ -138,7 +138,7 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
s"""
| merge into $tableName as t0
| using (
| select 1 as id, 'a1' as name, 10 as price, 1000L as ts, '2021-03-21' as dt
| select 1 as id, 'a1' as name, 10 as price, 1000 as ts, '2021-03-21' as dt
| ) as s0
| on t0.id = s0.id
| when matched and s0.id % 2 = 1 then update set id = s0.id, name = s0.name,
Expand Down Expand Up @@ -1168,7 +1168,7 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
withRecordType()(withTempDir { tmp =>
Seq("cow", "mor").foreach { tableType =>
Seq("COMMIT_TIME_ORDERING", "EVENT_TIME_ORDERING").foreach { mergeMode =>
withSparkSqlSessionConfig(DataSourceWriteOptions.ENABLE_MERGE_INTO_PARTIAL_UPDATES.key -> "false") {
withSparkSqlSessionConfig(DataSourceWriteOptions.ENABLE_MERGE_INTO_PARTIAL_UPDATES.key -> "true") {
val tableName = generateTableName
spark.sql(
s"""
Expand Down
34 changes: 17 additions & 17 deletions ...-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeModeCommitTimeOrdering.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ class TestMergeModeCommitTimeOrdering extends HoodieSparkSqlTestBase {
| id int,
| name string,
| price double,
| ts long
| ts int
| ) using hudi
| tblproperties (
| $writeTableVersionClause
Expand Down Expand Up @@ -242,7 +242,7 @@ class TestMergeModeCommitTimeOrdering extends HoodieSparkSqlTestBase {
| id int,
| name string,
| price double,
| ts long
| ts int
| ) using hudi
| tblproperties (
| $writeTableVersionClause
Expand All @@ -259,13 +259,13 @@ class TestMergeModeCommitTimeOrdering extends HoodieSparkSqlTestBase {
spark.sql(
s"""
| insert into $tableName
| select 1 as id, 'A' as name, 10.0 as price, 100L as ts union all
| select 0, 'X', 20.0, 100L union all
| select 2, 'B', 20.0, 100L union all
| select 3, 'C', 30.0, 100L union all
| select 4, 'D', 40.0, 100L union all
| select 5, 'E', 50.0, 100L union all
| select 6, 'F', 60.0, 100L
| select 1 as id, 'A' as name, 10.0 as price, 100 as ts union all
| select 0, 'X', 20.0, 100 union all
| select 2, 'B', 20.0, 100 union all
| select 3, 'C', 30.0, 100 union all
| select 4, 'D', 40.0, 100 union all
| select 5, 'E', 50.0, 100 union all
| select 6, 'F', 60.0, 100
""".stripMargin)
validateTableConfig(
storage, tmp.getCanonicalPath, expectedMergeConfigs, nonExistentConfigs)
Expand All @@ -278,9 +278,9 @@ class TestMergeModeCommitTimeOrdering extends HoodieSparkSqlTestBase {
s"""
| merge into $tableName t
| using (
| select 1 as id, 'B2' as name, 25.0 as price, 101L as ts union all
| select 2, '', 55.0, 99L as ts union all
| select 0, '', 55.0, 100L as ts
| select 1 as id, 'B2' as name, 25.0 as price, 101 as ts union all
| select 2, '', 55.0, 99 as ts union all
| select 0, '', 55.0, 100 as ts
| ) s
| on t.id = s.id
| when matched then delete
Expand All @@ -292,9 +292,9 @@ class TestMergeModeCommitTimeOrdering extends HoodieSparkSqlTestBase {
s"""
| merge into $tableName t
| using (
| select 4 as id, 'D2' as name, 45.0 as price, 101L as ts union all
| select 5, 'E2', 55.0, 99L as ts union all
| select 6, 'F2', 65.0, 100L as ts
| select 4 as id, 'D2' as name, 45.0 as price, 101 as ts union all
| select 5, 'E2', 55.0, 99 as ts union all
| select 6, 'F2', 65.0, 100 as ts
| ) s
| on t.id = s.id
| when matched then update set *
Expand Down Expand Up @@ -323,8 +323,8 @@ class TestMergeModeCommitTimeOrdering extends HoodieSparkSqlTestBase {
s"""
| merge into $tableName t
| using (
| select 7 as id, 'D2' as name, 45.0 as price, 100L as ts union all
| select 8, 'E2', 55.0, 100L as ts
| select 7 as id, 'D2' as name, 45.0 as price, 100 as ts union all
| select 8, 'E2', 55.0, 100 as ts
| ) s
| on t.id = s.id
| when not matched then insert *
Expand Down
14 changes: 7 additions & 7 deletions ...di-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestPartialUpdateForMergeInto.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
| id int,
| name string,
| price double,
| _ts long,
| _ts int,
| description string
|) using hudi
|tblproperties(
Expand Down Expand Up @@ -144,7 +144,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
| id int,
| name string,
| price double,
| _ts long,
| _ts int,
| description string
|) using hudi
|tblproperties(
Expand Down Expand Up @@ -205,7 +205,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
| id int,
| name string,
| price double,
| _ts long,
| _ts int,
| description string
|) using hudi
|tblproperties(
Expand All @@ -219,7 +219,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
StructField("id", IntegerType, nullable = true),
StructField("name", StringType, nullable = true),
StructField("price", DoubleType, nullable = true),
StructField("_ts", LongType, nullable = true),
StructField("_ts", IntegerType, nullable = true),
StructField("description", StringType, nullable = true))

spark.sql(s"insert into $tableName values (1, 'a1', 10, 1000, 'a1: desc1')," +
Expand Down Expand Up @@ -274,7 +274,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
| id int,
| name string,
| price double,
| _ts long,
| _ts int,
| description string
|) using hudi
|tblproperties(
Expand All @@ -288,7 +288,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
StructField("id", IntegerType, nullable = true),
StructField("name", StringType, nullable = true),
StructField("price", DoubleType, nullable = true),
StructField("_ts", LongType, nullable = true),
StructField("_ts", IntegerType, nullable = true),
StructField("description", StringType, nullable = true))
spark.sql(s"insert into $tableName values (1, 'a1', 10, 1000, 'a1: desc1')," +
"(2, 'a2', 20, 1200, 'a2: desc2'), (3, 'a3', 30, 1250, 'a3: desc3')")
Expand Down Expand Up @@ -444,7 +444,7 @@ class TestPartialUpdateForMergeInto extends HoodieSparkSqlTestBase {
| id int,
| name string,
| price double,
| _ts long,
| _ts int,
| description string
|) using hudi
|tblproperties(
Expand Down

0 comments on commit 3f42e47

Please sign in to comment.