Skip to content

Commit

Permalink
[SPARK-29045][SQL][TESTS] Drop table to avoid test failure in SQLMetr…
Browse files Browse the repository at this point in the history
…icsSuite
  • Loading branch information
LantaoJin committed Sep 11, 2019
1 parent 7f36cd2 commit 3dcd611
Showing 1 changed file with 24 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -115,29 +115,31 @@ trait SQLMetricsTestUtils extends SQLTestUtils {
provider: String,
dataFormat: String,
tableName: String): Unit = {
withTempPath { dir =>
spark.sql(
s"""
|CREATE TABLE $tableName(a int, b int)
|USING $provider
|PARTITIONED BY(a)
|LOCATION '${dir.toURI}'
""".stripMargin)
val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
assert(table.location == makeQualifiedPath(dir.getAbsolutePath))

val df = spark.range(start = 0, end = 40, step = 1, numPartitions = 1)
.selectExpr("id a", "id b")

// 40 files, 80 rows, 40 dynamic partitions.
verifyWriteDataMetrics(Seq(40, 40, 80)) {
df.union(df).repartition(2, $"a")
.write
.format(dataFormat)
.mode("overwrite")
.insertInto(tableName)
withTable(tableName) {
withTempPath { dir =>
spark.sql(
s"""
|CREATE TABLE $tableName(a int, b int)
|USING $provider
|PARTITIONED BY(a)
|LOCATION '${dir.toURI}'
""".stripMargin)
val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
assert(table.location == makeQualifiedPath(dir.getAbsolutePath))

val df = spark.range(start = 0, end = 40, step = 1, numPartitions = 1)
.selectExpr("id a", "id b")

// 40 files, 80 rows, 40 dynamic partitions.
verifyWriteDataMetrics(Seq(40, 40, 80)) {
df.union(df).repartition(2, $"a")
.write
.format(dataFormat)
.mode("overwrite")
.insertInto(tableName)
}
assert(TestUtils.recursiveList(dir).count(_.getName.startsWith("part-")) == 40)
}
assert(TestUtils.recursiveList(dir).count(_.getName.startsWith("part-")) == 40)
}
}

Expand Down

0 comments on commit 3dcd611

Please sign in to comment.