Skip to content

Commit

Permalink
fix: NPE on dumpShards (#1612)
Browse files Browse the repository at this point in the history
Fixes #1601 

## Test Plan
> How do we know the code works?

When use  ```--dump-shards```, flank should create dump shard file without errors. 

## Checklist

- [X] Unit tested
- [X] Integration tests updated
  • Loading branch information
adamfilipow92 authored Feb 18, 2021
1 parent d476321 commit 68166db
Show file tree
Hide file tree
Showing 11 changed files with 150 additions and 6 deletions.
87 changes: 87 additions & 0 deletions integration_tests/src/test/kotlin/integration/DumpShardsIT.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package integration

import FlankCommand
import com.google.common.truth.Truth
import flank.common.isWindows
import org.junit.Assume
import org.junit.Test
import run
import utils.containsAll
import utils.loadAndroidDumpShards
import utils.loadIosDumpShards
import java.io.File

class DumpShardsIT {
private val name = this::class.java.simpleName

@Test
fun `dump shards - android`() {
val name = "$name-android"
val result = FlankCommand(
flankPath = FLANK_JAR_PATH,
ymlPath = "$CONFIGS_PATH/dump_shards_android.yml",
params = androidRunCommands + "--dump-shards"
).run(
workingDirectory = "./",
testSuite = name
)

assertExitCode(result, 0)

val resOutput = result.output.removeUnicode()
Truth.assertThat(resOutput).containsMatch(findInCompare(name))
assertNoOutcomeSummary(resOutput)

val matrix = File("android_shards.json").loadAndroidDumpShards()

Truth.assertThat(matrix.shards.count()).isEqualTo(2)

Truth.assertThat(matrix.shards.values.flatten()).containsAll(
"class com.example.test_app.parametrized.EspressoParametrizedClassParameterizedNamed",
"class com.example.test_app.parametrized.EspressoParametrizedClassTestParameterized",
"class com.example.test_app.ParameterizedTest",
"class com.example.test_app.parametrized.EspressoParametrizedMethodTestJUnitParamsRunner",
)

Truth.assertThat(matrix.junitIgnored.count()).isEqualTo(4)
Truth.assertThat(matrix.junitIgnored).containsNoDuplicates()

Truth.assertThat(matrix.junitIgnored)
.containsExactly(
"class com.example.test_app.InstrumentedTest#ignoredTestWitSuppress",
"class com.example.test_app.InstrumentedTest#ignoredTestWithIgnore",
"class com.example.test_app.bar.BarInstrumentedTest#ignoredTestBar",
"class com.example.test_app.foo.FooInstrumentedTest#ignoredTestFoo"
)
}

@Test
fun `dump shards - ios`() {
Assume.assumeFalse(isWindows)
val name = "$name-ios"
val result = FlankCommand(
flankPath = FLANK_JAR_PATH,
ymlPath = "$CONFIGS_PATH/dump_shards_ios.yml",
params = iosRunCommands + "--dump-shards"
).run(
workingDirectory = "./",
testSuite = name
)

assertExitCode(result, 0)

val resOutput = result.output.removeUnicode()
Truth.assertThat(resOutput).containsMatch(findInCompare(name))
assertNoOutcomeSummary(resOutput)

val shards = File("ios_shards.json").loadIosDumpShards()

Truth.assertThat(shards.count()).isEqualTo(2)

shards.first().let { firstShard ->
Truth.assertThat(firstShard.count()).isEqualTo(8)
Truth.assertThat(firstShard)
.contains("EarlGreyExampleSwiftTests/testWithCustomFailureHandler")
}
}
}
10 changes: 10 additions & 0 deletions integration_tests/src/test/kotlin/utils/LoadTestSuites.kt
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
package utils

import com.fasterxml.jackson.databind.json.JsonMapper
import com.fasterxml.jackson.dataformat.xml.XmlMapper
import com.fasterxml.jackson.module.kotlin.KotlinModule
import com.fasterxml.jackson.module.kotlin.readValue
import utils.testResults.TestSuites
import java.io.File

fun File.loadAsTestSuite(): TestSuites =
XmlMapper().registerModule(KotlinModule()).readValue(this, TestSuites::class.java)

fun File.loadAndroidDumpShards() = jsonMapper
.readValue<AndroidMatrixTestShards>(this).entries.first().value

fun File.loadIosDumpShards() = jsonMapper
.readValue<IosMatrixTestShards>(this)

private val jsonMapper by lazy { JsonMapper().registerModule(KotlinModule()) }
15 changes: 15 additions & 0 deletions integration_tests/src/test/kotlin/utils/MatrixTestShards.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package utils

import com.fasterxml.jackson.annotation.JsonProperty

typealias AndroidMatrixTestShards = Map<String, AndroidTestShards>

data class AndroidTestShards(
val app: String,
val test: String,
val shards: Map<String, List<String>> = emptyMap(),
@JsonProperty("junit-ignored")
val junitIgnored: List<String> = emptyList()
)

typealias IosMatrixTestShards = List<List<String>>
5 changes: 5 additions & 0 deletions integration_tests/src/test/kotlin/utils/OutputHelper.kt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package utils

import com.google.common.truth.IterableSubject
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import utils.testResults.TestSuites
Expand Down Expand Up @@ -31,3 +32,7 @@ fun TestSuites.assertTestFail(tests: List<String>) = assertEquals(
tests.count(),
testSuites.flatMap { it.testCases }.filter { it.name in tests && it.failure != null }.distinctBy { it.name }.count()
)

fun IterableSubject.containsAll(vararg args: Any) = args.forEach { arg ->
contains(arg)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
gcloud:
app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk
test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-multiple-success-debug-androidTest.apk
use-orchestrator: false

flank:
disable-sharding: false
max-test-shards: 2
output-style: single
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
gcloud:
test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExample.zip
xctestrun-file: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/ios/EarlGreyExample/EarlGreyExampleSwiftTests.xctestrun

flank:
disable-sharding: false
max-test-shards: 2
output-style: single
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Saved 2 shards to android_shards.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Saved 2 shards to ios_shards.json
Original file line number Diff line number Diff line change
Expand Up @@ -68,18 +68,22 @@ class AndroidRunCommand : CommonRunCommand(), Runnable {

AndroidArgs.load(Paths.get(configPath), cli = this).apply {
setupLogLevel()
logLn(this)

outputReport.configure(toOutputReportConfiguration())
outputReport.log(this)
setCrashReportTag(
DEVICE_SYSTEM to "android",
TEST_TYPE to type?.name.orEmpty()
)
sendConfiguration()
}.validate().run {
}.validate().also { args ->
runBlocking {
if (dumpShards) dumpShards()
else newTestRun()
if (dumpShards)
args.dumpShards()
else {
logLn(args)
args.newTestRun()
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,14 @@ class IosRunCommand : CommonRunCommand(), Runnable {

IosArgs.load(Paths.get(configPath), cli = this).apply {
setupLogLevel()
logLn(this)
outputReport.configure(toOutputReportConfiguration())
outputReport.log(this)
setCrashReportTag(
DEVICE_SYSTEM to "ios",
TEST_TYPE to type?.name.orEmpty()
)
sendConfiguration()
if (dumpShards.not()) logLn(this)
}.validate().run {
if (dumpShards) dumpShards()
else runBlocking { newTestRun() }
Expand Down
6 changes: 5 additions & 1 deletion test_runner/src/main/kotlin/ftl/run/DumpShards.kt
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,11 @@ fun saveShardChunks(
)
logLn("${FtlConstants.indent}Saved $size shards to $shardFilePath", OutputLogLevel.DETAILED)
}
private fun String.createDirectories() = File(this).parent.toFile().mkdirs()

private fun String.createDirectories() {
if (this !in listOf(ANDROID_SHARD_FILE, IOS_SHARD_FILE))
File(this).parent.toFile().mkdirs()
}

private fun getGson(obfuscatedOutput: Boolean) =
if (obfuscatedOutput) obfuscatePrettyPrinter else prettyPrint
Expand Down

0 comments on commit 68166db

Please sign in to comment.