test: [IT] Add test cases

.github/pull_request_template.md

@@ -9,3 +9,4 @@ Fixes #
 
 - [ ] Documented
 - [ ] Unit tested
+- [ ] Integration tests updated

.gitignore

@@ -13,3 +13,5 @@ results
 xcuserdata/
 test_projects/ios/*/build*
 android_shards.json
+ios_shards.json
+*.log

integration_tests/build.gradle.kts

@@ -1,3 +1,6 @@
+import org.gradle.api.tasks.testing.logging.TestExceptionFormat
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
 plugins {
     java
     kotlin(Plugins.Kotlin.PLUGIN_JVM)
@@ -31,6 +34,7 @@ detekt {
 dependencies {
     implementation(kotlin("stdlib"))
     testImplementation(Dependencies.JUNIT)
+    testImplementation(Dependencies.TRUTH)
     detektPlugins(Dependencies.DETEKT_FORMATTING)
 }
 
@@ -44,4 +48,25 @@ tasks.test {
     systemProperty("working-directory", System.getProperty("working-directory"))
     systemProperty("output-pattern", System.getProperty("output-pattern"))
     systemProperty("expected-output-code", System.getProperty("expected-output-code"))
+    filter {
+        excludeTestsMatching("*IT")
+    }
+}
+
+tasks.register<Test>("integrationTests") {
+    group = "Verification"
+    description = "Runs flank integration tests"
+    filter {
+        includeTestsMatching("*IT")
+    }
+    testLogging {
+        events("skipped", "failed")
+        exceptionFormat = TestExceptionFormat.FULL
+    }
+    maxParallelForks = Runtime.getRuntime().availableProcessors() / 2
+}
+
+val compileTestKotlin: KotlinCompile by tasks
+compileTestKotlin.kotlinOptions {
+    freeCompilerArgs = listOf("-Xinline-classes")
 }

integration_tests/src/test/kotlin/FlankCommand.kt

@@ -5,4 +5,4 @@ data class FlankCommand(val flankPath: String, val ymlPath: String, val params:
 
 private fun FlankCommand.create() = "java -jar $flankPath ${params.joinToString(separator = " ")} -c=$ymlPath"
 
-fun FlankCommand.run(workingDirectory: String) = create().runCommand(File(workingDirectory))
+fun FlankCommand.run(workingDirectory: String, testSuite: String = "") = create().runCommand(File(workingDirectory), testSuite)

integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt

@@ -0,0 +1,50 @@
+package integration
+
+import FlankCommand
+import com.google.common.truth.Truth.assertThat
+import org.junit.Assume.assumeFalse
+import run
+import org.junit.Test
+
+class AllTestFilteredIT {
+    private val name = this::class.java.simpleName
+
+    @Test
+    fun `filter all tests - android`() {
+        val name = "$name-android"
+        val result = FlankCommand(
+            flankPath = FLANK_JAR_PATH,
+            ymlPath = "$CONFIGS_PATH/all_test_filtered_android.yml",
+            params = androidRunCommands
+        ).run(
+            workingDirectory = "./",
+            testSuite = name
+        )
+
+        assertExitCode(result, 1)
+
+        val resOutput = result.output.removeUnicode()
+        assertThat(resOutput).containsMatch(findInCompare(name))
+        assertNoOutcomeSummary(resOutput)
+    }
+
+    @Test
+    fun `filter all tests - ios`() {
+        assumeFalse(isWindows)
+        val name = "$name-ios"
+        val result = FlankCommand(
+            flankPath = FLANK_JAR_PATH,
+            ymlPath = "$CONFIGS_PATH/all_test_filtered_ios.yml",
+            params = iosRunCommands
+        ).run(
+            workingDirectory = "./",
+            testSuite = name
+        )
+
+        assertExitCode(result, 0)
+
+        val resOutput = result.output.removeUnicode()
+        assertThat(resOutput).containsMatch(findInCompare(name))
+        assertNoOutcomeSummary(resOutput)
+    }
+}

integration_tests/src/test/kotlin/integration/IgnoreFailedIT.kt

@@ -0,0 +1,29 @@
+package integration
+
+import FlankCommand
+import com.google.common.truth.Truth.assertThat
+import org.junit.Test
+import run
+
+class IgnoreFailedIT {
+    private val name = this::class.java.simpleName
+    @Test
+    fun `return with exit code 0 for failed tests`() {
+        val result = FlankCommand(
+            flankPath = FLANK_JAR_PATH,
+            ymlPath = "$CONFIGS_PATH/flank_android_ignore_failed.yml",
+            params = androidRunCommands
+        ).run(
+            workingDirectory = "./",
+            testSuite = name
+        )
+
+        assertExitCode(result, 0)
+
+        val resOutput = result.output.removeUnicode()
+        assertThat(resOutput).containsMatch(findInCompare(name))
+        assertContainsOutcomeSummary(resOutput) {
+            failure = 1
+        }
+    }
+}

integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt

@@ -0,0 +1,85 @@
+package integration
+
+import com.google.common.truth.Truth.assertThat
+import org.junit.Assert.assertEquals
+import utils.ProcessResult
+import java.io.File
+
+const val FLANK_JAR_PATH = "../test_runner/build/libs/flank.jar"
+const val CONFIGS_PATH = "./src/test/resources/cases"
+
+val androidRunCommands = listOf("firebase", "test", "android", "run")
+val iosRunCommands = listOf("firebase", "test", "ios", "run")
+
+fun assertExitCode(result: ProcessResult, expectedExitCode: Int) = assertEquals(
+    """
+    Exit code:
+      expected $expectedExitCode
+      actual   ${result.exitCode}
+    Output:
+      ${result.output}
+""".trimIndent(),
+    expectedExitCode,
+    result.exitCode
+)
+
+@Suppress("SetterBackingFieldAssignment")
+data class OutcomeSummary(val matcher: MutableMap<TestOutcome, Int> = mutableMapOf<TestOutcome, Int>().withDefault { 0 }) {
+    var success: Int = 0
+        set(value) {
+            matcher[TestOutcome.SUCCESS] = value
+        }
+
+    var failure: Int = 0
+        set(value) {
+            matcher[TestOutcome.FAILURE] = value
+        }
+
+    var flaky: Int = 0
+        set(value) {
+            matcher[TestOutcome.FLAKY] = value
+        }
+}
+
+fun assertContainsUploads(input: String, vararg uploads: String) = uploads.forEach {
+    assertThat(input).contains("Uploading $it")
+}
+
+fun assertContainsOutcomeSummary(input: String, block: OutcomeSummary.() -> Unit) =
+    OutcomeSummary().apply(block).matcher.entries.forEach { (outcome, times) ->
+        val actual = outcome.regex.findAll(input).toList().size
+        if (actual != times) throw AssertionError("""
+             |Incorrect number of ${outcome.name}
+             |  expected: $times
+             |  but was:  $actual
+             |Output:
+             |${"┌[\\s\\S]*┘".toRegex().find(input)?.value?.trimIndent()}
+         """.trimMargin()
+        )
+    }
+
+fun assertNoOutcomeSummary(input: String) {
+    if ("┌[\\s\\S]*┘".toRegex().matches(input)) throw AssertionError("There should be no outcome table.")
+}
+
+enum class TestOutcome(val regex: Regex) {
+    SUCCESS(fromCommon("success")),
+    FLAKY(fromCommon("flaky")),
+    FAILURE(fromCommon("failure")),
+}
+
+private val fromCommon =
+    { outcome: String ->
+        if (isWindows) "\\?\\s$outcome\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex()
+        else "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex()
+    }
+
+fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "").trimIndent()
+
+fun findInCompare(name: String) = File("./src/test/resources/compare/$name-compare").readText().trimIndent()
+
+private val osName = System.getProperty("os.name")?.toLowerCase() ?: ""
+
+val isWindows: Boolean by lazy {
+    osName.indexOf("win") >= 0
+}

integration_tests/src/test/kotlin/integration/MultipleApksIT.kt

@@ -0,0 +1,34 @@
+package integration
+
+import FlankCommand
+import com.google.common.truth.Truth.assertThat
+import org.junit.Test
+import run
+
+class MultipleApksIT {
+    private val name = this::class.java.simpleName
+
+    @Test
+    fun `flank full option run`() {
+        val result = FlankCommand(
+            flankPath = FLANK_JAR_PATH,
+            ymlPath = "$CONFIGS_PATH/flank_android_multiple_apk.yml",
+            params = androidRunCommands
+        ).run("./", name)
+
+        assertExitCode(result, 10)
+
+        val resOutput = result.output.removeUnicode()
+        assertThat(resOutput).containsMatch(findInCompare(name))
+        assertContainsUploads(resOutput,
+            "app-multiple-success-debug-androidTest.apk",
+            "app-multiple-error-debug-androidTest.apk",
+            "MainActivity_robo_script.json"
+        )
+
+        assertContainsOutcomeSummary(resOutput) {
+            success = 3
+            failure = 1
+        }
+    }
+}

integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt

@@ -0,0 +1,37 @@
+package integration
+
+import FlankCommand
+import com.google.common.truth.Truth.assertThat
+import run
+import org.junit.Test
+
+class MultipleDevicesIT {
+    private val name = this::class.java.simpleName
+
+    @Test
+    fun `run tests on multiple devices - android`() {
+        val name = "$name-android"
+        val result = FlankCommand(
+            flankPath = FLANK_JAR_PATH,
+            ymlPath = "$CONFIGS_PATH/flank_android_multiple_devices.yml",
+            params = androidRunCommands
+        ).run(
+            workingDirectory = "./",
+            testSuite = name
+        )
+
+        assertExitCode(result, 10)
+
+        val resOutput = result.output.removeUnicode()
+        assertThat(resOutput).containsMatch(findInCompare(name))
+        assertContainsUploads(resOutput,
+            "app-multiple-success-debug-androidTest.apk",
+            "app-multiple-error-debug-androidTest.apk",
+            "performanceMetrics.json"
+        )
+        assertContainsOutcomeSummary(resOutput) {
+            success = 6
+            failure = 3
+        }
+    }
+}

integration_tests/src/test/kotlin/integration/RunTimeoutIT.kt

@@ -0,0 +1,27 @@
+package integration
+
+import FlankCommand
+import com.google.common.truth.Truth.assertThat
+import org.junit.Test
+import run
+
+class RunTimeoutIT {
+    private val name = this::class.java.simpleName
+    @Test
+    fun `cancel test run on timeout`() {
+        val result = FlankCommand(
+            flankPath = FLANK_JAR_PATH,
+            ymlPath = "$CONFIGS_PATH/flank_android_run_timeout.yml",
+            params = androidRunCommands
+        ).run(
+            workingDirectory = "./",
+            testSuite = name
+        )
+
+        assertExitCode(result, 1)
+
+        val resOutput = result.output.removeUnicode()
+        assertThat(resOutput).containsMatch(findInCompare(name))
+        assertNoOutcomeSummary(resOutput)
+    }
+}

integration_tests/src/test/kotlin/integration/SanityRoboIT.kt

@@ -0,0 +1,30 @@
+package integration
+
+import FlankCommand
+import com.google.common.truth.Truth.assertThat
+import org.junit.Test
+import run
+
+class SanityRoboIT {
+    private val name = this::class.java.simpleName
+
+    @Test
+    fun `sanity robo`() {
+        val result = FlankCommand(
+            flankPath = FLANK_JAR_PATH,
+            ymlPath = "$CONFIGS_PATH/sanity_robo.yml",
+            params = androidRunCommands
+        ).run(
+            workingDirectory = "./",
+            testSuite = this::class.java.simpleName
+        )
+
+        assertExitCode(result, 0)
+
+        val resOutput = result.output.removeUnicode()
+        assertThat(resOutput).containsMatch(findInCompare(name))
+        assertContainsOutcomeSummary(resOutput) {
+            success = 1
+        }
+    }
+}

integration_tests/src/test/kotlin/integration/TestFilteringIT.kt

@@ -0,0 +1,30 @@
+package integration
+
+import FlankCommand
+import com.google.common.truth.Truth.assertThat
+import org.junit.Test
+import run
+
+class TestFilteringIT {
+    private val name = this::class.java.simpleName
+
+    @Test
+    fun `run test from only one apk`() {
+        val result = FlankCommand(
+            flankPath = FLANK_JAR_PATH,
+            ymlPath = "$CONFIGS_PATH/test_filtering_android.yml",
+            params = androidRunCommands
+        ).run(
+            workingDirectory = "./",
+            testSuite = name
+        )
+
+        assertExitCode(result, 0)
+
+        val resOutput = result.output.removeUnicode()
+        assertThat(resOutput).containsMatch(findInCompare(name))
+        assertContainsOutcomeSummary(resOutput) {
+            success = 1
+        }
+    }
+}