feat: Add logic to verify xml results (#1362)

Fixes #1315 ## Test Plan > How do we know the code works? 1. run ```./gradlew integrationTests``` 1. integration tests should pass ## Description Flank should also verify XML results for android and ios integration tests. This ticket should be merged after #1316 when we update all tests. ```filter all tests - ios``` is disabled and should be updated in #1388 ## Checklist - [X] Add verification for basic android test - [X] Add verification for tests from #1316
Flank · Dec 11, 2020 · 354213d · 354213d
1 parent 9d1ba8d
commit 354213d
Show file tree

Hide file tree

Showing 15 changed files with 196 additions and 20 deletions.
diff --git a/integration_tests/build.gradle.kts b/integration_tests/build.gradle.kts
@@ -34,6 +34,8 @@ detekt {
 dependencies {
     implementation(kotlin("stdlib"))
     testImplementation(Dependencies.JUNIT)
+    testImplementation(Dependencies.JACKSON_XML)
+    testImplementation(Dependencies.JACKSON_KOTLIN)
     testImplementation(Dependencies.TRUTH)
     detektPlugins(Dependencies.DETEKT_FORMATTING)
 }

diff --git a/integration_tests/buildSrc/src/main/kotlin/Deps.kt b/integration_tests/buildSrc/src/main/kotlin/Deps.kt
@@ -11,6 +11,10 @@ object Versions {
 
     // https://github.com/mockk/mockk
     const val MOCKK = "1.10.0"
+
+    // https://github.com/FasterXML/jackson-core/releases
+    // https://github.com/FasterXML/jackson-dataformat-xml/releases
+    const val JACKSON = "2.11.3"
 }
 
 object Libs {
@@ -22,6 +26,11 @@ object Libs {
     const val DETEKT_FORMATTING = "io.gitlab.arturbosch.detekt:detekt-formatting:${Versions.DETEKT}"
     //endregion
 
+    //region Jackson
+    const val JACKSON_KOTLIN = "com.fasterxml.jackson.module:jackson-module-kotlin:${Versions.JACKSON}"
+    const val JACKSON_XML = "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${Versions.JACKSON}"
+    //endregion
+
     //region Test Dependencies
     const val JUNIT = "junit:junit:${Versions.JUNIT}"
     const val MOCKK = "io.mockk:mockk:${Versions.MOCKK}"

diff --git a/integration_tests/src/test/kotlin/IntegrationTests.kt b/integration_tests/src/test/kotlin/IntegrationTests.kt
@@ -2,10 +2,16 @@ import org.junit.Assert.assertEquals
 import org.junit.Assert.assertTrue
 import org.junit.Ignore
 import org.junit.Test
+import utils.assertCountOfSkippedTests
+import utils.assertTestResultContainsWebLinks
+import utils.findTestDirectoryFromOutput
+import utils.loadAsTestSuite
+import utils.toJUnitXmlFile
 import utils.toStringMap
+import java.io.File
 
-class IntegrationTests {
 
+class IntegrationTests {
     @Test
     fun shouldMatchAndroidSuccessExitCodeAndPattern() {
         val testParameters = System.getProperties().toStringMap().toAndroidTestParameters()
@@ -24,6 +30,11 @@ class IntegrationTests {
             "Output don't match pattern, actual output: ${actual.output}",
             expectedOutput.find(actual.output)?.value.orEmpty().isNotBlank()
         )
+
+        actual.output.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run {
+            assertCountOfSkippedTests(3)
+            assertTestResultContainsWebLinks()
+        }
     }
 
     @Ignore("iOS has only physical devices, whit current configuration flank's project hits quota limit extremely fast")
@@ -41,6 +52,7 @@ class IntegrationTests {
         val expectedOutput = testParameters.outputPattern.toRegex(
             setOf(RegexOption.DOT_MATCHES_ALL)
         )
+        File("test.log").writeText(actual.output)
         assertTrue(
             "Output don't match pattern, actual output: ${actual.output}",
             expectedOutput.find(actual.output)?.value.orEmpty().isNotBlank()

diff --git a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt
@@ -3,6 +3,7 @@ package integration
 import FlankCommand
 import com.google.common.truth.Truth.assertThat
 import org.junit.Assume.assumeFalse
+import org.junit.Ignore
 import run
 import org.junit.Test
 
@@ -29,6 +30,7 @@ class AllTestFilteredIT {
     }
 
     @Test
+    @Ignore("Should be fixed in https://github.com/Flank/flank/issues/1388")
     fun `filter all tests - ios`() {
         assumeFalse(isWindows)
         val name = "$name-ios"

diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt
@@ -11,6 +11,20 @@ const val CONFIGS_PATH = "./src/test/resources/cases"
 val androidRunCommands = listOf("firebase", "test", "android", "run")
 val iosRunCommands = listOf("firebase", "test", "ios", "run")
 
+val multipleSuccessfulTests = listOf(
+    "test", "testFoo",
+    "test0", "test1", "test2",
+    "clickRightButton[0]", "clickRightButton[1]", "clickRightButton[2]",
+    "shouldHopefullyPass[0]", "shouldHopefullyPass[1]", "shouldHopefullyPass[2]",
+    "clickRightButtonFromMethod(toast, toast) [0]", "clickRightButtonFromMethod(alert, alert) [1]",
+    "clickRightButtonFromMethod(exception, exception) [2]", "clickRightButtonFromAnnotation(toast, toast) [0]",
+    "clickRightButtonFromAnnotation(alert, alert) [1]", "clickRightButtonFromAnnotation(exception, exception) [2]",
+    "clickRightButton[0: toast toast]", "clickRightButton[1: alert alert]", "clickRightButton[2: exception exception]"
+)
+val multipleFailedTests = listOf(
+    "testFoo", "test0", "test1", "test2", "testBar"
+)
+
 fun assertExitCode(result: ProcessResult, expectedExitCode: Int) = assertEquals(
     """
     Exit code:

diff --git a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt
@@ -4,6 +4,12 @@ import FlankCommand
 import com.google.common.truth.Truth.assertThat
 import org.junit.Test
 import run
+import utils.assertTestFail
+import utils.assertTestPass
+import utils.assertTestResultContainsWebLinks
+import utils.findTestDirectoryFromOutput
+import utils.loadAsTestSuite
+import utils.toJUnitXmlFile
 
 class MultipleApksIT {
     private val name = this::class.java.simpleName
@@ -30,5 +36,11 @@ class MultipleApksIT {
             success = 3
             failure = 1
         }
+
+        resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run {
+            assertTestResultContainsWebLinks()
+            assertTestPass(multipleSuccessfulTests)
+            assertTestFail(multipleFailedTests)
+        }
     }
 }
diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt
@@ -4,6 +4,12 @@ import FlankCommand
 import com.google.common.truth.Truth.assertThat
 import run
 import org.junit.Test
+import utils.assertTestFail
+import utils.assertTestPass
+import utils.assertTestResultContainsWebLinks
+import utils.findTestDirectoryFromOutput
+import utils.loadAsTestSuite
+import utils.toJUnitXmlFile
 
 class MultipleDevicesIT {
     private val name = this::class.java.simpleName
@@ -33,5 +39,10 @@ class MultipleDevicesIT {
             success = 6
             failure = 3
         }
+        resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run {
+            assertTestResultContainsWebLinks()
+            assertTestPass(multipleSuccessfulTests)
+            assertTestFail(multipleFailedTests)
+        }
     }
 }
diff --git a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt
@@ -4,6 +4,11 @@ import FlankCommand
 import com.google.common.truth.Truth.assertThat
 import org.junit.Test
 import run
+import utils.assertCountOfFailedTests
+import utils.assertTestResultContainsWebLinks
+import utils.findTestDirectoryFromOutput
+import utils.loadAsTestSuite
+import utils.toJUnitXmlFile
 
 class SanityRoboIT {
     private val name = this::class.java.simpleName
@@ -26,5 +31,9 @@ class SanityRoboIT {
         assertContainsOutcomeSummary(resOutput) {
             success = 1
         }
+        resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run {
+            assertTestResultContainsWebLinks()
+            assertCountOfFailedTests(0)
+        }
     }
 }
diff --git a/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt b/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt
@@ -4,6 +4,12 @@ import FlankCommand
 import com.google.common.truth.Truth.assertThat
 import org.junit.Test
 import run
+import utils.assertCountOfFailedTests
+import utils.assertTestPass
+import utils.assertTestResultContainsWebLinks
+import utils.findTestDirectoryFromOutput
+import utils.loadAsTestSuite
+import utils.toJUnitXmlFile
 
 class TestFilteringIT {
     private val name = this::class.java.simpleName
@@ -26,5 +32,10 @@ class TestFilteringIT {
         assertContainsOutcomeSummary(resOutput) {
             success = 1
         }
+        resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run {
+            assertTestResultContainsWebLinks()
+            assertCountOfFailedTests(0)
+            assertTestPass(listOf("test2"))
+        }
     }
 }
diff --git a/integration_tests/src/test/kotlin/utils/LoadTestSuites.kt b/integration_tests/src/test/kotlin/utils/LoadTestSuites.kt
@@ -0,0 +1,9 @@
+package utils
+
+import com.fasterxml.jackson.dataformat.xml.XmlMapper
+import com.fasterxml.jackson.module.kotlin.KotlinModule
+import utils.testResults.TestSuites
+import java.io.File
+
+fun File.loadAsTestSuite(): TestSuites =
+    XmlMapper().registerModule(KotlinModule()).readValue(this, TestSuites::class.java)
diff --git a/integration_tests/src/test/kotlin/utils/OutputHelper.kt b/integration_tests/src/test/kotlin/utils/OutputHelper.kt
@@ -0,0 +1,33 @@
+package utils
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import utils.testResults.TestSuites
+import java.io.File
+import java.nio.file.Paths
+
+fun String.findTestDirectoryFromOutput() =
+    "results-dir:\\s.*\\s".toRegex().find(this)?.value.orEmpty().trim().replace("results-dir: ", "")
+
+fun String.toJUnitXmlFile(): File = Paths.get("./", "results", this, "JUnitReport.xml").toFile()
+
+fun TestSuites.assertTestResultContainsWebLinks() =
+    testSuites.flatMap { it.testCases }.filter { it.skipped == null }.forEach {
+        assertFalse(it.webLink.isNullOrBlank())
+    }
+
+fun TestSuites.assertCountOfSkippedTests(expectedCount: Int) =
+    assertEquals(expectedCount, testSuites.sumBy { it.skipped })
+
+fun TestSuites.assertCountOfFailedTests(expectedCount: Int) =
+    assertEquals(expectedCount, testSuites.count { it.failures > 0 })
+
+fun TestSuites.assertTestPass(tests: List<String>) = assertEquals(
+    tests.count(),
+    testSuites.flatMap { it.testCases }.filter { it.name in tests && it.failure == null }.distinctBy { it.name }.count()
+)
+
+fun TestSuites.assertTestFail(tests: List<String>) = assertEquals(
+    tests.count(),
+    testSuites.flatMap { it.testCases }.filter { it.name in tests && it.failure != null }.distinctBy { it.name }.count()
+)
diff --git a/integration_tests/src/test/kotlin/utils/TestsConsts.kt b/integration_tests/src/test/kotlin/utils/TestsConsts.kt
@@ -13,15 +13,11 @@ const val defaultAndroidOutputPattern = "AndroidArgs\\s*" +
     "FetchArtifacts[\\s\\S]*" +
     "Updating matrix file"
 
-const val defaultIosOutputPattern = "IosArgs.*?" +
-    "gcloud:.*?" +
-    "flank:.*?" +
-    "RunTests.*?" +
-    "Matrices webLink.*?" +
-    "matrix-.*?" +
-    "FetchArtifacts.*?" +
-    "Updating matrix file.*?" +
-    "CostReport.*?" +
-    "MatrixResultsReport.*?" +
-    "test cases passed.*?" +
-    "Uploading JUnitReport.xml ."
+const val defaultIosOutputPattern = "IosArgs[\\s\\S]*" +
+    "flank:[\\s\\S]*" +
+    "RunTests[\\s\\S]*" +
+    "Matrices webLink[\\s\\S]*" +
+    "matrix-[\\s\\S]*" +
+    "CostReport[\\s\\S]*" +
+    "MatrixResultsReport[\\s\\S]*" +
+    "[\\s\\S]*test cases passed[\\s\\S]*"
diff --git a/integration_tests/src/test/kotlin/utils/testResults/TestSuite.kt b/integration_tests/src/test/kotlin/utils/testResults/TestSuite.kt
@@ -0,0 +1,58 @@
+package utils.testResults
+
+import com.fasterxml.jackson.annotation.JsonSetter
+import com.fasterxml.jackson.annotation.Nulls
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement
+
+@JacksonXmlRootElement(localName = "testsuites")
+data class TestSuites(
+    @JacksonXmlElementWrapper(useWrapping = false)
+    @JacksonXmlProperty(localName = "testsuite")
+    val testSuites: List<TestSuite> = emptyList()
+)
+
+data class TestSuite(
+    @JacksonXmlProperty(isAttribute = true)
+    val hostname: String = "",
+    @JacksonXmlProperty(isAttribute = true)
+    val failures: Int = 0,
+    @JacksonXmlProperty(isAttribute = true)
+    val flakes: Int = 0,
+    @JacksonXmlProperty(isAttribute = true)
+    val tests: Int = 0,
+    @JacksonXmlProperty(isAttribute = true)
+    val name: String = "",
+    @JacksonXmlProperty(isAttribute = true)
+    val time: String = "",
+    @JacksonXmlProperty(isAttribute = true)
+    val errors: String = "",
+
+    @JacksonXmlElementWrapper(useWrapping = false)
+    @JacksonXmlProperty(localName = "testcase")
+    val testCases: List<TestCase> = emptyList(),
+
+    @JacksonXmlProperty(isAttribute = true)
+    val skipped: Int = 0,
+    @JacksonXmlProperty(isAttribute = true)
+    val timestamp: String = ""
+)
+
+data class TestCase(
+    @JacksonXmlProperty(isAttribute = true)
+    val classname: String?,
+
+    val webLink: String?,
+
+    @JacksonXmlProperty(isAttribute = true)
+    val name: String?,
+
+    @JacksonXmlProperty(isAttribute = true)
+    val time: String?,
+
+    @JsonSetter(nulls = Nulls.AS_EMPTY)
+    val skipped: String?,
+
+    val failure: String?
+)
diff --git a/integration_tests/src/test/resources/cases/flank_android_run_timeout.yml b/integration_tests/src/test/resources/cases/flank_android_run_timeout.yml
@@ -1,6 +1,6 @@
 gcloud:
-  app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk
-  test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk
+  app: gs://flank-open-source.appspot.com/integration/app-debug.apk
+  test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk
 
 flank:
   disable-results-upload: true

diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare
@@ -9,8 +9,8 @@ AndroidArgs
       network-profile: null
       results-history-name: null
       # Android gcloud
-      app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk
-      test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk
+      app: [0-9a-zA-Z\\\/_.:-]*[\\\/]flank-open-source.appspot.com[\\\/]integration[\\\/]app-debug.apk
+      test: [0-9a-zA-Z\\\/_.:-]*[\\\/]flank-open-source.appspot.com[\\\/]integration[\\\/]app-single-success-debug-androidTest.apk
       additional-apks:
       auto-google-login: false
       use-orchestrator: true
@@ -63,9 +63,7 @@ AndroidArgs
 
 RunTests
 Saved 1 shards to android_shards.json
-  Uploading app-debug.apk \.*
-  Uploading app-single-success-debug-androidTest.apk \.*
-  1 test / 1 shard
+  1 test \/ 1 shard
 
   1 matrix ids created in \d{1,2}m \d{1,2}s
   https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*