From 354213db9c0c80eb1dc6f72a33d736d8d6ab882c Mon Sep 17 00:00:00 2001 From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com> Date: Fri, 11 Dec 2020 16:49:46 +0100 Subject: [PATCH] feat: Add logic to verify xml results (#1362) Fixes #1315 ## Test Plan > How do we know the code works? 1. run ```./gradlew integrationTests``` 1. integration tests should pass ## Description Flank should also verify XML results for android and ios integration tests. This ticket should be merged after #1316 when we update all tests. ```filter all tests - ios``` is disabled and should be updated in https://github.com/Flank/flank/issues/1388 ## Checklist - [X] Add verification for basic android test - [X] Add verification for tests from #1316 --- integration_tests/build.gradle.kts | 2 + .../buildSrc/src/main/kotlin/Deps.kt | 9 +++ .../src/test/kotlin/IntegrationTests.kt | 14 ++++- .../kotlin/integration/AllTestFilteredIT.kt | 2 + .../integration/IntergrationTestsUtils.kt | 14 +++++ .../test/kotlin/integration/MultipleApksIT.kt | 12 ++++ .../kotlin/integration/MultipleDevicesIT.kt | 11 ++++ .../test/kotlin/integration/SanityRoboIT.kt | 9 +++ .../kotlin/integration/TestFilteringIT.kt | 11 ++++ .../src/test/kotlin/utils/LoadTestSuites.kt | 9 +++ .../src/test/kotlin/utils/OutputHelper.kt | 33 +++++++++++ .../src/test/kotlin/utils/TestsConsts.kt | 20 +++---- .../kotlin/utils/testResults/TestSuite.kt | 58 +++++++++++++++++++ .../cases/flank_android_run_timeout.yml | 4 +- .../resources/compare/RunTimeoutIT-compare | 8 +-- 15 files changed, 196 insertions(+), 20 deletions(-) create mode 100644 integration_tests/src/test/kotlin/utils/LoadTestSuites.kt create mode 100644 integration_tests/src/test/kotlin/utils/OutputHelper.kt create mode 100644 integration_tests/src/test/kotlin/utils/testResults/TestSuite.kt diff --git a/integration_tests/build.gradle.kts b/integration_tests/build.gradle.kts index 96e0b548ac..d31a6dc6d7 100644 --- a/integration_tests/build.gradle.kts +++ b/integration_tests/build.gradle.kts @@ -34,6 +34,8 @@ detekt { dependencies { implementation(kotlin("stdlib")) testImplementation(Dependencies.JUNIT) + testImplementation(Dependencies.JACKSON_XML) + testImplementation(Dependencies.JACKSON_KOTLIN) testImplementation(Dependencies.TRUTH) detektPlugins(Dependencies.DETEKT_FORMATTING) } diff --git a/integration_tests/buildSrc/src/main/kotlin/Deps.kt b/integration_tests/buildSrc/src/main/kotlin/Deps.kt index ea1414853f..861e16f947 100644 --- a/integration_tests/buildSrc/src/main/kotlin/Deps.kt +++ b/integration_tests/buildSrc/src/main/kotlin/Deps.kt @@ -11,6 +11,10 @@ object Versions { // https://github.com/mockk/mockk const val MOCKK = "1.10.0" + + // https://github.com/FasterXML/jackson-core/releases + // https://github.com/FasterXML/jackson-dataformat-xml/releases + const val JACKSON = "2.11.3" } object Libs { @@ -22,6 +26,11 @@ object Libs { const val DETEKT_FORMATTING = "io.gitlab.arturbosch.detekt:detekt-formatting:${Versions.DETEKT}" //endregion + //region Jackson + const val JACKSON_KOTLIN = "com.fasterxml.jackson.module:jackson-module-kotlin:${Versions.JACKSON}" + const val JACKSON_XML = "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${Versions.JACKSON}" + //endregion + //region Test Dependencies const val JUNIT = "junit:junit:${Versions.JUNIT}" const val MOCKK = "io.mockk:mockk:${Versions.MOCKK}" diff --git a/integration_tests/src/test/kotlin/IntegrationTests.kt b/integration_tests/src/test/kotlin/IntegrationTests.kt index 21ede14160..d7b412b88d 100644 --- a/integration_tests/src/test/kotlin/IntegrationTests.kt +++ b/integration_tests/src/test/kotlin/IntegrationTests.kt @@ -2,10 +2,16 @@ import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Ignore import org.junit.Test +import utils.assertCountOfSkippedTests +import utils.assertTestResultContainsWebLinks +import utils.findTestDirectoryFromOutput +import utils.loadAsTestSuite +import utils.toJUnitXmlFile import utils.toStringMap +import java.io.File -class IntegrationTests { +class IntegrationTests { @Test fun shouldMatchAndroidSuccessExitCodeAndPattern() { val testParameters = System.getProperties().toStringMap().toAndroidTestParameters() @@ -24,6 +30,11 @@ class IntegrationTests { "Output don't match pattern, actual output: ${actual.output}", expectedOutput.find(actual.output)?.value.orEmpty().isNotBlank() ) + + actual.output.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run { + assertCountOfSkippedTests(3) + assertTestResultContainsWebLinks() + } } @Ignore("iOS has only physical devices, whit current configuration flank's project hits quota limit extremely fast") @@ -41,6 +52,7 @@ class IntegrationTests { val expectedOutput = testParameters.outputPattern.toRegex( setOf(RegexOption.DOT_MATCHES_ALL) ) + File("test.log").writeText(actual.output) assertTrue( "Output don't match pattern, actual output: ${actual.output}", expectedOutput.find(actual.output)?.value.orEmpty().isNotBlank() diff --git a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt index ee95a5ad96..17e84ea68c 100644 --- a/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt +++ b/integration_tests/src/test/kotlin/integration/AllTestFilteredIT.kt @@ -3,6 +3,7 @@ package integration import FlankCommand import com.google.common.truth.Truth.assertThat import org.junit.Assume.assumeFalse +import org.junit.Ignore import run import org.junit.Test @@ -29,6 +30,7 @@ class AllTestFilteredIT { } @Test + @Ignore("Should be fixed in https://github.com/Flank/flank/issues/1388") fun `filter all tests - ios`() { assumeFalse(isWindows) val name = "$name-ios" diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index ac97af7f35..89c8f3a91b 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -11,6 +11,20 @@ const val CONFIGS_PATH = "./src/test/resources/cases" val androidRunCommands = listOf("firebase", "test", "android", "run") val iosRunCommands = listOf("firebase", "test", "ios", "run") +val multipleSuccessfulTests = listOf( + "test", "testFoo", + "test0", "test1", "test2", + "clickRightButton[0]", "clickRightButton[1]", "clickRightButton[2]", + "shouldHopefullyPass[0]", "shouldHopefullyPass[1]", "shouldHopefullyPass[2]", + "clickRightButtonFromMethod(toast, toast) [0]", "clickRightButtonFromMethod(alert, alert) [1]", + "clickRightButtonFromMethod(exception, exception) [2]", "clickRightButtonFromAnnotation(toast, toast) [0]", + "clickRightButtonFromAnnotation(alert, alert) [1]", "clickRightButtonFromAnnotation(exception, exception) [2]", + "clickRightButton[0: toast toast]", "clickRightButton[1: alert alert]", "clickRightButton[2: exception exception]" +) +val multipleFailedTests = listOf( + "testFoo", "test0", "test1", "test2", "testBar" +) + fun assertExitCode(result: ProcessResult, expectedExitCode: Int) = assertEquals( """ Exit code: diff --git a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt index fec202a434..845accff24 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt @@ -4,6 +4,12 @@ import FlankCommand import com.google.common.truth.Truth.assertThat import org.junit.Test import run +import utils.assertTestFail +import utils.assertTestPass +import utils.assertTestResultContainsWebLinks +import utils.findTestDirectoryFromOutput +import utils.loadAsTestSuite +import utils.toJUnitXmlFile class MultipleApksIT { private val name = this::class.java.simpleName @@ -30,5 +36,11 @@ class MultipleApksIT { success = 3 failure = 1 } + + resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run { + assertTestResultContainsWebLinks() + assertTestPass(multipleSuccessfulTests) + assertTestFail(multipleFailedTests) + } } } diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt index 6be6bf2921..1132e252be 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt @@ -4,6 +4,12 @@ import FlankCommand import com.google.common.truth.Truth.assertThat import run import org.junit.Test +import utils.assertTestFail +import utils.assertTestPass +import utils.assertTestResultContainsWebLinks +import utils.findTestDirectoryFromOutput +import utils.loadAsTestSuite +import utils.toJUnitXmlFile class MultipleDevicesIT { private val name = this::class.java.simpleName @@ -33,5 +39,10 @@ class MultipleDevicesIT { success = 6 failure = 3 } + resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run { + assertTestResultContainsWebLinks() + assertTestPass(multipleSuccessfulTests) + assertTestFail(multipleFailedTests) + } } } diff --git a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt index 0e7e811889..700582f9ba 100644 --- a/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt +++ b/integration_tests/src/test/kotlin/integration/SanityRoboIT.kt @@ -4,6 +4,11 @@ import FlankCommand import com.google.common.truth.Truth.assertThat import org.junit.Test import run +import utils.assertCountOfFailedTests +import utils.assertTestResultContainsWebLinks +import utils.findTestDirectoryFromOutput +import utils.loadAsTestSuite +import utils.toJUnitXmlFile class SanityRoboIT { private val name = this::class.java.simpleName @@ -26,5 +31,9 @@ class SanityRoboIT { assertContainsOutcomeSummary(resOutput) { success = 1 } + resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run { + assertTestResultContainsWebLinks() + assertCountOfFailedTests(0) + } } } diff --git a/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt b/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt index e81e259cc5..69f73dba3b 100644 --- a/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt +++ b/integration_tests/src/test/kotlin/integration/TestFilteringIT.kt @@ -4,6 +4,12 @@ import FlankCommand import com.google.common.truth.Truth.assertThat import org.junit.Test import run +import utils.assertCountOfFailedTests +import utils.assertTestPass +import utils.assertTestResultContainsWebLinks +import utils.findTestDirectoryFromOutput +import utils.loadAsTestSuite +import utils.toJUnitXmlFile class TestFilteringIT { private val name = this::class.java.simpleName @@ -26,5 +32,10 @@ class TestFilteringIT { assertContainsOutcomeSummary(resOutput) { success = 1 } + resOutput.findTestDirectoryFromOutput().toJUnitXmlFile().loadAsTestSuite().run { + assertTestResultContainsWebLinks() + assertCountOfFailedTests(0) + assertTestPass(listOf("test2")) + } } } diff --git a/integration_tests/src/test/kotlin/utils/LoadTestSuites.kt b/integration_tests/src/test/kotlin/utils/LoadTestSuites.kt new file mode 100644 index 0000000000..04bdca365f --- /dev/null +++ b/integration_tests/src/test/kotlin/utils/LoadTestSuites.kt @@ -0,0 +1,9 @@ +package utils + +import com.fasterxml.jackson.dataformat.xml.XmlMapper +import com.fasterxml.jackson.module.kotlin.KotlinModule +import utils.testResults.TestSuites +import java.io.File + +fun File.loadAsTestSuite(): TestSuites = + XmlMapper().registerModule(KotlinModule()).readValue(this, TestSuites::class.java) diff --git a/integration_tests/src/test/kotlin/utils/OutputHelper.kt b/integration_tests/src/test/kotlin/utils/OutputHelper.kt new file mode 100644 index 0000000000..fdbe5864aa --- /dev/null +++ b/integration_tests/src/test/kotlin/utils/OutputHelper.kt @@ -0,0 +1,33 @@ +package utils + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import utils.testResults.TestSuites +import java.io.File +import java.nio.file.Paths + +fun String.findTestDirectoryFromOutput() = + "results-dir:\\s.*\\s".toRegex().find(this)?.value.orEmpty().trim().replace("results-dir: ", "") + +fun String.toJUnitXmlFile(): File = Paths.get("./", "results", this, "JUnitReport.xml").toFile() + +fun TestSuites.assertTestResultContainsWebLinks() = + testSuites.flatMap { it.testCases }.filter { it.skipped == null }.forEach { + assertFalse(it.webLink.isNullOrBlank()) + } + +fun TestSuites.assertCountOfSkippedTests(expectedCount: Int) = + assertEquals(expectedCount, testSuites.sumBy { it.skipped }) + +fun TestSuites.assertCountOfFailedTests(expectedCount: Int) = + assertEquals(expectedCount, testSuites.count { it.failures > 0 }) + +fun TestSuites.assertTestPass(tests: List) = assertEquals( + tests.count(), + testSuites.flatMap { it.testCases }.filter { it.name in tests && it.failure == null }.distinctBy { it.name }.count() +) + +fun TestSuites.assertTestFail(tests: List) = assertEquals( + tests.count(), + testSuites.flatMap { it.testCases }.filter { it.name in tests && it.failure != null }.distinctBy { it.name }.count() +) diff --git a/integration_tests/src/test/kotlin/utils/TestsConsts.kt b/integration_tests/src/test/kotlin/utils/TestsConsts.kt index d19a67fa79..78d5535a99 100644 --- a/integration_tests/src/test/kotlin/utils/TestsConsts.kt +++ b/integration_tests/src/test/kotlin/utils/TestsConsts.kt @@ -13,15 +13,11 @@ const val defaultAndroidOutputPattern = "AndroidArgs\\s*" + "FetchArtifacts[\\s\\S]*" + "Updating matrix file" -const val defaultIosOutputPattern = "IosArgs.*?" + - "gcloud:.*?" + - "flank:.*?" + - "RunTests.*?" + - "Matrices webLink.*?" + - "matrix-.*?" + - "FetchArtifacts.*?" + - "Updating matrix file.*?" + - "CostReport.*?" + - "MatrixResultsReport.*?" + - "test cases passed.*?" + - "Uploading JUnitReport.xml ." +const val defaultIosOutputPattern = "IosArgs[\\s\\S]*" + + "flank:[\\s\\S]*" + + "RunTests[\\s\\S]*" + + "Matrices webLink[\\s\\S]*" + + "matrix-[\\s\\S]*" + + "CostReport[\\s\\S]*" + + "MatrixResultsReport[\\s\\S]*" + + "[\\s\\S]*test cases passed[\\s\\S]*" diff --git a/integration_tests/src/test/kotlin/utils/testResults/TestSuite.kt b/integration_tests/src/test/kotlin/utils/testResults/TestSuite.kt new file mode 100644 index 0000000000..86b11d244e --- /dev/null +++ b/integration_tests/src/test/kotlin/utils/testResults/TestSuite.kt @@ -0,0 +1,58 @@ +package utils.testResults + +import com.fasterxml.jackson.annotation.JsonSetter +import com.fasterxml.jackson.annotation.Nulls +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement + +@JacksonXmlRootElement(localName = "testsuites") +data class TestSuites( + @JacksonXmlElementWrapper(useWrapping = false) + @JacksonXmlProperty(localName = "testsuite") + val testSuites: List = emptyList() +) + +data class TestSuite( + @JacksonXmlProperty(isAttribute = true) + val hostname: String = "", + @JacksonXmlProperty(isAttribute = true) + val failures: Int = 0, + @JacksonXmlProperty(isAttribute = true) + val flakes: Int = 0, + @JacksonXmlProperty(isAttribute = true) + val tests: Int = 0, + @JacksonXmlProperty(isAttribute = true) + val name: String = "", + @JacksonXmlProperty(isAttribute = true) + val time: String = "", + @JacksonXmlProperty(isAttribute = true) + val errors: String = "", + + @JacksonXmlElementWrapper(useWrapping = false) + @JacksonXmlProperty(localName = "testcase") + val testCases: List = emptyList(), + + @JacksonXmlProperty(isAttribute = true) + val skipped: Int = 0, + @JacksonXmlProperty(isAttribute = true) + val timestamp: String = "" +) + +data class TestCase( + @JacksonXmlProperty(isAttribute = true) + val classname: String?, + + val webLink: String?, + + @JacksonXmlProperty(isAttribute = true) + val name: String?, + + @JacksonXmlProperty(isAttribute = true) + val time: String?, + + @JsonSetter(nulls = Nulls.AS_EMPTY) + val skipped: String?, + + val failure: String? +) diff --git a/integration_tests/src/test/resources/cases/flank_android_run_timeout.yml b/integration_tests/src/test/resources/cases/flank_android_run_timeout.yml index f629b86964..4bca0e2c47 100644 --- a/integration_tests/src/test/resources/cases/flank_android_run_timeout.yml +++ b/integration_tests/src/test/resources/cases/flank_android_run_timeout.yml @@ -1,6 +1,6 @@ gcloud: - app: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk - test: ../test_runner/src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + app: gs://flank-open-source.appspot.com/integration/app-debug.apk + test: gs://flank-open-source.appspot.com/integration/app-single-success-debug-androidTest.apk flank: disable-results-upload: true diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare index 68308a85c8..150417b54d 100644 --- a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare +++ b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare @@ -9,8 +9,8 @@ AndroidArgs network-profile: null results-history-name: null # Android gcloud - app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk - test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-success-debug-androidTest.apk + app: [0-9a-zA-Z\\\/_.:-]*[\\\/]flank-open-source.appspot.com[\\\/]integration[\\\/]app-debug.apk + test: [0-9a-zA-Z\\\/_.:-]*[\\\/]flank-open-source.appspot.com[\\\/]integration[\\\/]app-single-success-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true @@ -63,9 +63,7 @@ AndroidArgs RunTests Saved 1 shards to android_shards.json - Uploading app-debug.apk \.* - Uploading app-single-success-debug-androidTest.apk \.* - 1 test / 1 shard + 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*