diff --git a/.github/workflows/full_suite_integration_tests.yml b/.github/workflows/full_suite_integration_tests.yml index f2e718723e..0187472ec6 100644 --- a/.github/workflows/full_suite_integration_tests.yml +++ b/.github/workflows/full_suite_integration_tests.yml @@ -75,7 +75,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ macos-latest, windows-latest, ubuntu-latest ] + os: [ macos-latest, windows-latest, ubuntu-18.04 ] fail-fast: false outputs: job_status: ${{ job.status }} diff --git a/test_runner/src/main/kotlin/ftl/adapter/DownloadAsJunitXML.kt b/test_runner/src/main/kotlin/ftl/adapter/DownloadAsJunitXML.kt index ea9679a2f7..a7520193de 100644 --- a/test_runner/src/main/kotlin/ftl/adapter/DownloadAsJunitXML.kt +++ b/test_runner/src/main/kotlin/ftl/adapter/DownloadAsJunitXML.kt @@ -2,11 +2,11 @@ package ftl.adapter import ftl.adapter.google.toApiModel import ftl.api.FileReference +import ftl.api.JUnitTest import ftl.client.google.downloadAsJunitXml -import ftl.reports.xml.model.JUnitTestResult object DownloadAsJunitXML : FileReference.DownloadAsXML, - (FileReference) -> JUnitTestResult by { fileReference -> + (FileReference) -> JUnitTest.Result by { fileReference -> downloadAsJunitXml(fileReference).toApiModel() } diff --git a/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestFetch.kt b/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestFetch.kt new file mode 100644 index 0000000000..a661f86616 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestFetch.kt @@ -0,0 +1,14 @@ +package ftl.adapter + +import ftl.adapter.google.toApiModel +import ftl.api.JUnitTest +import ftl.client.google.fetchMatrices +import ftl.client.junit.createJUnitTestResult + +object GoogleJUnitTestFetch : + JUnitTest.Result.GenerateFromApi, + (JUnitTest.Result.ApiIdentity) -> JUnitTest.Result by { (projectId, matrixIds) -> + fetchMatrices(matrixIds, projectId) + .createJUnitTestResult() + .toApiModel() + } diff --git a/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestParse.kt b/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestParse.kt new file mode 100644 index 0000000000..7835652e50 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestParse.kt @@ -0,0 +1,19 @@ +package ftl.adapter + +import ftl.adapter.google.toApiModel +import ftl.api.JUnitTest +import ftl.client.junit.parseJUnit +import ftl.client.junit.parseLegacyJUnit +import java.io.File + +object GoogleJUnitTestParse : + JUnitTest.Result.ParseFromFiles, + (File) -> JUnitTest.Result by { + it.parseJUnit().toApiModel() + } + +object GoogleLegacyJunitTestParse : + JUnitTest.Result.ParseFromFiles, + (File) -> JUnitTest.Result by { + it.parseLegacyJUnit().toApiModel() + } diff --git a/test_runner/src/main/kotlin/ftl/adapter/google/FileReferenceAdapter.kt b/test_runner/src/main/kotlin/ftl/adapter/google/FileReferenceAdapter.kt index be5701c239..b55d48f9fd 100644 --- a/test_runner/src/main/kotlin/ftl/adapter/google/FileReferenceAdapter.kt +++ b/test_runner/src/main/kotlin/ftl/adapter/google/FileReferenceAdapter.kt @@ -1,8 +1,5 @@ package ftl.adapter.google import ftl.api.FileReference -import ftl.reports.xml.model.JUnitTestResult internal fun String.toApiModel(fileReference: FileReference) = fileReference.copy(local = this) - -internal fun JUnitTestResult?.toApiModel() = JUnitTestResult(testsuites = this?.testsuites) diff --git a/test_runner/src/main/kotlin/ftl/adapter/google/JUnitTestResultAdapter.kt b/test_runner/src/main/kotlin/ftl/adapter/google/JUnitTestResultAdapter.kt new file mode 100644 index 0000000000..99af0f121e --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/adapter/google/JUnitTestResultAdapter.kt @@ -0,0 +1,42 @@ +package ftl.adapter.google + +import ftl.api.JUnitTest +import ftl.client.junit.JUnitTestCase +import ftl.client.junit.JUnitTestResult +import ftl.client.junit.JUnitTestSuite + +fun JUnitTestResult?.toApiModel(): JUnitTest.Result = JUnitTest.Result( + this?.testsuites?.map { it.toApiModel() }.orEmpty().toMutableList() +) + +private fun JUnitTestSuite.toApiModel(): JUnitTest.Suite { + return JUnitTest.Suite( + name = name, + tests = tests, + failures = failures, + flakes = flakes, + errors = errors, + skipped = skipped, + time = time, + timestamp = timestamp, + hostname = hostname, + testLabExecutionId = testLabExecutionId, + testcases = testcases?.toApiModel(), + properties = properties, + systemOut = systemOut, + systemErr = systemErr + ) +} + +private fun MutableCollection.toApiModel(): MutableCollection { + return map(JUnitTestCase::toApiModel).toMutableList() +} + +private fun JUnitTestCase.toApiModel() = JUnitTest.Case( + name = name, + classname = classname, + time = time, + failures = failures, + errors = errors, + skipped = skipped +) diff --git a/test_runner/src/main/kotlin/ftl/api/FileReference.kt b/test_runner/src/main/kotlin/ftl/api/FileReference.kt index b6216a9c2e..71b2140496 100644 --- a/test_runner/src/main/kotlin/ftl/api/FileReference.kt +++ b/test_runner/src/main/kotlin/ftl/api/FileReference.kt @@ -2,7 +2,6 @@ package ftl.api import ftl.adapter.DownloadAsJunitXML import ftl.adapter.GcStorageDownload -import ftl.reports.xml.model.JUnitTestResult val downloadFileReference: FileReference.Download get() = GcStorageDownload val downloadAsJunitXML: FileReference.DownloadAsXML get() = DownloadAsJunitXML @@ -13,5 +12,5 @@ data class FileReference( ) { interface Download : (FileReference, Boolean, Boolean) -> FileReference - interface DownloadAsXML : (FileReference) -> JUnitTestResult + interface DownloadAsXML : (FileReference) -> JUnitTest.Result } diff --git a/test_runner/src/main/kotlin/ftl/api/JUnitTestResult.kt b/test_runner/src/main/kotlin/ftl/api/JUnitTestResult.kt new file mode 100644 index 0000000000..b31d0df8d4 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/api/JUnitTestResult.kt @@ -0,0 +1,128 @@ +package ftl.api + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement +import ftl.adapter.GoogleJUnitTestFetch +import ftl.adapter.GoogleJUnitTestParse +import ftl.adapter.GoogleLegacyJunitTestParse +import java.io.File + +val generateJUnitTestResultFromApi: JUnitTest.Result.GenerateFromApi get() = GoogleJUnitTestFetch +val parseJUnitTestResultFromFile: JUnitTest.Result.ParseFromFiles get() = GoogleJUnitTestParse +val parseJUnitLegacyTestResultFromFile: JUnitTest.Result.ParseFromFiles get() = GoogleLegacyJunitTestParse + +object JUnitTest { + + @JacksonXmlRootElement(localName = "testsuites") + data class Result( + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @JacksonXmlProperty(localName = "testsuite") + var testsuites: MutableList? = null + ) { + data class ApiIdentity( + val projectId: String, + val matrixIds: List + ) + + interface GenerateFromApi : (ApiIdentity) -> Result + + interface ParseFromFiles : (File) -> Result + } + + data class Suite( + @JacksonXmlProperty(isAttribute = true) + var name: String, + + @JacksonXmlProperty(isAttribute = true) + var tests: String, // Int + + @JacksonXmlProperty(isAttribute = true) + var failures: String, // Int + + @JacksonXmlProperty(isAttribute = true) + var flakes: Int? = null, + + @JacksonXmlProperty(isAttribute = true) + var errors: String, // Int + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(isAttribute = true) + var skipped: String?, // Int. Android only + + @JacksonXmlProperty(isAttribute = true) + var time: String, // Double + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(isAttribute = true) + val timestamp: String?, // String. Android only + + @JacksonXmlProperty(isAttribute = true) + val hostname: String? = "localhost", // String. + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(isAttribute = true) + val testLabExecutionId: String? = null, // String. + + @JacksonXmlProperty(localName = "testcase") + var testcases: MutableCollection?, + + // not used + @JsonInclude(JsonInclude.Include.NON_NULL) + val properties: Any? = null, // + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(localName = "system-out") + val systemOut: Any? = null, // + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(localName = "system-err") + val systemErr: Any? = null // + ) + + data class Case( + // name, classname, and time are always present except for empty test cases + @JacksonXmlProperty(isAttribute = true) + val name: String?, + + @JacksonXmlProperty(isAttribute = true) + val classname: String?, + + @JacksonXmlProperty(isAttribute = true) + val time: String?, + + // iOS contains multiple failures for a single test. + // JUnit XML allows arbitrary amounts of failure/error tags + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(localName = "failure") + val failures: List? = null, + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(localName = "error") + val errors: List? = null, + + @JsonInclude(JsonInclude.Include.CUSTOM, valueFilter = FilterNotNull::class) + val skipped: String? = "absent" // used by FilterNotNull to filter out absent `skipped` values + ) { + + // Consider to move all properties to constructor if will doesn't conflict with parser + @JsonInclude(JsonInclude.Include.NON_NULL) + var webLink: String? = null + + @JacksonXmlProperty(isAttribute = true) + var flaky: Boolean? = null // use null instead of false + } + + @Suppress("UnusedPrivateClass") + private class FilterNotNull { + override fun equals(other: Any?): Boolean { + // other is null = present + // other is not null = absent (default value) + return other != null + } + + override fun hashCode(): Int { + return javaClass.hashCode() + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt b/test_runner/src/main/kotlin/ftl/client/google/FetchTestMatrices.kt similarity index 62% rename from test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt rename to test_runner/src/main/kotlin/ftl/client/google/FetchTestMatrices.kt index 00a59d6c95..df4c1c3be8 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/FetchTestMatrices.kt @@ -1,18 +1,15 @@ -package ftl.reports.api +package ftl.client.google import com.google.testing.model.TestExecution import com.google.testing.model.TestMatrix -import ftl.args.IArgs -import ftl.gc.GcTestMatrix -import ftl.json.MatrixMap import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.runBlocking -fun refreshMatricesAndGetExecutions(matrices: MatrixMap, args: IArgs): List = refreshTestMatrices( - matrixIds = matrices.map.values.map { it.matrixId }, - projectId = args.project +fun fetchMatrices(matricesIds: List, projectId: String): List = refreshTestMatrices( + matrixIds = matricesIds, + projectId = projectId ).getTestExecutions() private fun refreshTestMatrices( @@ -29,6 +26,6 @@ private fun refreshTestMatrices( }.awaitAll() } -private fun List.getTestExecutions(): List = this - .mapNotNull(TestMatrix::getTestExecutions) - .flatten() +private fun List.getTestExecutions(): List = + mapNotNull(TestMatrix::getTestExecutions) + .flatten() diff --git a/test_runner/src/main/kotlin/ftl/client/google/FileReference.kt b/test_runner/src/main/kotlin/ftl/client/google/FileReference.kt index e9a38ac371..4c0f38f794 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/FileReference.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/FileReference.kt @@ -1,8 +1,8 @@ package ftl.client.google import ftl.api.FileReference -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.parseAllSuitesXml +import ftl.client.junit.JUnitTestResult +import ftl.client.junit.parseAllSuitesXml import ftl.run.exception.FlankGeneralError import java.nio.file.Paths diff --git a/test_runner/src/main/kotlin/ftl/client/google/GcStorage.kt b/test_runner/src/main/kotlin/ftl/client/google/GcStorage.kt index 31d53f98cd..a7a3971a0b 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/GcStorage.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/GcStorage.kt @@ -14,9 +14,6 @@ import ftl.args.IArgs import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_PREFIX import ftl.config.FtlConstants.GCS_STORAGE_LINK -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.parseAllSuitesXml -import ftl.reports.xml.xmlToString import ftl.run.exception.FlankGeneralError import ftl.util.runWithProgress import java.io.File @@ -63,26 +60,19 @@ object GcStorage { ) } - fun uploadJunitXml(testResult: JUnitTestResult, args: IArgs) { + fun uploadJunitXml(testResultXml: String, args: IArgs) { if (args.smartFlankGcsPath.isBlank() || args.smartFlankDisableUpload) return // bucket/path/to/object val rawPath = args.smartFlankGcsPath.drop(GCS_PREFIX.length) - testResult.xmlToString().toByteArray().uploadWithProgress( + testResultXml.toByteArray().uploadWithProgress( bucket = rawPath.substringBefore('/'), path = rawPath.substringAfter('/'), name = "smart flank XML" ) } - // junit xml may not exist. ignore error if it doesn't exist - private fun downloadJunitXml( - args: IArgs - ): JUnitTestResult? = download(args.smartFlankGcsPath, ignoreError = true) - .takeIf { it.isNotEmpty() } - ?.let { parseAllSuitesXml(Paths.get(it)) } - private val duplicatedGcsPathCounter = ConcurrentHashMap() @VisibleForTesting diff --git a/test_runner/src/main/kotlin/ftl/gc/GcTestMatrix.kt b/test_runner/src/main/kotlin/ftl/client/google/GcTestMatrix.kt similarity index 98% rename from test_runner/src/main/kotlin/ftl/gc/GcTestMatrix.kt rename to test_runner/src/main/kotlin/ftl/client/google/GcTestMatrix.kt index 6c63e59b75..4871aa515c 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/GcTestMatrix.kt @@ -1,7 +1,8 @@ -package ftl.gc +package ftl.client.google import com.google.testing.model.CancelTestMatrixResponse import com.google.testing.model.TestMatrix +import ftl.gc.GcTesting import ftl.http.executeWithRetry import ftl.run.exception.FlankGeneralError import kotlinx.coroutines.Dispatchers diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt similarity index 95% rename from test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt rename to test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt index bca6cd5454..d0b452c8f7 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt @@ -1,9 +1,10 @@ -package ftl.reports.api +package ftl.client.junit import com.google.api.services.toolresults.model.StackTrace import com.google.api.services.toolresults.model.TestCase import com.google.testing.model.ToolResultsStep -import ftl.reports.xml.model.JUnitTestCase +import ftl.reports.api.format +import ftl.reports.api.millis internal fun createJUnitTestCases( testCases: List, diff --git a/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestResult.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestResult.kt new file mode 100644 index 0000000000..34a65919a9 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestResult.kt @@ -0,0 +1,13 @@ +package ftl.client.junit + +import com.google.testing.model.TestExecution + +internal fun List.createJUnitTestResult() = JUnitTestResult( + testsuites = filterNullToolResultsStep() + .createTestExecutionDataListAsync() + .prepareForJUnitResult() + .createJUnitTestSuites() + .toMutableList() +) + +private fun List.filterNullToolResultsStep() = filter { it.toolResultsStep != null } diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt similarity index 90% rename from test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt rename to test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt index 64b7df7d33..64f1a36e47 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt @@ -1,9 +1,8 @@ -package ftl.reports.api +package ftl.client.junit -import ftl.reports.api.data.TestExecutionData import ftl.reports.api.data.TestSuiteOverviewData +import ftl.reports.api.format import ftl.reports.outcome.axisValue -import ftl.reports.xml.model.JUnitTestSuite internal fun List.createJUnitTestSuites() = mapNotNull { data: TestExecutionData -> data.createTestSuiteOverviewData()?.let { overviewData -> diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateTestExecutionData.kt similarity index 96% rename from test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt rename to test_runner/src/main/kotlin/ftl/client/junit/CreateTestExecutionData.kt index 2a020e2048..156a6b5f47 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestExecutionData.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/CreateTestExecutionData.kt @@ -1,4 +1,4 @@ -package ftl.reports.api +package ftl.client.junit import com.google.api.services.toolresults.model.Step import com.google.api.services.toolresults.model.TestCase @@ -6,7 +6,6 @@ import com.google.api.services.toolresults.model.Timestamp import com.google.testing.model.TestExecution import com.google.testing.model.ToolResultsStep import ftl.gc.GcToolResults -import ftl.reports.api.data.TestExecutionData import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuiteOverviewData.kt b/test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt similarity index 94% rename from test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuiteOverviewData.kt rename to test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt index e3aa875b9f..97564c0263 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateTestSuiteOverviewData.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/CreateTestSuiteOverviewData.kt @@ -1,9 +1,9 @@ -package ftl.reports.api +package ftl.client.junit import com.google.api.services.toolresults.model.TestCase import com.google.api.services.toolresults.model.TestSuiteOverview -import ftl.reports.api.data.TestExecutionData import ftl.reports.api.data.TestSuiteOverviewData +import ftl.reports.api.millis internal fun TestExecutionData.createTestSuiteOverviewData(): TestSuiteOverviewData? = step .testExecutionStep diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestCase.kt b/test_runner/src/main/kotlin/ftl/client/junit/JUnitTestCase.kt similarity index 75% rename from test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestCase.kt rename to test_runner/src/main/kotlin/ftl/client/junit/JUnitTestCase.kt index 6604542d58..47b8b56cae 100644 --- a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestCase.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/JUnitTestCase.kt @@ -1,21 +1,8 @@ -package ftl.reports.xml.model +package ftl.client.junit import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty -@Suppress("UnusedPrivateClass") -private class FilterNotNull { - override fun equals(other: Any?): Boolean { - // other is null = present - // other is not null = absent (default value) - return other != null - } - - override fun hashCode(): Int { - return javaClass.hashCode() - } -} - // https://android.googlesource.com/platform/tools/base/+/tools_r22/ddmlib/src/main/java/com/android/ddmlib/testrunner/XmlTestRunListener.java#256 data class JUnitTestCase( // name, classname, and time are always present except for empty test cases @@ -47,25 +34,17 @@ data class JUnitTestCase( @JacksonXmlProperty(isAttribute = true) var flaky: Boolean? = null // use null instead of false +} - fun empty(): Boolean { - return name == null || classname == null || time == null - } - - /** Failed means there was a failure or an error. */ - fun failed(): Boolean { - return failures?.isNotEmpty() == true || errors?.isNotEmpty() == true - } - - fun skipped(): Boolean { - return skipped == null - } - - fun successful(): Boolean { - return failed().not().and(skipped().not()) +@Suppress("UnusedPrivateClass") +private class FilterNotNull { + override fun equals(other: Any?): Boolean { + // other is null = present + // other is not null = absent (default value) + return other != null } - fun stackTrace(): String { - return failures?.joinToString() + errors?.joinToString() + override fun hashCode(): Int { + return javaClass.hashCode() } } diff --git a/test_runner/src/main/kotlin/ftl/client/junit/JUnitTestResult.kt b/test_runner/src/main/kotlin/ftl/client/junit/JUnitTestResult.kt new file mode 100644 index 0000000000..69c8610844 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/junit/JUnitTestResult.kt @@ -0,0 +1,15 @@ +package ftl.client.junit + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement + +/** + * Firebase generates testsuites for iOS. + * .xctestrun file may contain multiple test bundles (each one is a testsuite) */ +@JacksonXmlRootElement(localName = "testsuites") +data class JUnitTestResult( + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @JacksonXmlProperty(localName = "testsuite") + var testsuites: MutableList? = null +) diff --git a/test_runner/src/main/kotlin/ftl/client/junit/JUnitTestSuite.kt b/test_runner/src/main/kotlin/ftl/client/junit/JUnitTestSuite.kt new file mode 100644 index 0000000000..fc65be5f1d --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/junit/JUnitTestSuite.kt @@ -0,0 +1,54 @@ +package ftl.client.junit + +import com.fasterxml.jackson.annotation.JsonInclude +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty + +data class JUnitTestSuite( + @JacksonXmlProperty(isAttribute = true) + var name: String, + + @JacksonXmlProperty(isAttribute = true) + var tests: String, // Int + + @JacksonXmlProperty(isAttribute = true) + var failures: String, // Int + + @JacksonXmlProperty(isAttribute = true) + var flakes: Int? = null, + + @JacksonXmlProperty(isAttribute = true) + var errors: String, // Int + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(isAttribute = true) + var skipped: String?, // Int. Android only + + @JacksonXmlProperty(isAttribute = true) + var time: String, // Double + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(isAttribute = true) + val timestamp: String?, // String. Android only + + @JacksonXmlProperty(isAttribute = true) + val hostname: String? = "localhost", // String. + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(isAttribute = true) + val testLabExecutionId: String? = null, // String. + + @JacksonXmlProperty(localName = "testcase") + var testcases: MutableCollection?, + + // not used + @JsonInclude(JsonInclude.Include.NON_NULL) + val properties: Any? = null, // + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(localName = "system-out") + val systemOut: Any? = null, // + + @JsonInclude(JsonInclude.Include.NON_NULL) + @JacksonXmlProperty(localName = "system-err") + val systemErr: Any? = null // +) diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/JUnitXml.kt b/test_runner/src/main/kotlin/ftl/client/junit/JUnitXml.kt similarity index 79% rename from test_runner/src/main/kotlin/ftl/reports/xml/JUnitXml.kt rename to test_runner/src/main/kotlin/ftl/client/junit/JUnitXml.kt index 7763617dce..7c5cdea6b6 100644 --- a/test_runner/src/main/kotlin/ftl/reports/xml/JUnitXml.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/JUnitXml.kt @@ -1,13 +1,10 @@ -package ftl.reports.xml +package ftl.client.junit import com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule import com.fasterxml.jackson.dataformat.xml.XmlMapper import com.fasterxml.jackson.dataformat.xml.deser.FromXmlParser.Feature.EMPTY_ELEMENT_AS_NULL import com.fasterxml.jackson.module.kotlin.KotlinModule -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.model.JUnitTestSuite -import ftl.reports.xml.preprocesor.fixHtmlCodes import ftl.run.exception.FlankGeneralError import java.io.File import java.nio.file.Files @@ -27,26 +24,18 @@ private fun xmlText(path: Path): String { return String(Files.readAllBytes(path)) } -fun JUnitTestResult?.xmlToString(): String { - if (this == null) return "" - val prefix = "\n" - return prefix + xmlPrettyWriter.writeValueAsString(this) +fun parseOneSuiteXml(path: File): JUnitTestResult { + return parseOneSuiteXml(xmlText(path.toPath())) } fun parseOneSuiteXml(path: Path): JUnitTestResult { return parseOneSuiteXml(xmlText(path)) } -fun parseOneSuiteXml(path: File): JUnitTestResult { - return parseOneSuiteXml(xmlText(path.toPath())) -} - -fun parseOneSuiteXml(data: String): JUnitTestResult { +private fun parseOneSuiteXml(data: String): JUnitTestResult { return JUnitTestResult(mutableListOf(xmlMapper.readValue(fixHtmlCodes(data), JUnitTestSuite::class.java))) } -// -- - fun parseAllSuitesXml(path: Path): JUnitTestResult { return parseAllSuitesXml(xmlText(path)) } @@ -55,7 +44,7 @@ fun parseAllSuitesXml(path: File): JUnitTestResult { return parseAllSuitesXml(path.toPath()) } -fun parseAllSuitesXml(data: String): JUnitTestResult = +private fun parseAllSuitesXml(data: String): JUnitTestResult = // This is workaround for flank being unable to parse into JUnitTesResults // We need to preserve configure(EMPTY_ELEMENT_AS_NULL, true) to skip empty elements // Once better solution is found, this should be fixed diff --git a/test_runner/src/main/kotlin/ftl/client/junit/JUnitXmlParser.kt b/test_runner/src/main/kotlin/ftl/client/junit/JUnitXmlParser.kt new file mode 100644 index 0000000000..91df9de2b5 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/junit/JUnitXmlParser.kt @@ -0,0 +1,37 @@ +package ftl.client.junit + +import com.google.common.annotations.VisibleForTesting +import ftl.run.exception.FlankGeneralError +import java.io.File + +fun File.parseJUnit(): JUnitTestResult = parse(::parseAllSuitesXml) + +fun File.parseLegacyJUnit(): JUnitTestResult = parse(::parseOneSuiteXml) + +private fun File.parse( + process: (file: File) -> JUnitTestResult +): JUnitTestResult = runCatching(process) + .getOrElse { e -> throw FlankGeneralError("Cannot process xml file: $absolutePath", e) } + .updateTestSuites(deviceName = getDeviceString(parentFile.name)) + +private val deviceStringRgx = Regex("([^-]+-[^-]+-[^-]+-[^-]+).*") + +// NexusLowRes-28-en-portrait-rerun_1 => NexusLowRes-28-en-portrait +@VisibleForTesting +internal fun getDeviceString(deviceString: String): String { + val matchResult = deviceStringRgx.find(deviceString) + return matchResult?.groupValues?.last().orEmpty() +} + +private fun JUnitTestResult.updateTestSuites(deviceName: String) = apply { + testsuites?.forEach { testSuite -> + testSuite.name = "$deviceName#${testSuite.name}" + } +} + +/*todo move it */ +/* +* testSuite.testcases?.forEach { testCase -> + testCase.webLink = ReportManager.getWebLink(matrices, xmlFile) + } +* */ diff --git a/test_runner/src/main/kotlin/ftl/reports/api/PrepareForJUnitResult.kt b/test_runner/src/main/kotlin/ftl/client/junit/PrepareForJUnitResult.kt similarity index 97% rename from test_runner/src/main/kotlin/ftl/reports/api/PrepareForJUnitResult.kt rename to test_runner/src/main/kotlin/ftl/client/junit/PrepareForJUnitResult.kt index 7009628d21..f688c039d1 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/PrepareForJUnitResult.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/PrepareForJUnitResult.kt @@ -1,8 +1,7 @@ -package ftl.reports.api +package ftl.client.junit import com.google.api.services.toolresults.model.Step import com.google.api.services.toolresults.model.TestCase -import ftl.reports.api.data.TestExecutionData // List of TestExecutionData can contains also secondary steps from flaky tests reruns. // We need only primary steps, but we also prefer to display failed tests over successful, diff --git a/test_runner/src/main/kotlin/ftl/reports/api/data/TestExecutionData.kt b/test_runner/src/main/kotlin/ftl/client/junit/TestExecutionData.kt similarity index 92% rename from test_runner/src/main/kotlin/ftl/reports/api/data/TestExecutionData.kt rename to test_runner/src/main/kotlin/ftl/client/junit/TestExecutionData.kt index b3737c35dc..61cfa0b7a6 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/data/TestExecutionData.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/TestExecutionData.kt @@ -1,4 +1,4 @@ -package ftl.reports.api.data +package ftl.client.junit import com.google.api.services.toolresults.model.Step import com.google.api.services.toolresults.model.TestCase diff --git a/test_runner/src/main/kotlin/ftl/client/junit/Utils.kt b/test_runner/src/main/kotlin/ftl/client/junit/Utils.kt new file mode 100644 index 0000000000..103e2e76f8 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/junit/Utils.kt @@ -0,0 +1,12 @@ +package ftl.client.junit + +import com.google.api.services.toolresults.model.TestCase +import com.google.api.services.toolresults.model.Timestamp +import ftl.reports.api.utcDateFormat +import ftl.util.mutableMapProperty + +fun Timestamp.asUnixTimestamp() = (seconds ?: 0) * 1_000 + (nanos ?: 0) / 1_000_000 + +fun Long.formatUtcDate() = utcDateFormat.format(this)!! + +var TestCase.flaky: Boolean by mutableMapProperty { false } diff --git a/test_runner/src/main/kotlin/ftl/client/junit/XmlPreprocessor.kt b/test_runner/src/main/kotlin/ftl/client/junit/XmlPreprocessor.kt new file mode 100644 index 0000000000..22aac774f8 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/junit/XmlPreprocessor.kt @@ -0,0 +1,27 @@ +package ftl.client.junit + +import ftl.client.junit.UtfControlCharRanges.CONTROL_BOTTOM_END +import ftl.client.junit.UtfControlCharRanges.CONTROL_BOTTOM_START +import ftl.client.junit.UtfControlCharRanges.CONTROL_TOP_END +import ftl.client.junit.UtfControlCharRanges.CONTROL_TOP_START +import org.apache.commons.text.StringEscapeUtils + +fun fixHtmlCodes(data: String): String = listOf( + CONTROL_TOP_START.charValue..CONTROL_TOP_END.charValue, + CONTROL_BOTTOM_START.charValue..CONTROL_BOTTOM_END.charValue +).flatten() + .map { StringEscapeUtils.escapeXml11(it.toChar().toString()) } + .filter { it.startsWith("&#") } + .fold(data) { fixedStr: String, isoControlCode: String -> fixedStr.replace(isoControlCode, "") } + +/** + * Numbers come from ascii table https://www.utf8-chartable.de. + * and represents control chars. We need to avoid characters in ranges CONTROL_TOP_START..CONTROL_TOP_END and + * CONTROL_BOTTOM_START..CONTROL_BOTTOM_END because chars from that range escaped to html causing parsing errors. + * */ +private enum class UtfControlCharRanges(val charValue: Int) { + CONTROL_TOP_START(0x00), + CONTROL_TOP_END(0x1F), + CONTROL_BOTTOM_START(0x7F), + CONTROL_BOTTOM_END(0x9F) +} diff --git a/test_runner/src/main/kotlin/ftl/domain/junit/JUnitTestMerge.kt b/test_runner/src/main/kotlin/ftl/domain/junit/JUnitTestMerge.kt new file mode 100644 index 0000000000..40bd98f321 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/domain/junit/JUnitTestMerge.kt @@ -0,0 +1,122 @@ +package ftl.domain.junit + +import ftl.api.JUnitTest +import ftl.run.exception.FlankGeneralError +import ftl.util.stripNotNumbers +import java.util.Locale + +fun JUnitTest.Result.mergeTestTimes(other: JUnitTest.Result?): JUnitTest.Result { + if (other == null) return this + if (this.testsuites == null) this.testsuites = mutableListOf() + + // newTestResult.mergeTestTimes(oldTestResult) + // + // for each new JUnitTestSuite, check if it exists on old + // if JUnitTestSuite exists on both then merge test times + this.testsuites?.forEach { testSuite -> + val oldSuite = other.testsuites?.firstOrNull { it.name == testSuite.name } + if (oldSuite != null) testSuite.mergeTestTimes(oldSuite) + } + + return this +} + +fun JUnitTest.Result.merge(other: JUnitTest.Result?): JUnitTest.Result { + if (other == null) return this + if (testsuites == null) testsuites = mutableListOf() + + other.testsuites?.forEach { testSuite -> + val mergeCandidate = this.testsuites?.firstOrNull { it.name == testSuite.name } + + if (mergeCandidate == null) { + this.testsuites?.add(testSuite) + } else { + mergeCandidate.merge(testSuite) + } + } + + return this +} + +fun JUnitTest.Suite.merge(other: JUnitTest.Suite): JUnitTest.Suite { + if (this.name != other.name) throw FlankGeneralError("Attempted to merge ${other.name} into ${this.name}") + + // tests, failures, errors + this.tests = mergeInt(this.tests, other.tests) + this.failures = mergeInt(this.failures, other.failures) + this.errors = mergeInt(this.errors, other.errors) + this.skipped = mergeInt(this.skipped, other.skipped) + this.time = mergeDouble(this.time, other.time) + + if (this.testcases == null) this.testcases = mutableListOf() + if (other.testcases?.isNotEmpty() == true) { + this.testcases?.addAll(other.testcases!!) + } + + return this +} + +fun JUnitTest.Suite.mergeTestTimes(other: JUnitTest.Suite): JUnitTest.Suite { + if (this.name != other.name) throw FlankGeneralError("Attempted to merge ${other.name} into ${this.name}") + + // For each new JUnitTestCase: + // * if it failed then pull timing info from old + // * remove if not successful in either new or old + + // if we ran no test cases then don't bother merging old times. + if (this.testcases == null) return this + + val mergedTestCases = mutableListOf() + var mergedTime = 0.0 + + this.testcases?.forEach { testcase -> + // if test was skipped or empty, then continue to skip it. + if (testcase.skipped() || testcase.empty()) return@forEach + val testcaseTime = testcase.time.stripNotNumbers() + + // if the test succeeded, use the new time value + if (testcase.successful() && testcase.time != null) { + mergedTime += testcaseTime.toDouble() + mergedTestCases.add( + JUnitTest.Case( + name = testcase.name, + classname = testcase.classname, + time = testcaseTime + ) + ) + return@forEach + } + + // if the test we ran failed, copy timing from the last successful run + val lastSuccessfulRun = other.testcases?.firstOrNull { + it.successful() && it.name == testcase.name && it.classname == testcase.classname + } ?: return@forEach + + val lastSuccessfulRunTime = lastSuccessfulRun.time.stripNotNumbers() + if (lastSuccessfulRun.time != null) mergedTime += lastSuccessfulRunTime.toDouble() + mergedTestCases.add( + JUnitTest.Case( + name = testcase.name, + classname = testcase.classname, + time = lastSuccessfulRunTime + ) + ) + } + + this.testcases = mergedTestCases + this.tests = mergedTestCases.size.toString() + this.failures = "0" + this.errors = "0" + this.skipped = "0" + this.time = mergedTime.toString() + + return this +} + +private fun mergeInt(a: String?, b: String?): String { + return (a.stripNotNumbers().toInt() + b.stripNotNumbers().toInt()).toString() +} + +fun mergeDouble(a: String?, b: String?): String { + return "%.3f".format(Locale.ROOT, (a.stripNotNumbers().toDouble() + b.stripNotNumbers().toDouble())) +} diff --git a/test_runner/src/main/kotlin/ftl/domain/junit/JUnitTestStatus.kt b/test_runner/src/main/kotlin/ftl/domain/junit/JUnitTestStatus.kt new file mode 100644 index 0000000000..64657588e9 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/domain/junit/JUnitTestStatus.kt @@ -0,0 +1,37 @@ +package ftl.domain.junit + +import ftl.api.JUnitTest + +fun JUnitTest.Case.empty(): Boolean { + return name == null || classname == null || time == null +} + +/** Failed means there was a failure or an error. */ +fun JUnitTest.Case.failed(): Boolean { + return failures?.isNotEmpty() == true || errors?.isNotEmpty() == true +} + +fun JUnitTest.Case.skipped(): Boolean { + return skipped == null +} + +fun JUnitTest.Case.successful(): Boolean { + return failed().not().and(skipped().not()) +} + +fun JUnitTest.Result.successful(): Boolean { + var successful = true + testsuites?.forEach { suite -> + if (suite.failed()) successful = false + } + + return successful +} + +fun JUnitTest.Suite.successful(): Boolean { + return failures == "0" && errors == "0" +} + +fun JUnitTest.Suite.failed(): Boolean { + return successful().not() +} diff --git a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt index ca1fd24a4e..ac98cbbe8d 100644 --- a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt @@ -1,12 +1,12 @@ package ftl.reports import flank.common.println +import ftl.api.JUnitTest import ftl.args.IArgs import ftl.config.FtlConstants.indent import ftl.json.MatrixMap import ftl.reports.util.IReport import ftl.reports.util.ReportManager -import ftl.reports.xml.model.JUnitTestResult import ftl.util.estimateCosts import java.io.StringWriter @@ -36,7 +36,7 @@ object CostReport : IReport { } } - override fun run(matrices: MatrixMap, result: JUnitTestResult?, printToStdout: Boolean, args: IArgs) { + override fun run(matrices: MatrixMap, result: JUnitTest.Result?, printToStdout: Boolean, args: IArgs) { val output = generate(matrices) if (printToStdout) print(output) write(matrices, output, args) diff --git a/test_runner/src/main/kotlin/ftl/reports/FullJUnitReport.kt b/test_runner/src/main/kotlin/ftl/reports/FullJUnitReport.kt index f76791a1fa..bb7e3b5f28 100644 --- a/test_runner/src/main/kotlin/ftl/reports/FullJUnitReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/FullJUnitReport.kt @@ -1,18 +1,17 @@ package ftl.reports import flank.common.log +import ftl.api.JUnitTest import ftl.args.IArgs import ftl.json.MatrixMap import ftl.reports.util.IReport import ftl.reports.util.ReportManager -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.xmlToString object FullJUnitReport : IReport { override val extension = ".xml" - override fun run(matrices: MatrixMap, result: JUnitTestResult?, printToStdout: Boolean, args: IArgs) { - val output = result.xmlToString() + override fun run(matrices: MatrixMap, result: JUnitTest.Result?, printToStdout: Boolean, args: IArgs) { + val output = result.toXmlString() if (printToStdout) { log(output) } else { diff --git a/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt b/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt index a6780688a9..b441314d6a 100644 --- a/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt @@ -1,13 +1,12 @@ package ftl.reports import com.google.gson.Gson +import ftl.api.JUnitTest import ftl.args.IArgs +import ftl.domain.junit.failed import ftl.json.MatrixMap import ftl.reports.util.IReport import ftl.reports.util.ReportManager -import ftl.reports.xml.model.JUnitTestCase -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.model.JUnitTestSuite import ftl.util.readTextResource import java.nio.file.Files import java.nio.file.Paths @@ -24,7 +23,7 @@ object HtmlErrorReport : IReport { override fun run( matrices: MatrixMap, - result: JUnitTestResult?, + result: JUnitTest.Result?, printToStdout: Boolean, args: IArgs ) { @@ -37,42 +36,45 @@ object HtmlErrorReport : IReport { } } -internal fun List.process(): List = this - .getFailures() +internal fun List.process(): List = getFailures() .groupByLabel() .createGroups() -private fun List.getFailures(): List>> = +private fun List.getFailures(): List>> = mapNotNull { suite -> suite.testcases?.let { testCases -> suite.name to testCases.filter { it.failed() } } } -private fun List>>.groupByLabel(): Map> = this - .map { (suiteName, testCases) -> +private fun List>>.groupByLabel(): Map> = + map { (suiteName, testCases) -> testCases.map { testCase -> "$suiteName ${testCase.classname}#${testCase.name}".trim() to testCase } } - .flatten() - .groupBy({ (label: String, _) -> label }) { (_, useCase) -> useCase } + .flatten() + .groupBy({ (label: String, _) -> label }) { (_, useCase) -> useCase } -private fun Map>.createGroups(): List = - map { (label, testCases: List) -> +private fun Map>.createGroups(): List = + map { (label, testCases: List) -> HtmlErrorReport.Group( label = label, items = testCases.createItems() ) } -private fun List.createItems(): List = map { testCase -> +private fun List.createItems(): List = map { testCase -> HtmlErrorReport.Item( label = testCase.stackTrace().split("\n").firstOrNull() ?: "", url = testCase.webLink ?: "" ) } +private fun JUnitTest.Case.stackTrace(): String { + return failures?.joinToString() + errors?.joinToString() +} + private fun List.createHtmlReport(): String = readTextResource("inline.html").replace( oldValue = "\"INJECT-DATA-HERE\"", diff --git a/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt b/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt index e89b8e884f..b9d933c8ab 100644 --- a/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/JUnitReport.kt @@ -1,26 +1,25 @@ package ftl.reports import flank.common.log +import ftl.api.JUnitTest import ftl.args.IArgs import ftl.json.MatrixMap import ftl.reports.util.IReport import ftl.reports.util.ReportManager -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.xmlToString object JUnitReport : IReport { override val extension = ".xml" - override fun run(matrices: MatrixMap, result: JUnitTestResult?, printToStdout: Boolean, args: IArgs) { + override fun run(matrices: MatrixMap, result: JUnitTest.Result?, printToStdout: Boolean, args: IArgs) { if (result == null) { return } - val output = result.xmlToString() + val output = result.toXmlString() if (printToStdout) { log(output) } else { write(matrices, output, args) } - ReportManager.uploadReportResult(result.xmlToString(), args, fileName()) + ReportManager.uploadReportResult(result.toXmlString(), args, fileName()) } } diff --git a/test_runner/src/main/kotlin/ftl/reports/JUnitTestResultToXml.kt b/test_runner/src/main/kotlin/ftl/reports/JUnitTestResultToXml.kt new file mode 100644 index 0000000000..c1708adc5b --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/reports/JUnitTestResultToXml.kt @@ -0,0 +1,10 @@ +package ftl.reports + +import ftl.api.JUnitTest +import ftl.client.junit.xmlPrettyWriter + +fun JUnitTest.Result?.toXmlString(): String { + if (this == null) return "" + val prefix = "\n" + return prefix + xmlPrettyWriter.writeValueAsString(this) +} diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt index 11347fde8b..71e2381958 100644 --- a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt @@ -3,6 +3,7 @@ package ftl.reports import flank.common.log import flank.common.println import flank.common.startWithNewLine +import ftl.api.JUnitTest import ftl.args.IArgs import ftl.config.FtlConstants.indent import ftl.json.MatrixMap @@ -13,7 +14,6 @@ import ftl.reports.output.log import ftl.reports.output.outputReport import ftl.reports.util.IReport import ftl.reports.util.ReportManager -import ftl.reports.xml.model.JUnitTestResult import java.io.StringWriter import java.text.DecimalFormat import java.text.DecimalFormatSymbols @@ -34,6 +34,13 @@ object MatrixResultsReport : IReport { private val percentFormat by lazy { DecimalFormat("#0.00", DecimalFormatSymbols(Locale.US)) } + override fun run(matrices: MatrixMap, result: JUnitTest.Result?, printToStdout: Boolean, args: IArgs) { + val output = generate(matrices) + if (printToStdout) log(output) + write(matrices, output, args) + ReportManager.uploadReportResult(output, args, fileName()) + } + private fun generate(matrices: MatrixMap): String { val total = matrices.map.size // unfinished matrix will not be reported as failed since it's still running @@ -72,11 +79,4 @@ object MatrixResultsReport : IReport { forEach { writer.println(it.webLinkWithoutExecutionDetails.orEmpty()) } writer.println() } - - override fun run(matrices: MatrixMap, result: JUnitTestResult?, printToStdout: Boolean, args: IArgs) { - val output = generate(matrices) - if (printToStdout) log(output) - write(matrices, output, args) - ReportManager.uploadReportResult(output, args, fileName()) - } } diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestResult.kt b/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestResult.kt deleted file mode 100644 index b071db7dd1..0000000000 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestResult.kt +++ /dev/null @@ -1,21 +0,0 @@ -package ftl.reports.api - -import com.google.testing.model.TestExecution -import ftl.reports.xml.model.JUnitTestResult - -internal fun List.createJUnitTestResult( - withStackTraces: Boolean = false -) = JUnitTestResult( - testsuites = this - .filterNullToolResultsStep() - .createTestExecutionDataListAsync() - .prepareForJUnitResult() - .let { executionDataList -> - if (withStackTraces) executionDataList - else executionDataList.removeStackTraces() - } - .createJUnitTestSuites() - .toMutableList() -) - -private fun List.filterNullToolResultsStep() = filter { it.toolResultsStep != null } diff --git a/test_runner/src/main/kotlin/ftl/reports/api/Utils.kt b/test_runner/src/main/kotlin/ftl/reports/api/Utils.kt index 9d00ad41a0..4de497e8e9 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/Utils.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/Utils.kt @@ -1,9 +1,6 @@ package ftl.reports.api import com.google.api.services.toolresults.model.Duration -import com.google.api.services.toolresults.model.TestCase -import com.google.api.services.toolresults.model.Timestamp -import ftl.util.mutableMapProperty import java.text.SimpleDateFormat import java.util.Locale import java.util.TimeZone @@ -28,9 +25,3 @@ private fun nanosToSeconds(nanos: Int?): Double = val utcDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").apply { timeZone = TimeZone.getTimeZone("UTC") } - -fun Timestamp.asUnixTimestamp() = (seconds ?: 0) * 1_000 + (nanos ?: 0) / 1_000_000 - -fun Long.formatUtcDate() = utcDateFormat.format(this)!! - -var TestCase.flaky: Boolean by mutableMapProperty { false } diff --git a/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt b/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt index a552cea644..b13f25e11c 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/IReport.kt @@ -1,14 +1,14 @@ package ftl.reports.util import flank.common.write +import ftl.api.JUnitTest import ftl.args.IArgs import ftl.json.MatrixMap -import ftl.reports.xml.model.JUnitTestResult import ftl.util.resolveLocalRunPath import java.nio.file.Paths interface IReport { - fun run(matrices: MatrixMap, result: JUnitTestResult?, printToStdout: Boolean = false, args: IArgs) + fun run(matrices: MatrixMap, result: JUnitTest.Result?, printToStdout: Boolean = false, args: IArgs) fun reportName(): String { return this::class.java.simpleName diff --git a/test_runner/src/main/kotlin/ftl/reports/util/JUnitDedupe.kt b/test_runner/src/main/kotlin/ftl/reports/util/JUnitDedupe.kt index dd49fdf5a1..8006dbf18c 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/JUnitDedupe.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/JUnitDedupe.kt @@ -1,20 +1,23 @@ package ftl.reports.util -import ftl.reports.xml.model.JUnitTestCase -import ftl.reports.xml.model.JUnitTestResult +import ftl.api.JUnitTest +import ftl.domain.junit.mergeDouble +import ftl.domain.junit.skipped +import ftl.domain.junit.successful +import ftl.util.stripNotNumbers // Read in JUnitReport.xml and remove duplicate results when `num-flaky-test-attempts` is > 0 // for each test `name="testFails" classname="com.example.app.ExampleUiTest"` // Keep first result. If next result for the same test is successful, keep last successful result. object JUnitDedupe { - private fun JUnitTestCase.key(): String { + private fun JUnitTest.Case.key(): String { return "${this.classname}#${this.name}" } - fun modify(testResult: JUnitTestResult?) { + fun modify(testResult: JUnitTest.Result?) { testResult?.testsuites?.forEach { suite -> - val testCaseMap = mutableMapOf() + val testCaseMap = mutableMapOf() suite.testcases?.forEach { testcase -> if (testCaseMap[testcase.key()] == null || testcase.successful()) { @@ -26,4 +29,13 @@ object JUnitDedupe { suite.updateTestStats() } } + + /** Call after setting testcases manually to update the statistics (error count, skip count, etc.) */ + private fun JUnitTest.Suite.updateTestStats() { + this.tests = testcases?.size.toString() + this.failures = testcases?.count { it.failures?.isNotEmpty() == true }.toString() + this.errors = testcases?.count { it.errors?.isNotEmpty() == true }.toString() + this.skipped = testcases?.count { it.skipped() }.toString() + this.time = testcases?.fold("0") { acc, test -> mergeDouble(acc, test.time.stripNotNumbers()) } ?: "0" + } } diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt index e365be2bf4..ecd415a7c1 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt @@ -3,8 +3,12 @@ package ftl.reports.util import com.google.common.annotations.VisibleForTesting import com.google.testing.model.TestExecution import flank.common.logLn +import ftl.api.JUnitTest import ftl.api.RemoteStorage import ftl.api.downloadAsJunitXML +import ftl.api.generateJUnitTestResultFromApi +import ftl.api.parseJUnitLegacyTestResultFromFile +import ftl.api.parseJUnitTestResultFromFile import ftl.api.uploadToRemoteStorage import ftl.args.AndroidArgs import ftl.args.IArgs @@ -13,6 +17,8 @@ import ftl.args.IosArgs import ftl.args.ShardChunks import ftl.client.google.GcStorage import ftl.config.FtlConstants +import ftl.domain.junit.merge +import ftl.domain.junit.mergeTestTimes import ftl.json.MatrixMap import ftl.json.isAllSuccessful import ftl.reports.CostReport @@ -20,16 +26,10 @@ import ftl.reports.FullJUnitReport import ftl.reports.HtmlErrorReport import ftl.reports.JUnitReport import ftl.reports.MatrixResultsReport -import ftl.reports.api.createJUnitTestResult import ftl.reports.api.getAndUploadPerformanceMetrics -import ftl.reports.api.refreshMatricesAndGetExecutions -import ftl.reports.xml.model.JUnitTestCase -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.model.getSkippedJUnitTestSuite -import ftl.reports.xml.parseAllSuitesXml -import ftl.reports.xml.parseOneSuiteXml +import ftl.reports.api.utcDateFormat +import ftl.reports.toXmlString import ftl.run.common.getMatrixFilePath -import ftl.run.exception.FlankGeneralError import ftl.shard.createTestMethodDurationMap import ftl.util.Artifacts import ftl.util.getSmartFlankGCSPathAsFileReference @@ -37,67 +37,49 @@ import ftl.util.resolveLocalRunPath import java.io.File import java.nio.file.Files import java.nio.file.Paths +import java.util.Date import kotlin.math.roundToInt object ReportManager { - private fun findXmlFiles( + private fun Pair.toApiIdentity(): JUnitTest.Result.ApiIdentity { + val (args, matrices) = this + return JUnitTest.Result.ApiIdentity( + projectId = args.project, + matrixIds = matrices.map.values.map { it.matrixId } + ) + } + + @VisibleForTesting + internal fun processXmlFromFile( matrices: MatrixMap, - args: IArgs - ) = File(resolveLocalRunPath(matrices, args)).walk() - .filter { it.name.matches(Artifacts.testResultRgx) } - .fold(listOf()) { xmlFiles, file -> - xmlFiles + file - } + args: IArgs, + parsingFunction: (File) -> JUnitTest.Result + ): JUnitTest.Result? = findXmlFiles(matrices, args) + .map { xmlFile -> parsingFunction(xmlFile).updateWebLink(getWebLink(matrices, xmlFile)) } + .reduceOrNull { acc, result -> result.merge(acc) } private fun getWebLink(matrices: MatrixMap, xmlFile: File): String = xmlFile.getMatrixPath(matrices.runPath) ?.findMatrixPath(matrices) ?: "".also { logLn("WARNING: Matrix path not found in JSON.") } - private val deviceStringRgx = Regex("([^-]+-[^-]+-[^-]+-[^-]+).*") - - // NexusLowRes-28-en-portrait-rerun_1 => NexusLowRes-28-en-portrait - fun getDeviceString(deviceString: String): String { - val matchResult = deviceStringRgx.find(deviceString) - return matchResult?.groupValues?.last().orEmpty() - } - - @VisibleForTesting - internal fun processXmlFromFile( + private fun findXmlFiles( matrices: MatrixMap, - args: IArgs, - process: (file: File) -> JUnitTestResult - ): JUnitTestResult? { - var mergedXml: JUnitTestResult? = null - - findXmlFiles(matrices, args).forEach { xmlFile -> - val parsedXml = try { - process(xmlFile) - } catch (e: Throwable) { - throw FlankGeneralError("Cannot process xml file: ${xmlFile.absolutePath}", e) - } - val webLink = getWebLink(matrices, xmlFile) - val deviceName = getDeviceString(xmlFile.parentFile.name) - - parsedXml.testsuites?.forEach { testSuite -> - testSuite.name = "$deviceName#${testSuite.name}" - testSuite.testcases?.forEach { testCase -> - testCase.webLink = webLink - } + args: IArgs + ) = File(resolveLocalRunPath(matrices, args)) + .walk() + .filter { it.name.matches(Artifacts.testResultRgx) } + .fold(listOf()) { xmlFiles, file -> xmlFiles + file } + + private fun JUnitTest.Result.updateWebLink( + webLink: String + ) = apply { + testsuites?.forEach { testSuite -> + testSuite.testcases?.forEach { testCase -> + testCase.webLink = webLink } - - mergedXml = parsedXml.merge(mergedXml) } - - return mergedXml - } - - private fun parseTestSuite(matrices: MatrixMap, args: IArgs): JUnitTestResult? = when { - // ios supports only legacy parsing - args is IosArgs -> processXmlFromFile(matrices, args, ::parseAllSuitesXml) - args.useLegacyJUnitResult -> processXmlFromFile(matrices, args, ::parseOneSuiteXml) - else -> refreshMatricesAndGetExecutions(matrices, args).createJUnitTestResult() } /** Returns true if there were no test failures */ @@ -107,7 +89,7 @@ object ReportManager { testShardChunks: ShardChunks, ignoredTestCases: IgnoredTestCases = listOf() ) { - val testSuite: JUnitTestResult? = parseTestSuite(matrices, args) + val testSuite: JUnitTest.Result? = parseTestSuite(matrices, args) if (args.useLegacyJUnitResult) { val useFlakyTests = args.flakyTestAttempts > 0 if (useFlakyTests) JUnitDedupe.modify(testSuite) @@ -131,12 +113,20 @@ object ReportManager { args = args ) - val testExecutions = refreshMatricesAndGetExecutions(matrices, args) - processJunitResults(args, matrices, testSuite, testShardChunks, testExecutions) - createAndUploadPerformanceMetricsForAndroid(args, testExecutions, matrices) + val testsResult = generateJUnitTestResultFromApi((args to matrices).toApiIdentity()) + processJunitResults(args, matrices, testSuite, testShardChunks, testsResult) + // TODO move it to next #1756 + // createAndUploadPerformanceMetricsForAndroid(args, testsResult, matrices) uploadMatricesId(args, matrices) } + private fun parseTestSuite(matrices: MatrixMap, args: IArgs): JUnitTest.Result? = when { + // ios supports only legacy parsing + args is IosArgs -> processXmlFromFile(matrices, args, parseJUnitTestResultFromFile) + args.useLegacyJUnitResult -> processXmlFromFile(matrices, args, parseJUnitLegacyTestResultFromFile) + else -> generateJUnitTestResultFromApi((args to matrices).toApiIdentity()) + } + @VisibleForTesting internal fun uploadReportResult(testResult: String, args: IArgs, fileName: String) { if (args.resultsBucket.isBlank() || args.resultsDir.isBlank() || args.disableResultsUpload) return @@ -149,12 +139,12 @@ object ReportManager { private fun processJunitResults( args: IArgs, matrices: MatrixMap, - testSuite: JUnitTestResult?, + testSuite: JUnitTest.Result?, testShardChunks: ShardChunks, - testExecutions: List + testsResult: JUnitTest.Result ) { when { - args.fullJUnitResult -> processFullJunitResult(args, matrices, testShardChunks, testExecutions) + args.fullJUnitResult -> processFullJunitResult(args, matrices, testShardChunks, testsResult) args.useLegacyJUnitResult -> processJunitXml(testSuite, args, testShardChunks) else -> processJunitXml(testSuite, args, testShardChunks) } @@ -179,7 +169,7 @@ object ReportManager { private fun IgnoredTestCases.toJunitTestsResults() = getSkippedJUnitTestSuite( map { - JUnitTestCase( + JUnitTest.Case( classname = it.split("#").first().replace("class ", ""), name = it.split("#").last(), time = "0.0", @@ -188,13 +178,24 @@ object ReportManager { } ) + @VisibleForTesting + internal fun getSkippedJUnitTestSuite(listOfJUnitTestCase: List) = JUnitTest.Suite( + name = "junit-ignored", + tests = listOfJUnitTestCase.size.toString(), + errors = "0", + failures = "0", + skipped = listOfJUnitTestCase.size.toString(), + time = "0.0", + timestamp = utcDateFormat.format(Date()), + testcases = listOfJUnitTestCase.toMutableList() + ) + private fun processFullJunitResult( args: IArgs, matrices: MatrixMap, testShardChunks: ShardChunks, - testExecutions: List + testSuite: JUnitTest.Result ) { - val testSuite = testExecutions.createJUnitTestResult(withStackTraces = true) FullJUnitReport.run(matrices, testSuite, printToStdout = false, args = args) processJunitXml(testSuite, args, testShardChunks) } @@ -207,8 +208,8 @@ object ReportManager { ) fun createShardEfficiencyList( - oldResult: JUnitTestResult, - newResult: JUnitTestResult, + oldResult: JUnitTest.Result, + newResult: JUnitTest.Result, args: IArgs, testShardChunks: ShardChunks ): List { @@ -230,8 +231,8 @@ object ReportManager { } private fun printActual( - oldResult: JUnitTestResult, - newResult: JUnitTestResult, + oldResult: JUnitTest.Result, + newResult: JUnitTest.Result, args: IArgs, testShardChunks: ShardChunks ) { @@ -245,7 +246,7 @@ object ReportManager { } private fun processJunitXml( - newTestResult: JUnitTestResult?, + newTestResult: JUnitTest.Result?, args: IArgs, testShardChunks: ShardChunks ) { @@ -259,25 +260,26 @@ object ReportManager { printActual(oldTestResult, newTestResult, args, testShardChunks) - GcStorage.uploadJunitXml(newTestResult, args) + GcStorage.uploadJunitXml(newTestResult.toXmlString(), args) } -} -fun uploadMatricesId(args: IArgs, matrixMap: MatrixMap) { - if (args.disableResultsUpload) return - val file = args.getMatrixFilePath(matrixMap).toString() - if (file.startsWith(FtlConstants.GCS_PREFIX)) return + private fun uploadMatricesId(args: IArgs, matrixMap: MatrixMap) { + if (args.disableResultsUpload) return + val file = args.getMatrixFilePath(matrixMap).toString() + if (file.startsWith(FtlConstants.GCS_PREFIX)) return - uploadToRemoteStorage( - RemoteStorage.Dir(args.resultsBucket, args.resultsDir), - RemoteStorage.Data(file, Files.readAllBytes(Paths.get(file))) - ) -} + uploadToRemoteStorage( + RemoteStorage.Dir(args.resultsBucket, args.resultsDir), + RemoteStorage.Data(file, Files.readAllBytes(Paths.get(file))) + ) + } -private fun String.findMatrixPath(matrices: MatrixMap) = matrices.map.values - .firstOrNull { savedMatrix -> savedMatrix.gcsPath.endsWithTextWithOptionalSlashAtTheEnd(this) } - ?.webLink - ?: "".also { logLn("WARNING: Matrix path not found in JSON. $this") } + private fun String.findMatrixPath(matrices: MatrixMap) = matrices.map.values + .firstOrNull { savedMatrix -> savedMatrix.gcsPath.endsWithTextWithOptionalSlashAtTheEnd(this) } + ?.webLink + ?: "".also { logLn("WARNING: Matrix path not found in JSON. $this") } -@VisibleForTesting -internal fun String.endsWithTextWithOptionalSlashAtTheEnd(text: String) = "($text)/*$".toRegex().containsMatchIn(this) + @VisibleForTesting + internal fun String.endsWithTextWithOptionalSlashAtTheEnd(text: String) = + "($text)/*$".toRegex().containsMatchIn(this) +} diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestResult.kt b/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestResult.kt deleted file mode 100644 index 0361753702..0000000000 --- a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestResult.kt +++ /dev/null @@ -1,57 +0,0 @@ -package ftl.reports.xml.model - -import com.fasterxml.jackson.annotation.JsonInclude -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement - -/** - * Firebase generates testsuites for iOS. - * .xctestrun file may contain multiple test bundles (each one is a testsuite) */ -@JacksonXmlRootElement(localName = "testsuites") -data class JUnitTestResult( - @JsonInclude(JsonInclude.Include.NON_EMPTY) - @JacksonXmlProperty(localName = "testsuite") - var testsuites: MutableList? = null -) { - fun successful(): Boolean { - var successful = true - testsuites?.forEach { suite -> - if (suite.failed()) successful = false - } - - return successful - } - - fun mergeTestTimes(other: JUnitTestResult?): JUnitTestResult { - if (other == null) return this - if (this.testsuites == null) this.testsuites = mutableListOf() - - // newTestResult.mergeTestTimes(oldTestResult) - // - // for each new JUnitTestSuite, check if it exists on old - // if JUnitTestSuite exists on both then merge test times - this.testsuites?.forEach { testSuite -> - val oldSuite = other.testsuites?.firstOrNull { it.name == testSuite.name } - if (oldSuite != null) testSuite.mergeTestTimes(oldSuite) - } - - return this - } - - fun merge(other: JUnitTestResult?): JUnitTestResult { - if (other == null) return this - if (this.testsuites == null) this.testsuites = mutableListOf() - - other.testsuites?.forEach { testSuite -> - val mergeCandidate = this.testsuites?.firstOrNull { it.name == testSuite.name } - - if (mergeCandidate == null) { - this.testsuites?.add(testSuite) - } else { - mergeCandidate.merge(testSuite) - } - } - - return this - } -} diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestSuite.kt b/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestSuite.kt deleted file mode 100644 index 74f19878f5..0000000000 --- a/test_runner/src/main/kotlin/ftl/reports/xml/model/JUnitTestSuite.kt +++ /dev/null @@ -1,167 +0,0 @@ -package ftl.reports.xml.model - -import com.fasterxml.jackson.annotation.JsonInclude -import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty -import ftl.run.exception.FlankGeneralError -import java.util.Locale - -data class JUnitTestSuite( - @JacksonXmlProperty(isAttribute = true) - var name: String, - - @JacksonXmlProperty(isAttribute = true) - var tests: String, // Int - - @JacksonXmlProperty(isAttribute = true) - var failures: String, // Int - - @JacksonXmlProperty(isAttribute = true) - var flakes: Int? = null, - - @JacksonXmlProperty(isAttribute = true) - var errors: String, // Int - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JacksonXmlProperty(isAttribute = true) - var skipped: String?, // Int. Android only - - @JacksonXmlProperty(isAttribute = true) - var time: String, // Double - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JacksonXmlProperty(isAttribute = true) - val timestamp: String?, // String. Android only - - @JacksonXmlProperty(isAttribute = true) - val hostname: String? = "localhost", // String. - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JacksonXmlProperty(isAttribute = true) - val testLabExecutionId: String? = null, // String. - - @JacksonXmlProperty(localName = "testcase") - var testcases: MutableCollection?, - - // not used - @JsonInclude(JsonInclude.Include.NON_NULL) - val properties: Any? = null, // - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JacksonXmlProperty(localName = "system-out") - val systemOut: Any? = null, // - - @JsonInclude(JsonInclude.Include.NON_NULL) - @JacksonXmlProperty(localName = "system-err") - val systemErr: Any? = null // -) { - - fun successful(): Boolean { - return failures == "0" && errors == "0" - } - - fun failed(): Boolean { - return successful().not() - } - - /** Call after setting testcases manually to update the statistics (error count, skip count, etc.) */ - fun updateTestStats() { - this.tests = testcases?.size.toString() - this.failures = testcases?.count { it.failures?.isNotEmpty() == true }.toString() - this.errors = testcases?.count { it.errors?.isNotEmpty() == true }.toString() - this.skipped = testcases?.count { it.skipped() }.toString() - this.time = testcases?.fold("0") { acc, test -> mergeDouble(acc, test.time.clean()) } ?: "0" - } - - /** - * Strips all characters except numbers and a period - * Returns 0 when the string is null or blank - * - * Example: z1,23.45 => 123.45 */ - private fun String?.clean(): String { - if (this.isNullOrBlank()) return "0" - return this.replace(Regex("""[^0-9\\.]"""), "") - } - - private fun mergeInt(a: String?, b: String?): String { - return (a.clean().toInt() + b.clean().toInt()).toString() - } - - private fun mergeDouble(a: String?, b: String?): String { - return "%.3f".format(Locale.ROOT, (a.clean().toDouble() + b.clean().toDouble())) - } - - fun merge(other: JUnitTestSuite): JUnitTestSuite { - if (this.name != other.name) throw FlankGeneralError("Attempted to merge ${other.name} into ${this.name}") - - // tests, failures, errors - this.tests = mergeInt(this.tests, other.tests) - this.failures = mergeInt(this.failures, other.failures) - this.errors = mergeInt(this.errors, other.errors) - this.skipped = mergeInt(this.skipped, other.skipped) - this.time = mergeDouble(this.time, other.time) - - if (this.testcases == null) this.testcases = mutableListOf() - if (other.testcases?.isNotEmpty() == true) { - this.testcases?.addAll(other.testcases!!) - } - - return this - } - - fun mergeTestTimes(other: JUnitTestSuite): JUnitTestSuite { - if (this.name != other.name) throw FlankGeneralError("Attempted to merge ${other.name} into ${this.name}") - - // For each new JUnitTestCase: - // * if it failed then pull timing info from old - // * remove if not successful in either new or old - - // if we ran no test cases then don't bother merging old times. - if (this.testcases == null) return this - - val mergedTestCases = mutableListOf() - var mergedTime = 0.0 - - this.testcases?.forEach { testcase -> - // if test was skipped or empty, then continue to skip it. - if (testcase.skipped() || testcase.empty()) return@forEach - val testcaseTime = testcase.time.clean() - - // if the test succeeded, use the new time value - if (testcase.successful() && testcase.time != null) { - mergedTime += testcaseTime.toDouble() - mergedTestCases.add( - JUnitTestCase( - name = testcase.name, - classname = testcase.classname, - time = testcaseTime - ) - ) - return@forEach - } - - // if the test we ran failed, copy timing from the last successful run - val lastSuccessfulRun = other.testcases?.firstOrNull { - it.successful() && it.name == testcase.name && it.classname == testcase.classname - } ?: return@forEach - - val lastSuccessfulRunTime = lastSuccessfulRun.time.clean() - if (lastSuccessfulRun.time != null) mergedTime += lastSuccessfulRunTime.toDouble() - mergedTestCases.add( - JUnitTestCase( - name = testcase.name, - classname = testcase.classname, - time = lastSuccessfulRunTime - ) - ) - } - - this.testcases = mergedTestCases - this.tests = mergedTestCases.size.toString() - this.failures = "0" - this.errors = "0" - this.skipped = "0" - this.time = mergedTime.toString() - - return this - } -} diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/model/SkippedTestJUnitTestSuite.kt b/test_runner/src/main/kotlin/ftl/reports/xml/model/SkippedTestJUnitTestSuite.kt deleted file mode 100644 index bfd9ad9155..0000000000 --- a/test_runner/src/main/kotlin/ftl/reports/xml/model/SkippedTestJUnitTestSuite.kt +++ /dev/null @@ -1,15 +0,0 @@ -package ftl.reports.xml.model - -import ftl.reports.api.utcDateFormat -import java.util.Date - -fun getSkippedJUnitTestSuite(listOfJUnitTestCase: List) = JUnitTestSuite( - name = "junit-ignored", - tests = listOfJUnitTestCase.size.toString(), - errors = "0", - failures = "0", - skipped = listOfJUnitTestCase.size.toString(), - time = "0.0", - timestamp = utcDateFormat.format(Date()), - testcases = listOfJUnitTestCase.toMutableList() -) diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/preprocesor/UtfControlCharRanges.kt b/test_runner/src/main/kotlin/ftl/reports/xml/preprocesor/UtfControlCharRanges.kt deleted file mode 100644 index 47948d1a68..0000000000 --- a/test_runner/src/main/kotlin/ftl/reports/xml/preprocesor/UtfControlCharRanges.kt +++ /dev/null @@ -1,13 +0,0 @@ -package ftl.reports.xml.preprocesor - -/** - * Numbers come from ascii table https://www.utf8-chartable.de. - * and represents control chars. We need to avoid characters in ranges CONTROL_TOP_START..CONTROL_TOP_END and - * CONTROL_BOTTOM_START..CONTROL_BOTTOM_END because chars from that range escaped to html causing parsing errors. - * */ -enum class UtfControlCharRanges(val charValue: Int) { - CONTROL_TOP_START(0x00), - CONTROL_TOP_END(0x1F), - CONTROL_BOTTOM_START(0x7F), - CONTROL_BOTTOM_END(0x9F) -} diff --git a/test_runner/src/main/kotlin/ftl/reports/xml/preprocesor/XmlPreprocessor.kt b/test_runner/src/main/kotlin/ftl/reports/xml/preprocesor/XmlPreprocessor.kt deleted file mode 100644 index 288dba1e23..0000000000 --- a/test_runner/src/main/kotlin/ftl/reports/xml/preprocesor/XmlPreprocessor.kt +++ /dev/null @@ -1,11 +0,0 @@ -package ftl.reports.xml.preprocesor - -import org.apache.commons.text.StringEscapeUtils - -fun fixHtmlCodes(data: String): String = listOf( - UtfControlCharRanges.CONTROL_TOP_START.charValue..UtfControlCharRanges.CONTROL_TOP_END.charValue, - UtfControlCharRanges.CONTROL_BOTTOM_START.charValue..UtfControlCharRanges.CONTROL_BOTTOM_END.charValue -).flatten() - .map { StringEscapeUtils.escapeXml11(it.toChar().toString()) } - .filter { it.startsWith("&#") } - .fold(data) { fixedStr: String, isoControlCode: String -> fixedStr.replace(isoControlCode, "") } diff --git a/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt b/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt index de952d9ba5..dd4dced54c 100644 --- a/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt @@ -2,8 +2,8 @@ package ftl.run import flank.common.logLn import ftl.args.IArgs +import ftl.client.google.GcTestMatrix import ftl.config.FtlConstants -import ftl.gc.GcTestMatrix import ftl.json.SavedMatrix import ftl.run.common.getLastArgs import ftl.run.common.getLastMatrices diff --git a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt index 6e78df298d..b07e21e8de 100644 --- a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt @@ -4,8 +4,8 @@ import com.google.testing.model.TestMatrix import flank.common.logLn import ftl.args.IArgs import ftl.args.ShardChunks +import ftl.client.google.GcTestMatrix import ftl.config.FtlConstants -import ftl.gc.GcTestMatrix import ftl.json.MatrixMap import ftl.json.needsUpdate import ftl.json.updateMatrixMap diff --git a/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt b/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt index 8a67bf70fc..1d588ffd61 100644 --- a/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt +++ b/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt @@ -5,7 +5,7 @@ package ftl.run.common import com.google.testing.model.TestMatrix import flank.common.logLn import ftl.args.IArgs -import ftl.gc.GcTestMatrix +import ftl.client.google.GcTestMatrix import ftl.run.status.TestMatrixStatusPrinter import ftl.util.MatrixState import kotlinx.coroutines.coroutineScope diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt index f176adf61b..6ecb80780b 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt @@ -6,9 +6,9 @@ import flank.common.startWithNewLine import ftl.api.RemoteStorage import ftl.api.uploadToRemoteStorage import ftl.args.IArgs +import ftl.client.google.GcTestMatrix import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_STORAGE_LINK -import ftl.gc.GcTestMatrix import ftl.json.MatrixMap import ftl.json.createSavedMatrix import ftl.reports.addStepTime diff --git a/test_runner/src/main/kotlin/ftl/shard/Shard.kt b/test_runner/src/main/kotlin/ftl/shard/Shard.kt index 4a3f3b113e..e9a99c271e 100644 --- a/test_runner/src/main/kotlin/ftl/shard/Shard.kt +++ b/test_runner/src/main/kotlin/ftl/shard/Shard.kt @@ -1,10 +1,10 @@ package ftl.shard import flank.common.logLn +import ftl.api.JUnitTest import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs -import ftl.reports.xml.model.JUnitTestResult import ftl.run.exception.FlankConfigurationError import ftl.util.FlankTestMethod import kotlin.math.roundToInt @@ -51,7 +51,7 @@ fun AndroidArgs.createShardsByTestForShards(): List = testTargetsForShard // take in the XML with timing info then return list of shards based on the amount of shards to use fun createShardsByShardCount( testsToRun: List, - oldTestResult: JUnitTestResult, + oldTestResult: JUnitTest.Result, args: IArgs, forcedShardCount: Int = -1 ): List { diff --git a/test_runner/src/main/kotlin/ftl/shard/ShardCount.kt b/test_runner/src/main/kotlin/ftl/shard/ShardCount.kt index 057e064612..0fc7016321 100644 --- a/test_runner/src/main/kotlin/ftl/shard/ShardCount.kt +++ b/test_runner/src/main/kotlin/ftl/shard/ShardCount.kt @@ -1,8 +1,8 @@ package ftl.shard +import ftl.api.JUnitTest import ftl.args.IArgs import ftl.args.IArgs.Companion.AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE -import ftl.reports.xml.model.JUnitTestResult import ftl.run.exception.FlankConfigurationError import ftl.util.FlankTestMethod import kotlin.math.ceil @@ -14,7 +14,7 @@ private const val NO_LIMIT = -1 // take in the XML with timing info then return the shard count based on execution time fun shardCountByTime( testsToRun: List, - oldTestResult: JUnitTestResult, + oldTestResult: JUnitTest.Result, args: IArgs ): Int = when { args.shardTime == NO_LIMIT -> NO_LIMIT @@ -24,7 +24,7 @@ fun shardCountByTime( private fun calculateShardCount( testsToRun: List, - oldTestResult: JUnitTestResult, + oldTestResult: JUnitTest.Result, args: IArgs ): Int { val previousMethodDurations = createTestMethodDurationMap(oldTestResult, args) diff --git a/test_runner/src/main/kotlin/ftl/shard/TestMethodDuration.kt b/test_runner/src/main/kotlin/ftl/shard/TestMethodDuration.kt index 29d13348a5..fafa1a2b89 100644 --- a/test_runner/src/main/kotlin/ftl/shard/TestMethodDuration.kt +++ b/test_runner/src/main/kotlin/ftl/shard/TestMethodDuration.kt @@ -1,11 +1,11 @@ package ftl.shard +import ftl.api.JUnitTest import ftl.args.AndroidArgs import ftl.args.IArgs -import ftl.reports.xml.model.JUnitTestCase -import ftl.reports.xml.model.JUnitTestResult +import ftl.domain.junit.empty -fun createTestMethodDurationMap(junitResult: JUnitTestResult, args: IArgs): Map { +fun createTestMethodDurationMap(junitResult: JUnitTest.Result, args: IArgs): Map { val junitMap = mutableMapOf() // Create a map with information from previous junit run @@ -23,11 +23,11 @@ fun createTestMethodDurationMap(junitResult: JUnitTestResult, args: IArgs): Map< return junitMap } -private fun JUnitTestCase.androidKey(): String { +private fun JUnitTest.Case.androidKey(): String { return "class $classname#$name" } -private fun JUnitTestCase.iosKey(): String { +private fun JUnitTest.Case.iosKey(): String { // FTL iOS XML appends `()` to each test name. ex: `testBasicSelection()` // xctestrun file requires classname/name with no `()` val testName = name?.substringBefore('(') diff --git a/test_runner/src/main/kotlin/ftl/util/Utils.kt b/test_runner/src/main/kotlin/ftl/util/Utils.kt index 3a6c8455f2..c98db6df5c 100644 --- a/test_runner/src/main/kotlin/ftl/util/Utils.kt +++ b/test_runner/src/main/kotlin/ftl/util/Utils.kt @@ -103,6 +103,16 @@ fun , T> mutableMapProperty( thisRef.set(name ?: property.name, value as Any) } +/** + * Strips all characters except numbers and a period + * Returns 0 when the string is null or blank + * + * Example: z1,23.45 => 123.45 */ +fun String?.stripNotNumbers(): String { + if (this.isNullOrBlank()) return "0" + return this.replace(Regex("""[^0-9\\.]"""), "") +} + /** * Used to validate values from yml config file. * Should be used only on properties with [JsonProperty] annotation. diff --git a/test_runner/src/test/kotlin/ftl/reports/xml/JUnitXmlTest.kt b/test_runner/src/test/kotlin/ftl/client/xml/JUnitXmlTest.kt similarity index 90% rename from test_runner/src/test/kotlin/ftl/reports/xml/JUnitXmlTest.kt rename to test_runner/src/test/kotlin/ftl/client/xml/JUnitXmlTest.kt index 5fcba75dda..621f890b13 100644 --- a/test_runner/src/test/kotlin/ftl/reports/xml/JUnitXmlTest.kt +++ b/test_runner/src/test/kotlin/ftl/client/xml/JUnitXmlTest.kt @@ -1,11 +1,18 @@ -package ftl.reports.xml +package ftl.client.xml import com.google.common.truth.Truth.assertThat import flank.common.normalizeLineEnding +import ftl.adapter.google.toApiModel +import ftl.client.junit.parseAllSuitesXml +import ftl.client.junit.parseOneSuiteXml import ftl.doctor.assertEqualsIgnoreNewlineStyle +import ftl.domain.junit.merge +import ftl.domain.junit.mergeTestTimes +import ftl.reports.toXmlString import ftl.run.exception.FlankGeneralError import org.junit.Assert import org.junit.Test +import java.io.File import java.nio.file.Paths class JUnitXmlTest { @@ -38,22 +45,23 @@ class JUnitXmlTest { """.trimIndent() - parseAllSuitesXml(xml) + parseAllSuitesXml(xml.writeToTempFile()) } @Test fun `empty testcase -- infrastructure error result`() { val xml = "" - parseAllSuitesXml(xml).run { + parseAllSuitesXml(xml.writeToTempFile()).run { assertThat(testsuites).isNull() } } @Test fun `merge android`() { - val mergedXml = parseOneSuiteXml(androidPassXml).merge(parseOneSuiteXml(androidFailXml)) - val merged = mergedXml.xmlToString().normalizeLineEnding() + val mergedXml = parseOneSuiteXml(androidPassXml).toApiModel() + .merge(parseOneSuiteXml(androidFailXml).toApiModel()) + val merged = mergedXml.toXmlString().normalizeLineEnding() val testSuite = mergedXml.testsuites?.first() ?: throw java.lang.RuntimeException("no test suite") assertThat(testSuite.name).isEqualTo("") @@ -86,7 +94,10 @@ junit.framework.Assert.fail(Assert.java:50) @Test fun `merge ios`() { val merged = - parseAllSuitesXml(iosPassXml).merge(parseAllSuitesXml(iosFailXml)).xmlToString().normalizeLineEnding() + parseAllSuitesXml(iosPassXml).toApiModel() + .merge(parseAllSuitesXml(iosFailXml).toApiModel()) + .toXmlString() + .normalizeLineEnding() val expected = """ @@ -108,7 +119,10 @@ junit.framework.Assert.fail(Assert.java:50) @Test fun `Merge iOS large time`() { val merged = - parseAllSuitesXml(iosLargeNum).merge(parseAllSuitesXml(iosLargeNum)).xmlToString().normalizeLineEnding() + parseAllSuitesXml(iosLargeNum).toApiModel() + .merge(parseAllSuitesXml(iosLargeNum).toApiModel()) + .toXmlString() + .normalizeLineEnding() val expected = """ @@ -125,15 +139,15 @@ junit.framework.Assert.fail(Assert.java:50) @Test fun `parse androidSkipped`() { - val parsed = parseOneSuiteXml(androidSkipped) + val parsed = parseOneSuiteXml(androidSkipped.writeToTempFile()) assertThat(parsed.testsuites?.first()?.testcases?.first()?.skipped).isNull() } @Test fun `merge androidSkipped`() { - val merged = parseOneSuiteXml(androidSkipped) + val merged = parseOneSuiteXml(androidSkipped.writeToTempFile()).toApiModel() merged.merge(merged) - val actual = merged.xmlToString().normalizeLineEnding() + val actual = merged.toXmlString().normalizeLineEnding() assertThat(actual).isEqualTo( """ @@ -176,14 +190,14 @@ junit.framework.Assert.fail(Assert.java:50) """.trimIndent() - val parsed = parseAllSuitesXml(unknownXml).xmlToString() + val parsed = parseAllSuitesXml(unknownXml.writeToTempFile()).toApiModel().toXmlString() assertEqualsIgnoreNewlineStyle(parsed, expected) } @Test fun `junitXmlToString androidPassXml`() { - val parsed = parseOneSuiteXml(androidPassXml).xmlToString().normalizeLineEnding() + val parsed = parseOneSuiteXml(androidPassXml).toApiModel().toXmlString().normalizeLineEnding() val expected = """ @@ -199,7 +213,7 @@ junit.framework.Assert.fail(Assert.java:50) @Test fun `junitXmlToString androidFailXml`() { - val parsed = parseOneSuiteXml(androidFailXml).xmlToString().normalizeLineEnding() + val parsed = parseOneSuiteXml(androidFailXml).toApiModel().toXmlString().normalizeLineEnding() val expected = """ @@ -219,7 +233,7 @@ junit.framework.Assert.fail(Assert.java:50) @Test fun `junitXmlToString iosPassXml`() { - val parsed = parseAllSuitesXml(iosPassXml).xmlToString().normalizeLineEnding() + val parsed = parseAllSuitesXml(iosPassXml).toApiModel().toXmlString().normalizeLineEnding() val expected = """ @@ -236,7 +250,7 @@ junit.framework.Assert.fail(Assert.java:50) @Test fun `junitXmlToString iosFailXml`() { - val parsed = parseAllSuitesXml(iosFailXml).xmlToString().normalizeLineEnding() + val parsed = parseAllSuitesXml(iosFailXml).toApiModel().toXmlString().normalizeLineEnding() val expected = """ @@ -437,7 +451,8 @@ junit.framework.Assert.fail(Assert.java:50) // * d() was skipped in newRun and successful in oldRun. d() is excluded from the merged result val merged = - parseAllSuitesXml(newRun).mergeTestTimes(parseAllSuitesXml(oldRun)).xmlToString().normalizeLineEnding() + parseAllSuitesXml(newRun.writeToTempFile()).toApiModel() + .mergeTestTimes(parseAllSuitesXml(oldRun.writeToTempFile()).toApiModel()).toXmlString().normalizeLineEnding() val expected = """ @@ -479,7 +494,7 @@ junit.framework.Assert.fail(Assert.java:50) """.trimIndent() - val allSuitesXml = parseAllSuitesXml(crashingAllSuitesMessage).xmlToString().trimIndent() + val allSuitesXml = parseAllSuitesXml(crashingAllSuitesMessage.writeToTempFile()).toApiModel().toXmlString().trimIndent() Assert.assertEquals("All Suite Messages should be the same!", expectedAllSuitesMessage, allSuitesXml) } @@ -508,7 +523,10 @@ junit.framework.Assert.fail(Assert.java:50) """.trimIndent() - val oneSuiteXml = parseOneSuiteXml(crashingOneSuiteMessage).xmlToString().trimIndent() + val oneSuiteXml = parseOneSuiteXml(crashingOneSuiteMessage.writeToTempFile()).toApiModel().toXmlString().trimIndent() Assert.assertEquals("One Suite Messages should be the same!", expectedOneSuiteMessage, oneSuiteXml) } } + +private fun String.writeToTempFile(): File = File.createTempFile("temp", "test") + .apply { writeText(this@writeToTempFile) } diff --git a/test_runner/src/test/kotlin/ftl/reports/xml/model/SkippedTestJUnitTestSuiteTest.kt b/test_runner/src/test/kotlin/ftl/client/xml/model/SkippedTestJUnitTestSuiteTest.kt similarity index 76% rename from test_runner/src/test/kotlin/ftl/reports/xml/model/SkippedTestJUnitTestSuiteTest.kt rename to test_runner/src/test/kotlin/ftl/client/xml/model/SkippedTestJUnitTestSuiteTest.kt index 75df853854..76252c362b 100644 --- a/test_runner/src/test/kotlin/ftl/reports/xml/model/SkippedTestJUnitTestSuiteTest.kt +++ b/test_runner/src/test/kotlin/ftl/client/xml/model/SkippedTestJUnitTestSuiteTest.kt @@ -1,6 +1,8 @@ -package ftl.reports.xml.model +package ftl.client.xml.model import com.google.common.truth.Truth.assertThat +import ftl.api.JUnitTest +import ftl.reports.util.ReportManager.getSkippedJUnitTestSuite import io.mockk.mockk import org.junit.Test @@ -9,7 +11,7 @@ internal class SkippedTestJUnitTestSuiteTest { @Test fun `should created JUnitTestSuite with constant fields for skipped junit test case`() { // given - val testsList = listOf(mockk()) + val testsList = listOf(mockk()) // when val actual = getSkippedJUnitTestSuite(testsList) diff --git a/test_runner/src/test/kotlin/ftl/gc/GcTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcTestMatrixTest.kt index 149986837d..1a3f626b41 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcTestMatrixTest.kt @@ -1,6 +1,7 @@ package ftl.gc import ftl.args.IArgs +import ftl.client.google.GcTestMatrix import ftl.test.util.FlankTestRunner import io.mockk.every import io.mockk.mockk diff --git a/test_runner/src/test/kotlin/ftl/reports/HtmlErrorReportTest.kt b/test_runner/src/test/kotlin/ftl/reports/HtmlErrorReportTest.kt index 4db934530c..b28ab48c8a 100644 --- a/test_runner/src/test/kotlin/ftl/reports/HtmlErrorReportTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/HtmlErrorReportTest.kt @@ -1,9 +1,11 @@ package ftl.reports -import ftl.reports.xml.JUnitXmlTest -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.parseAllSuitesXml -import ftl.reports.xml.parseOneSuiteXml +import ftl.adapter.google.toApiModel +import ftl.api.JUnitTest +import ftl.client.junit.parseAllSuitesXml +import ftl.client.junit.parseOneSuiteXml +import ftl.client.xml.JUnitXmlTest +import ftl.domain.junit.merge import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test @@ -12,13 +14,13 @@ class HtmlErrorReportTest { @Test fun `reactJson androidPassXml`() { - val results = parseOneSuiteXml(JUnitXmlTest.androidPassXml).testsuites!!.process() + val results = parseOneSuiteXml(JUnitXmlTest.androidPassXml).toApiModel().testsuites!!.process() assertTrue(results.isEmpty()) } @Test fun `reactJson androidFailXml`() { - val results: List = parseOneSuiteXml(JUnitXmlTest.androidFailXml).testsuites!!.process() + val results: List = parseOneSuiteXml(JUnitXmlTest.androidFailXml).toApiModel().testsuites!!.process() val group = results.first() assertEquals(1, results.size) @@ -34,7 +36,7 @@ class HtmlErrorReportTest { @Test fun `reactJson androidFailXml merged`() { // 4 tests - 2 pass, 2 fail. we should have 2 failures in the report - val mergedXml: JUnitTestResult = parseOneSuiteXml(JUnitXmlTest.androidFailXml) + val mergedXml: JUnitTest.Result = parseOneSuiteXml(JUnitXmlTest.androidFailXml).toApiModel() mergedXml.merge(mergedXml) assertEquals(4, mergedXml.testsuites?.first()?.testcases?.size) @@ -55,13 +57,13 @@ class HtmlErrorReportTest { @Test fun `reactJson iosPassXml`() { - val results = parseAllSuitesXml(JUnitXmlTest.iosPassXml).testsuites!!.process() + val results = parseAllSuitesXml(JUnitXmlTest.iosPassXml).toApiModel().testsuites!!.process() assertTrue(results.isEmpty()) } @Test fun `reactJson iosFailXml`() { - val results = parseAllSuitesXml(JUnitXmlTest.iosFailXml).testsuites!!.process() + val results = parseAllSuitesXml(JUnitXmlTest.iosFailXml).toApiModel().testsuites!!.process() val group = results.first() assertEquals(1, results.size) diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt index a2bdb7ff6a..24259fc935 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt @@ -5,8 +5,10 @@ import com.google.api.services.toolresults.model.StackTrace import com.google.api.services.toolresults.model.TestCase import com.google.api.services.toolresults.model.TestCaseReference import com.google.testing.model.ToolResultsStep -import ftl.reports.xml.model.JUnitTestCase -import ftl.reports.xml.xmlPrettyWriter +import ftl.client.junit.JUnitTestCase +import ftl.client.junit.createJUnitTestCases +import ftl.client.junit.flaky +import ftl.client.junit.xmlPrettyWriter import org.junit.Assert.assertArrayEquals import org.junit.Test diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt index 2c0e4d9143..8580b3b9db 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestSuiteKtTest.kt @@ -7,11 +7,14 @@ import com.google.api.services.toolresults.model.TestExecutionStep import com.google.api.services.toolresults.model.Timestamp import com.google.testing.model.TestExecution import com.google.testing.model.ToolResultsStep -import ftl.reports.api.data.TestExecutionData +import ftl.client.junit.JUnitTestCase +import ftl.client.junit.JUnitTestSuite +import ftl.client.junit.TestExecutionData +import ftl.client.junit.createJUnitTestCases +import ftl.client.junit.createJUnitTestSuites +import ftl.client.junit.createTestSuiteOverviewData +import ftl.client.junit.xmlPrettyWriter import ftl.reports.api.data.TestSuiteOverviewData -import ftl.reports.xml.model.JUnitTestCase -import ftl.reports.xml.model.JUnitTestSuite -import ftl.reports.xml.xmlPrettyWriter import io.mockk.every import io.mockk.mockkStatic import io.mockk.unmockkAll @@ -25,8 +28,8 @@ class CreateJUnitTestSuiteKtTest { @Before fun setUp() { mockkStatic( - "ftl.reports.api.CreateTestSuiteOverviewDataKt", - "ftl.reports.api.CreateJUnitTestCaseKt" + "ftl.client.junit.CreateTestSuiteOverviewDataKt", + "ftl.client.junit.CreateJUnitTestCaseKt" ) } diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestExecutionDataKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestExecutionDataKtTest.kt index d20eda9206..e14fbb899a 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestExecutionDataKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestExecutionDataKtTest.kt @@ -6,8 +6,9 @@ import com.google.api.services.toolresults.model.TestCase import com.google.api.services.toolresults.model.Timestamp import com.google.testing.model.TestExecution import com.google.testing.model.ToolResultsStep +import ftl.client.junit.TestExecutionData +import ftl.client.junit.createTestExecutionDataListAsync import ftl.gc.GcToolResults -import ftl.reports.api.data.TestExecutionData import io.mockk.every import io.mockk.mockkObject import io.mockk.unmockkObject diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt index b5468c8a6b..1a2611858c 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateTestSuiteOverviewDataKtTest.kt @@ -7,7 +7,9 @@ import com.google.api.services.toolresults.model.TestExecutionStep import com.google.api.services.toolresults.model.TestSuiteOverview import com.google.api.services.toolresults.model.Timestamp import com.google.testing.model.TestExecution -import ftl.reports.api.data.TestExecutionData +import ftl.client.junit.TestExecutionData +import ftl.client.junit.createTestSuiteOverviewData +import ftl.client.junit.flaky import ftl.reports.api.data.TestSuiteOverviewData import org.junit.Assert.assertEquals import org.junit.Test diff --git a/test_runner/src/test/kotlin/ftl/reports/api/PrepareForJUnitResultKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/PrepareForJUnitResultKtTest.kt index 15dd8f0a24..d60b787ccd 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/PrepareForJUnitResultKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/PrepareForJUnitResultKtTest.kt @@ -6,7 +6,10 @@ import com.google.api.services.toolresults.model.Step import com.google.api.services.toolresults.model.TestCase import com.google.api.services.toolresults.model.Timestamp import com.google.testing.model.TestExecution -import ftl.reports.api.data.TestExecutionData +import ftl.client.junit.TestExecutionData +import ftl.client.junit.flaky +import ftl.client.junit.prepareForJUnitResult +import ftl.client.junit.removeStackTraces import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse diff --git a/test_runner/src/test/kotlin/ftl/reports/util/EndsWithTextWithOptionalSlashAtTheEndTest.kt b/test_runner/src/test/kotlin/ftl/reports/util/EndsWithTextWithOptionalSlashAtTheEndTest.kt index 6563431403..7dfae2b9a3 100644 --- a/test_runner/src/test/kotlin/ftl/reports/util/EndsWithTextWithOptionalSlashAtTheEndTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/util/EndsWithTextWithOptionalSlashAtTheEndTest.kt @@ -1,5 +1,6 @@ package ftl.reports.util +import ftl.reports.util.ReportManager.endsWithTextWithOptionalSlashAtTheEnd import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test diff --git a/test_runner/src/test/kotlin/ftl/reports/utils/JUnitDedupeTest.kt b/test_runner/src/test/kotlin/ftl/reports/utils/JUnitDedupeTest.kt index f04698d2ee..b1afb59d55 100644 --- a/test_runner/src/test/kotlin/ftl/reports/utils/JUnitDedupeTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/utils/JUnitDedupeTest.kt @@ -2,12 +2,14 @@ package ftl.reports.utils import com.google.common.truth.Truth.assertThat import flank.common.normalizeLineEnding +import ftl.adapter.google.toApiModel +import ftl.client.junit.parseAllSuitesXml +import ftl.reports.toXmlString import ftl.reports.util.JUnitDedupe -import ftl.reports.xml.parseAllSuitesXml -import ftl.reports.xml.xmlToString import ftl.test.util.FlankTestRunner import org.junit.Test import org.junit.runner.RunWith +import java.io.File @RunWith(FlankTestRunner::class) class JUnitDedupeTest { @@ -50,22 +52,21 @@ class JUnitDedupeTest { junit.framework.AssertionFailedError - matrices/7494574344413871385 - - - matrices/7494574344413871385 - - - matrices/7494574344413871385 + + """.trimIndent() - val suites = parseAllSuitesXml(inputXml) + val suites = parseAllSuitesXml( + File.createTempFile("test", "file").apply { writeText(inputXml) } + ).toApiModel() JUnitDedupe.modify(suites) - assertThat(suites.xmlToString().normalizeLineEnding()).isEqualTo(expectedXml) + val x = suites.toXmlString().normalizeLineEnding() + val y = expectedXml + assertThat(x).isEqualTo(y) } } diff --git a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt index ffc5fb0070..1d283e52fc 100644 --- a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt @@ -1,19 +1,19 @@ package ftl.reports.utils import com.google.common.truth.Truth.assertThat -import com.google.testing.model.TestExecution import flank.common.isWindows +import ftl.adapter.GoogleJUnitTestFetch +import ftl.api.JUnitTest +import ftl.api.generateJUnitTestResultFromApi +import ftl.api.parseJUnitLegacyTestResultFromFile +import ftl.api.parseJUnitTestResultFromFile import ftl.args.AndroidArgs import ftl.client.google.GcStorage +import ftl.client.junit.getDeviceString import ftl.json.validate -import ftl.reports.api.createJUnitTestResult -import ftl.reports.api.refreshMatricesAndGetExecutions +import ftl.reports.toXmlString import ftl.reports.util.ReportManager import ftl.reports.util.getMatrixPath -import ftl.reports.xml.model.JUnitTestCase -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.model.JUnitTestSuite -import ftl.reports.xml.parseOneSuiteXml import ftl.run.common.matrixPathToObj import ftl.run.exception.FTLError import ftl.test.util.FlankTestRunner @@ -54,8 +54,8 @@ class ReportManagerTest { val mockArgs = mockk(relaxed = true) every { mockArgs.smartFlankGcsPath } returns "" every { mockArgs.useLegacyJUnitResult } returns true - mockkStatic("ftl.reports.api.ProcessFromApiKt") - every { refreshMatricesAndGetExecutions(any(), any()) } returns emptyList() + mockkObject(GoogleJUnitTestFetch) + every { generateJUnitTestResultFromApi(any()) } returns JUnitTest.Result(mutableListOf()) ReportManager.generate(matrix, mockArgs, emptyList()) matrix.validate() } @@ -66,8 +66,8 @@ class ReportManagerTest { val mockArgs = mockk(relaxed = true) every { mockArgs.smartFlankGcsPath } returns "" every { mockArgs.useLegacyJUnitResult } returns true - mockkStatic("ftl.reports.api.ProcessFromApiKt") - every { refreshMatricesAndGetExecutions(any(), any()) } returns emptyList() + mockkObject(GoogleJUnitTestFetch) + every { generateJUnitTestResultFromApi(any()) } returns JUnitTest.Result(mutableListOf()) ReportManager.generate(matrix, mockArgs, emptyList()) } @@ -113,9 +113,9 @@ class ReportManagerTest { fun `uploadJunitXml should be called`() { val matrix = matrixPathToObj("./src/test/kotlin/ftl/fixtures/success_result", AndroidArgs.default()) val mockArgs = prepareMockAndroidArgs() - val junitTestResult = ReportManager.processXmlFromFile(matrix, mockArgs, ::parseOneSuiteXml) + val junitTestResult = ReportManager.processXmlFromFile(matrix, mockArgs, parseJUnitTestResultFromFile) ReportManager.generate(matrix, mockArgs, emptyList()) - verify { GcStorage.uploadJunitXml(junitTestResult!!, mockArgs) } + verify { GcStorage.uploadJunitXml(junitTestResult.toXmlString(), mockArgs) } } @Test @@ -125,15 +125,13 @@ class ReportManagerTest { every { mockArgs.useLegacyJUnitResult } returns false every { mockArgs.project } returns "projecId" - val executions = emptyList() - mockkStatic("ftl.reports.api.ProcessFromApiKt") - mockkStatic("ftl.reports.api.CreateJUnitTestResultKt") - every { refreshMatricesAndGetExecutions(any(), any()) } returns executions - every { executions.createJUnitTestResult(any()) } returns JUnitTestResult(mutableListOf(suite)) + mockkObject(GoogleJUnitTestFetch) + mockkStatic("ftl.client.junit.CreateJUnitTestResultKt") + every { generateJUnitTestResultFromApi(any()) } returns JUnitTest.Result(mutableListOf(suite)) - val junitTestResult = ReportManager.processXmlFromFile(matrix, mockArgs, ::parseOneSuiteXml) + val junitTestResult = ReportManager.processXmlFromFile(matrix, mockArgs, parseJUnitLegacyTestResultFromFile) ReportManager.generate(matrix, mockArgs, emptyList()) - verify { GcStorage.uploadJunitXml(junitTestResult!!, mockArgs) } + verify { GcStorage.uploadJunitXml(junitTestResult.toXmlString(), mockArgs) } } @Test @@ -144,36 +142,34 @@ class ReportManagerTest { every { mockArgs.fullJUnitResult } returns true every { mockArgs.project } returns "projecId" - val executions = emptyList() - mockkStatic("ftl.reports.api.ProcessFromApiKt") - mockkStatic("ftl.reports.api.CreateJUnitTestResultKt") - every { refreshMatricesAndGetExecutions(any(), any()) } returns executions - every { executions.createJUnitTestResult(any()) } returns JUnitTestResult(mutableListOf(suite)) + mockkObject(GoogleJUnitTestFetch) + mockkStatic("ftl.client.junit.CreateJUnitTestResultKt") + every { generateJUnitTestResultFromApi(any()) } returns JUnitTest.Result(mutableListOf(suite)) - val junitTestResult = ReportManager.processXmlFromFile(matrix, mockArgs, ::parseOneSuiteXml) + val junitTestResult = ReportManager.processXmlFromFile(matrix, mockArgs, parseJUnitLegacyTestResultFromFile) ReportManager.generate(matrix, mockArgs, emptyList()) - verify { GcStorage.uploadJunitXml(junitTestResult!!, mockArgs) } + verify { GcStorage.uploadJunitXml(junitTestResult.toXmlString(), mockArgs) } } @Test fun createShardEfficiencyListTest() { val oldRunTestCases = mutableListOf( - JUnitTestCase("a", "a", "10.0"), - JUnitTestCase("b", "b", "20.0"), - JUnitTestCase("c", "c", "30.0") + JUnitTest.Case("a", "a", "10.0"), + JUnitTest.Case("b", "b", "20.0"), + JUnitTest.Case("c", "c", "30.0") ) val oldRunSuite = - JUnitTestSuite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", oldRunTestCases, null, null, null) - val oldTestResult = JUnitTestResult(mutableListOf(oldRunSuite)) + JUnitTest.Suite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", oldRunTestCases, null, null, null) + val oldTestResult = JUnitTest.Result(mutableListOf(oldRunSuite)) val newRunTestCases = mutableListOf( - JUnitTestCase("a", "a", "9.0"), - JUnitTestCase("b", "b", "21.0"), - JUnitTestCase("c", "c", "30.0") + JUnitTest.Case("a", "a", "9.0"), + JUnitTest.Case("b", "b", "21.0"), + JUnitTest.Case("c", "c", "30.0") ) val newRunSuite = - JUnitTestSuite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", newRunTestCases, null, null, null) - val newTestResult = JUnitTestResult(mutableListOf(newRunSuite)) + JUnitTest.Suite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", newRunTestCases, null, null, null) + val newTestResult = JUnitTest.Result(mutableListOf(newRunSuite)) val mockArgs = mockk() @@ -191,13 +187,13 @@ class ReportManagerTest { @Test fun `Test getDeviceString`() { - assertThat(ReportManager.getDeviceString("NexusLowRes-28-en-portrait-rerun_1")) + assertThat(getDeviceString("NexusLowRes-28-en-portrait-rerun_1")) .isEqualTo("NexusLowRes-28-en-portrait") - assertThat(ReportManager.getDeviceString("NexusLowRes-28-en-portrait")) + assertThat(getDeviceString("NexusLowRes-28-en-portrait")) .isEqualTo("NexusLowRes-28-en-portrait") - assertThat(ReportManager.getDeviceString("")) + assertThat(getDeviceString("")) .isEqualTo("") } @@ -221,8 +217,8 @@ class ReportManagerTest { assertEquals("test_dir/shard_0", path.getMatrixPath("test_dir")) } - private val suite: JUnitTestSuite - get() = JUnitTestSuite( + private val suite: JUnitTest.Suite + get() = JUnitTest.Suite( name = "any", tests = "2", failures = "0", diff --git a/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt b/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt index c118ffa321..16d6b74145 100644 --- a/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt +++ b/test_runner/src/test/kotlin/ftl/shard/ShardTest.kt @@ -1,12 +1,10 @@ package ftl.shard import com.google.common.truth.Truth.assertThat +import ftl.api.JUnitTest import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs -import ftl.reports.xml.model.JUnitTestCase -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.model.JUnitTestSuite import ftl.run.exception.FlankConfigurationError import ftl.test.util.FlankTestRunner import ftl.util.FlankTestMethod @@ -67,7 +65,7 @@ class ShardTest { @Test fun firstRun() { val testsToRun = listOfFlankTestMethod("a", "b", "c") - val result = createShardsByShardCount(testsToRun, JUnitTestResult(null), mockArgs(2)) + val result = createShardsByShardCount(testsToRun, JUnitTest.Result(null), mockArgs(2)) assertThat(result.size).isEqualTo(2) assertThat(result.sumByDouble { it.time }).isEqualTo(3 * DEFAULT_TEST_TIME_SEC) @@ -110,7 +108,7 @@ class ShardTest { while (attempt++ < maxAttempts && ms >= maxMs) { ms = measureTimeMillis { - createShardsByShardCount(testsToRun, JUnitTestResult(null), arg) + createShardsByShardCount(testsToRun, JUnitTest.Result(null), arg) } println("Shards calculated in $ms ms, attempt: $attempt") } @@ -127,10 +125,10 @@ class ShardTest { ) } - private fun newSuite(testCases: MutableList): JUnitTestResult { - return JUnitTestResult( + private fun newSuite(testCases: MutableList): JUnitTest.Result { + return JUnitTest.Result( mutableListOf( - JUnitTestSuite( + JUnitTest.Suite( "", "3", "0", @@ -153,9 +151,9 @@ class ShardTest { private fun shardCountByTime(shardTime: Int): Int { val testsToRun = listOfFlankTestMethod("a/a", "b/b", "c/c") val testCases = mutableListOf( - JUnitTestCase("a", "a", "0.001"), - JUnitTestCase("b", "b", "0.0"), - JUnitTestCase("c", "c", "0.0") + JUnitTest.Case("a", "a", "0.001"), + JUnitTest.Case("b", "b", "0.0"), + JUnitTest.Case("c", "c", "0.0") ) val oldTestResult = newSuite(testCases) @@ -192,7 +190,7 @@ class ShardTest { @Test(expected = FlankConfigurationError::class) fun `should terminate with exit status == 3 test targets is 0 and maxTestShards == -1`() { - createShardsByShardCount(emptyList(), JUnitTestResult(mutableListOf()), mockArgs(-1)) + createShardsByShardCount(emptyList(), JUnitTest.Result(mutableListOf()), mockArgs(-1)) } @Test @@ -209,9 +207,9 @@ class ShardTest { val oldTestResult = newSuite( mutableListOf( - JUnitTestCase("a", "a", "5.0"), - JUnitTestCase("b", "b", "10.0"), - JUnitTestCase("c", "c", "10.0") + JUnitTest.Case("a", "a", "5.0"), + JUnitTest.Case("b", "b", "10.0"), + JUnitTest.Case("c", "c", "10.0") ) ) diff --git a/test_runner/src/test/kotlin/ftl/shard/ShardTestCommon.kt b/test_runner/src/test/kotlin/ftl/shard/ShardTestCommon.kt index bc2408e133..011bebfc75 100644 --- a/test_runner/src/test/kotlin/ftl/shard/ShardTestCommon.kt +++ b/test_runner/src/test/kotlin/ftl/shard/ShardTestCommon.kt @@ -1,29 +1,27 @@ package ftl.shard +import ftl.api.JUnitTest import ftl.args.IArgs import ftl.args.IosArgs -import ftl.reports.xml.model.JUnitTestCase -import ftl.reports.xml.model.JUnitTestResult -import ftl.reports.xml.model.JUnitTestSuite import ftl.util.FlankTestMethod import io.mockk.every import io.mockk.mockk -internal fun sample(): JUnitTestResult { +internal fun sample(): JUnitTest.Result { val testCases = mutableListOf( - JUnitTestCase("a", "a", "1.0"), - JUnitTestCase("b", "b", "2.0"), - JUnitTestCase("c", "c", "4.0"), - JUnitTestCase("d", "d", "6.0"), - JUnitTestCase("e", "e", "0.5"), - JUnitTestCase("f", "f", "2.0"), - JUnitTestCase("g", "g", "1.0") + JUnitTest.Case("a", "a", "1.0"), + JUnitTest.Case("b", "b", "2.0"), + JUnitTest.Case("c", "c", "4.0"), + JUnitTest.Case("d", "d", "6.0"), + JUnitTest.Case("e", "e", "0.5"), + JUnitTest.Case("f", "f", "2.0"), + JUnitTest.Case("g", "g", "1.0") ) - val suite1 = JUnitTestSuite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", testCases, null, null, null) - val suite2 = JUnitTestSuite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", mutableListOf(), null, null, null) + val suite1 = JUnitTest.Suite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", testCases, null, null, null) + val suite2 = JUnitTest.Suite("", "-1", "-1", -1, "-1", "-1", "-1", "-1", "-1", "-1", mutableListOf(), null, null, null) - return JUnitTestResult(mutableListOf(suite1, suite2)) + return JUnitTest.Result(mutableListOf(suite1, suite2)) } internal fun listOfFlankTestMethod(vararg args: String) = listOf(*args).map { FlankTestMethod(it) }