refactor: Refactor data scratch-Junit test result (#1884)

.github/workflows/full_suite_integration_tests.yml

@@ -75,7 +75,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ macos-latest, windows-latest, ubuntu-latest ]
+        os: [ macos-latest, windows-latest, ubuntu-18.04 ]
       fail-fast: false
     outputs:
       job_status: ${{ job.status }}

test_runner/src/main/kotlin/ftl/adapter/DownloadAsJunitXML.kt

@@ -2,11 +2,11 @@ package ftl.adapter
 
 import ftl.adapter.google.toApiModel
 import ftl.api.FileReference
+import ftl.api.JUnitTest
 import ftl.client.google.downloadAsJunitXml
-import ftl.reports.xml.model.JUnitTestResult
 
 object DownloadAsJunitXML :
     FileReference.DownloadAsXML,
-    (FileReference) -> JUnitTestResult by { fileReference ->
+    (FileReference) -> JUnitTest.Result by { fileReference ->
         downloadAsJunitXml(fileReference).toApiModel()
     }

test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestFetch.kt

@@ -0,0 +1,14 @@
+package ftl.adapter
+
+import ftl.adapter.google.toApiModel
+import ftl.api.JUnitTest
+import ftl.client.google.fetchMatrices
+import ftl.client.junit.createJUnitTestResult
+
+object GoogleJUnitTestFetch :
+    JUnitTest.Result.GenerateFromApi,
+    (JUnitTest.Result.ApiIdentity) -> JUnitTest.Result by { (projectId, matrixIds) ->
+        fetchMatrices(matrixIds, projectId)
+            .createJUnitTestResult()
+            .toApiModel()
+    }

test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestParse.kt

@@ -0,0 +1,19 @@
+package ftl.adapter
+
+import ftl.adapter.google.toApiModel
+import ftl.api.JUnitTest
+import ftl.client.junit.parseJUnit
+import ftl.client.junit.parseLegacyJUnit
+import java.io.File
+
+object GoogleJUnitTestParse :
+    JUnitTest.Result.ParseFromFiles,
+    (File) -> JUnitTest.Result by {
+        it.parseJUnit().toApiModel()
+    }
+
+object GoogleLegacyJunitTestParse :
+    JUnitTest.Result.ParseFromFiles,
+    (File) -> JUnitTest.Result by {
+        it.parseLegacyJUnit().toApiModel()
+    }

test_runner/src/main/kotlin/ftl/adapter/google/FileReferenceAdapter.kt

@@ -1,8 +1,5 @@
 package ftl.adapter.google
 
 import ftl.api.FileReference
-import ftl.reports.xml.model.JUnitTestResult
 
 internal fun String.toApiModel(fileReference: FileReference) = fileReference.copy(local = this)
-
-internal fun JUnitTestResult?.toApiModel() = JUnitTestResult(testsuites = this?.testsuites)

test_runner/src/main/kotlin/ftl/adapter/google/JUnitTestResultAdapter.kt

@@ -0,0 +1,45 @@
+package ftl.adapter.google
+
+import ftl.api.JUnitTest
+import ftl.client.junit.JUnitTestCase
+import ftl.client.junit.JUnitTestResult
+import ftl.client.junit.JUnitTestSuite
+
+fun JUnitTestResult?.toApiModel(): JUnitTest.Result = JUnitTest.Result(
+    this?.testsuites?.map { it.toApiModel() }.orEmpty().toMutableList()
+)
+
+private fun JUnitTestSuite.toApiModel(): JUnitTest.Suite {
+    return JUnitTest.Suite(
+        name = name,
+        tests = tests,
+        failures = failures,
+        flakes = flakes,
+        errors = errors,
+        skipped = skipped,
+        time = time,
+        timestamp = timestamp,
+        hostname = hostname,
+        testLabExecutionId = testLabExecutionId,
+        testcases = testcases?.toApiModel(),
+        properties = properties,
+        systemOut = systemOut,
+        systemErr = systemErr
+    )
+}
+
+private fun MutableCollection<JUnitTestCase>.toApiModel(): MutableCollection<JUnitTest.Case> {
+    return map(JUnitTestCase::toApiModel).toMutableList()
+}
+
+private fun JUnitTestCase.toApiModel() = JUnitTest.Case(
+    name = name,
+    classname = classname,
+    time = time,
+    failures = failures,
+    errors = errors,
+    skipped = skipped,
+    flaky = flaky
+).also {
+    it.webLink = webLink
+}

test_runner/src/main/kotlin/ftl/api/FileReference.kt

@@ -2,7 +2,6 @@ package ftl.api
 
 import ftl.adapter.DownloadAsJunitXML
 import ftl.adapter.GcStorageDownload
-import ftl.reports.xml.model.JUnitTestResult
 
 val downloadFileReference: FileReference.Download get() = GcStorageDownload
 val downloadAsJunitXML: FileReference.DownloadAsXML get() = DownloadAsJunitXML
@@ -13,5 +12,5 @@ data class FileReference(
 ) {
 
     interface Download : (FileReference, Boolean, Boolean) -> FileReference
-    interface DownloadAsXML : (FileReference) -> JUnitTestResult
+    interface DownloadAsXML : (FileReference) -> JUnitTest.Result
 }

test_runner/src/main/kotlin/ftl/api/JUnitTestResult.kt

@@ -0,0 +1,128 @@
+package ftl.api
+
+import com.fasterxml.jackson.annotation.JsonInclude
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty
+import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement
+import ftl.adapter.GoogleJUnitTestFetch
+import ftl.adapter.GoogleJUnitTestParse
+import ftl.adapter.GoogleLegacyJunitTestParse
+import java.io.File
+
+val generateJUnitTestResultFromApi: JUnitTest.Result.GenerateFromApi get() = GoogleJUnitTestFetch
+val parseJUnitTestResultFromFile: JUnitTest.Result.ParseFromFiles get() = GoogleJUnitTestParse
+val parseJUnitLegacyTestResultFromFile: JUnitTest.Result.ParseFromFiles get() = GoogleLegacyJunitTestParse
+
+object JUnitTest {
+
+    @JacksonXmlRootElement(localName = "testsuites")
+    data class Result(
+        @JsonInclude(JsonInclude.Include.NON_EMPTY)
+        @JacksonXmlProperty(localName = "testsuite")
+        var testsuites: MutableList<Suite>? = null
+    ) {
+        data class ApiIdentity(
+            val projectId: String,
+            val matrixIds: List<String>
+        )
+
+        interface GenerateFromApi : (ApiIdentity) -> Result
+
+        interface ParseFromFiles : (File) -> Result
+    }
+
+    data class Suite(
+        @JacksonXmlProperty(isAttribute = true)
+        var name: String,
+
+        @JacksonXmlProperty(isAttribute = true)
+        var tests: String, // Int
+
+        @JacksonXmlProperty(isAttribute = true)
+        var failures: String, // Int
+
+        @JacksonXmlProperty(isAttribute = true)
+        var flakes: Int? = null,
+
+        @JacksonXmlProperty(isAttribute = true)
+        var errors: String, // Int
+
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        @JacksonXmlProperty(isAttribute = true)
+        var skipped: String?, // Int. Android only
+
+        @JacksonXmlProperty(isAttribute = true)
+        var time: String, // Double
+
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        @JacksonXmlProperty(isAttribute = true)
+        val timestamp: String?, // String. Android only
+
+        @JacksonXmlProperty(isAttribute = true)
+        val hostname: String? = "localhost", // String.
+
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        @JacksonXmlProperty(isAttribute = true)
+        val testLabExecutionId: String? = null, // String.
+
+        @JacksonXmlProperty(localName = "testcase")
+        var testcases: MutableCollection<Case>?,
+
+        // not used
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        val properties: Any? = null, // <properties />
+
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        @JacksonXmlProperty(localName = "system-out")
+        val systemOut: Any? = null, // <system-out />
+
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        @JacksonXmlProperty(localName = "system-err")
+        val systemErr: Any? = null // <system-err />
+    )
+
+    data class Case(
+        // name, classname, and time are always present except for empty test cases <testcase/>
+        @JacksonXmlProperty(isAttribute = true)
+        val name: String?,
+
+        @JacksonXmlProperty(isAttribute = true)
+        val classname: String?,
+
+        @JacksonXmlProperty(isAttribute = true)
+        val time: String?,
+
+        // iOS contains multiple failures for a single test.
+        // JUnit XML allows arbitrary amounts of failure/error tags
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        @JacksonXmlProperty(localName = "failure")
+        val failures: List<String>? = null,
+
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        @JacksonXmlProperty(localName = "error")
+        val errors: List<String>? = null,
+
+        @JsonInclude(JsonInclude.Include.CUSTOM, valueFilter = FilterNotNull::class)
+        val skipped: String? = "absent", // used by FilterNotNull to filter out absent `skipped` values
+
+        @JacksonXmlProperty(isAttribute = true)
+        val flaky: Boolean? = null // use null instead of false
+    ) {
+
+        // Consider to move all properties to constructor if will doesn't conflict with parser
+        @JsonInclude(JsonInclude.Include.NON_NULL)
+        var webLink: String? = null
+    }
+
+    @Suppress("UnusedPrivateClass")
+    private class FilterNotNull {
+        override fun equals(other: Any?): Boolean {
+            // other is null     = present
+            // other is not null = absent (default value)
+            return other != null
+        }
+
+        override fun hashCode(): Int {
+            return javaClass.hashCode()
+        }
+    }
+}

test_runner/src/main/kotlin/ftl/reports/api/ProcessFromApi.kt → test_runner/src/main/kotlin/ftl/client/google/FetchTestMatrices.kt

@@ -1,18 +1,15 @@
-package ftl.reports.api
+package ftl.client.google
 
 import com.google.testing.model.TestExecution
 import com.google.testing.model.TestMatrix
-import ftl.args.IArgs
-import ftl.gc.GcTestMatrix
-import ftl.json.MatrixMap
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.runBlocking
 
-fun refreshMatricesAndGetExecutions(matrices: MatrixMap, args: IArgs): List<TestExecution> = refreshTestMatrices(
-    matrixIds = matrices.map.values.map { it.matrixId },
-    projectId = args.project
+fun fetchMatrices(matricesIds: List<String>, projectId: String): List<TestExecution> = refreshTestMatrices(
+    matrixIds = matricesIds,
+    projectId = projectId
 ).getTestExecutions()
 
 private fun refreshTestMatrices(
@@ -29,6 +26,6 @@ private fun refreshTestMatrices(
     }.awaitAll()
 }
 
-private fun List<TestMatrix>.getTestExecutions(): List<TestExecution> = this
-    .mapNotNull(TestMatrix::getTestExecutions)
-    .flatten()
+private fun List<TestMatrix>.getTestExecutions(): List<TestExecution> =
+    mapNotNull(TestMatrix::getTestExecutions)
+        .flatten()

test_runner/src/main/kotlin/ftl/client/google/FileReference.kt

@@ -1,8 +1,8 @@
 package ftl.client.google
 
 import ftl.api.FileReference
-import ftl.reports.xml.model.JUnitTestResult
-import ftl.reports.xml.parseAllSuitesXml
+import ftl.client.junit.JUnitTestResult
+import ftl.client.junit.parseAllSuitesXml
 import ftl.run.exception.FlankGeneralError
 import java.nio.file.Paths
 

test_runner/src/main/kotlin/ftl/client/google/GcStorage.kt

@@ -14,9 +14,6 @@ import ftl.args.IArgs
 import ftl.config.FtlConstants
 import ftl.config.FtlConstants.GCS_PREFIX
 import ftl.config.FtlConstants.GCS_STORAGE_LINK
-import ftl.reports.xml.model.JUnitTestResult
-import ftl.reports.xml.parseAllSuitesXml
-import ftl.reports.xml.xmlToString
 import ftl.run.exception.FlankGeneralError
 import ftl.util.runWithProgress
 import java.io.File
@@ -63,26 +60,19 @@ object GcStorage {
         )
     }
 
-    fun uploadJunitXml(testResult: JUnitTestResult, args: IArgs) {
+    fun uploadJunitXml(testResultXml: String, args: IArgs) {
         if (args.smartFlankGcsPath.isBlank() || args.smartFlankDisableUpload) return
 
         // bucket/path/to/object
         val rawPath = args.smartFlankGcsPath.drop(GCS_PREFIX.length)
 
-        testResult.xmlToString().toByteArray().uploadWithProgress(
+        testResultXml.toByteArray().uploadWithProgress(
             bucket = rawPath.substringBefore('/'),
             path = rawPath.substringAfter('/'),
             name = "smart flank XML"
         )
     }
 
-    // junit xml may not exist. ignore error if it doesn't exist
-    private fun downloadJunitXml(
-        args: IArgs
-    ): JUnitTestResult? = download(args.smartFlankGcsPath, ignoreError = true)
-        .takeIf { it.isNotEmpty() }
-        ?.let { parseAllSuitesXml(Paths.get(it)) }
-
     private val duplicatedGcsPathCounter = ConcurrentHashMap<String, Int>()
 
     @VisibleForTesting

test_runner/src/main/kotlin/ftl/gc/GcTestMatrix.kt → test_runner/src/main/kotlin/ftl/client/google/GcTestMatrix.kt

@@ -1,7 +1,8 @@
-package ftl.gc
+package ftl.client.google
 
 import com.google.testing.model.CancelTestMatrixResponse
 import com.google.testing.model.TestMatrix
+import ftl.gc.GcTesting
 import ftl.http.executeWithRetry
 import ftl.run.exception.FlankGeneralError
 import kotlinx.coroutines.Dispatchers

test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt → test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestCase.kt

@@ -1,9 +1,10 @@
-package ftl.reports.api
+package ftl.client.junit
 
 import com.google.api.services.toolresults.model.StackTrace
 import com.google.api.services.toolresults.model.TestCase
 import com.google.testing.model.ToolResultsStep
-import ftl.reports.xml.model.JUnitTestCase
+import ftl.reports.api.format
+import ftl.reports.api.millis
 
 internal fun createJUnitTestCases(
     testCases: List<TestCase>,

test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestResult.kt

@@ -0,0 +1,13 @@
+package ftl.client.junit
+
+import com.google.testing.model.TestExecution
+
+internal fun List<TestExecution>.createJUnitTestResult() = JUnitTestResult(
+    testsuites = filterNullToolResultsStep()
+        .createTestExecutionDataListAsync()
+        .prepareForJUnitResult()
+        .createJUnitTestSuites()
+        .toMutableList()
+)
+
+private fun List<TestExecution>.filterNullToolResultsStep() = filter { it.toolResultsStep != null }

test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestSuite.kt → test_runner/src/main/kotlin/ftl/client/junit/CreateJUnitTestSuite.kt

@@ -1,9 +1,8 @@
-package ftl.reports.api
+package ftl.client.junit
 
-import ftl.reports.api.data.TestExecutionData
 import ftl.reports.api.data.TestSuiteOverviewData
+import ftl.reports.api.format
 import ftl.reports.outcome.axisValue
-import ftl.reports.xml.model.JUnitTestSuite
 
 internal fun List<TestExecutionData>.createJUnitTestSuites() = mapNotNull { data: TestExecutionData ->
     data.createTestSuiteOverviewData()?.let { overviewData ->