From 7a5cbbcc01b027dd27ecaae01618e4b98a6c8b37 Mon Sep 17 00:00:00 2001 From: pawelpasterz <32893017+pawelpasterz@users.noreply.github.com> Date: Mon, 22 Mar 2021 09:26:35 +0100 Subject: [PATCH] feat: Add app name to the test result (#1704) --- .../integration/IntergrationTestsUtils.kt | 4 +- .../resources/compare/SanityRoboIT-compare | 6 +- .../src/main/kotlin/ftl/json/SavedMatrix.kt | 34 +++++++- .../kotlin/ftl/json/SavedMatrixTableUtil.kt | 5 ++ .../ftl/reports/output/OutputReportLoggers.kt | 5 +- .../test/kotlin/ftl/json/SavedMatrixTest.kt | 83 ++++++++++++++++++- .../reports/output/OutputReportLoggersTest.kt | 7 +- 7 files changed, 131 insertions(+), 13 deletions(-) diff --git a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt index d44de7c878..85a23516a3 100644 --- a/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt @@ -86,8 +86,8 @@ enum class TestOutcome(val regex: Regex) { private val fromCommon = { outcome: String -> - if (isWindows) "\\?\\s$outcome\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex() - else "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex() + if (isWindows) "\\?\\s$outcome\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9._-]*\\s*\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex() + else "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9._-]*\\s*│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex() } fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "").trimIndent() diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare index 99813ebb78..a2674f54ed 100644 --- a/integration_tests/src/test/resources/compare/SanityRoboIT-compare +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -83,11 +83,7 @@ CostReport MatrixResultsReport 1 \/ 1 \(100\.00\%\) -.* -[│\?] OUTCOME [│\?] MATRIX ID [│\?] TEST AXIS VALUE [│\?] TEST DETAILS [│\?] -.* -[│\?] success [│\?] matrix-[a-zA-Z0-9\s]*[│\?] NexusLowRes-28-en-portrait [│\?] --- [│\?] -.* +[\s\S]* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt index 9a1afb2d3b..79377ae132 100644 --- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt @@ -1,11 +1,14 @@ package ftl.json +import com.google.testing.model.FileReference import com.google.testing.model.TestMatrix +import ftl.analytics.toJSONObject import ftl.environment.orUnknown import ftl.reports.outcome.BillableMinutes import ftl.reports.outcome.TestOutcome import ftl.reports.outcome.createMatrixOutcomeSummary import ftl.reports.outcome.fetchTestOutcomeContext +import ftl.run.common.prettyPrint import ftl.util.MatrixState.FINISHED import ftl.util.MatrixState.INVALID import ftl.util.StepOutcome @@ -32,11 +35,14 @@ data class SavedMatrix( val gcsPathWithoutRootBucket: String = "", val gcsRootBucket: String = "", val webLinkWithoutExecutionDetails: String? = "", - val testAxises: List = emptyList() + val testAxises: List = emptyList(), + val appFileName: String = fallbackAppName ) { val outcome = testAxises.maxByOrNull { StepOutcome.order.indexOf(it.outcome) }?.outcome.orEmpty() } +private const val fallbackAppName = "N/A" + fun createSavedMatrix(testMatrix: TestMatrix) = SavedMatrix().updateWithMatrix(testMatrix) fun SavedMatrix.canceledByUser() = testAxises.any { it.details == ABORTED_BY_USER_MESSAGE } @@ -97,9 +103,24 @@ private fun SavedMatrix.updateProperties(newMatrix: TestMatrix) = copy( clientDetails = newMatrix.getClientDetails(), gcsPathWithoutRootBucket = newMatrix.getGcsPathWithoutRootBucket(), gcsRootBucket = newMatrix.getGcsRootBucket(), - webLinkWithoutExecutionDetails = newMatrix.webLinkWithoutExecutionDetails() + webLinkWithoutExecutionDetails = newMatrix.webLinkWithoutExecutionDetails(), + appFileName = newMatrix.extractAppFileName() ?: fallbackAppName ) +private fun TestMatrix.extractAppFileName() = testSpecification?.run { + listOf( + androidInstrumentationTest, + androidTestLoop, + androidRoboTest, + iosXcTest, + iosTestLoop + ) + .firstOrNull { it != null } + ?.toJSONObject() + ?.let { prettyPrint.fromJson(it.toString(), AppPath::class.java).gcsPath } + ?.substringAfterLast('/') +} + private fun SavedMatrix.updateBillableMinutes(billableMinutes: BillableMinutes) = copy( billablePhysicalMinutes = billableMinutes.physical, billableVirtualMinutes = billableMinutes.virtual, @@ -113,3 +134,12 @@ private fun TestMatrix.invalidTestOutcome() = TestOutcome( outcome = INVALID, details = invalidMatrixDetails.orUnknown() ) + +private data class AppPath( + private val appApk: FileReference?, + private val testsZip: FileReference?, + private val appIpa: FileReference? +) { + val gcsPath: String? + get() = (appApk ?: testsZip ?: appIpa)?.gcsPath +} diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt index 8b98af9306..7e51c0d39a 100644 --- a/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt +++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt @@ -20,6 +20,10 @@ fun List.asPrintableTable(): String = buildTable( header = MATRIX_ID_COLUMN_HEADER, data = flatMapTestAxis { matrix -> matrix.matrixId } ), + TableColumn( + header = APP_NAME_COLUMN_HEADER, + data = flatMapTestAxis { matrix -> matrix.appFileName } + ), TableColumn( header = TEST_AXIS_VALUE_HEADER, data = flatMapTestAxis { device } @@ -43,5 +47,6 @@ private val TestOutcome.outcomeColor private const val OUTCOME_COLUMN_HEADER = "OUTCOME" private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID" +private const val APP_NAME_COLUMN_HEADER = "APP NAME" private const val TEST_AXIS_VALUE_HEADER = "TEST AXIS VALUE" private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS" diff --git a/test_runner/src/main/kotlin/ftl/reports/output/OutputReportLoggers.kt b/test_runner/src/main/kotlin/ftl/reports/output/OutputReportLoggers.kt index 06d85d52e5..aa067a3f0b 100644 --- a/test_runner/src/main/kotlin/ftl/reports/output/OutputReportLoggers.kt +++ b/test_runner/src/main/kotlin/ftl/reports/output/OutputReportLoggers.kt @@ -17,7 +17,10 @@ internal fun OutputReport.log(matrices: Collection) { add( "test_results", matrices.map { - it.matrixId to it.testAxises + it.matrixId to mapOf( + "app" to it.appFileName, + "test-axises" to it.testAxises + ) }.toMap() ) } diff --git a/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt b/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt index 77d053ac91..872d4bda34 100644 --- a/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt @@ -2,14 +2,17 @@ package ftl.json import com.google.common.truth.Truth.assertThat import com.google.testing.model.Environment +import com.google.testing.model.FileReference import com.google.testing.model.GoogleCloudStorage import com.google.testing.model.ResultStorage import com.google.testing.model.TestExecution import com.google.testing.model.TestMatrix +import com.google.testing.model.TestSpecification import com.google.testing.model.ToolResultsExecution import com.google.testing.model.ToolResultsStep import ftl.config.Device import ftl.gc.GcAndroidDevice +import ftl.reports.outcome.make import ftl.test.util.FlankTestRunner import ftl.util.MatrixState.FINISHED import ftl.util.MatrixState.INVALID @@ -60,9 +63,10 @@ class SavedMatrixTest { } } - fun testMatrix() = TestMatrix().also { + fun testMatrix(block: TestMatrix.() -> Unit = {}) = TestMatrix().also { it.projectId = projectId it.testMatrixId = testMatrixId + it.block() } } @@ -235,4 +239,81 @@ class SavedMatrixTest { savedMatrix.outcome ) } + + @Test + fun `SavedMatrix should be updated with apk file name - android`() { + val appName = "any-test_app.apk" + + val specs = listOf Unit>( + { androidInstrumentationTest = make { appApk = ref { "gs://any/path/to/app/$appName" } } }, + { androidTestLoop = make { appApk = ref { "gs://any/path/to/app/$appName" } } }, + { androidRoboTest = make { appApk = ref { "gs://any/path/to/app/$appName" } } }, + ) + + val getNewTestMatrix = { + testMatrix { + state = PENDING + resultStorage = createResultsStorage() + testExecutions = listOf(createStepExecution(1, "NexusLowRes")) + } + } + + specs.forEach { spec -> + val testMatrix = getNewTestMatrix() + val savedMatrix = createSavedMatrix(testMatrix) + + testMatrix.state = FINISHED + testMatrix.testSpecification = make(spec) + val updatedMatrix = savedMatrix.updateWithMatrix(testMatrix) + assertEquals(appName, updatedMatrix.appFileName) + } + } + + @Test + fun `SavedMatrix should be updated with apk file name - ios`() { + val appName = "any-test_app.zip" + + val specs = listOf Unit>( + { iosXcTest = make { testsZip = ref { "gs://any/path/to/app/$appName" } } }, + { iosTestLoop = make { appIpa = ref { "gs://any/path/to/app/$appName" } } }, + ) + + val getNewTestMatrix = { + testMatrix { + state = PENDING + resultStorage = createResultsStorage() + testExecutions = listOf(createStepExecution(1, "iPhone6")) + } + } + + specs.forEach { spec -> + val testMatrix = getNewTestMatrix() + val savedMatrix = createSavedMatrix(testMatrix) + + testMatrix.state = FINISHED + testMatrix.testSpecification = make(spec) + val updatedMatrix = savedMatrix.updateWithMatrix(testMatrix) + assertEquals(appName, updatedMatrix.appFileName) + } + } + + @Test + fun `SavedMatrix should be updated with NA file name if none is available`() { + val getNewTestMatrix = { + testMatrix { + state = PENDING + resultStorage = createResultsStorage() + testExecutions = listOf(createStepExecution(1, "iPhone6")) + } + } + + val testMatrix = getNewTestMatrix() + val savedMatrix = createSavedMatrix(testMatrix) + + testMatrix.state = FINISHED + val updatedMatrix = savedMatrix.updateWithMatrix(testMatrix) + assertEquals("N/A", updatedMatrix.appFileName) + } } + +private inline fun ref(path: () -> String) = FileReference().apply { gcsPath = path() } diff --git a/test_runner/src/test/kotlin/ftl/reports/output/OutputReportLoggersTest.kt b/test_runner/src/test/kotlin/ftl/reports/output/OutputReportLoggersTest.kt index 0244828a10..ed01299219 100644 --- a/test_runner/src/test/kotlin/ftl/reports/output/OutputReportLoggersTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/output/OutputReportLoggersTest.kt @@ -43,10 +43,13 @@ class OutputReportLoggersTest { fun `should log test_results`() { // given val matrices = listOf( - SavedMatrix(matrixId = "1", testAxises = listOf(TestOutcome("device1"))) + SavedMatrix(matrixId = "1", testAxises = listOf(TestOutcome("device1")), appFileName = "any.apk") ) val testResults = matrices.map { - it.matrixId to it.testAxises + it.matrixId to mapOf( + "app" to it.appFileName, + "test-axises" to it.testAxises + ) }.toMap() // when