Skip to content

Commit

Permalink
feat: Add app name to the test result (#1704)
Browse files Browse the repository at this point in the history
  • Loading branch information
pawelpasterz authored Mar 22, 2021
1 parent 4371bcf commit 7a5cbbc
Showing 7 changed files with 131 additions and 13 deletions.
Original file line number Diff line number Diff line change
@@ -86,8 +86,8 @@ enum class TestOutcome(val regex: Regex) {

private val fromCommon =
{ outcome: String ->
if (isWindows) "\\?\\s$outcome\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex()
else "\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex()
if (isWindows) "\\?\\s$outcome\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9._-]*\\s*\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex()
else "\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9._-]*\\s*│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex()
}

fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "").trimIndent()
Original file line number Diff line number Diff line change
@@ -83,11 +83,7 @@ CostReport

MatrixResultsReport
1 \/ 1 \(100\.00\%\)
.*
[│\?] OUTCOME [│\?] MATRIX ID [│\?] TEST AXIS VALUE [│\?] TEST DETAILS [│\?]
.*
[│\?] success [│\?] matrix-[a-zA-Z0-9\s]*[│\?] NexusLowRes-28-en-portrait [│\?] --- [│\?]
.*
[\s\S]*
Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*
Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*
Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*
34 changes: 32 additions & 2 deletions test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package ftl.json

import com.google.testing.model.FileReference
import com.google.testing.model.TestMatrix
import ftl.analytics.toJSONObject
import ftl.environment.orUnknown
import ftl.reports.outcome.BillableMinutes
import ftl.reports.outcome.TestOutcome
import ftl.reports.outcome.createMatrixOutcomeSummary
import ftl.reports.outcome.fetchTestOutcomeContext
import ftl.run.common.prettyPrint
import ftl.util.MatrixState.FINISHED
import ftl.util.MatrixState.INVALID
import ftl.util.StepOutcome
@@ -32,11 +35,14 @@ data class SavedMatrix(
val gcsPathWithoutRootBucket: String = "",
val gcsRootBucket: String = "",
val webLinkWithoutExecutionDetails: String? = "",
val testAxises: List<TestOutcome> = emptyList()
val testAxises: List<TestOutcome> = emptyList(),
val appFileName: String = fallbackAppName
) {
val outcome = testAxises.maxByOrNull { StepOutcome.order.indexOf(it.outcome) }?.outcome.orEmpty()
}

private const val fallbackAppName = "N/A"

fun createSavedMatrix(testMatrix: TestMatrix) = SavedMatrix().updateWithMatrix(testMatrix)

fun SavedMatrix.canceledByUser() = testAxises.any { it.details == ABORTED_BY_USER_MESSAGE }
@@ -97,9 +103,24 @@ private fun SavedMatrix.updateProperties(newMatrix: TestMatrix) = copy(
clientDetails = newMatrix.getClientDetails(),
gcsPathWithoutRootBucket = newMatrix.getGcsPathWithoutRootBucket(),
gcsRootBucket = newMatrix.getGcsRootBucket(),
webLinkWithoutExecutionDetails = newMatrix.webLinkWithoutExecutionDetails()
webLinkWithoutExecutionDetails = newMatrix.webLinkWithoutExecutionDetails(),
appFileName = newMatrix.extractAppFileName() ?: fallbackAppName
)

private fun TestMatrix.extractAppFileName() = testSpecification?.run {
listOf(
androidInstrumentationTest,
androidTestLoop,
androidRoboTest,
iosXcTest,
iosTestLoop
)
.firstOrNull { it != null }
?.toJSONObject()
?.let { prettyPrint.fromJson(it.toString(), AppPath::class.java).gcsPath }
?.substringAfterLast('/')
}

private fun SavedMatrix.updateBillableMinutes(billableMinutes: BillableMinutes) = copy(
billablePhysicalMinutes = billableMinutes.physical,
billableVirtualMinutes = billableMinutes.virtual,
@@ -113,3 +134,12 @@ private fun TestMatrix.invalidTestOutcome() = TestOutcome(
outcome = INVALID,
details = invalidMatrixDetails.orUnknown()
)

private data class AppPath(
private val appApk: FileReference?,
private val testsZip: FileReference?,
private val appIpa: FileReference?
) {
val gcsPath: String?
get() = (appApk ?: testsZip ?: appIpa)?.gcsPath
}
5 changes: 5 additions & 0 deletions test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt
Original file line number Diff line number Diff line change
@@ -20,6 +20,10 @@ fun List<SavedMatrix>.asPrintableTable(): String = buildTable(
header = MATRIX_ID_COLUMN_HEADER,
data = flatMapTestAxis { matrix -> matrix.matrixId }
),
TableColumn(
header = APP_NAME_COLUMN_HEADER,
data = flatMapTestAxis { matrix -> matrix.appFileName }
),
TableColumn(
header = TEST_AXIS_VALUE_HEADER,
data = flatMapTestAxis { device }
@@ -43,5 +47,6 @@ private val TestOutcome.outcomeColor

private const val OUTCOME_COLUMN_HEADER = "OUTCOME"
private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID"
private const val APP_NAME_COLUMN_HEADER = "APP NAME"
private const val TEST_AXIS_VALUE_HEADER = "TEST AXIS VALUE"
private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS"
Original file line number Diff line number Diff line change
@@ -17,7 +17,10 @@ internal fun OutputReport.log(matrices: Collection<SavedMatrix>) {
add(
"test_results",
matrices.map {
it.matrixId to it.testAxises
it.matrixId to mapOf(
"app" to it.appFileName,
"test-axises" to it.testAxises
)
}.toMap()
)
}
83 changes: 82 additions & 1 deletion test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt
Original file line number Diff line number Diff line change
@@ -2,14 +2,17 @@ package ftl.json

import com.google.common.truth.Truth.assertThat
import com.google.testing.model.Environment
import com.google.testing.model.FileReference
import com.google.testing.model.GoogleCloudStorage
import com.google.testing.model.ResultStorage
import com.google.testing.model.TestExecution
import com.google.testing.model.TestMatrix
import com.google.testing.model.TestSpecification
import com.google.testing.model.ToolResultsExecution
import com.google.testing.model.ToolResultsStep
import ftl.config.Device
import ftl.gc.GcAndroidDevice
import ftl.reports.outcome.make
import ftl.test.util.FlankTestRunner
import ftl.util.MatrixState.FINISHED
import ftl.util.MatrixState.INVALID
@@ -60,9 +63,10 @@ class SavedMatrixTest {
}
}

fun testMatrix() = TestMatrix().also {
fun testMatrix(block: TestMatrix.() -> Unit = {}) = TestMatrix().also {
it.projectId = projectId
it.testMatrixId = testMatrixId
it.block()
}
}

@@ -235,4 +239,81 @@ class SavedMatrixTest {
savedMatrix.outcome
)
}

@Test
fun `SavedMatrix should be updated with apk file name - android`() {
val appName = "any-test_app.apk"

val specs = listOf<TestSpecification.() -> Unit>(
{ androidInstrumentationTest = make { appApk = ref { "gs://any/path/to/app/$appName" } } },
{ androidTestLoop = make { appApk = ref { "gs://any/path/to/app/$appName" } } },
{ androidRoboTest = make { appApk = ref { "gs://any/path/to/app/$appName" } } },
)

val getNewTestMatrix = {
testMatrix {
state = PENDING
resultStorage = createResultsStorage()
testExecutions = listOf(createStepExecution(1, "NexusLowRes"))
}
}

specs.forEach { spec ->
val testMatrix = getNewTestMatrix()
val savedMatrix = createSavedMatrix(testMatrix)

testMatrix.state = FINISHED
testMatrix.testSpecification = make(spec)
val updatedMatrix = savedMatrix.updateWithMatrix(testMatrix)
assertEquals(appName, updatedMatrix.appFileName)
}
}

@Test
fun `SavedMatrix should be updated with apk file name - ios`() {
val appName = "any-test_app.zip"

val specs = listOf<TestSpecification.() -> Unit>(
{ iosXcTest = make { testsZip = ref { "gs://any/path/to/app/$appName" } } },
{ iosTestLoop = make { appIpa = ref { "gs://any/path/to/app/$appName" } } },
)

val getNewTestMatrix = {
testMatrix {
state = PENDING
resultStorage = createResultsStorage()
testExecutions = listOf(createStepExecution(1, "iPhone6"))
}
}

specs.forEach { spec ->
val testMatrix = getNewTestMatrix()
val savedMatrix = createSavedMatrix(testMatrix)

testMatrix.state = FINISHED
testMatrix.testSpecification = make(spec)
val updatedMatrix = savedMatrix.updateWithMatrix(testMatrix)
assertEquals(appName, updatedMatrix.appFileName)
}
}

@Test
fun `SavedMatrix should be updated with NA file name if none is available`() {
val getNewTestMatrix = {
testMatrix {
state = PENDING
resultStorage = createResultsStorage()
testExecutions = listOf(createStepExecution(1, "iPhone6"))
}
}

val testMatrix = getNewTestMatrix()
val savedMatrix = createSavedMatrix(testMatrix)

testMatrix.state = FINISHED
val updatedMatrix = savedMatrix.updateWithMatrix(testMatrix)
assertEquals("N/A", updatedMatrix.appFileName)
}
}

private inline fun ref(path: () -> String) = FileReference().apply { gcsPath = path() }
Original file line number Diff line number Diff line change
@@ -43,10 +43,13 @@ class OutputReportLoggersTest {
fun `should log test_results`() {
// given
val matrices = listOf(
SavedMatrix(matrixId = "1", testAxises = listOf(TestOutcome("device1")))
SavedMatrix(matrixId = "1", testAxises = listOf(TestOutcome("device1")), appFileName = "any.apk")
)
val testResults = matrices.map {
it.matrixId to it.testAxises
it.matrixId to mapOf(
"app" to it.appFileName,
"test-axises" to it.testAxises
)
}.toMap()

// when

0 comments on commit 7a5cbbc

Please sign in to comment.