From 66e796ce23906a903b24c7c39251314538a2c6be Mon Sep 17 00:00:00 2001 From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com> Date: Tue, 11 May 2021 09:43:40 +0200 Subject: [PATCH] refactor: Data scratch - test matrix (#1901) Fixes #1756 ## Test Plan > How do we know the code works? * All tests pass * Flank works as before ## Checklist - [X] Unit tests updated --- .../kotlin/integration/CustomShardingIT.kt | 4 +- .../test/kotlin/integration/MultipleApksIT.kt | 4 +- .../kotlin/integration/MultipleDevicesIT.kt | 4 +- .../kotlin/utils/IntergrationTestsUtils.kt | 2 +- .../test/kotlin/utils/OutputReportParser.kt | 32 +-- .../ftl/adapter/GoogleJUnitTestFetch.kt | 6 +- .../kotlin/ftl/adapter/TestMatrixCancel.kt | 13 ++ .../kotlin/ftl/adapter/TestMatrixFetch.kt | 11 + .../kotlin/ftl/adapter/TestMatrixRefresh.kt | 14 ++ .../adapter/google/GoogleTestMatrixAdapter.kt | 105 +++++++++ .../ftl/adapter/google/SummaryAdapter.kt | 15 ++ .../main/kotlin/ftl/api/JUnitTestResult.kt | 6 +- .../src/main/kotlin/ftl/api/TestMatrix.kt | 96 +++++++++ .../google}/BillableMinutes.kt | 9 +- .../google}/CreateTestSuiteOverviewData.kt | 0 .../ftl/client/google/FetchTestMatrices.kt | 16 +- .../client/google/GoogleTestMatrixCancel.kt | 5 + .../ftl/client/google/PerformanceMetrics.kt | 22 ++ .../ftl/client/google/TestMatrixRefresh.kt | 8 + .../kotlin/ftl/client/google/TestOutcome.kt | 10 + .../ftl/client/google/TestOutcomeContext.kt | 95 ++++++++ .../ftl/client/junit/TestExecutionData.kt | 6 + .../ftl/domain/testmatrix/UpdateTestMatrix.kt | 62 ++++++ .../src/main/kotlin/ftl/json/MatrixMap.kt | 58 +++-- .../src/main/kotlin/ftl/json/SavedMatrix.kt | 147 ------------- .../kotlin/ftl/json/SavedMatrixTableUtil.kt | 12 +- .../src/main/kotlin/ftl/reports/CostReport.kt | 4 +- .../kotlin/ftl/reports/MatrixResultsReport.kt | 4 +- .../outcome/CreateMatrixOutcomeSummary.kt | 18 -- .../kotlin/ftl/reports/outcome/TestOutcome.kt | 51 ----- .../ftl/reports/outcome/TestOutcomeContext.kt | 50 ----- .../ftl/reports/output/OutputReportLoggers.kt | 6 +- .../kotlin/ftl/reports/util/ReportManager.kt | 27 +-- .../src/main/kotlin/ftl/run/CancelLastRun.kt | 8 +- .../src/main/kotlin/ftl/run/NewTestRun.kt | 4 +- .../src/main/kotlin/ftl/run/RefreshLastRun.kt | 14 +- .../ftl/run/common/FetchAllArtifacts.kt | 6 +- .../kotlin/ftl/run/common/GetLastMatrices.kt | 4 +- .../kotlin/ftl/run/common/PollMatrices.kt | 21 +- .../ftl/run/exception/ExceptionHandler.kt | 4 +- .../ftl/run/exception/FlankException.kt | 8 +- .../ftl/run/platform/RunAndroidTests.kt | 4 +- .../kotlin/ftl/run/platform/RunIosTests.kt | 3 +- .../ftl/run/platform/common/AfterRunTests.kt | 15 +- .../run/status/ExecutionStatusListPrinter.kt | 36 ++-- .../ftl/run/status/TestMatrixStatusPrinter.kt | 11 +- .../src/main/kotlin/ftl/util/MatrixState.kt | 5 +- .../TestMatrixTest.kt} | 202 +++++++++--------- .../src/test/kotlin/ftl/json/MatrixMapTest.kt | 41 ++-- .../kotlin/ftl/json/TestMatrixTestUtils.kt | 8 + .../reports/outcome/BillableMinutesTest.kt | 2 + .../CreateMatrixOutcomeSummaryKtTest.kt | 5 +- .../reports/output/OutputReportLoggersTest.kt | 13 +- .../status/ExecutionStatusListPrinterTest.kt | 13 +- .../run/status/TestMatrixStatusPrinterTest.kt | 9 +- .../src/test/kotlin/ftl/util/UtilsTest.kt | 44 ++-- 56 files changed, 816 insertions(+), 586 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/adapter/TestMatrixCancel.kt create mode 100644 test_runner/src/main/kotlin/ftl/adapter/TestMatrixFetch.kt create mode 100644 test_runner/src/main/kotlin/ftl/adapter/TestMatrixRefresh.kt create mode 100644 test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt create mode 100644 test_runner/src/main/kotlin/ftl/adapter/google/SummaryAdapter.kt create mode 100644 test_runner/src/main/kotlin/ftl/api/TestMatrix.kt rename test_runner/src/main/kotlin/ftl/{reports/outcome => client/google}/BillableMinutes.kt (80%) rename test_runner/src/main/kotlin/ftl/{reports/outcome => client/google}/CreateTestSuiteOverviewData.kt (100%) create mode 100644 test_runner/src/main/kotlin/ftl/client/google/GoogleTestMatrixCancel.kt create mode 100644 test_runner/src/main/kotlin/ftl/client/google/PerformanceMetrics.kt create mode 100644 test_runner/src/main/kotlin/ftl/client/google/TestMatrixRefresh.kt create mode 100644 test_runner/src/main/kotlin/ftl/client/google/TestOutcome.kt create mode 100644 test_runner/src/main/kotlin/ftl/client/google/TestOutcomeContext.kt create mode 100644 test_runner/src/main/kotlin/ftl/domain/testmatrix/UpdateTestMatrix.kt delete mode 100644 test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt delete mode 100644 test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt delete mode 100644 test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt delete mode 100644 test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcomeContext.kt rename test_runner/src/test/kotlin/ftl/{json/SavedMatrixTest.kt => api/TestMatrixTest.kt} (56%) create mode 100644 test_runner/src/test/kotlin/ftl/json/TestMatrixTestUtils.kt diff --git a/integration_tests/src/test/kotlin/integration/CustomShardingIT.kt b/integration_tests/src/test/kotlin/integration/CustomShardingIT.kt index cfe554bf50..de54b66037 100644 --- a/integration_tests/src/test/kotlin/integration/CustomShardingIT.kt +++ b/integration_tests/src/test/kotlin/integration/CustomShardingIT.kt @@ -101,8 +101,8 @@ class CustomShardingIT { .map { it.testAxises } .flatten() - assertThat(testsResults.sumBy { it.testSuiteOverview.failures }).isEqualTo(5) - assertThat(testsResults.sumBy { it.testSuiteOverview.total }).isEqualTo(41) + assertThat(testsResults.sumBy { it.suiteOverview.failures }).isEqualTo(5) + assertThat(testsResults.sumBy { it.suiteOverview.total }).isEqualTo(41) } } diff --git a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt index 7fcc853e29..d0b2ada63a 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleApksIT.kt @@ -69,7 +69,7 @@ class MultipleApksIT { .map { it.testAxises } .flatten() - assertThat(testsResults.sumBy { it.testSuiteOverview.failures }).isEqualTo(5) - assertThat(testsResults.sumBy { it.testSuiteOverview.total }).isEqualTo(41) + assertThat(testsResults.sumBy { it.suiteOverview.failures }).isEqualTo(5) + assertThat(testsResults.sumBy { it.suiteOverview.total }).isEqualTo(41) } } diff --git a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt index 7b262d1b20..9a1937c478 100644 --- a/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt +++ b/integration_tests/src/test/kotlin/integration/MultipleDevicesIT.kt @@ -48,7 +48,7 @@ class MultipleDevicesIT { .map { it.testAxises } .flatten() - assertThat(testsResults.sumBy { it.testSuiteOverview.failures }).isEqualTo(15) - assertThat(testsResults.sumBy { it.testSuiteOverview.total }).isEqualTo(123) + assertThat(testsResults.sumBy { it.suiteOverview.failures }).isEqualTo(15) + assertThat(testsResults.sumBy { it.suiteOverview.total }).isEqualTo(123) } } diff --git a/integration_tests/src/test/kotlin/utils/IntergrationTestsUtils.kt b/integration_tests/src/test/kotlin/utils/IntergrationTestsUtils.kt index 8fd2c0da49..0f9d29dcad 100644 --- a/integration_tests/src/test/kotlin/utils/IntergrationTestsUtils.kt +++ b/integration_tests/src/test/kotlin/utils/IntergrationTestsUtils.kt @@ -60,7 +60,7 @@ fun assertContainsUploads(input: String, vararg uploads: String) = uploads.forEa assertThat(input).contains("Uploading [$it]") } -fun TestSuiteOverview.assertTestCountMatches( +fun SuiteOverview.assertTestCountMatches( total: Int = 0, errors: Int = 0, failures: Int = 0, diff --git a/integration_tests/src/test/kotlin/utils/OutputReportParser.kt b/integration_tests/src/test/kotlin/utils/OutputReportParser.kt index 283b48e0a1..69aa7b7999 100644 --- a/integration_tests/src/test/kotlin/utils/OutputReportParser.kt +++ b/integration_tests/src/test/kotlin/utils/OutputReportParser.kt @@ -10,8 +10,8 @@ private val jsonMapper by lazy { JsonMapper().registerModule(KotlinModule()) } fun String.asOutputReport() = jsonMapper.readValue(this) -val OutputReport.firstTestSuiteOverview: TestSuiteOverview - get() = testResults.values.first().testAxises.first().testSuiteOverview +val OutputReport.firstTestSuiteOverview: SuiteOverview + get() = testResults.values.first().testAxises.first().suiteOverview data class OutputReport( val args: Any, @@ -23,22 +23,22 @@ data class OutputReport( data class Matrix( val app: String = "", - @JsonProperty("test-axises") val testAxises: List + @JsonProperty("test-axises") val testAxises: List = emptyList() ) -data class TextAxis( - val device: String, - val outcome: String, - val details: String, - val testSuiteOverview: TestSuiteOverview +data class Outcome( + val device: String = "", + val outcome: String = "", + val details: String = "", + val suiteOverview: SuiteOverview = SuiteOverview() ) -data class TestSuiteOverview( - val total: Int, - val errors: Int, - val failures: Int, - val flakes: Int, - val skipped: Int, - val elapsedTime: Double, - val overheadTime: Double +data class SuiteOverview( + val total: Int = 0, + val errors: Int = 0, + val failures: Int = 0, + val flakes: Int = 0, + val skipped: Int = 0, + val elapsedTime: Double = 0.0, + val overheadTime: Double = 0.0 ) diff --git a/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestFetch.kt b/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestFetch.kt index a661f86616..5c694d75d6 100644 --- a/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestFetch.kt +++ b/test_runner/src/main/kotlin/ftl/adapter/GoogleJUnitTestFetch.kt @@ -2,13 +2,15 @@ package ftl.adapter import ftl.adapter.google.toApiModel import ftl.api.JUnitTest +import ftl.client.google.createAndUploadPerformanceMetricsForAndroid import ftl.client.google.fetchMatrices import ftl.client.junit.createJUnitTestResult object GoogleJUnitTestFetch : JUnitTest.Result.GenerateFromApi, - (JUnitTest.Result.ApiIdentity) -> JUnitTest.Result by { (projectId, matrixIds) -> - fetchMatrices(matrixIds, projectId) + (JUnitTest.Result.ApiIdentity) -> JUnitTest.Result by { (args, matrixMap) -> + fetchMatrices(matrixMap.map.values.map { it.matrixId }, args.project) + .createAndUploadPerformanceMetricsForAndroid(args, matrixMap) .createJUnitTestResult() .toApiModel() } diff --git a/test_runner/src/main/kotlin/ftl/adapter/TestMatrixCancel.kt b/test_runner/src/main/kotlin/ftl/adapter/TestMatrixCancel.kt new file mode 100644 index 0000000000..729b294471 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/adapter/TestMatrixCancel.kt @@ -0,0 +1,13 @@ +package ftl.adapter + +import ftl.api.TestMatrix +import ftl.client.google.cancelMatrices +import kotlinx.coroutines.runBlocking + +object TestMatrixCancel : + TestMatrix.Cancel, + (TestMatrix.Identity) -> Unit by { identity -> + runBlocking { + cancelMatrices(identity.matrixId, identity.projectId) + } + } diff --git a/test_runner/src/main/kotlin/ftl/adapter/TestMatrixFetch.kt b/test_runner/src/main/kotlin/ftl/adapter/TestMatrixFetch.kt new file mode 100644 index 0000000000..823b0b398c --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/adapter/TestMatrixFetch.kt @@ -0,0 +1,11 @@ +package ftl.adapter + +import ftl.adapter.google.toApiModel +import ftl.api.TestMatrix +import ftl.client.google.fetchMatrixOutcome + +object TestMatrixFetch : + TestMatrix.Summary.Fetch, + (TestMatrix.Data) -> TestMatrix.Summary by { + fetchMatrixOutcome(it).toApiModel() + } diff --git a/test_runner/src/main/kotlin/ftl/adapter/TestMatrixRefresh.kt b/test_runner/src/main/kotlin/ftl/adapter/TestMatrixRefresh.kt new file mode 100644 index 0000000000..b22379f007 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/adapter/TestMatrixRefresh.kt @@ -0,0 +1,14 @@ +package ftl.adapter + +import ftl.adapter.google.toApiModel +import ftl.api.TestMatrix +import ftl.client.google.refreshMatrix +import kotlinx.coroutines.runBlocking + +object TestMatrixRefresh : + TestMatrix.Refresh, + (TestMatrix.Identity) -> TestMatrix.Data by { identity -> + runBlocking { + refreshMatrix(identity.matrixId, identity.projectId).toApiModel(identity) + } + } diff --git a/test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt b/test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt new file mode 100644 index 0000000000..b5b3bb90ba --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt @@ -0,0 +1,105 @@ +package ftl.adapter.google + +import com.google.testing.model.FileReference +import com.google.testing.model.TestExecution +import com.google.testing.model.TestMatrix +import ftl.analytics.toJSONObject +import ftl.api.TestMatrix.Data +import ftl.api.TestMatrix.Outcome +import ftl.api.TestMatrix.SuiteOverview +import ftl.client.google.TestOutcome +import ftl.environment.orUnknown +import ftl.run.common.prettyPrint +import ftl.util.MatrixState +import ftl.util.getClientDetails +import ftl.util.getGcsPath +import ftl.util.getGcsPathWithoutRootBucket +import ftl.util.getGcsRootBucket +import ftl.util.timeoutToSeconds +import ftl.util.webLink +import ftl.util.webLinkWithoutExecutionDetails + +fun TestMatrix.toApiModel(identity: ftl.api.TestMatrix.Identity? = null) = Data( + projectId = projectId.orEmpty(), + matrixId = testMatrixId.orEmpty(), + gcsPath = getGcsPath(), + webLink = webLink(), + downloaded = false, + clientDetails = getClientDetails(), + gcsPathWithoutRootBucket = getGcsPathWithoutRootBucket(), + gcsRootBucket = getGcsRootBucket(), + webLinkWithoutExecutionDetails = webLinkWithoutExecutionDetails(), + appFileName = extractAppFileName() ?: fallbackAppName, + isCompleted = MatrixState.completed(state) && + testExecutions?.all { MatrixState.completed(it.state.orEmpty()) } ?: true, + testExecutions = testExecutions?.toApiModel().orEmpty(), + testTimeout = testTimeout(), + isRoboTest = isRoboTest(), + historyId = resultStorage?.toolResultsExecution?.historyId ?: identity?.historyId.orEmpty(), + executionId = resultStorage?.toolResultsExecution?.executionId ?: identity?.executionId.orEmpty(), + invalidMatrixDetails = invalidMatrixDetails.orUnknown(), + state = state.orEmpty(), +) + +private fun TestMatrix.testTimeout() = timeoutToSeconds( + testExecutions + ?.firstOrNull { it?.testSpecification?.testTimeout != null } + ?.testSpecification + ?.testTimeout + ?: "0s" +) + +private fun TestMatrix.isRoboTest() = testExecutions.orEmpty().any { it?.testSpecification?.androidRoboTest != null } + +private const val fallbackAppName = "N/A" + +fun TestOutcome.toApiModel() = Outcome( + device, outcome, details, + SuiteOverview( + testSuiteOverview.total, + testSuiteOverview.errors, + testSuiteOverview.failures, + testSuiteOverview.flakes, + testSuiteOverview.skipped, + testSuiteOverview.elapsedTime, + testSuiteOverview.overheadTime + ) +) + +fun List.toApiModel() = map(TestExecution::toApiModel) + +fun TestExecution.toApiModel() = ftl.api.TestMatrix.TestExecution( + id = id.orEmpty(), + modelId = environment?.androidDevice?.androidModelId + ?: environment?.iosDevice?.iosModelId ?: "", + deviceVersion = environment?.androidDevice?.androidVersionId + ?: environment?.iosDevice?.iosVersionId ?: "", + shardIndex = shard?.shardIndex, + state = state.orUnknown(), + errorMessage = testDetails?.errorMessage.orEmpty(), + progress = testDetails?.progressMessages ?: emptyList(), + toolResultsStep = toolResultsStep +) + +private fun TestMatrix.extractAppFileName() = testSpecification?.run { + listOf( + androidInstrumentationTest, + androidTestLoop, + androidRoboTest, + iosXcTest, + iosTestLoop + ) + .firstOrNull { it != null } + ?.toJSONObject() + ?.let { prettyPrint.fromJson(it.toString(), AppPath::class.java).gcsPath } + ?.substringAfterLast('/') +} + +private data class AppPath( + private val appApk: FileReference?, + private val testsZip: FileReference?, + private val appIpa: FileReference? +) { + val gcsPath: String? + get() = (appApk ?: testsZip ?: appIpa)?.gcsPath +} diff --git a/test_runner/src/main/kotlin/ftl/adapter/google/SummaryAdapter.kt b/test_runner/src/main/kotlin/ftl/adapter/google/SummaryAdapter.kt new file mode 100644 index 0000000000..da09676bda --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/adapter/google/SummaryAdapter.kt @@ -0,0 +1,15 @@ +package ftl.adapter.google + +import ftl.api.TestMatrix +import ftl.client.google.BillableMinutes +import ftl.client.google.TestOutcome + +fun Pair>.toApiModel(): TestMatrix.Summary { + val (billableMinutes, outcomes) = this + return TestMatrix.Summary( + billableMinutes = billableMinutes.toApiModel(), + axes = outcomes.map(TestOutcome::toApiModel) + ) +} + +private fun BillableMinutes.toApiModel() = TestMatrix.BillableMinutes(virtual, physical) diff --git a/test_runner/src/main/kotlin/ftl/api/JUnitTestResult.kt b/test_runner/src/main/kotlin/ftl/api/JUnitTestResult.kt index f61e02f19b..eff5f3f4a0 100644 --- a/test_runner/src/main/kotlin/ftl/api/JUnitTestResult.kt +++ b/test_runner/src/main/kotlin/ftl/api/JUnitTestResult.kt @@ -6,6 +6,8 @@ import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement import ftl.adapter.GoogleJUnitTestFetch import ftl.adapter.GoogleJUnitTestParse import ftl.adapter.GoogleLegacyJunitTestParse +import ftl.args.IArgs +import ftl.json.MatrixMap import java.io.File val generateJUnitTestResultFromApi: JUnitTest.Result.GenerateFromApi get() = GoogleJUnitTestFetch @@ -21,8 +23,8 @@ object JUnitTest { var testsuites: MutableList? = null ) { data class ApiIdentity( - val projectId: String, - val matrixIds: List + val args: IArgs, + val matrixMap: MatrixMap ) interface GenerateFromApi : (ApiIdentity) -> Result diff --git a/test_runner/src/main/kotlin/ftl/api/TestMatrix.kt b/test_runner/src/main/kotlin/ftl/api/TestMatrix.kt new file mode 100644 index 0000000000..249a3e1f73 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/api/TestMatrix.kt @@ -0,0 +1,96 @@ +package ftl.api + +import com.google.testing.model.ToolResultsStep +import ftl.adapter.TestMatrixCancel +import ftl.adapter.TestMatrixFetch +import ftl.adapter.TestMatrixRefresh +import ftl.util.StepOutcome + +val refreshTestMatrix: TestMatrix.Refresh get() = TestMatrixRefresh +val cancelTestMatrix: TestMatrix.Cancel get() = TestMatrixCancel +val fetchTestSummary: TestMatrix.Summary.Fetch get() = TestMatrixFetch + +private const val fallbackAppName = "N/A" + +object TestMatrix { + + data class Result( + val runPath: String, + val map: Map, + ) + + data class Data( + val projectId: String = "", + val matrixId: String = "", + val state: String = "", + val gcsPath: String = "", + val webLink: String = "", + val downloaded: Boolean = false, + val billableMinutes: BillableMinutes = BillableMinutes(), + val clientDetails: Map? = null, + val gcsPathWithoutRootBucket: String = "", + val gcsRootBucket: String = "", + val webLinkWithoutExecutionDetails: String? = "", + val axes: List = emptyList(), + val appFileName: String = fallbackAppName, + val isCompleted: Boolean = false, + val testExecutions: List = emptyList(), + val testTimeout: Long = 0, + val isRoboTest: Boolean = false, + val historyId: String = "", + val executionId: String = "", + val invalidMatrixDetails: String = "" + ) { + val outcome = axes.maxByOrNull { StepOutcome.order.indexOf(it.outcome) }?.outcome.orEmpty() + } + + data class TestExecution( + val id: String, + val modelId: String, + val deviceVersion: String, + val shardIndex: Int?, + val state: String, + val errorMessage: String = "", + val progress: List = emptyList(), + val toolResultsStep: ToolResultsStep? + ) + + data class Outcome( + val device: String = "", + val outcome: String = "", + val details: String = "", + val suiteOverview: SuiteOverview = SuiteOverview() + ) + + data class SuiteOverview( + val total: Int = 0, + val errors: Int = 0, + val failures: Int = 0, + val flakes: Int = 0, + val skipped: Int = 0, + val elapsedTime: Double = 0.0, + val overheadTime: Double = 0.0 + ) + + data class BillableMinutes( + val virtual: Long = 0, + val physical: Long = 0 + ) + + data class Summary( + val billableMinutes: BillableMinutes, + val axes: List, + ) { + interface Fetch : (Data) -> Summary + } + + data class Identity( + val matrixId: String, + val projectId: String, + val historyId: String = "", + val executionId: String = "", + ) + + interface Cancel : (Identity) -> Unit + interface Refresh : (Identity) -> Data +} diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/BillableMinutes.kt b/test_runner/src/main/kotlin/ftl/client/google/BillableMinutes.kt similarity index 80% rename from test_runner/src/main/kotlin/ftl/reports/outcome/BillableMinutes.kt rename to test_runner/src/main/kotlin/ftl/client/google/BillableMinutes.kt index d811f34828..7d66db9ec5 100644 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/BillableMinutes.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/BillableMinutes.kt @@ -1,8 +1,6 @@ -package ftl.reports.outcome +package ftl.client.google import com.google.api.services.toolresults.model.Step -import ftl.client.google.AndroidCatalog -import ftl.client.google.DeviceType import ftl.environment.orUnknown import ftl.util.billableMinutes import kotlin.math.min @@ -21,12 +19,11 @@ fun List.calculateAndroidBillableMinutes(projectId: String, timeoutValue: ) } -private fun Step.deviceModel() = dimensionValue.find { it.key.equals("Model", ignoreCase = true) }?.value.orUnknown() +private fun Step.deviceModel() = dimensionValue?.find { it.key.equals("Model", ignoreCase = true) }?.value.orUnknown() private fun List.sumBillableMinutes(timeout: Long) = this .mapNotNull { it.getBillableSeconds(default = timeout) } - .map { billableMinutes(it) } - .sum() + .sumOf { billableMinutes(it) } private fun Step.getBillableSeconds(default: Long) = testExecutionStep?.testTiming?.testProcessDuration?.seconds?.let { diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/CreateTestSuiteOverviewData.kt b/test_runner/src/main/kotlin/ftl/client/google/CreateTestSuiteOverviewData.kt similarity index 100% rename from test_runner/src/main/kotlin/ftl/reports/outcome/CreateTestSuiteOverviewData.kt rename to test_runner/src/main/kotlin/ftl/client/google/CreateTestSuiteOverviewData.kt diff --git a/test_runner/src/main/kotlin/ftl/client/google/FetchTestMatrices.kt b/test_runner/src/main/kotlin/ftl/client/google/FetchTestMatrices.kt index df4c1c3be8..9d77f503a2 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/FetchTestMatrices.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/FetchTestMatrices.kt @@ -7,7 +7,10 @@ import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.runBlocking -fun fetchMatrices(matricesIds: List, projectId: String): List = refreshTestMatrices( +fun fetchMatrices( + matricesIds: List, + projectId: String +): List = refreshTestMatrices( matrixIds = matricesIds, projectId = projectId ).getTestExecutions() @@ -18,14 +21,11 @@ private fun refreshTestMatrices( ): List = runBlocking { matrixIds.map { matrixId -> async(Dispatchers.IO) { - GcTestMatrix.refresh( - matrixId, - projectId - ) + GcTestMatrix.refresh(matrixId, projectId) } }.awaitAll() } -private fun List.getTestExecutions(): List = - mapNotNull(TestMatrix::getTestExecutions) - .flatten() +private fun List.getTestExecutions(): List = this + .mapNotNull(TestMatrix::getTestExecutions) + .flatten() diff --git a/test_runner/src/main/kotlin/ftl/client/google/GoogleTestMatrixCancel.kt b/test_runner/src/main/kotlin/ftl/client/google/GoogleTestMatrixCancel.kt new file mode 100644 index 0000000000..c4e3bb3d43 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/google/GoogleTestMatrixCancel.kt @@ -0,0 +1,5 @@ +package ftl.client.google + +suspend fun cancelMatrices(matrixId: String, projectId: String) { + GcTestMatrix.cancel(matrixId, projectId) +} diff --git a/test_runner/src/main/kotlin/ftl/client/google/PerformanceMetrics.kt b/test_runner/src/main/kotlin/ftl/client/google/PerformanceMetrics.kt new file mode 100644 index 0000000000..3af66b9d59 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/google/PerformanceMetrics.kt @@ -0,0 +1,22 @@ +package ftl.client.google + +import com.google.testing.model.TestExecution +import ftl.args.AndroidArgs +import ftl.args.IArgs +import ftl.json.MatrixMap +import ftl.reports.api.getAndUploadPerformanceMetrics + +fun List.createAndUploadPerformanceMetricsForAndroid( + args: IArgs, + matrices: MatrixMap +) = also { + takeIf { args is AndroidArgs } + ?.run { withGcsStoragePath(matrices, args.resultsDir).getAndUploadPerformanceMetrics(args) } +} + +private fun List.withGcsStoragePath( + matrices: MatrixMap, + defaultResultDir: String +) = map { testExecution -> + testExecution to (matrices.map[testExecution.matrixId]?.gcsPathWithoutRootBucket ?: defaultResultDir) +} diff --git a/test_runner/src/main/kotlin/ftl/client/google/TestMatrixRefresh.kt b/test_runner/src/main/kotlin/ftl/client/google/TestMatrixRefresh.kt new file mode 100644 index 0000000000..7e377a5b14 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/google/TestMatrixRefresh.kt @@ -0,0 +1,8 @@ +package ftl.client.google + +import com.google.testing.model.TestMatrix + +suspend fun refreshMatrix( + matrixId: String, + projectId: String +): TestMatrix = GcTestMatrix.refresh(matrixId, projectId) diff --git a/test_runner/src/main/kotlin/ftl/client/google/TestOutcome.kt b/test_runner/src/main/kotlin/ftl/client/google/TestOutcome.kt new file mode 100644 index 0000000000..b2ea57c7e7 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/google/TestOutcome.kt @@ -0,0 +1,10 @@ +package ftl.client.google + +import ftl.reports.api.data.TestSuiteOverviewData + +data class TestOutcome( + val device: String = "", + val outcome: String = "", + val details: String = "", + val testSuiteOverview: TestSuiteOverviewData = TestSuiteOverviewData() +) diff --git a/test_runner/src/main/kotlin/ftl/client/google/TestOutcomeContext.kt b/test_runner/src/main/kotlin/ftl/client/google/TestOutcomeContext.kt new file mode 100644 index 0000000000..71a90dea4d --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/client/google/TestOutcomeContext.kt @@ -0,0 +1,95 @@ +package ftl.client.google + +import com.google.api.services.toolresults.model.Environment +import com.google.api.services.toolresults.model.Outcome +import com.google.api.services.toolresults.model.Step +import com.google.common.annotations.VisibleForTesting +import com.google.testing.model.ToolResultsExecution +import flank.common.logLn +import ftl.api.TestMatrix +import ftl.gc.GcToolResults +import ftl.json.getDetails +import ftl.reports.outcome.axisValue +import ftl.reports.outcome.createTestSuiteOverviewData +import ftl.util.StepOutcome + +data class TestOutcomeContext( + val matrixId: String, + val projectId: String, + val environments: List, + val steps: List, + val testTimeout: Long, + val isRoboTest: Boolean +) + +fun fetchMatrixOutcome(newMatrix: TestMatrix.Data) = newMatrix.fetchTestOutcomeContext().createMatrixOutcomeSummary() + +private fun TestMatrix.Data.fetchTestOutcomeContext(): TestOutcomeContext = getToolResultsIds().let { ids -> + TestOutcomeContext( + projectId = projectId, + matrixId = matrixId, + environments = GcToolResults.listAllEnvironments(ids), + steps = GcToolResults.listAllSteps(ids), + testTimeout = testTimeout, + isRoboTest = isRoboTest + ) +} + +private fun TestMatrix.Data.getToolResultsIds(): ToolResultsExecution = ToolResultsExecution() + .setProjectId(projectId) + .setHistoryId(historyId) + .setExecutionId(executionId) + +@VisibleForTesting +internal fun TestOutcomeContext.createMatrixOutcomeSummary(): Pair> = + billableMinutes() to outcomeSummary() + +private fun TestOutcomeContext.billableMinutes() = steps.calculateAndroidBillableMinutes(projectId, testTimeout) + +private fun TestOutcomeContext.outcomeSummary(): List = + if (environments.hasOutcome()) + createMatrixOutcomeSummaryUsingEnvironments() + else { + if (steps.isEmpty()) logLn("No test results found, something went wrong. Try re-running the tests.") + createMatrixOutcomeSummaryUsingSteps() + } + +private fun List.hasOutcome() = isNotEmpty() && all { it.environmentResult?.outcome?.summary != null } + +private fun TestOutcomeContext.createMatrixOutcomeSummaryUsingEnvironments(): List = environments + .map { environment -> + TestOutcome( + device = environment.axisValue(), + outcome = environment.outcomeSummary, + details = environment.getOutcomeDetails(isRoboTest), + testSuiteOverview = environment.createTestSuiteOverviewData() + ) + } + +private val Environment.outcomeSummary + get() = environmentResult?.outcome?.summary ?: UNKNOWN_OUTCOME + +private fun Environment.getOutcomeDetails(isRoboTest: Boolean) = + environmentResult?.outcome.getDetails(createTestSuiteOverviewData(), isRoboTest) + +private fun TestOutcomeContext.createMatrixOutcomeSummaryUsingSteps() = steps + .groupBy(Step::axisValue) + .map { (device, steps) -> + TestOutcome( + device = device, + outcome = steps.getOutcomeSummary(), + details = steps.getOutcomeDetails(isRoboTest), + testSuiteOverview = steps.createTestSuiteOverviewData() + ) + } + +private fun List.getOutcomeSummary() = getOutcomeFromSteps()?.summary ?: UNKNOWN_OUTCOME + +private fun List.getOutcomeDetails(isRoboTest: Boolean) = + getOutcomeFromSteps().getDetails(createTestSuiteOverviewData(), isRoboTest) + +private fun List.getOutcomeFromSteps(): Outcome? = maxByOrNull { + StepOutcome.order.indexOf(it.outcome?.summary) +}?.outcome + +private const val UNKNOWN_OUTCOME = "Unknown" diff --git a/test_runner/src/main/kotlin/ftl/client/junit/TestExecutionData.kt b/test_runner/src/main/kotlin/ftl/client/junit/TestExecutionData.kt index 61cfa0b7a6..167cd66599 100644 --- a/test_runner/src/main/kotlin/ftl/client/junit/TestExecutionData.kt +++ b/test_runner/src/main/kotlin/ftl/client/junit/TestExecutionData.kt @@ -11,3 +11,9 @@ data class TestExecutionData( val step: Step, val timestamp: Timestamp ) + +data class TestCasesWithStep( + val testCases: List, + val step: Step, + val timestamp: Timestamp +) diff --git a/test_runner/src/main/kotlin/ftl/domain/testmatrix/UpdateTestMatrix.kt b/test_runner/src/main/kotlin/ftl/domain/testmatrix/UpdateTestMatrix.kt new file mode 100644 index 0000000000..dc329f07dd --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/domain/testmatrix/UpdateTestMatrix.kt @@ -0,0 +1,62 @@ +package ftl.domain.testmatrix + +import ftl.api.TestMatrix +import ftl.api.fetchTestSummary +import ftl.environment.orUnknown +import ftl.util.MatrixState + +internal fun TestMatrix.Data.updateWithMatrix(newMatrix: TestMatrix.Data): TestMatrix.Data = + if (needsUpdate(newMatrix)) updatedSavedMatrix(newMatrix) + else this + +fun TestMatrix.Data.needsUpdate(newMatrix: TestMatrix.Data): Boolean { + val newState = newMatrix.state + val newLink = newMatrix.webLink + val changedState = state != newState + val changedLink = webLink != newLink + return (changedState || changedLink) +} + +private fun TestMatrix.Data.updatedSavedMatrix( + newMatrix: TestMatrix.Data +): TestMatrix.Data = when (newMatrix.state) { + state -> this + MatrixState.FINISHED -> { + val (billableMinutes, outcomes) = fetchTestSummary(newMatrix) + updateProperties(newMatrix) + .updateOutcome(outcomes) + .updateBillableMinutes(billableMinutes) + } + + MatrixState.INVALID -> updateProperties(newMatrix).updateOutcome(listOf(newMatrix.invalidTestOutcome())) + else -> updateProperties(newMatrix) +} + +private fun TestMatrix.Data.updateProperties(newMatrix: TestMatrix.Data) = copy( + matrixId = newMatrix.matrixId, + state = newMatrix.state, + gcsPath = newMatrix.gcsPath, + webLink = newMatrix.webLink, + downloaded = false, + clientDetails = newMatrix.clientDetails, + gcsPathWithoutRootBucket = newMatrix.gcsPathWithoutRootBucket, + gcsRootBucket = newMatrix.gcsRootBucket, + webLinkWithoutExecutionDetails = newMatrix.webLinkWithoutExecutionDetails, + appFileName = newMatrix.appFileName, + isCompleted = MatrixState.completed(state) && + newMatrix.testExecutions.all { MatrixState.completed(it.state) }, + testExecutions = newMatrix.testExecutions +) + +private fun TestMatrix.Data.updateBillableMinutes(billableMinutes: TestMatrix.BillableMinutes) = copy( + billableMinutes = TestMatrix.BillableMinutes(billableMinutes.virtual, billableMinutes.physical) +) + +private fun TestMatrix.Data.updateOutcome(outcome: List) = copy( + axes = outcome +) + +private fun TestMatrix.Data.invalidTestOutcome() = TestMatrix.Outcome( + outcome = MatrixState.INVALID, + details = invalidMatrixDetails.orUnknown() +) diff --git a/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt b/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt index 4a871cd454..6bf8b00020 100644 --- a/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt +++ b/test_runner/src/main/kotlin/ftl/json/MatrixMap.kt @@ -1,6 +1,7 @@ package ftl.json -import com.google.testing.model.TestMatrix +import ftl.api.TestMatrix +import ftl.domain.testmatrix.updateWithMatrix import ftl.environment.orUnknown import ftl.reports.outcome.getOutcomeMessageByKey import ftl.run.exception.FTLError @@ -10,21 +11,24 @@ import ftl.run.exception.InfrastructureError import ftl.run.exception.MatrixCanceledError import ftl.run.exception.MatrixValidationError import ftl.util.MatrixState +import ftl.util.StepOutcome -data class MatrixMap(val map: Map, val runPath: String) { +data class MatrixMap(val map: Map, val runPath: String) { private val mutableMap - get() = map as MutableMap + get() = map as MutableMap - fun update(id: String, savedMatrix: SavedMatrix) { + fun update(id: String, savedMatrix: TestMatrix.Data) { mutableMap[id] = savedMatrix } } -fun MatrixMap.isAllSuccessful() = map.values.any(SavedMatrix::isFailed).not() +fun MatrixMap.isAllSuccessful() = map.values.any(TestMatrix.Data::isFailed).not() -fun Iterable.updateMatrixMap(matrixMap: MatrixMap) = forEach { matrix -> - matrixMap.map[matrix.testMatrixId]?.updateWithMatrix(matrix)?.let { - matrixMap.update(matrix.testMatrixId, it) +fun Iterable.updateMatrixMap(matrixMap: MatrixMap) { + forEach { matrix -> + matrixMap.map[matrix.matrixId]?.updateWithMatrix(matrix)?.let { + matrixMap.update(matrix.matrixId, it) + } } } @@ -52,10 +56,10 @@ fun Iterable.updateMatrixMap(matrixMap: MatrixMap) = forEach { matri fun MatrixMap.validate(shouldIgnore: Boolean = false) { map.values.run { - firstOrNull { it.canceledByUser() }?.run { throw MatrixCanceledError(outcomeDetails) } - firstOrNull { it.infrastructureFail() }?.run { throw InfrastructureError(outcomeDetails) } - firstOrNull { it.incompatibleFail() }?.run { throw IncompatibleTestDimensionError(outcomeDetails) } - firstOrNull { it.invalid() }?.run { throw MatrixValidationError(errorMessage()) } + firstOrNull { it.canceledByUser }?.run { throw MatrixCanceledError(outcomeDetails) } + firstOrNull { it.infrastructureFail }?.run { throw InfrastructureError(outcomeDetails) } + firstOrNull { it.incompatibleFail }?.run { throw IncompatibleTestDimensionError(outcomeDetails) } + firstOrNull { it.invalid }?.run { throw MatrixValidationError(errorMessage()) } firstOrNull { it.state != MatrixState.FINISHED }?.let { throw FTLError(it) } filter { it.isFailed() }.let { if (it.isNotEmpty()) throw FailedMatrixError( @@ -66,6 +70,32 @@ fun MatrixMap.validate(shouldIgnore: Boolean = false) { } } -private val SavedMatrix.outcomeDetails get() = testAxises.firstOrNull()?.details.orEmpty() +fun TestMatrix.Data.isFailed() = when (outcome) { + StepOutcome.failure -> true + StepOutcome.skipped -> true + StepOutcome.inconclusive -> true + MatrixState.INVALID -> true + else -> false +} + +private val TestMatrix.Data.canceledByUser: Boolean + get() = axes.any { it.details == ABORTED_BY_USER_MESSAGE } + +private val TestMatrix.Data.infrastructureFail: Boolean + get() = axes.any { it.details == INFRASTRUCTURE_FAILURE_MESSAGE } + +private val TestMatrix.Data.incompatibleFail: Boolean + get() = axes.map { it.details }.intersect(incompatibleFails).isNotEmpty() + +private val TestMatrix.Data.invalid: Boolean + get() = axes.any { it.outcome == MatrixState.INVALID } + +private val incompatibleFails = setOf( + INCOMPATIBLE_APP_VERSION_MESSAGE, + INCOMPATIBLE_ARCHITECTURE_MESSAGE, + INCOMPATIBLE_DEVICE_MESSAGE +) + +private val TestMatrix.Data.outcomeDetails get() = axes.firstOrNull()?.details.orEmpty() -private fun SavedMatrix.errorMessage() = "Matrix: [$matrixId] failed: ".plus(getOutcomeMessageByKey(testAxises.firstOrNull()?.details.orUnknown())) +private fun TestMatrix.Data.errorMessage() = "Matrix: [$matrixId] failed: ".plus(getOutcomeMessageByKey(axes.firstOrNull()?.details.orUnknown())) diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt deleted file mode 100644 index 4906cca860..0000000000 --- a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt +++ /dev/null @@ -1,147 +0,0 @@ -package ftl.json - -import com.google.testing.model.FileReference -import com.google.testing.model.TestMatrix -import ftl.analytics.toJSONObject -import ftl.environment.orUnknown -import ftl.reports.outcome.BillableMinutes -import ftl.reports.outcome.TestOutcome -import ftl.reports.outcome.createMatrixOutcomeSummary -import ftl.reports.outcome.fetchTestOutcomeContext -import ftl.run.common.prettyPrint -import ftl.util.MatrixState.FINISHED -import ftl.util.MatrixState.INVALID -import ftl.util.StepOutcome -import ftl.util.StepOutcome.failure -import ftl.util.StepOutcome.inconclusive -import ftl.util.StepOutcome.skipped -import ftl.util.getClientDetails -import ftl.util.getGcsPath -import ftl.util.getGcsPathWithoutRootBucket -import ftl.util.getGcsRootBucket -import ftl.util.webLink -import ftl.util.webLinkWithoutExecutionDetails - -// execution gcs paths aren't API accessible. -data class SavedMatrix( - val matrixId: String = "", - val state: String = "", - val gcsPath: String = "", - val webLink: String = "", - // this is variable intentionally, this is workaround to pass changes while fetching artifacts - // todo think about different solution - var downloaded: Boolean = false, - val billableVirtualMinutes: Long = 0, - val billablePhysicalMinutes: Long = 0, - val clientDetails: Map? = null, - val gcsPathWithoutRootBucket: String = "", - val gcsRootBucket: String = "", - val webLinkWithoutExecutionDetails: String? = "", - val testAxises: List = emptyList(), - val appFileName: String = fallbackAppName -) { - val outcome = testAxises.maxByOrNull { StepOutcome.order.indexOf(it.outcome) }?.outcome.orEmpty() -} - -private const val fallbackAppName = "N/A" - -fun createSavedMatrix(testMatrix: TestMatrix) = SavedMatrix().updateWithMatrix(testMatrix) - -fun SavedMatrix.canceledByUser() = testAxises.any { it.details == ABORTED_BY_USER_MESSAGE } - -fun SavedMatrix.infrastructureFail() = testAxises.any { it.details == INFRASTRUCTURE_FAILURE_MESSAGE } - -fun SavedMatrix.incompatibleFail() = testAxises.map { it.details }.intersect(incompatibleFails).isNotEmpty() - -fun SavedMatrix.invalid() = testAxises.any { it.outcome == INVALID } - -private val incompatibleFails = setOf( - INCOMPATIBLE_APP_VERSION_MESSAGE, - INCOMPATIBLE_ARCHITECTURE_MESSAGE, - INCOMPATIBLE_DEVICE_MESSAGE -) - -fun SavedMatrix.isFailed() = when (outcome) { - failure -> true - skipped -> true - inconclusive -> true - INVALID -> true - else -> false -} - -fun SavedMatrix.needsUpdate(newMatrix: TestMatrix): Boolean { - val newState = newMatrix.state - val newLink = newMatrix.webLink() - val changedState = state != newState - val changedLink = webLink != newLink - return (changedState || changedLink) -} - -internal fun SavedMatrix.updateWithMatrix(newMatrix: TestMatrix): SavedMatrix = - if (needsUpdate(newMatrix)) updatedSavedMatrix(newMatrix) - else this - -private fun SavedMatrix.updatedSavedMatrix( - newMatrix: TestMatrix -): SavedMatrix = when (newMatrix.state) { - state -> this - - FINISHED -> - newMatrix.fetchTestOutcomeContext().createMatrixOutcomeSummary().let { (billableMinutes, outcomes) -> - updateProperties(newMatrix).updateOutcome(outcomes).updateBillableMinutes(billableMinutes) - } - - INVALID -> updateProperties(newMatrix).updateOutcome(listOf(newMatrix.invalidTestOutcome())) - - else -> updateProperties(newMatrix) -} - -private fun SavedMatrix.updateProperties(newMatrix: TestMatrix) = copy( - matrixId = newMatrix.testMatrixId, - state = newMatrix.state, - gcsPath = newMatrix.getGcsPath(), - webLink = newMatrix.webLink(), - downloaded = false, - clientDetails = newMatrix.getClientDetails(), - gcsPathWithoutRootBucket = newMatrix.getGcsPathWithoutRootBucket(), - gcsRootBucket = newMatrix.getGcsRootBucket(), - webLinkWithoutExecutionDetails = newMatrix.webLinkWithoutExecutionDetails(), - appFileName = newMatrix.extractAppFileName() ?: fallbackAppName -) - -private fun TestMatrix.extractAppFileName() = testSpecification?.run { - listOf( - androidInstrumentationTest, - androidTestLoop, - androidRoboTest, - iosXcTest, - iosTestLoop - ) - .firstOrNull { it != null } - ?.toJSONObject() - ?.let { prettyPrint.fromJson(it.toString(), AppPath::class.java).gcsPath } - ?.substringAfterLast('/') -} - -private fun SavedMatrix.updateBillableMinutes(billableMinutes: BillableMinutes) = copy( - billablePhysicalMinutes = billableMinutes.physical, - billableVirtualMinutes = billableMinutes.virtual, -) - -private fun SavedMatrix.updateOutcome(outcome: List) = copy( - testAxises = outcome -) - -private fun TestMatrix.invalidTestOutcome() = TestOutcome( - outcome = INVALID, - details = invalidMatrixDetails.orUnknown() -) - -private data class AppPath( - private val appApk: FileReference?, - private val testsZip: FileReference?, - private val appIpa: FileReference? -) { - val gcsPath: String? - get() = (appApk ?: testsZip ?: appIpa)?.gcsPath -} diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt index 7e51c0d39a..4575e61c4c 100644 --- a/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt +++ b/test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt @@ -1,6 +1,6 @@ package ftl.json -import ftl.reports.outcome.TestOutcome +import ftl.api.TestMatrix import ftl.util.StepOutcome.failure import ftl.util.StepOutcome.flaky import ftl.util.StepOutcome.success @@ -8,9 +8,9 @@ import ftl.util.SystemOutColor import ftl.util.TableColumn import ftl.util.buildTable -fun SavedMatrix.asPrintableTable(): String = listOf(this).asPrintableTable() +fun TestMatrix.Data.asPrintableTable(): String = listOf(this).asPrintableTable() -fun List.asPrintableTable(): String = buildTable( +fun List.asPrintableTable(): String = buildTable( TableColumn( header = OUTCOME_COLUMN_HEADER, data = flatMapTestAxis { outcome }, @@ -34,10 +34,10 @@ fun List.asPrintableTable(): String = buildTable( ) ) -private fun List.flatMapTestAxis(transform: TestOutcome.(SavedMatrix) -> T) = - flatMap { matrix -> matrix.testAxises.map { axis -> axis.transform(matrix) } } +private fun List.flatMapTestAxis(transform: TestMatrix.Outcome.(TestMatrix.Data) -> T) = + flatMap { matrix -> matrix.axes.map { axis -> axis.transform(matrix) } } -private val TestOutcome.outcomeColor +private val TestMatrix.Outcome.outcomeColor get() = when (outcome) { failure -> SystemOutColor.RED success -> SystemOutColor.GREEN diff --git a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt index ac98cbbe8d..0553cea28b 100644 --- a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt @@ -20,8 +20,8 @@ object CostReport : IReport { var totalBillablePhysicalMinutes = 0L matrices.map.values.forEach { - totalBillableVirtualMinutes += it.billableVirtualMinutes - totalBillablePhysicalMinutes += it.billablePhysicalMinutes + totalBillableVirtualMinutes += it.billableMinutes.virtual + totalBillablePhysicalMinutes += it.billableMinutes.physical } return estimateCosts(totalBillableVirtualMinutes, totalBillablePhysicalMinutes) diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt index 71e2381958..982b2be13a 100644 --- a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt @@ -4,10 +4,10 @@ import flank.common.log import flank.common.println import flank.common.startWithNewLine import ftl.api.JUnitTest +import ftl.api.TestMatrix import ftl.args.IArgs import ftl.config.FtlConstants.indent import ftl.json.MatrixMap -import ftl.json.SavedMatrix import ftl.json.asPrintableTable import ftl.json.isFailed import ftl.reports.output.log @@ -71,7 +71,7 @@ object MatrixResultsReport : IReport { } } - private fun Collection.printMatricesLinks(writer: StringWriter) = this + private fun Collection.printMatricesLinks(writer: StringWriter) = this .filter { it.isFailed() } .takeIf { it.isNotEmpty() } ?.run { diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt deleted file mode 100644 index 25eaa4f11b..0000000000 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummary.kt +++ /dev/null @@ -1,18 +0,0 @@ -package ftl.reports.outcome - -import com.google.api.services.toolresults.model.Environment -import flank.common.logLn - -fun TestOutcomeContext.createMatrixOutcomeSummary() = billableMinutes() to outcomeSummary() - -private fun TestOutcomeContext.billableMinutes() = steps.calculateAndroidBillableMinutes(projectId, testTimeout) - -private fun TestOutcomeContext.outcomeSummary() = - if (environments.hasOutcome()) - createMatrixOutcomeSummaryUsingEnvironments() - else { - if (steps.isEmpty()) logLn("No test results found, something went wrong. Try re-running the tests.") - createMatrixOutcomeSummaryUsingSteps() - } - -private fun List.hasOutcome() = isNotEmpty() && all { it.environmentResult?.outcome?.summary != null } diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt deleted file mode 100644 index 9f06d6e19a..0000000000 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcome.kt +++ /dev/null @@ -1,51 +0,0 @@ -package ftl.reports.outcome - -import com.google.api.services.toolresults.model.Environment -import com.google.api.services.toolresults.model.Outcome -import com.google.api.services.toolresults.model.Step -import ftl.json.getDetails -import ftl.reports.api.data.TestSuiteOverviewData -import ftl.util.StepOutcome - -data class TestOutcome( - val device: String = "", - val outcome: String = "", - val details: String = "", - val testSuiteOverview: TestSuiteOverviewData = TestSuiteOverviewData() -) - -fun TestOutcomeContext.createMatrixOutcomeSummaryUsingEnvironments(): List = environments - .map { environment -> - TestOutcome( - device = environment.axisValue(), - outcome = environment.outcomeSummary, - details = environment.getOutcomeDetails(isRoboTest), - testSuiteOverview = environment.createTestSuiteOverviewData() - ) - } - -private val Environment.outcomeSummary - get() = environmentResult?.outcome?.summary ?: UNKNOWN_OUTCOME - -private fun Environment.getOutcomeDetails(isRoboTest: Boolean) = environmentResult?.outcome.getDetails(createTestSuiteOverviewData(), isRoboTest) - -fun TestOutcomeContext.createMatrixOutcomeSummaryUsingSteps() = steps - .groupBy(Step::axisValue) - .map { (device, steps) -> - TestOutcome( - device = device, - outcome = steps.getOutcomeSummary(), - details = steps.getOutcomeDetails(isRoboTest), - testSuiteOverview = steps.createTestSuiteOverviewData() - ) - } - -private fun List.getOutcomeSummary() = getOutcomeFromSteps()?.summary ?: UNKNOWN_OUTCOME - -private fun List.getOutcomeDetails(isRoboTest: Boolean) = getOutcomeFromSteps().getDetails(createTestSuiteOverviewData(), isRoboTest) - -private fun List.getOutcomeFromSteps(): Outcome? = maxByOrNull { - StepOutcome.order.indexOf(it.outcome?.summary) -}?.outcome - -private const val UNKNOWN_OUTCOME = "Unknown" diff --git a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcomeContext.kt b/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcomeContext.kt deleted file mode 100644 index 7a534064f3..0000000000 --- a/test_runner/src/main/kotlin/ftl/reports/outcome/TestOutcomeContext.kt +++ /dev/null @@ -1,50 +0,0 @@ -package ftl.reports.outcome - -import com.google.api.services.toolresults.model.Environment -import com.google.api.services.toolresults.model.Step -import com.google.testing.model.TestMatrix -import com.google.testing.model.ToolResultsExecution -import ftl.gc.GcToolResults -import ftl.json.SavedMatrix -import ftl.json.createSavedMatrix -import ftl.run.exception.FTLError -import ftl.util.timeoutToSeconds - -data class TestOutcomeContext( - val matrixId: String, - val projectId: String, - val environments: List, - val steps: List, - val testTimeout: Long, - val isRoboTest: Boolean -) - -fun TestMatrix.fetchTestOutcomeContext() = getToolResultsIds().let { ids -> - TestOutcomeContext( - projectId = projectId, - matrixId = testMatrixId, - environments = GcToolResults.listAllEnvironments(ids), - steps = GcToolResults.listAllSteps(ids), - testTimeout = testTimeout(), - isRoboTest = isRoboTest() - ) -} - -private fun TestMatrix.getToolResultsIds(): ToolResultsExecution = ToolResultsExecution() - .setProjectId(projectId) - .setHistoryId(resultStorage?.toolResultsExecution?.historyId ?: throw badMatrixError()) - .setExecutionId(resultStorage?.toolResultsExecution?.executionId ?: throw badMatrixError()) - -private fun TestMatrix.badMatrixError() = BadMatrixError(createSavedMatrix(this)) - -class BadMatrixError(matrix: SavedMatrix) : FTLError(matrix) - -private fun TestMatrix.testTimeout() = timeoutToSeconds( - testExecutions - .firstOrNull { it?.testSpecification?.testTimeout != null } - ?.testSpecification - ?.testTimeout - ?: "0s" -) - -private fun TestMatrix.isRoboTest() = testExecutions.orEmpty().any { it?.testSpecification?.androidRoboTest != null } diff --git a/test_runner/src/main/kotlin/ftl/reports/output/OutputReportLoggers.kt b/test_runner/src/main/kotlin/ftl/reports/output/OutputReportLoggers.kt index e16876ed8b..6773cfb15b 100644 --- a/test_runner/src/main/kotlin/ftl/reports/output/OutputReportLoggers.kt +++ b/test_runner/src/main/kotlin/ftl/reports/output/OutputReportLoggers.kt @@ -5,9 +5,9 @@ import flank.common.OUTPUT_COST import flank.common.OUTPUT_TEST_RESULTS import flank.common.OUTPUT_WEBLINKS import flank.common.OutputReportCostNode +import ftl.api.TestMatrix import ftl.args.IArgs import ftl.json.MatrixMap -import ftl.json.SavedMatrix import java.math.BigDecimal internal fun OutputReport.log(args: IArgs) { @@ -18,13 +18,13 @@ internal fun OutputReport.log(matrixMap: MatrixMap) { add(OUTPUT_WEBLINKS, matrixMap.map.values.map { it.webLink }) } -internal fun OutputReport.log(matrices: Collection) { +internal fun OutputReport.log(matrices: Collection) { add( OUTPUT_TEST_RESULTS, matrices.map { it.matrixId to mapOf( "app" to it.appFileName, - "test-axises" to it.testAxises + "test-axises" to it.axes ) }.toMap() ) diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt index 47dfd78910..fb5901fc03 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt @@ -1,7 +1,6 @@ package ftl.reports.util import com.google.common.annotations.VisibleForTesting -import com.google.testing.model.TestExecution import flank.common.logLn import ftl.api.JUnitTest import ftl.api.RemoteStorage @@ -10,7 +9,6 @@ import ftl.api.generateJUnitTestResultFromApi import ftl.api.parseJUnitLegacyTestResultFromFile import ftl.api.parseJUnitTestResultFromFile import ftl.api.uploadToRemoteStorage -import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IgnoredTestCases import ftl.args.IosArgs @@ -26,7 +24,6 @@ import ftl.reports.FullJUnitReport import ftl.reports.HtmlErrorReport import ftl.reports.JUnitReport import ftl.reports.MatrixResultsReport -import ftl.reports.api.getAndUploadPerformanceMetrics import ftl.reports.api.utcDateFormat import ftl.reports.toXmlString import ftl.run.common.getMatrixFilePath @@ -75,16 +72,15 @@ object ReportManager { val testsResult = generateJUnitTestResultFromApi((args to matrices).toApiIdentity()) processJunitResults(args, matrices, testSuite, testShardChunks, testsResult) - // TODO move it to next #1756 - // createAndUploadPerformanceMetricsForAndroid(args, testsResult, matrices) + uploadMatricesId(args, matrices) } private fun Pair.toApiIdentity(): JUnitTest.Result.ApiIdentity { val (args, matrices) = this return JUnitTest.Result.ApiIdentity( - projectId = args.project, - matrixIds = matrices.map.values.map { it.matrixId } + args = args, + matrixMap = matrices ) } @@ -159,23 +155,6 @@ object ReportManager { } } - private fun createAndUploadPerformanceMetricsForAndroid( - args: IArgs, - testExecutions: List, - matrices: MatrixMap - ) { - testExecutions - .takeIf { args is AndroidArgs } - ?.run { withGcsStoragePath(matrices, args.resultsDir).getAndUploadPerformanceMetrics(args) } - } - - private fun List.withGcsStoragePath( - matrices: MatrixMap, - defaultResultDir: String - ) = map { testExecution -> - testExecution to (matrices.map[testExecution.matrixId]?.gcsPathWithoutRootBucket ?: defaultResultDir) - } - private fun IgnoredTestCases.toJunitTestsResults() = getSkippedJUnitTestSuite( map { JUnitTest.Case( diff --git a/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt b/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt index 4392d94261..b68a5536e8 100644 --- a/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/CancelLastRun.kt @@ -1,9 +1,9 @@ package ftl.run import flank.common.logLn +import ftl.api.TestMatrix +import ftl.api.cancelTestMatrix import ftl.args.IArgs -import ftl.client.google.GcTestMatrix -import ftl.json.SavedMatrix import ftl.run.common.getLastArgs import ftl.run.common.getLastMatrices import ftl.util.MatrixState @@ -21,7 +21,7 @@ fun cancelLastRun(args: IArgs): MatrixCancelStatus = runBlocking { /** Cancel all in progress matrices in parallel **/ suspend fun cancelMatrices( - matrixMap: Map, + matrixMap: Map, project: String ): MatrixCancelStatus = coroutineScope { logLn("CancelMatrices") @@ -29,7 +29,7 @@ suspend fun cancelMatrices( return@coroutineScope matrixMap .filter { matrix -> MatrixState.inProgress(matrix.value.state) } .map { (matrixKey, _) -> matrixKey } - .onEach { matrixKey -> launch { GcTestMatrix.cancel(matrixKey, project) } } + .onEach { matrixKey -> launch { cancelTestMatrix(TestMatrix.Identity(matrixKey, project)) } } .toMatrixCancelStatus() } diff --git a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt index cdcffb4335..692359949a 100644 --- a/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/NewTestRun.kt @@ -1,9 +1,9 @@ package ftl.run +import ftl.api.TestMatrix import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs -import ftl.json.SavedMatrix import ftl.json.updateMatrixMap import ftl.json.validate import ftl.reports.addStepTime @@ -57,7 +57,7 @@ private suspend fun IArgs.runTests(): TestResult = private suspend fun cancelTestsOnTimeout( projectId: String, - savedMatrix: Map? = null, + savedMatrix: Map? = null, block: suspend () -> T ) = try { block() diff --git a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt index ab18b1892b..75ba6595f7 100644 --- a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt @@ -1,15 +1,15 @@ package ftl.run -import com.google.testing.model.TestMatrix import flank.common.logLn +import ftl.api.TestMatrix +import ftl.api.refreshTestMatrix import ftl.args.IArgs import ftl.args.ShardChunks -import ftl.client.google.GcTestMatrix import ftl.config.FtlConstants +import ftl.domain.testmatrix.needsUpdate +import ftl.domain.testmatrix.updateWithMatrix import ftl.json.MatrixMap -import ftl.json.needsUpdate import ftl.json.updateMatrixMap -import ftl.json.updateWithMatrix import ftl.json.validate import ftl.reports.util.ReportManager import ftl.run.common.fetchAllTestRunArtifacts @@ -43,14 +43,14 @@ suspend fun refreshLastRun(currentArgs: IArgs, testShardChunks: ShardChunks) { private suspend fun refreshMatrices(matrixMap: MatrixMap, args: IArgs) = coroutineScope { logLn("RefreshMatrices") - val jobs = arrayListOf>() + val jobs = arrayListOf>() val map = matrixMap.map var matrixCount = 0 map.forEach { matrix -> // Only refresh unfinished if (MatrixState.inProgress(matrix.value.state)) { matrixCount += 1 - jobs += async(Dispatchers.IO) { GcTestMatrix.refresh(matrix.key, args.project) } + jobs += async(Dispatchers.IO) { refreshTestMatrix(TestMatrix.Identity(matrix.key, args.project, matrix.value.historyId, matrix.value.executionId)) } } } @@ -60,7 +60,7 @@ private suspend fun refreshMatrices(matrixMap: MatrixMap, args: IArgs) = corouti var dirty = false jobs.awaitAll().forEach { matrix -> - val matrixId = matrix.testMatrixId + val matrixId = matrix.matrixId logLn(FtlConstants.indent + "${matrix.state} $matrixId") diff --git a/test_runner/src/main/kotlin/ftl/run/common/FetchAllArtifacts.kt b/test_runner/src/main/kotlin/ftl/run/common/FetchAllArtifacts.kt index 5f447b722a..961a1fb946 100644 --- a/test_runner/src/main/kotlin/ftl/run/common/FetchAllArtifacts.kt +++ b/test_runner/src/main/kotlin/ftl/run/common/FetchAllArtifacts.kt @@ -43,9 +43,13 @@ internal suspend fun fetchAllTestRunArtifacts(matrixMap: MatrixMap, args: IArgs) } .map { fetchArtifacts(it) } .forEach { (matrixId, _) -> - matrixMap.map[matrixId]?.downloaded = true + matrixMap.matrixDownloaded(matrixId) } logLn(FtlConstants.indent + "Updating matrix file", level = OutputLogLevel.DETAILED) args.updateMatrixFile(matrixMap) } + +private fun MatrixMap.matrixDownloaded(matrixId: String) = map[matrixId]?.let { matrix -> + update(matrixId, matrix.copy(downloaded = true)) +} diff --git a/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt b/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt index bbd4c5502c..462a038d42 100644 --- a/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt +++ b/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt @@ -1,10 +1,10 @@ package ftl.run.common import flank.common.logLn +import ftl.api.TestMatrix import ftl.args.IArgs import ftl.config.FtlConstants import ftl.json.MatrixMap -import ftl.json.SavedMatrix import ftl.run.exception.FlankGeneralError import java.nio.file.Paths @@ -24,7 +24,7 @@ internal fun matrixPathToObj(path: String, args: IArgs): MatrixMap { } val json = filePath.readText() - val map: MutableMap = fromJson(json) + val map: MutableMap = fromJson(json) return MatrixMap(map, path) } diff --git a/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt b/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt index 1d588ffd61..b519b8864c 100644 --- a/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt +++ b/test_runner/src/main/kotlin/ftl/run/common/PollMatrices.kt @@ -2,12 +2,11 @@ package ftl.run.common -import com.google.testing.model.TestMatrix import flank.common.logLn +import ftl.api.TestMatrix +import ftl.api.refreshTestMatrix import ftl.args.IArgs -import ftl.client.google.GcTestMatrix import ftl.run.status.TestMatrixStatusPrinter -import ftl.util.MatrixState import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow @@ -20,11 +19,11 @@ import kotlinx.coroutines.flow.onEach suspend fun pollMatrices( testMatricesIds: Iterable, args: IArgs, - printMatrixStatus: (TestMatrix) -> Unit = TestMatrixStatusPrinter( + printMatrixStatus: (TestMatrix.Data) -> Unit = TestMatrixStatusPrinter( args = args, testMatricesIds = testMatricesIds ) -): Collection = coroutineScope { +): Collection = coroutineScope { testMatricesIds.asFlow().flatMapMerge { testMatrixId -> matrixChangesFlow( testMatrixId = testMatrixId, @@ -32,8 +31,8 @@ suspend fun pollMatrices( ) }.onEach { printMatrixStatus(it) - }.fold(emptyMap()) { matrices, next -> - matrices + (next.testMatrixId to next) + }.fold(emptyMap()) { matrices, next -> + matrices + (next.matrixId to next) }.values.also { logLn() } @@ -42,14 +41,10 @@ suspend fun pollMatrices( private fun matrixChangesFlow( testMatrixId: String, projectId: String -): Flow = flow { +): Flow = flow { while (true) { - val matrix = GcTestMatrix.refresh(testMatrixId, projectId) + val matrix = refreshTestMatrix(TestMatrix.Identity(testMatrixId, projectId)) emit(matrix) if (matrix.isCompleted) break else delay(5_000) } } - -private val TestMatrix.isCompleted: Boolean - get() = MatrixState.completed(state) && - testExecutions?.all { MatrixState.completed(it.state) } ?: true diff --git a/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt b/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt index 15e106d5cb..28ff1b4ebe 100644 --- a/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt +++ b/test_runner/src/main/kotlin/ftl/run/exception/ExceptionHandler.kt @@ -1,7 +1,7 @@ package ftl.run.exception import flank.common.logLn -import ftl.json.SavedMatrix +import ftl.api.TestMatrix import ftl.reports.output.add import ftl.reports.output.generate import ftl.reports.output.outputReport @@ -97,6 +97,6 @@ private fun printError(string: String?) { string?.let { outputReport.add("error", it) } } -private fun SavedMatrix.logError() { +private fun TestMatrix.Data.logError() { logLn("Matrix is $state") } diff --git a/test_runner/src/main/kotlin/ftl/run/exception/FlankException.kt b/test_runner/src/main/kotlin/ftl/run/exception/FlankException.kt index c9adf6bbdb..815273b4d8 100644 --- a/test_runner/src/main/kotlin/ftl/run/exception/FlankException.kt +++ b/test_runner/src/main/kotlin/ftl/run/exception/FlankException.kt @@ -1,6 +1,6 @@ package ftl.run.exception -import ftl.json.SavedMatrix +import ftl.api.TestMatrix import java.io.IOException import java.lang.Exception @@ -16,7 +16,7 @@ sealed class FlankException(message: String? = null, cause: Throwable? = null) : * * @param matrices [List]<[SavedMatrix]> List of failed matrices */ -class FailedMatrixError(val matrices: List, val ignoreFailed: Boolean = false) : FlankException() +class FailedMatrixError(val matrices: List, val ignoreFailed: Boolean = false) : FlankException() /** * Thrown when at least one matrix is not finished. @@ -26,7 +26,7 @@ class FailedMatrixError(val matrices: List, val ignoreFailed: Boole * * @param matrix [SavedMatrix] Not finished matrix (with matrix state different then FINISHED) */ -open class FTLError(val matrix: SavedMatrix) : FlankException() +open class FTLError(val matrix: TestMatrix.Data) : FlankException() /** * Thrown when doctor command found an error in yml fail and wa unable to fix it @@ -43,7 +43,7 @@ class YmlValidationError(message: String? = null) : FlankException(message) * @param map [Map]<[String], [SavedMatrix]>? map of matrices to be cancelled * @param projectId [String] id of flank's project */ -class FlankTimeoutError(val map: Map?, val projectId: String) : FlankException() +class FlankTimeoutError(val map: Map?, val projectId: String) : FlankException() /** * Usually indicates missing or wrong usage of flags, incorrect parameters, errors in config files. diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 71668bf1d2..87c0293438 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -4,6 +4,7 @@ import com.google.testing.Testing import com.google.testing.model.TestMatrix import flank.common.join import flank.common.logLn +import ftl.adapter.google.toApiModel import ftl.api.RemoteStorage import ftl.api.uploadToRemoteStorage import ftl.args.AndroidArgs @@ -80,7 +81,8 @@ internal suspend fun AndroidArgs.runAndroidTests(): TestResult = coroutineScope logLn(beforeRunMessage(allTestShardChunks)) TestResult( - matrixMap = afterRunTests(testMatrices.awaitAll(), stopwatch), + // TDODO: https://github.com/Flank/flank/issues/1754 + matrixMap = afterRunTests(testMatrices.awaitAll().map { it.toApiModel() }, stopwatch), shardChunks = allTestShardChunks.testCases, ignoredTests = ignoredTestsShardChunks.flatten() ) diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt index 777b83fc01..82a7e3a3bc 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunIosTests.kt @@ -2,6 +2,7 @@ package ftl.run.platform import flank.common.join import flank.common.logLn +import ftl.adapter.google.toApiModel import ftl.api.RemoteStorage import ftl.api.uploadToRemoteStorage import ftl.args.IosArgs @@ -70,7 +71,7 @@ internal suspend fun IosArgs.runIosTests(): TestResult = coroutineScope { TestResult( matrixMap = afterRunTests( - testMatrices = result, + testMatrices = result.map { it.toApiModel() }, stopwatch = stopwatch ), shardChunks = testShardChunks.testCases diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt index 6ecb80780b..7fb5b5a822 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt @@ -1,16 +1,15 @@ package ftl.run.platform.common -import com.google.testing.model.TestMatrix import flank.common.logLn import flank.common.startWithNewLine import ftl.api.RemoteStorage +import ftl.api.TestMatrix +import ftl.api.refreshTestMatrix import ftl.api.uploadToRemoteStorage import ftl.args.IArgs -import ftl.client.google.GcTestMatrix import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_STORAGE_LINK import ftl.json.MatrixMap -import ftl.json.createSavedMatrix import ftl.reports.addStepTime import ftl.run.common.SESSION_ID_FILE import ftl.run.common.saveSessionId @@ -27,7 +26,7 @@ import java.nio.file.Files import java.nio.file.Paths internal suspend fun IArgs.afterRunTests( - testMatrices: List, + testMatrices: List, stopwatch: StopWatch, ) = MatrixMap( map = testMatrices.toSavedMatrixMap(), @@ -44,8 +43,8 @@ internal suspend fun IArgs.afterRunTests( addStepTime("Running tests", stopwatch.check()) } -private fun List.toSavedMatrixMap() = - associate { matrix -> matrix.testMatrixId to createSavedMatrix(matrix) } +private fun List.toSavedMatrixMap() = + associate { matrix -> matrix.matrixId to matrix } private fun IArgs.saveConfigFile(matrixMap: MatrixMap) { val configFilePath = if (useLocalResultDir()) @@ -66,10 +65,10 @@ internal suspend inline fun MatrixMap.printMatricesWebLinks(project: String) = c logLn() } -private tailrec suspend fun getOrUpdateWebLink(link: String, project: String, matrixId: String): String = +private tailrec fun getOrUpdateWebLink(link: String, project: String, matrixId: String): String = if (link.isNotBlank()) link else getOrUpdateWebLink( - link = GcTestMatrix.refresh(matrixId, project).run { if (isInvalid()) "Unable to get web link" else webLink() }, + link = refreshTestMatrix(TestMatrix.Identity(matrixId, project)).run { if (isInvalid) "Unable to get web link" else webLink }, project = project, matrixId = matrixId ) diff --git a/test_runner/src/main/kotlin/ftl/run/status/ExecutionStatusListPrinter.kt b/test_runner/src/main/kotlin/ftl/run/status/ExecutionStatusListPrinter.kt index 2c62cd1915..7d4dcf5074 100644 --- a/test_runner/src/main/kotlin/ftl/run/status/ExecutionStatusListPrinter.kt +++ b/test_runner/src/main/kotlin/ftl/run/status/ExecutionStatusListPrinter.kt @@ -1,23 +1,22 @@ package ftl.run.status -import com.google.testing.model.Environment -import com.google.testing.model.TestExecution +import ftl.api.TestMatrix import ftl.args.IArgs class ExecutionStatusListPrinter( private val args: IArgs, private val executionsCache: LinkedHashMap = LinkedHashMap(), private val printExecutionStatues: (List) -> Unit = createExecutionStatusPrinter(args) -) : (String, List?) -> Unit { +) : (String, List) -> Unit { override fun invoke( time: String, - executions: List? + executions: List ) = getChanges(time, executions).let(printExecutionStatues) private fun getChanges( time: String, - executions: List? - ): List = executions?.map { execution -> + executions: List + ): List = executions.map { execution -> ExecutionStatus.Change( name = execution.formatName(args), previous = executionsCache[execution.id] ?: ExecutionStatus(), @@ -26,22 +25,19 @@ class ExecutionStatusListPrinter( }, time = time ) - } ?: emptyList() + } } -private fun TestExecution.formatName(args: IArgs): String { - val matrixExecutionId = id?.split("_") - val matrixId = matrixExecutionId?.first() - val executionId = matrixExecutionId?.takeIf { args.flakyTestAttempts > 0 }?.getOrNull(1)?.let { " $it" } ?: "" - val env: Environment? = environment - val device = env?.androidDevice?.androidModelId ?: env?.iosDevice?.iosModelId - val deviceVersion = env?.androidDevice?.androidVersionId ?: env?.iosDevice?.iosVersionId - val shard = shard?.takeUnless { args.disableSharding }?.run { " shard-${(shardIndex ?: 0)}" } ?: "" - return "$matrixId $device-$deviceVersion$shard$executionId" +private fun TestMatrix.TestExecution.formatName(args: IArgs): String { + val matrixExecutionId = id.split("_") + val matrixId = matrixExecutionId.first() + val executionId = matrixExecutionId.takeIf { args.flakyTestAttempts > 0 }?.getOrNull(1)?.let { " $it" } ?: "" + val shard = shardIndex?.takeUnless { args.disableSharding }?.run { " shard-${(shardIndex)}" } ?: "" + return "$matrixId $modelId-$deviceVersion$shard$executionId" } -private fun TestExecution.executionStatus() = ExecutionStatus( - state = state ?: "UNKNOWN", - error = testDetails?.errorMessage, - progress = testDetails?.progressMessages ?: emptyList() +private fun TestMatrix.TestExecution.executionStatus() = ExecutionStatus( + state = state, + error = errorMessage, + progress = progress ) diff --git a/test_runner/src/main/kotlin/ftl/run/status/TestMatrixStatusPrinter.kt b/test_runner/src/main/kotlin/ftl/run/status/TestMatrixStatusPrinter.kt index aeccd15eca..60da919d12 100644 --- a/test_runner/src/main/kotlin/ftl/run/status/TestMatrixStatusPrinter.kt +++ b/test_runner/src/main/kotlin/ftl/run/status/TestMatrixStatusPrinter.kt @@ -1,8 +1,7 @@ package ftl.run.status -import com.google.testing.model.TestExecution -import com.google.testing.model.TestMatrix import flank.common.logLn +import ftl.api.TestMatrix import ftl.args.IArgs import ftl.config.FtlConstants import ftl.reports.addStepTime @@ -14,8 +13,8 @@ class TestMatrixStatusPrinter( private val args: IArgs, testMatricesIds: Iterable, private val stopWatch: StopWatch = StopWatch(), - private val printExecutionStatusList: (String, List?) -> Unit = ExecutionStatusListPrinter(args) -) : (TestMatrix) -> Unit { + private val printExecutionStatusList: (String, List) -> Unit = ExecutionStatusListPrinter(args) +) : (TestMatrix.Data) -> Unit { init { stopWatch.start() } @@ -23,10 +22,10 @@ class TestMatrixStatusPrinter( private val cache = testMatricesIds.associateWith { MatrixState.PENDING }.toMutableMap() private val allMatricesCompleted get() = cache.values.all(MatrixState::completed) - override fun invoke(matrix: TestMatrix) { + override fun invoke(matrix: TestMatrix.Data) { val time = stopWatch.check() printExecutionStatusList(time.formatted(alignSeconds = true), matrix.testExecutions) - cache[matrix.testMatrixId] = matrix.state + cache[matrix.matrixId] = matrix.state if (allMatricesCompleted) { printTestMatrixStatusList(time.formatted(alignSeconds = true)) addStepTime("Executing matrices", time) diff --git a/test_runner/src/main/kotlin/ftl/util/MatrixState.kt b/test_runner/src/main/kotlin/ftl/util/MatrixState.kt index c3d8cdda26..77e2becd87 100644 --- a/test_runner/src/main/kotlin/ftl/util/MatrixState.kt +++ b/test_runner/src/main/kotlin/ftl/util/MatrixState.kt @@ -1,6 +1,6 @@ package ftl.util -import com.google.testing.model.TestMatrix +import ftl.api.TestMatrix object MatrixState { // https://github.com/bootstraponline/gcloud_cli/blob/0752e88b155a417a18d244c242b4ab3fb9aa1c1f/google-cloud-sdk/lib/googlecloudsdk/third_party/apis/testing_v1.json#L171 @@ -47,4 +47,5 @@ object MatrixState { } } -fun TestMatrix.isInvalid() = MatrixState.isInvalid(this.state) +val TestMatrix.Data.isInvalid + get() = MatrixState.isInvalid(this.state) diff --git a/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt b/test_runner/src/test/kotlin/ftl/api/TestMatrixTest.kt similarity index 56% rename from test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt rename to test_runner/src/test/kotlin/ftl/api/TestMatrixTest.kt index 872d4bda34..d74af0029e 100644 --- a/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/api/TestMatrixTest.kt @@ -1,4 +1,4 @@ -package ftl.json +package ftl.api import com.google.common.truth.Truth.assertThat import com.google.testing.model.Environment @@ -10,8 +10,11 @@ import com.google.testing.model.TestMatrix import com.google.testing.model.TestSpecification import com.google.testing.model.ToolResultsExecution import com.google.testing.model.ToolResultsStep +import ftl.adapter.google.toApiModel import ftl.config.Device +import ftl.domain.testmatrix.updateWithMatrix import ftl.gc.GcAndroidDevice +import ftl.json.createAndUpdateMatrix import ftl.reports.outcome.make import ftl.test.util.FlankTestRunner import ftl.util.MatrixState.FINISHED @@ -22,8 +25,9 @@ import org.junit.Assert.assertEquals import org.junit.Test import org.junit.runner.RunWith +// TODO: fix naming @RunWith(FlankTestRunner::class) -class SavedMatrixTest { +class TestMatrixTest { companion object { const val projectId = "1" @@ -63,7 +67,7 @@ class SavedMatrixTest { } } - fun testMatrix(block: TestMatrix.() -> Unit = {}) = TestMatrix().also { + fun ftlTestMatrix(block: TestMatrix.() -> Unit = {}) = TestMatrix().also { it.projectId = projectId it.testMatrixId = testMatrixId it.block() @@ -71,7 +75,7 @@ class SavedMatrixTest { } @Test - fun `savedMatrix failureOutcome`() { + fun `testMatrix failureOutcome`() { // Verify that if we have two executions: failure then success // the SavedMatrix outcome is correctly recorded as failure val testExecutions = listOf( @@ -81,7 +85,7 @@ class SavedMatrixTest { val matrixId = "123" val matrixState = FINISHED - val testMatrix = testMatrix() + val testMatrix = ftlTestMatrix() testMatrix.testMatrixId = matrixId testMatrix.state = matrixState testMatrix.resultStorage = createResultsStorage().apply { @@ -89,24 +93,24 @@ class SavedMatrixTest { } testMatrix.testExecutions = testExecutions - val savedMatrix = createSavedMatrix(testMatrix) - assertThat(savedMatrix.outcome).isEqualTo("failure") + val testMatrixData = createAndUpdateMatrix(testMatrix) + assertThat(testMatrixData.outcome).isEqualTo("failure") // assert other properties - assertThat(savedMatrix.matrixId).isEqualTo(matrixId) - assertThat(savedMatrix.state).isEqualTo(matrixState) - assertThat(savedMatrix.gcsPath).isEqualTo(mockGcsPath) - assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/1/testlab/histories/2/matrices/-1") - assertThat(savedMatrix.downloaded).isFalse() - assertThat(savedMatrix.billableVirtualMinutes).isEqualTo(1) - assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1) - assertThat(savedMatrix.gcsPathWithoutRootBucket).isEqualTo(mockFileName) - assertThat(savedMatrix.gcsRootBucket).isEqualTo(mockBucket) - assertThat(savedMatrix.testAxises.first().details).isNotEmpty() + assertThat(testMatrixData.matrixId).isEqualTo(matrixId) + assertThat(testMatrixData.state).isEqualTo(matrixState) + assertThat(testMatrixData.gcsPath).isEqualTo(mockGcsPath) + assertThat(testMatrixData.webLink).isEqualTo("https://console.firebase.google.com/project/1/testlab/histories/2/matrices/-1") + assertThat(testMatrixData.downloaded).isFalse() + assertThat(testMatrixData.billableMinutes.virtual).isEqualTo(1) + assertThat(testMatrixData.billableMinutes.physical).isEqualTo(1) + assertThat(testMatrixData.gcsPathWithoutRootBucket).isEqualTo(mockFileName) + assertThat(testMatrixData.gcsRootBucket).isEqualTo(mockBucket) + assertThat(testMatrixData.axes.first().details).isNotEmpty() } @Test - fun `savedMatrix skippedOutcome`() { + fun `testMatrix skippedOutcome`() { // Verify that if we have two executions: skipped // the SavedMatrix outcome is correctly recorded as skipped val testExecutions = listOf( @@ -115,95 +119,95 @@ class SavedMatrixTest { val matrixId = "123" val matrixState = FINISHED - val testMatrix = testMatrix() - testMatrix.testMatrixId = matrixId - testMatrix.state = matrixState - testMatrix.resultStorage = createResultsStorage().apply { + val ftlTestMatrix = ftlTestMatrix() + ftlTestMatrix.testMatrixId = matrixId + ftlTestMatrix.state = matrixState + ftlTestMatrix.resultStorage = createResultsStorage().apply { toolResultsExecution.executionId = "-3" } - testMatrix.testExecutions = testExecutions + ftlTestMatrix.testExecutions = testExecutions - val savedMatrix = createSavedMatrix(testMatrix) - assertThat(savedMatrix.outcome).isEqualTo("skipped") + val testMatrix = createAndUpdateMatrix(ftlTestMatrix) + assertThat(testMatrix.outcome).isEqualTo("skipped") // assert other properties - assertThat(savedMatrix.matrixId).isEqualTo(matrixId) - assertThat(savedMatrix.state).isEqualTo(matrixState) - assertThat(savedMatrix.gcsPath).isEqualTo(mockGcsPath) - assertThat(savedMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/1/testlab/histories/2/matrices/-3/executions/-3") - assertThat(savedMatrix.downloaded).isFalse() - assertThat(savedMatrix.billableVirtualMinutes).isEqualTo(1) - assertThat(savedMatrix.billablePhysicalMinutes).isEqualTo(1) - assertThat(savedMatrix.gcsPathWithoutRootBucket).isEqualTo(mockFileName) - assertThat(savedMatrix.gcsRootBucket).isEqualTo(mockBucket) - assertThat(savedMatrix.testAxises.first().details).isNotEmpty() + assertThat(testMatrix.matrixId).isEqualTo(matrixId) + assertThat(testMatrix.state).isEqualTo(matrixState) + assertThat(testMatrix.gcsPath).isEqualTo(mockGcsPath) + assertThat(testMatrix.webLink).isEqualTo("https://console.firebase.google.com/project/1/testlab/histories/2/matrices/-3/executions/-3") + assertThat(testMatrix.downloaded).isFalse() + assertThat(testMatrix.billableMinutes.virtual).isEqualTo(1) + assertThat(testMatrix.billableMinutes.physical).isEqualTo(1) + assertThat(testMatrix.gcsPathWithoutRootBucket).isEqualTo(mockFileName) + assertThat(testMatrix.gcsRootBucket).isEqualTo(mockBucket) + assertThat(testMatrix.axes.first().details).isNotEmpty() } @Test - fun `savedMatrix update`() { + fun `testMatrix update`() { val testExecutions = listOf( createStepExecution(1, "shamu"), createStepExecution(1, "NexusLowRes") ) val matrixId = "123" - val testMatrix = testMatrix() - testMatrix.testMatrixId = matrixId - testMatrix.state = PENDING - testMatrix.resultStorage = createResultsStorage() - testMatrix.testExecutions = testExecutions - - var savedMatrix = createSavedMatrix(testMatrix) - - assert(savedMatrix.state != FINISHED) - testMatrix.state = FINISHED - testMatrix.webLink() - savedMatrix = savedMatrix.updateWithMatrix(testMatrix) - assert(savedMatrix.state == FINISHED) + val ftlTestMatrix = ftlTestMatrix() + ftlTestMatrix.testMatrixId = matrixId + ftlTestMatrix.state = PENDING + ftlTestMatrix.resultStorage = createResultsStorage() + ftlTestMatrix.testExecutions = testExecutions + + var testMatrix = ftlTestMatrix.toApiModel() + + assert(testMatrix.state != FINISHED) + ftlTestMatrix.state = FINISHED + ftlTestMatrix.webLink() + testMatrix = testMatrix.updateWithMatrix(ftlTestMatrix.toApiModel()) + assert(testMatrix.state == FINISHED) } @Test - fun `savedMatrix on finish should calculate cost when state != ERROR`() { + fun `testMatrix on finish should calculate cost when state != ERROR`() { val testExecutions = listOf( createStepExecution(1, "shamu"), createStepExecution(1, "NexusLowRes") ) - val testMatrix = testMatrix() - testMatrix.projectId = projectId - testMatrix.testMatrixId = "123" - testMatrix.state = PENDING - testMatrix.resultStorage = createResultsStorage() - testMatrix.testExecutions = testExecutions - - var savedMatrix = createSavedMatrix(testMatrix) - - testMatrix.state = FINISHED - testMatrix.webLink() - savedMatrix = savedMatrix.updateWithMatrix(testMatrix) - assertEquals(1, savedMatrix.billableVirtualMinutes) - assertEquals(1, savedMatrix.billablePhysicalMinutes) + val ftlTestMatrix = ftlTestMatrix() + ftlTestMatrix.projectId = projectId + ftlTestMatrix.testMatrixId = "123" + ftlTestMatrix.state = PENDING + ftlTestMatrix.resultStorage = createResultsStorage() + ftlTestMatrix.testExecutions = testExecutions + + var testMatrix = ftlTestMatrix.toApiModel() + + ftlTestMatrix.state = FINISHED + ftlTestMatrix.webLink() + testMatrix = testMatrix.updateWithMatrix(ftlTestMatrix.toApiModel()) + assertEquals(1, testMatrix.billableMinutes.virtual) + assertEquals(1, testMatrix.billableMinutes.physical) } @Test - fun `savedMatrix should have outcome and outcome details properly filled when state is INVALID`() { + fun `testMatrix should have outcome and outcome details properly filled when state is INVALID`() { val expectedOutcome = "INVALID" val expectedOutcomeDetails = "UNKNOWN" - val testMatrix = testMatrix() - testMatrix.testMatrixId = "123" - testMatrix.state = PENDING - testMatrix.resultStorage = createResultsStorage() - - var savedMatrix = createSavedMatrix(testMatrix) - - testMatrix.state = INVALID - savedMatrix = savedMatrix.updateWithMatrix(testMatrix) - assertEquals(expectedOutcome, savedMatrix.outcome) - assertEquals(expectedOutcomeDetails, savedMatrix.testAxises.first().details) - assertEquals(INVALID, savedMatrix.state) + val ftlTestMatrix = ftlTestMatrix() + ftlTestMatrix.testMatrixId = "123" + ftlTestMatrix.state = PENDING + ftlTestMatrix.resultStorage = createResultsStorage() + + var testMatrix = ftlTestMatrix.toApiModel() + + ftlTestMatrix.state = INVALID + testMatrix = testMatrix.updateWithMatrix(ftlTestMatrix.toApiModel()) + assertEquals(expectedOutcome, testMatrix.outcome) + assertEquals(expectedOutcomeDetails, testMatrix.axes.first().details) + assertEquals(INVALID, testMatrix.state) } @Test - fun `savedMatrix should have failed outcome when at least one test is failed`() { + fun `testMatrix should have failed outcome when at least one test is failed`() { val expectedOutcome = "failure" val successStepExecution = createStepExecution(1) // success val failedStepExecution = createStepExecution(-1) // failure @@ -222,7 +226,7 @@ class SavedMatrixTest { flakyOutcomeComparedStepExecution ) - val testMatrix = testMatrix().apply { + val ftlTestMatrix = ftlTestMatrix().apply { testMatrixId = "123" state = FINISHED resultStorage = createResultsStorage().apply { @@ -231,17 +235,17 @@ class SavedMatrixTest { testExecutions = executions } - val savedMatrix = createSavedMatrix(testMatrix) + val testMatrix = createAndUpdateMatrix(ftlTestMatrix) assertEquals( "Does not return failed outcome when last execution is flaky", expectedOutcome, - savedMatrix.outcome + testMatrix.outcome ) } @Test - fun `SavedMatrix should be updated with apk file name - android`() { + fun `testMatrix should be updated with apk file name - android`() { val appName = "any-test_app.apk" val specs = listOf Unit>( @@ -251,7 +255,7 @@ class SavedMatrixTest { ) val getNewTestMatrix = { - testMatrix { + ftlTestMatrix { state = PENDING resultStorage = createResultsStorage() testExecutions = listOf(createStepExecution(1, "NexusLowRes")) @@ -259,12 +263,12 @@ class SavedMatrixTest { } specs.forEach { spec -> - val testMatrix = getNewTestMatrix() - val savedMatrix = createSavedMatrix(testMatrix) + val ftlTestMatrix = getNewTestMatrix() + val testMatrix = ftlTestMatrix.toApiModel() - testMatrix.state = FINISHED - testMatrix.testSpecification = make(spec) - val updatedMatrix = savedMatrix.updateWithMatrix(testMatrix) + ftlTestMatrix.state = FINISHED + ftlTestMatrix.testSpecification = make(spec) + val updatedMatrix = testMatrix.updateWithMatrix(ftlTestMatrix.toApiModel()) assertEquals(appName, updatedMatrix.appFileName) } } @@ -279,7 +283,7 @@ class SavedMatrixTest { ) val getNewTestMatrix = { - testMatrix { + ftlTestMatrix { state = PENDING resultStorage = createResultsStorage() testExecutions = listOf(createStepExecution(1, "iPhone6")) @@ -287,12 +291,12 @@ class SavedMatrixTest { } specs.forEach { spec -> - val testMatrix = getNewTestMatrix() - val savedMatrix = createSavedMatrix(testMatrix) + val ftlTestMatrix = getNewTestMatrix() + val testMatrix = ftlTestMatrix.toApiModel() - testMatrix.state = FINISHED - testMatrix.testSpecification = make(spec) - val updatedMatrix = savedMatrix.updateWithMatrix(testMatrix) + ftlTestMatrix.state = FINISHED + ftlTestMatrix.testSpecification = make(spec) + val updatedMatrix = testMatrix.updateWithMatrix(ftlTestMatrix.toApiModel()) assertEquals(appName, updatedMatrix.appFileName) } } @@ -300,18 +304,18 @@ class SavedMatrixTest { @Test fun `SavedMatrix should be updated with NA file name if none is available`() { val getNewTestMatrix = { - testMatrix { + ftlTestMatrix { state = PENDING resultStorage = createResultsStorage() testExecutions = listOf(createStepExecution(1, "iPhone6")) } } - val testMatrix = getNewTestMatrix() - val savedMatrix = createSavedMatrix(testMatrix) + val ftlTestMatrix = getNewTestMatrix() + val testMatrix = ftlTestMatrix.toApiModel() - testMatrix.state = FINISHED - val updatedMatrix = savedMatrix.updateWithMatrix(testMatrix) + ftlTestMatrix.state = FINISHED + val updatedMatrix = testMatrix.updateWithMatrix(ftlTestMatrix.toApiModel()) assertEquals("N/A", updatedMatrix.appFileName) } } diff --git a/test_runner/src/test/kotlin/ftl/json/MatrixMapTest.kt b/test_runner/src/test/kotlin/ftl/json/MatrixMapTest.kt index f82b17a0e2..27c635031d 100644 --- a/test_runner/src/test/kotlin/ftl/json/MatrixMapTest.kt +++ b/test_runner/src/test/kotlin/ftl/json/MatrixMapTest.kt @@ -1,9 +1,10 @@ package ftl.json import com.google.common.truth.Truth.assertThat -import ftl.json.SavedMatrixTest.Companion.createResultsStorage -import ftl.json.SavedMatrixTest.Companion.createStepExecution -import ftl.json.SavedMatrixTest.Companion.testMatrix +import ftl.api.TestMatrix +import ftl.api.TestMatrixTest.Companion.createResultsStorage +import ftl.api.TestMatrixTest.Companion.createStepExecution +import ftl.api.TestMatrixTest.Companion.ftlTestMatrix import ftl.run.exception.MatrixValidationError import ftl.test.util.FlankTestRunner import ftl.test.util.assertThrowsWithMessage @@ -25,23 +26,21 @@ class MatrixMapTest { assertThat(matrixMap.map).isNotNull() } - private fun matrixForExecution(executionId: Int): SavedMatrix { - return createSavedMatrix( - testMatrix = testMatrix() - .setResultStorage( - createResultsStorage().apply { - toolResultsExecution.executionId = executionId.toString() - } + private fun matrixForExecution(executionId: Int): TestMatrix.Data = createAndUpdateMatrix( + ftlTestMatrix() + .setResultStorage( + createResultsStorage().apply { + toolResultsExecution.executionId = executionId.toString() + } + ) + .setState(MatrixState.FINISHED) + .setTestMatrixId("123") + .setTestExecutions( + listOf( + createStepExecution(executionId, "") ) - .setState(MatrixState.FINISHED) - .setTestMatrixId("123") - .setTestExecutions( - listOf( - createStepExecution(executionId, "") - ) - ) - ) - } + ) + ) @Test fun `matrixMap failure`() { @@ -54,11 +53,11 @@ class MatrixMapTest { @Test fun `invalid matrix should contain a specific error message`() { - val testMatrix = testMatrix() + val testMatrix = ftlTestMatrix() testMatrix.testMatrixId = "123" testMatrix.state = MatrixState.INVALID - val savedMatrix = createSavedMatrix(testMatrix) + val savedMatrix = createAndUpdateMatrix(testMatrix) val expectedErrorMessage = "Matrix: [${testMatrix.testMatrixId}] failed: Unknown error" assertThrowsWithMessage(MatrixValidationError::class, expectedErrorMessage) { diff --git a/test_runner/src/test/kotlin/ftl/json/TestMatrixTestUtils.kt b/test_runner/src/test/kotlin/ftl/json/TestMatrixTestUtils.kt new file mode 100644 index 0000000000..6fff292fed --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/json/TestMatrixTestUtils.kt @@ -0,0 +1,8 @@ +package ftl.json + +import com.google.testing.model.TestMatrix +import ftl.adapter.google.toApiModel +import ftl.api.TestMatrix.Data +import ftl.domain.testmatrix.updateWithMatrix + +fun createAndUpdateMatrix(testMatrix: TestMatrix) = Data().updateWithMatrix(testMatrix.toApiModel()) diff --git a/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt b/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt index f2a5cd3016..f61b95b8c7 100644 --- a/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt @@ -2,7 +2,9 @@ package ftl.reports.outcome import com.google.api.services.toolresults.model.Step import com.google.testing.model.AndroidModel +import ftl.client.google.BillableMinutes import ftl.client.google.DeviceType +import ftl.client.google.calculateAndroidBillableMinutes import ftl.gc.GcTesting import ftl.http.executeWithRetry import io.mockk.every diff --git a/test_runner/src/test/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummaryKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummaryKtTest.kt index 38200e431f..740609dd71 100644 --- a/test_runner/src/test/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummaryKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/outcome/CreateMatrixOutcomeSummaryKtTest.kt @@ -2,6 +2,9 @@ package ftl.reports.outcome import com.google.api.services.toolresults.model.Environment import com.google.api.services.toolresults.model.Step +import ftl.client.google.TestOutcomeContext +import ftl.client.google.calculateAndroidBillableMinutes +import ftl.client.google.createMatrixOutcomeSummary import io.mockk.every import io.mockk.mockkStatic import io.mockk.unmockkAll @@ -16,7 +19,7 @@ class CreateMatrixOutcomeSummaryKtTest { @Before fun setUp() { mockkStatic("ftl.reports.outcome.UtilKt") - mockkStatic("ftl.reports.outcome.BillableMinutesKt") + mockkStatic("ftl.client.google.BillableMinutesKt") } @After diff --git a/test_runner/src/test/kotlin/ftl/reports/output/OutputReportLoggersTest.kt b/test_runner/src/test/kotlin/ftl/reports/output/OutputReportLoggersTest.kt index ed01299219..e219837f6b 100644 --- a/test_runner/src/test/kotlin/ftl/reports/output/OutputReportLoggersTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/output/OutputReportLoggersTest.kt @@ -1,9 +1,9 @@ package ftl.reports.output import com.google.common.truth.Truth.assertThat +import ftl.api.TestMatrix +import ftl.api.TestMatrixTest.Companion.historyId import ftl.args.AndroidArgs -import ftl.json.SavedMatrix -import ftl.reports.outcome.TestOutcome import org.junit.Before import org.junit.Test import java.math.BigDecimal @@ -43,12 +43,17 @@ class OutputReportLoggersTest { fun `should log test_results`() { // given val matrices = listOf( - SavedMatrix(matrixId = "1", testAxises = listOf(TestOutcome("device1")), appFileName = "any.apk") + TestMatrix.Data( + matrixId = "1", + axes = listOf(TestMatrix.Outcome("device1")), + appFileName = "any.apk", + historyId = historyId + ) ) val testResults = matrices.map { it.matrixId to mapOf( "app" to it.appFileName, - "test-axises" to it.testAxises + "test-axises" to it.axes ) }.toMap() diff --git a/test_runner/src/test/kotlin/ftl/run/status/ExecutionStatusListPrinterTest.kt b/test_runner/src/test/kotlin/ftl/run/status/ExecutionStatusListPrinterTest.kt index 9467058b39..7d0b375366 100644 --- a/test_runner/src/test/kotlin/ftl/run/status/ExecutionStatusListPrinterTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/status/ExecutionStatusListPrinterTest.kt @@ -2,6 +2,8 @@ package ftl.run.status import com.google.testing.model.TestDetails import com.google.testing.model.TestExecution +import ftl.adapter.google.toApiModel +import ftl.api.TestMatrix import ftl.args.IArgs import ftl.util.MatrixState import io.mockk.every @@ -15,7 +17,7 @@ class ExecutionStatusListPrinterTest { fun test() { // given val time = "time" - val executions = (0..1).map { size -> + val executions: List> = (0..1).map { size -> (0..size).map { index -> TestExecution().apply { id = "${size}_$index" @@ -26,10 +28,11 @@ class ExecutionStatusListPrinterTest { } } } - } + }.map { it.toApiModel() } val args = mockk { every { outputStyle } returns OutputStyle.Single every { flakyTestAttempts } returns 1 + every { disableSharding } returns false } val result = mutableListOf() val printExecutionStatues = { changes: List -> @@ -47,9 +50,9 @@ class ExecutionStatusListPrinterTest { progress = listOf("test progress") ) val expected = listOf( - ExecutionStatus.Change("0 null-null 0", previous, current, time), - ExecutionStatus.Change("1 null-null 0", previous, current, time), - ExecutionStatus.Change("1 null-null 1", previous, current, time) + ExecutionStatus.Change("0 - 0", previous, current, time), + ExecutionStatus.Change("1 - 0", previous, current, time), + ExecutionStatus.Change("1 - 1", previous, current, time) ) // when executions.forEach { execution -> diff --git a/test_runner/src/test/kotlin/ftl/run/status/TestMatrixStatusPrinterTest.kt b/test_runner/src/test/kotlin/ftl/run/status/TestMatrixStatusPrinterTest.kt index 4cf7905295..7a06cff19f 100644 --- a/test_runner/src/test/kotlin/ftl/run/status/TestMatrixStatusPrinterTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/status/TestMatrixStatusPrinterTest.kt @@ -1,7 +1,8 @@ package ftl.run.status -import com.google.testing.model.TestExecution import com.google.testing.model.TestMatrix +import ftl.adapter.google.toApiModel +import ftl.api.TestMatrix.TestExecution import ftl.args.IArgs import ftl.util.Duration import ftl.util.MatrixState @@ -14,6 +15,7 @@ import org.junit.Assert.assertEquals import org.junit.Rule import org.junit.Test import org.junit.contrib.java.lang.system.SystemOutRule +import com.google.testing.model.TestExecution as GoogleTestExecution class TestMatrixStatusPrinterTest { @Rule @@ -28,10 +30,11 @@ class TestMatrixStatusPrinterTest { val testMatricesIds = (0..1).map(Int::toString) val matrices = testMatricesIds.mapIndexed { index, s -> TestMatrix().apply { + projectId = "test" testMatrixId = s state = MatrixState.FINISHED - testExecutions = (0..index).map { TestExecution() } - } + testExecutions = (0..index).map { GoogleTestExecution() } + }.toApiModel() } val args = mockk { every { outputStyle } returns OutputStyle.Single diff --git a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt index 7ec253fffd..1c4dc12afe 100644 --- a/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt +++ b/test_runner/src/test/kotlin/ftl/util/UtilsTest.kt @@ -1,15 +1,15 @@ package ftl.util import com.google.common.truth.Truth.assertThat +import ftl.adapter.google.toApiModel +import ftl.api.TestMatrix +import ftl.api.TestMatrixTest.Companion.createResultsStorage +import ftl.api.TestMatrixTest.Companion.createStepExecution +import ftl.api.TestMatrixTest.Companion.ftlTestMatrix import ftl.config.FtlConstants import ftl.json.MatrixMap -import ftl.json.SavedMatrix -import ftl.json.SavedMatrixTest.Companion.createResultsStorage -import ftl.json.SavedMatrixTest.Companion.createStepExecution -import ftl.json.SavedMatrixTest.Companion.testMatrix -import ftl.json.createSavedMatrix +import ftl.json.createAndUpdateMatrix import ftl.json.validate -import ftl.reports.outcome.TestOutcome import ftl.run.MatrixCancelStatus import ftl.run.cancelMatrices import ftl.run.exception.CONFIGURATION_FAIL @@ -95,14 +95,14 @@ class UtilsTest { createStepExecution(1, "Success"), createStepExecution(-1, "Failed") ) - val testMatrix = testMatrix() + val testMatrix = ftlTestMatrix() testMatrix.testMatrixId = "123" testMatrix.state = MatrixState.FINISHED testMatrix.resultStorage = createResultsStorage().apply { toolResultsExecution.executionId = "-1" } testMatrix.testExecutions = testExecutions - val finishedMatrix = createSavedMatrix(testMatrix) + val finishedMatrix = createAndUpdateMatrix(testMatrix) MatrixMap(mutableMapOf("finishedMatrix" to finishedMatrix), "MockPath").validate() } @@ -111,12 +111,12 @@ class UtilsTest { val testExecutions = listOf( createStepExecution(1, "Success") ) - val testMatrix = testMatrix() + val testMatrix = ftlTestMatrix() testMatrix.testMatrixId = "123" testMatrix.state = MatrixState.FINISHED testMatrix.resultStorage = createResultsStorage() testMatrix.testExecutions = testExecutions - val finishedMatrix = createSavedMatrix(testMatrix) + val finishedMatrix = createAndUpdateMatrix(testMatrix) MatrixMap(mutableMapOf("" to finishedMatrix), "MockPath").validate() } @@ -125,14 +125,14 @@ class UtilsTest { val testExecutions = listOf( createStepExecution(-2, "Inconclusive") ) - val testMatrix = testMatrix() + val testMatrix = ftlTestMatrix() testMatrix.testMatrixId = "123" testMatrix.state = MatrixState.FINISHED testMatrix.resultStorage = createResultsStorage().apply { toolResultsExecution.executionId = "-2" } testMatrix.testExecutions = testExecutions - val finishedMatrix = createSavedMatrix(testMatrix) + val finishedMatrix = createAndUpdateMatrix(testMatrix) MatrixMap(mutableMapOf("" to finishedMatrix), "MockPath").validate() } @@ -142,12 +142,12 @@ class UtilsTest { createStepExecution(-2, "Inconclusive"), createStepExecution(-3, "Skipped") ) - val testMatrix = testMatrix() + val testMatrix = ftlTestMatrix() testMatrix.testMatrixId = "123" testMatrix.state = MatrixState.ERROR testMatrix.resultStorage = createResultsStorage() testMatrix.testExecutions = testExecutions - val errorMatrix = createSavedMatrix(testMatrix) + val errorMatrix = testMatrix.toApiModel() MatrixMap(mutableMapOf("errorMatrix" to errorMatrix), "MockPath").validate() } @@ -158,12 +158,12 @@ class UtilsTest { createStepExecution(1, "Success"), createStepExecution(-1, "Failed") ) - val testMatrix = testMatrix() + val testMatrix = ftlTestMatrix() testMatrix.testMatrixId = "123" testMatrix.state = MatrixState.FINISHED testMatrix.resultStorage = createResultsStorage() testMatrix.testExecutions = testExecutions - val finishedMatrix = createSavedMatrix(testMatrix) + val finishedMatrix = testMatrix.toApiModel() try { MatrixMap(mutableMapOf("" to finishedMatrix), "MockPath").validate(shouldIgnore) } catch (t: FailedMatrixError) { @@ -388,21 +388,21 @@ class UtilsTest { } } -private val testMatrix1 = mockk(relaxed = true) { +private val testMatrix1 = mockk(relaxed = true) { every { matrixId } returns "1" every { webLink } returns "www.flank.com/1" - every { testAxises } returns listOf( - TestOutcome( + every { axes } returns listOf( + TestMatrix.Outcome( outcome = "Failed", details = "Test failed to run" ) ) } -private val testMatrix2 = mockk(relaxed = true) { +private val testMatrix2 = mockk(relaxed = true) { every { matrixId } returns "2" every { webLink } returns "www.flank.com/2" - every { testAxises } returns listOf( - TestOutcome( + every { axes } returns listOf( + TestMatrix.Outcome( outcome = "Failed", details = "Test failed to run" )