-
Notifications
You must be signed in to change notification settings - Fork 119
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: Data scratch - test matrix (#1901)
Fixes #1756 ## Test Plan > How do we know the code works? * All tests pass * Flank works as before ## Checklist - [X] Unit tests updated
- Loading branch information
1 parent
1c5f14c
commit 66e796c
Showing
56 changed files
with
816 additions
and
586 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
13 changes: 13 additions & 0 deletions
13
test_runner/src/main/kotlin/ftl/adapter/TestMatrixCancel.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package ftl.adapter | ||
|
|
||
| import ftl.api.TestMatrix | ||
| import ftl.client.google.cancelMatrices | ||
| import kotlinx.coroutines.runBlocking | ||
|
|
||
| object TestMatrixCancel : | ||
| TestMatrix.Cancel, | ||
| (TestMatrix.Identity) -> Unit by { identity -> | ||
| runBlocking { | ||
| cancelMatrices(identity.matrixId, identity.projectId) | ||
| } | ||
| } |
11 changes: 11 additions & 0 deletions
11
test_runner/src/main/kotlin/ftl/adapter/TestMatrixFetch.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package ftl.adapter | ||
|
|
||
| import ftl.adapter.google.toApiModel | ||
| import ftl.api.TestMatrix | ||
| import ftl.client.google.fetchMatrixOutcome | ||
|
|
||
| object TestMatrixFetch : | ||
| TestMatrix.Summary.Fetch, | ||
| (TestMatrix.Data) -> TestMatrix.Summary by { | ||
| fetchMatrixOutcome(it).toApiModel() | ||
| } |
14 changes: 14 additions & 0 deletions
14
test_runner/src/main/kotlin/ftl/adapter/TestMatrixRefresh.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| package ftl.adapter | ||
|
|
||
| import ftl.adapter.google.toApiModel | ||
| import ftl.api.TestMatrix | ||
| import ftl.client.google.refreshMatrix | ||
| import kotlinx.coroutines.runBlocking | ||
|
|
||
| object TestMatrixRefresh : | ||
| TestMatrix.Refresh, | ||
| (TestMatrix.Identity) -> TestMatrix.Data by { identity -> | ||
| runBlocking { | ||
| refreshMatrix(identity.matrixId, identity.projectId).toApiModel(identity) | ||
| } | ||
| } |
105 changes: 105 additions & 0 deletions
105
test_runner/src/main/kotlin/ftl/adapter/google/GoogleTestMatrixAdapter.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| package ftl.adapter.google | ||
|
|
||
| import com.google.testing.model.FileReference | ||
| import com.google.testing.model.TestExecution | ||
| import com.google.testing.model.TestMatrix | ||
| import ftl.analytics.toJSONObject | ||
| import ftl.api.TestMatrix.Data | ||
| import ftl.api.TestMatrix.Outcome | ||
| import ftl.api.TestMatrix.SuiteOverview | ||
| import ftl.client.google.TestOutcome | ||
| import ftl.environment.orUnknown | ||
| import ftl.run.common.prettyPrint | ||
| import ftl.util.MatrixState | ||
| import ftl.util.getClientDetails | ||
| import ftl.util.getGcsPath | ||
| import ftl.util.getGcsPathWithoutRootBucket | ||
| import ftl.util.getGcsRootBucket | ||
| import ftl.util.timeoutToSeconds | ||
| import ftl.util.webLink | ||
| import ftl.util.webLinkWithoutExecutionDetails | ||
|
|
||
| fun TestMatrix.toApiModel(identity: ftl.api.TestMatrix.Identity? = null) = Data( | ||
| projectId = projectId.orEmpty(), | ||
| matrixId = testMatrixId.orEmpty(), | ||
| gcsPath = getGcsPath(), | ||
| webLink = webLink(), | ||
| downloaded = false, | ||
| clientDetails = getClientDetails(), | ||
| gcsPathWithoutRootBucket = getGcsPathWithoutRootBucket(), | ||
| gcsRootBucket = getGcsRootBucket(), | ||
| webLinkWithoutExecutionDetails = webLinkWithoutExecutionDetails(), | ||
| appFileName = extractAppFileName() ?: fallbackAppName, | ||
| isCompleted = MatrixState.completed(state) && | ||
| testExecutions?.all { MatrixState.completed(it.state.orEmpty()) } ?: true, | ||
| testExecutions = testExecutions?.toApiModel().orEmpty(), | ||
| testTimeout = testTimeout(), | ||
| isRoboTest = isRoboTest(), | ||
| historyId = resultStorage?.toolResultsExecution?.historyId ?: identity?.historyId.orEmpty(), | ||
| executionId = resultStorage?.toolResultsExecution?.executionId ?: identity?.executionId.orEmpty(), | ||
| invalidMatrixDetails = invalidMatrixDetails.orUnknown(), | ||
| state = state.orEmpty(), | ||
| ) | ||
|
|
||
| private fun TestMatrix.testTimeout() = timeoutToSeconds( | ||
| testExecutions | ||
| ?.firstOrNull { it?.testSpecification?.testTimeout != null } | ||
| ?.testSpecification | ||
| ?.testTimeout | ||
| ?: "0s" | ||
| ) | ||
|
|
||
| private fun TestMatrix.isRoboTest() = testExecutions.orEmpty().any { it?.testSpecification?.androidRoboTest != null } | ||
|
|
||
| private const val fallbackAppName = "N/A" | ||
|
|
||
| fun TestOutcome.toApiModel() = Outcome( | ||
| device, outcome, details, | ||
| SuiteOverview( | ||
| testSuiteOverview.total, | ||
| testSuiteOverview.errors, | ||
| testSuiteOverview.failures, | ||
| testSuiteOverview.flakes, | ||
| testSuiteOverview.skipped, | ||
| testSuiteOverview.elapsedTime, | ||
| testSuiteOverview.overheadTime | ||
| ) | ||
| ) | ||
|
|
||
| fun List<TestExecution>.toApiModel() = map(TestExecution::toApiModel) | ||
|
|
||
| fun TestExecution.toApiModel() = ftl.api.TestMatrix.TestExecution( | ||
| id = id.orEmpty(), | ||
| modelId = environment?.androidDevice?.androidModelId | ||
| ?: environment?.iosDevice?.iosModelId ?: "", | ||
| deviceVersion = environment?.androidDevice?.androidVersionId | ||
| ?: environment?.iosDevice?.iosVersionId ?: "", | ||
| shardIndex = shard?.shardIndex, | ||
| state = state.orUnknown(), | ||
| errorMessage = testDetails?.errorMessage.orEmpty(), | ||
| progress = testDetails?.progressMessages ?: emptyList(), | ||
| toolResultsStep = toolResultsStep | ||
| ) | ||
|
|
||
| private fun TestMatrix.extractAppFileName() = testSpecification?.run { | ||
| listOf( | ||
| androidInstrumentationTest, | ||
| androidTestLoop, | ||
| androidRoboTest, | ||
| iosXcTest, | ||
| iosTestLoop | ||
| ) | ||
| .firstOrNull { it != null } | ||
| ?.toJSONObject() | ||
| ?.let { prettyPrint.fromJson(it.toString(), AppPath::class.java).gcsPath } | ||
| ?.substringAfterLast('/') | ||
| } | ||
|
|
||
| private data class AppPath( | ||
| private val appApk: FileReference?, | ||
| private val testsZip: FileReference?, | ||
| private val appIpa: FileReference? | ||
| ) { | ||
| val gcsPath: String? | ||
| get() = (appApk ?: testsZip ?: appIpa)?.gcsPath | ||
| } |
15 changes: 15 additions & 0 deletions
15
test_runner/src/main/kotlin/ftl/adapter/google/SummaryAdapter.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| package ftl.adapter.google | ||
|
|
||
| import ftl.api.TestMatrix | ||
| import ftl.client.google.BillableMinutes | ||
| import ftl.client.google.TestOutcome | ||
|
|
||
| fun Pair<BillableMinutes, List<TestOutcome>>.toApiModel(): TestMatrix.Summary { | ||
| val (billableMinutes, outcomes) = this | ||
| return TestMatrix.Summary( | ||
| billableMinutes = billableMinutes.toApiModel(), | ||
| axes = outcomes.map(TestOutcome::toApiModel) | ||
| ) | ||
| } | ||
|
|
||
| private fun BillableMinutes.toApiModel() = TestMatrix.BillableMinutes(virtual, physical) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| package ftl.api | ||
|
|
||
| import com.google.testing.model.ToolResultsStep | ||
| import ftl.adapter.TestMatrixCancel | ||
| import ftl.adapter.TestMatrixFetch | ||
| import ftl.adapter.TestMatrixRefresh | ||
| import ftl.util.StepOutcome | ||
|
|
||
| val refreshTestMatrix: TestMatrix.Refresh get() = TestMatrixRefresh | ||
| val cancelTestMatrix: TestMatrix.Cancel get() = TestMatrixCancel | ||
| val fetchTestSummary: TestMatrix.Summary.Fetch get() = TestMatrixFetch | ||
|
|
||
| private const val fallbackAppName = "N/A" | ||
|
|
||
| object TestMatrix { | ||
|
|
||
| data class Result( | ||
| val runPath: String, | ||
| val map: Map<String, Data>, | ||
| ) | ||
|
|
||
| data class Data( | ||
| val projectId: String = "", | ||
| val matrixId: String = "", | ||
| val state: String = "", | ||
| val gcsPath: String = "", | ||
| val webLink: String = "", | ||
| val downloaded: Boolean = false, | ||
| val billableMinutes: BillableMinutes = BillableMinutes(), | ||
| val clientDetails: Map<String, String>? = null, | ||
| val gcsPathWithoutRootBucket: String = "", | ||
| val gcsRootBucket: String = "", | ||
| val webLinkWithoutExecutionDetails: String? = "", | ||
| val axes: List<Outcome> = emptyList(), | ||
| val appFileName: String = fallbackAppName, | ||
| val isCompleted: Boolean = false, | ||
| val testExecutions: List<TestExecution> = emptyList(), | ||
| val testTimeout: Long = 0, | ||
| val isRoboTest: Boolean = false, | ||
| val historyId: String = "", | ||
| val executionId: String = "", | ||
| val invalidMatrixDetails: String = "" | ||
| ) { | ||
| val outcome = axes.maxByOrNull { StepOutcome.order.indexOf(it.outcome) }?.outcome.orEmpty() | ||
| } | ||
|
|
||
| data class TestExecution( | ||
| val id: String, | ||
| val modelId: String, | ||
| val deviceVersion: String, | ||
| val shardIndex: Int?, | ||
| val state: String, | ||
| val errorMessage: String = "", | ||
| val progress: List<String> = emptyList(), | ||
| val toolResultsStep: ToolResultsStep? | ||
| ) | ||
|
|
||
| data class Outcome( | ||
| val device: String = "", | ||
| val outcome: String = "", | ||
| val details: String = "", | ||
| val suiteOverview: SuiteOverview = SuiteOverview() | ||
| ) | ||
|
|
||
| data class SuiteOverview( | ||
| val total: Int = 0, | ||
| val errors: Int = 0, | ||
| val failures: Int = 0, | ||
| val flakes: Int = 0, | ||
| val skipped: Int = 0, | ||
| val elapsedTime: Double = 0.0, | ||
| val overheadTime: Double = 0.0 | ||
| ) | ||
|
|
||
| data class BillableMinutes( | ||
| val virtual: Long = 0, | ||
| val physical: Long = 0 | ||
| ) | ||
|
|
||
| data class Summary( | ||
| val billableMinutes: BillableMinutes, | ||
| val axes: List<Outcome>, | ||
| ) { | ||
| interface Fetch : (Data) -> Summary | ||
| } | ||
|
|
||
| data class Identity( | ||
| val matrixId: String, | ||
| val projectId: String, | ||
| val historyId: String = "", | ||
| val executionId: String = "", | ||
| ) | ||
|
|
||
| interface Cancel : (Identity) -> Unit | ||
| interface Refresh : (Identity) -> Data | ||
| } |
Oops, something went wrong.