-
Notifications
You must be signed in to change notification settings - Fork 118
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: Refactor data scratch-Android run (#1941)
Fixes #1754 ## Test Plan > How do we know the code works? - code is refactored according to the description provided in #1754 with a change `interface Execute : (Config, Type) -> TestMatrix.Result` -> `interface Execute : (Config, List<Type>) -> <TestMatrix.Data>` - all runs of `flank firebase test android run ... ` work without any problems like previously ## Checklist - [x] Unit tested
- Loading branch information
1 parent
569df1c
commit 5a0acf2
Showing
35 changed files
with
496 additions
and
457 deletions.
There are no files selected for viewing
16 changes: 16 additions & 0 deletions
16
test_runner/src/main/kotlin/ftl/adapter/GoogleTestMatrixAndroid.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package ftl.adapter | ||
|
|
||
| import ftl.adapter.google.toApiModel | ||
| import ftl.api.TestMatrix | ||
| import ftl.api.TestMatrixAndroid | ||
| import ftl.client.google.run.android.executeAndroidTests | ||
| import kotlinx.coroutines.runBlocking | ||
| import com.google.testing.model.TestMatrix as GoogleTestMatrix | ||
|
|
||
| object GoogleTestMatrixAndroid : | ||
| TestMatrixAndroid.Execute, | ||
| (TestMatrixAndroid.Config, List<TestMatrixAndroid.Type>) -> List<TestMatrix.Data> by { config, types -> | ||
| runBlocking { | ||
| executeAndroidTests(config, types).map(GoogleTestMatrix::toApiModel) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| package ftl.api | ||
|
|
||
| import ftl.adapter.GoogleTestMatrixAndroid | ||
| import ftl.args.FlankRoboDirective | ||
| import ftl.config.Device | ||
|
|
||
| val executeTestMatrixAndroid: TestMatrixAndroid.Execute get() = GoogleTestMatrixAndroid | ||
|
|
||
| object TestMatrixAndroid { | ||
|
|
||
| data class Config( | ||
| // args | ||
| val clientDetails: Map<String, String>?, | ||
| val resultsBucket: String, | ||
| val autoGoogleLogin: Boolean, | ||
| val networkProfile: String?, | ||
| val directoriesToPull: List<String>, | ||
| val obbNames: List<String>, | ||
| val environmentVariables: Map<String, String>, | ||
| val autoGrantPermissions: Boolean, | ||
| val testTimeout: String, | ||
| val performanceMetrics: Boolean, | ||
| val recordVideo: Boolean, | ||
| val flakyTestAttempts: Int, | ||
| val failFast: Boolean, | ||
| val project: String, | ||
| val resultsHistoryName: String?, | ||
| val repeatTests: Int, | ||
|
|
||
| // build | ||
| val otherFiles: Map<String, String>, | ||
| val resultsDir: String, | ||
| val devices: List<Device>, | ||
| val additionalApkGcsPaths: List<String>, | ||
| val obbFiles: Map<String, String>, | ||
| ) | ||
|
|
||
| sealed class Type { | ||
| data class Instrumentation( | ||
| val appApkGcsPath: String, | ||
| val testApkGcsPath: String, | ||
| val testRunnerClass: String?, | ||
| val orchestratorOption: String?, | ||
| // sharding | ||
| val disableSharding: Boolean, | ||
| val testShards: ShardChunks, | ||
| val numUniformShards: Int?, | ||
| val keepTestTargetsEmpty: Boolean, | ||
| val environmentVariables: Map<String, String> = emptyMap(), | ||
| val testTargetsForShard: ShardChunks | ||
| ) : Type() | ||
|
|
||
| data class Robo( | ||
| val appApkGcsPath: String, | ||
| val flankRoboDirectives: List<FlankRoboDirective>?, | ||
| val roboScriptGcsPath: String? | ||
| ) : Type() | ||
|
|
||
| data class GameLoop( | ||
| val appApkGcsPath: String, | ||
| val testRunnerClass: String?, | ||
| val scenarioNumbers: List<String>, | ||
| val scenarioLabels: List<String> | ||
| ) : Type() | ||
| } | ||
|
|
||
| interface Execute : (Config, List<Type>) -> List<TestMatrix.Data> | ||
| } | ||
|
|
||
| typealias ShardChunks = List<List<String>> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
28 changes: 28 additions & 0 deletions
28
test_runner/src/main/kotlin/ftl/client/google/run/Utils.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package ftl.client.google.run | ||
|
|
||
| import com.google.testing.model.ClientInfoDetail | ||
| import com.google.testing.model.FileReference | ||
| import com.google.testing.model.IosDeviceFile | ||
|
|
||
| internal fun List<String>.mapGcsPathsToFileReference(): List<FileReference> = map { it.toFileReference() } | ||
|
|
||
| fun String.toFileReference(): FileReference = FileReference().setGcsPath(this) | ||
|
|
||
| internal fun Map<String, String>.mapToIosDeviceFiles(): List<IosDeviceFile> = map { (testDevicePath, gcsFilePath) -> toIosDeviceFile(testDevicePath, gcsFilePath) } | ||
|
|
||
| internal fun toIosDeviceFile(testDevicePath: String, gcsFilePath: String = "") = IosDeviceFile().apply { | ||
| if (testDevicePath.contains(":")) { | ||
| val (bundleIdSeparated, pathSeparated) = testDevicePath.split(":") | ||
| bundleId = bundleIdSeparated | ||
| devicePath = pathSeparated | ||
| } else { | ||
| devicePath = testDevicePath | ||
| } | ||
| content = FileReference().setGcsPath(gcsFilePath) | ||
| } | ||
|
|
||
| internal fun Map<String, String>.toClientInfoDetailList() = map { (key, value) -> | ||
| ClientInfoDetail() | ||
| .setKey(key) | ||
| .setValue(value) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 3 additions & 3 deletions
6
...n/ftl/gc/android/CreateAndroidLoopTest.kt → ...ogle/run/android/CreateAndroidLoopTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 3 additions & 3 deletions
6
.../ftl/gc/android/CreateAndroidRobotTest.kt → ...gle/run/android/CreateAndroidRobotTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
...src/main/kotlin/ftl/gc/GcAndroidDevice.kt → ...ent/google/run/android/GcAndroidDevice.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
144 changes: 144 additions & 0 deletions
144
test_runner/src/main/kotlin/ftl/client/google/run/android/GcAndroidTestMatrix.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| package ftl.client.google.run.android | ||
|
|
||
| import com.google.common.annotations.VisibleForTesting | ||
| import com.google.testing.Testing | ||
| import com.google.testing.model.Account | ||
| import com.google.testing.model.Apk | ||
| import com.google.testing.model.ClientInfo | ||
| import com.google.testing.model.DeviceFile | ||
| import com.google.testing.model.EnvironmentMatrix | ||
| import com.google.testing.model.FileReference | ||
| import com.google.testing.model.GoogleAuto | ||
| import com.google.testing.model.GoogleCloudStorage | ||
| import com.google.testing.model.ObbFile | ||
| import com.google.testing.model.RegularFile | ||
| import com.google.testing.model.ResultStorage | ||
| import com.google.testing.model.TestMatrix | ||
| import com.google.testing.model.TestSetup | ||
| import com.google.testing.model.TestSpecification | ||
| import com.google.testing.model.ToolResultsHistory | ||
| import flank.common.join | ||
| import ftl.api.TestMatrixAndroid | ||
| import ftl.client.google.GcTesting | ||
| import ftl.client.google.run.toClientInfoDetailList | ||
| import ftl.client.google.run.toFileReference | ||
| import ftl.http.executeWithRetry | ||
| import ftl.run.exception.FlankGeneralError | ||
| import ftl.util.timeoutToSeconds | ||
| import kotlinx.coroutines.Deferred | ||
| import kotlinx.coroutines.Dispatchers | ||
| import kotlinx.coroutines.async | ||
| import kotlinx.coroutines.awaitAll | ||
| import kotlinx.coroutines.coroutineScope | ||
|
|
||
| suspend fun executeAndroidTests( | ||
| config: TestMatrixAndroid.Config, | ||
| testMatrixTypes: List<TestMatrixAndroid.Type>, | ||
| ): List<TestMatrix> = testMatrixTypes | ||
| .foldIndexed(emptyList<Deferred<TestMatrix>>()) { testMatrixTypeIndex, testMatrices, testMatrixType -> | ||
| testMatrices + executeAndroidTestMatrix(testMatrixType, testMatrixTypeIndex, config) | ||
| }.awaitAll() | ||
|
|
||
| private suspend fun executeAndroidTestMatrix( | ||
| type: TestMatrixAndroid.Type, | ||
| typeIndex: Int, | ||
| config: TestMatrixAndroid.Config | ||
| ): List<Deferred<TestMatrix>> = coroutineScope { | ||
| (0 until config.repeatTests).map { runIndex -> | ||
| async(Dispatchers.IO) { | ||
| createAndroidTestMatrix(type, config, typeIndex, runIndex).executeWithRetry() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private fun createAndroidTestMatrix( | ||
| testMatrixType: TestMatrixAndroid.Type, | ||
| config: TestMatrixAndroid.Config, | ||
| contextIndex: Int, | ||
| runIndex: Int | ||
| ): Testing.Projects.TestMatrices.Create { | ||
|
|
||
| val testMatrix = TestMatrix() | ||
| .setClientInfo(config.clientInfo) | ||
| .setTestSpecification(getTestSpecification(testMatrixType, config)) | ||
| .setResultStorage(config.resultsStorage(contextIndex, runIndex)) | ||
| .setEnvironmentMatrix(config.environmentMatrix) | ||
| .setFlakyTestAttempts(config.flakyTestAttempts) | ||
| .setFailFast(config.failFast) | ||
|
|
||
| return runCatching { | ||
| GcTesting.get.projects().testMatrices().create(config.project, testMatrix) | ||
| }.getOrElse { e -> throw FlankGeneralError(e) } | ||
| } | ||
|
|
||
| // https://github.com/bootstraponline/studio-google-cloud-testing/blob/203ed2890c27a8078cd1b8f7ae12cf77527f426b/firebase-testing/src/com/google/gct/testing/launcher/CloudTestsLauncher.java#L120 | ||
| private val TestMatrixAndroid.Config.clientInfo | ||
| get() = ClientInfo() | ||
| .setName("Flank") | ||
| .setClientInfoDetails(clientDetails?.toClientInfoDetailList()) | ||
|
|
||
| private val TestMatrixAndroid.Config.environmentMatrix | ||
| get() = EnvironmentMatrix() | ||
| .setAndroidDeviceList(GcAndroidDevice.build(devices)) | ||
|
|
||
| private fun getTestSpecification( | ||
| testMatrixType: TestMatrixAndroid.Type, | ||
| config: TestMatrixAndroid.Config | ||
| ): TestSpecification = TestSpecification() | ||
| .setDisablePerformanceMetrics(!config.performanceMetrics) | ||
| .setDisableVideoRecording(!config.recordVideo) | ||
| .setTestTimeout("${timeoutToSeconds(config.testTimeout)}s") | ||
| .setTestSetup(getTestSetup(testMatrixType, config)) | ||
| .setupAndroidTest(testMatrixType) | ||
|
|
||
| private fun getTestSetup( | ||
| testMatrixType: TestMatrixAndroid.Type, | ||
| config: TestMatrixAndroid.Config | ||
| ): TestSetup = TestSetup() | ||
| .setAccount(config.account) | ||
| .setNetworkProfile(config.networkProfile) | ||
| .setDirectoriesToPull(config.directoriesToPull) | ||
| .setAdditionalApks(config.additionalApkGcsPaths.mapGcsPathsToApks()) | ||
| .setFilesToPush(config.otherFiles.mapToDeviceFiles() + config.obbFiles.mapToDeviceObbFiles(config.obbNames)) | ||
| .setDontAutograntPermissions(config.autoGrantPermissions.not()) | ||
| .setEnvironmentVariables(config.environmentVariables, testMatrixType) | ||
|
|
||
| internal fun List<String>.mapGcsPathsToApks(): List<Apk>? = this | ||
| .map { gcsPath -> Apk().setLocation(gcsPath.toFileReference()) } | ||
| .takeIf { it.isNotEmpty() } | ||
|
|
||
| private fun Map<String, String>.mapToDeviceFiles(): List<DeviceFile> = | ||
| map { (devicePath: String, gcsFilePath: String) -> | ||
| DeviceFile().setRegularFile( | ||
| RegularFile() | ||
| .setDevicePath(devicePath) | ||
| .setContent(gcsFilePath.toFileReference()) | ||
| ) | ||
| } | ||
|
|
||
| private fun Map<String, String>.mapToDeviceObbFiles(obbnames: List<String>): List<DeviceFile> { | ||
| return values.mapIndexed { index, gcsFilePath -> | ||
| DeviceFile().setObbFile( | ||
| ObbFile().setObb(FileReference().setGcsPath(gcsFilePath)).setObbFileName(obbnames[index]) | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| private fun TestMatrixAndroid.Config.resultsStorage( | ||
| contextIndex: Int, | ||
| runIndex: Int | ||
| ) = ResultStorage() | ||
| .setGoogleCloudStorage( | ||
| GoogleCloudStorage().setGcsPath(join(resultsBucket, resultsDir.createGcsPath(contextIndex, runIndex))) | ||
| ) | ||
| .setToolResultsHistory( | ||
| ToolResultsHistory().setHistoryId(resultsHistoryName).setProjectId(project) | ||
| ) | ||
|
|
||
| @VisibleForTesting | ||
| internal fun String.createGcsPath(contextIndex: Int, runIndex: Int) = | ||
| if (runIndex == 0) "$this/matrix_$contextIndex/" | ||
| else "$this/matrix_${contextIndex}_$runIndex/" | ||
|
|
||
| private val TestMatrixAndroid.Config.account | ||
| get() = if (autoGoogleLogin) Account().setGoogleAuto(GoogleAuto()) else null |
12 changes: 12 additions & 0 deletions
12
test_runner/src/main/kotlin/ftl/client/google/run/android/SetupAndroidTest.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| package ftl.client.google.run.android | ||
|
|
||
| import com.google.testing.model.TestSpecification | ||
| import ftl.api.TestMatrixAndroid | ||
|
|
||
| internal fun TestSpecification.setupAndroidTest(config: TestMatrixAndroid.Type) = apply { | ||
| when (config) { | ||
| is TestMatrixAndroid.Type.Instrumentation -> androidInstrumentationTest = createAndroidInstrumentationTest(config) | ||
| is TestMatrixAndroid.Type.Robo -> androidRoboTest = createAndroidRoboTest(config) | ||
| is TestMatrixAndroid.Type.GameLoop -> androidTestLoop = createGameLoopTest(config) | ||
| } | ||
| } |
15 changes: 7 additions & 8 deletions
15
...l/gc/android/SetupEnvironmentVariables.kt → .../run/android/SetupEnvironmentVariables.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.