diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index b0e1317f7f..745caf2cd1 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -223,6 +223,7 @@ object ArgsHelper { val shardCount = Shard.shardCountByTime(filteredTests, oldTestResult, args) + // TODO: manual sharding must be limited to 1..50 tests per shard val shards = Shard.createShardsByShardCount(filteredTests, oldTestResult, args, shardCount) return testMethodsAlwaysRun(shards.stringShards(), args) diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index 0b4bdfb228..59c7cc777a 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -1,20 +1,7 @@ package ftl.gc import com.google.api.services.testing.Testing -import com.google.api.services.testing.model.Account -import com.google.api.services.testing.model.AndroidDeviceList -import com.google.api.services.testing.model.AndroidInstrumentationTest -import com.google.api.services.testing.model.ClientInfo -import com.google.api.services.testing.model.EnvironmentMatrix -import com.google.api.services.testing.model.EnvironmentVariable -import com.google.api.services.testing.model.FileReference -import com.google.api.services.testing.model.GoogleAuto -import com.google.api.services.testing.model.GoogleCloudStorage -import com.google.api.services.testing.model.ResultStorage -import com.google.api.services.testing.model.TestMatrix -import com.google.api.services.testing.model.TestSetup -import com.google.api.services.testing.model.TestSpecification -import com.google.api.services.testing.model.ToolResultsHistory +import com.google.api.services.testing.model.* import ftl.args.AndroidArgs import ftl.util.ShardCounter import ftl.util.Utils.fatalError @@ -33,7 +20,7 @@ object GcAndroidTestMatrix { testApkGcsPath: String, runGcsPath: String, androidDeviceList: AndroidDeviceList, - testTargets: List, + testTargets: List>, args: AndroidArgs, shardCounter: ShardCounter, toolResultsHistory: ToolResultsHistory @@ -44,9 +31,17 @@ object GcAndroidTestMatrix { val matrixGcsPath = join(args.resultsBucket, runGcsPath, shardCounter.next()) + // ShardingOption().setUniformSharding(UniformSharding().setNumShards()) + val testTargetsForShard: List = testTargets.map { + TestTargetsForShard().setTestTargets(it) + } + val manualSharding = ManualSharding().setTestTargetsForShard(testTargetsForShard) + val shardingOption = ShardingOption().setManualSharding(manualSharding) + val androidInstrumentation = AndroidInstrumentationTest() .setAppApk(FileReference().setGcsPath(appApkGcsPath)) .setTestApk(FileReference().setGcsPath(testApkGcsPath)) + .setShardingOption(shardingOption) if (args.testRunnerClass != null) { androidInstrumentation.testRunnerClass = args.testRunnerClass @@ -56,8 +51,6 @@ object GcAndroidTestMatrix { androidInstrumentation.orchestratorOption = "USE_ORCHESTRATOR" } - androidInstrumentation.testTargets = testTargets - // --auto-google-login // https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run // https://github.com/bootstraponline/gcloud_cli/blob/e4b5e01610abad2e31d8a6edb20b17b2f84c5395/google-cloud-sdk/lib/googlecloudsdk/api_lib/firebase/test/android/matrix_creator.py#L174 diff --git a/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt b/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt index 6e29262316..f51a9edf8c 100644 --- a/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt +++ b/test_runner/src/main/kotlin/ftl/run/AndroidTestRunner.kt @@ -38,21 +38,19 @@ object AndroidTestRunner { val testShardChunks = AndroidTestShard.getTestShardChunks(androidArgs, apk.test) allTestShardChunks += testShardChunks repeat(runCount) { - testShardChunks.forEach { testTargets -> - // specify dispatcher to avoid inheriting main runBlocking context that runs in the main thread - // https://kotlinlang.org/docs/reference/coroutines/coroutine-context-and-dispatchers.html - jobs += async(Dispatchers.IO) { - GcAndroidTestMatrix.build( - appApkGcsPath = apk.app ?: androidArgs.appApk, - testApkGcsPath = apk.test, - runGcsPath = runGcsPath, - androidDeviceList = androidDeviceList, - testTargets = testTargets, - args = androidArgs, - shardCounter = shardCounter, - toolResultsHistory = history - ).executeWithRetry() - } + // specify dispatcher to avoid inheriting main runBlocking context that runs in the main thread + // https://kotlinlang.org/docs/reference/coroutines/coroutine-context-and-dispatchers.html + jobs += async(Dispatchers.IO) { + GcAndroidTestMatrix.build( + appApkGcsPath = apk.app ?: androidArgs.appApk, + testApkGcsPath = apk.test, + runGcsPath = runGcsPath, + androidDeviceList = androidDeviceList, + testTargets = testShardChunks, + args = androidArgs, + shardCounter = shardCounter, + toolResultsHistory = history + ).executeWithRetry() } } } @@ -73,7 +71,8 @@ object AndroidTestRunner { val result = mutableListOf() appTestApks.forEach { apks -> - val appApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.app ?: args.appApk, gcsBucket, runGcsPath) } + val appApkGcsPath = + async(Dispatchers.IO) { GcStorage.upload(apks.app ?: args.appApk, gcsBucket, runGcsPath) } val testApkGcsPath = async(Dispatchers.IO) { GcStorage.upload(apks.test, gcsBucket, runGcsPath) } result.add( diff --git a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt index ff8a243a0b..18a1ab0eb0 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcAndroidTestMatrixTest.kt @@ -37,7 +37,7 @@ class GcAndroidTestMatrixTest { testApkGcsPath = "", runGcsPath = "", androidDeviceList = AndroidDeviceList(), - testTargets = listOf(""), + testTargets = listOf(listOf("")), args = androidArgs, shardCounter = ShardCounter(), toolResultsHistory = createToolResultsHistory(androidArgs)