From 58420ba91ebf92b7e2843319bdf3943b2434ca42 Mon Sep 17 00:00:00 2001 From: Max Oliynick Date: Tue, 30 Nov 2021 13:34:39 +0200 Subject: [PATCH 1/3] Adds check for .aab format before querying apkDetails --- test_projects/android/bundle/app-debug.aab | 0 .../ftl/run/platform/RunAndroidTests.kt | 19 +++----- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 46 +++++++++++++++---- .../src/test/kotlin/ftl/run/TestRunnerTest.kt | 22 ++------- .../src/test/kotlin/ftl/util/MatricesMocks.kt | 33 +++++++++++++ 5 files changed, 79 insertions(+), 41 deletions(-) create mode 100644 test_projects/android/bundle/app-debug.aab create mode 100644 test_runner/src/test/kotlin/ftl/util/MatricesMocks.kt diff --git a/test_projects/android/bundle/app-debug.aab b/test_projects/android/bundle/app-debug.aab new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 9cef1cde40..0aee6be347 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -13,18 +13,8 @@ import ftl.args.shardsFilePath import ftl.client.google.getAndroidAppDetails import ftl.config.FtlConstants import ftl.run.exception.FlankGeneralError -import ftl.run.model.AndroidMatrixTestShards -import ftl.run.model.AndroidTestContext -import ftl.run.model.GameLoopContext -import ftl.run.model.InstrumentationTestContext -import ftl.run.model.RoboTestContext -import ftl.run.model.SanityRoboTestContext -import ftl.run.model.TestResult -import ftl.run.platform.android.asMatrixTestShards -import ftl.run.platform.android.createAndroidTestConfig -import ftl.run.platform.android.createAndroidTestContexts -import ftl.run.platform.android.createAndroidTestMatrixType -import ftl.run.platform.android.upload +import ftl.run.model.* +import ftl.run.platform.android.* import ftl.run.platform.common.afterRunTests import ftl.run.platform.common.beforeRunMessage import ftl.run.platform.common.beforeRunTests @@ -50,7 +40,10 @@ internal suspend fun AndroidArgs.runAndroidTests(): TestResult = coroutineScope ignoredTestsShardChunks += context.ignoredTestCases allTestShardChunks += context.shards } - context.reportPackageName() + + if (args.appApk?.endsWith(".aab") != true) { + context.reportPackageName() + } } .map { createTestSetup(it) } .run { executeTestMatrixAndroid(this) } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 1bda31327e..0ea55ec8f6 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -2,12 +2,14 @@ package ftl.args import com.google.common.truth.Truth.assertThat import com.google.testing.model.TestSpecification +import flank.tool.analytics.mixpanel.Mixpanel import ftl.args.IArgs.Companion.AVAILABLE_PHYSICAL_SHARD_COUNT_RANGE import ftl.args.IArgs.Companion.AVAILABLE_VIRTUAL_SHARD_COUNT_RANGE import ftl.args.yml.AppTestPair import ftl.args.yml.Type import ftl.client.google.GcStorage import ftl.client.google.GcStorage.exist +import ftl.client.google.getAndroidAppDetails import ftl.client.google.run.android.setupAndroidTest import ftl.config.Device import ftl.config.FtlConstants.defaultAndroidModel @@ -31,17 +33,12 @@ import ftl.test.util.TestHelper.getPath import ftl.test.util.TestHelper.getThrowable import ftl.test.util.assertThrowsWithMessage import ftl.util.asFileReference -import io.mockk.every -import io.mockk.mockkObject -import io.mockk.mockkStatic -import io.mockk.unmockkAll +import ftl.util.getMockedTestMatrix +import ftl.util.mockTestMatrices +import io.mockk.* import kotlinx.coroutines.runBlocking import org.junit.After -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertNotNull -import org.junit.Assert.assertTrue -import org.junit.Assert.fail +import org.junit.Assert.* import org.junit.Rule import org.junit.Test import org.junit.contrib.java.lang.system.SystemOutRule @@ -50,7 +47,7 @@ import picocli.CommandLine import java.io.StringReader import java.nio.file.Files import java.nio.file.Paths -import java.util.UUID +import java.util.* @Suppress("TooManyFunctions") @RunWith(FlankTestRunner::class) @@ -62,6 +59,7 @@ class AndroidArgsTest { private val empty = emptyList() private val appApk = "../test_projects/android/apks/app-debug.apk" + private val appAab = "../test_projects/android/bundle/app-debug.aab" private val invalidApk = "../test_projects/android/apks/invalid.apk" private val nonExistingApk = "../test_projects/android/apks/app-debug_non_existing.apk" private val testApk = "../test_projects/android/apks/app-debug-androidTest.apk" @@ -1717,6 +1715,34 @@ AndroidArgs } } + @Test + fun `should send no package name to mixpanel for aab format`() { + val yaml = """ + gcloud: + app: $appAab + test: $testApk + """.trimIndent() + + val parsedYml = AndroidArgs.load(yaml).validate() + + mockTestMatrices( + getMockedTestMatrix().apply { state = "RUNNING" }, + getMockedTestMatrix().apply { state = "RUNNING" }, + getMockedTestMatrix() + ) + + mockkStatic("ftl.client.google.AppDetailsKt") + every { + getAndroidAppDetails(any()) + } returns "com.example.test_app" + + mockkObject(Mixpanel) + + runBlocking { parsedYml.runAndroidTests() } + + verify(exactly = 0) { Mixpanel.add(any(), any()) } + } + @Test fun `results-dir (cloud directory) should not throw if it doesn't exist locally`() { val resultsDir = UUID.randomUUID().toString() diff --git a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt index 4ac1bce5ca..5e58497146 100644 --- a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt @@ -15,6 +15,8 @@ import ftl.client.google.getAndroidAppDetails import ftl.http.executeWithRetry import ftl.test.util.FlankTestRunner import ftl.test.util.LocalGcs +import ftl.util.getMockedTestMatrix +import ftl.util.mockTestMatrices import io.mockk.every import io.mockk.mockkStatic import io.mockk.unmockkAll @@ -155,10 +157,7 @@ class TestRunnerTest { @Test fun `flank should stop updating web link if matrix has invalid state`() { val localConfig = AndroidArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.local.yml")) - mockkStatic("ftl.http.ExecuteWithRetryKt") - every { - any().executeWithRetry() - } returnsMany listOf( + mockTestMatrices( getMockedTestMatrix().apply { state = "RUNNING" }, getMockedTestMatrix().apply { state = "RUNNING" }, getMockedTestMatrix() @@ -181,17 +180,4 @@ class TestRunnerTest { verify(exactly = 3) { any().executeWithRetry() } } - private fun getMockedTestMatrix() = TestMatrix().apply { - state = "INVALID" - testMatrixId = "matrix-12345" - testExecutions = listOf( - TestExecution().apply { - resultStorage = ResultStorage().apply { - googleCloudStorage = GoogleCloudStorage().apply { - gcsPath = "any/Path" - } - } - } - ) - } -} +} \ No newline at end of file diff --git a/test_runner/src/test/kotlin/ftl/util/MatricesMocks.kt b/test_runner/src/test/kotlin/ftl/util/MatricesMocks.kt new file mode 100644 index 0000000000..4097c06a06 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/util/MatricesMocks.kt @@ -0,0 +1,33 @@ +package ftl.util + +import com.google.testing.Testing +import com.google.testing.model.GoogleCloudStorage +import com.google.testing.model.ResultStorage +import com.google.testing.model.TestExecution +import com.google.testing.model.TestMatrix +import ftl.http.executeWithRetry +import io.mockk.every +import io.mockk.mockkStatic + +fun mockTestMatrices( + vararg mocks: TestMatrix +) { + mockkStatic("ftl.http.ExecuteWithRetryKt") + every { + any().executeWithRetry() + } returnsMany listOf(*mocks) +} + +fun getMockedTestMatrix() = TestMatrix().apply { + state = "INVALID" + testMatrixId = "matrix-12345" + testExecutions = listOf( + TestExecution().apply { + resultStorage = ResultStorage().apply { + googleCloudStorage = GoogleCloudStorage().apply { + gcsPath = "any/Path" + } + } + } + ) +} \ No newline at end of file From 0fba099bdfc6a7f036bf76818cc77569cf87874c Mon Sep 17 00:00:00 2001 From: Max Oliynick Date: Fri, 3 Dec 2021 15:09:11 +0200 Subject: [PATCH 2/3] PR fixes --- .../kotlin/ftl/client/google/AppDetails.kt | 8 ++++++-- .../kotlin/ftl/run/platform/RunAndroidTests.kt | 18 +++++++++++++----- .../test/kotlin/ftl/args/AndroidArgsTest.kt | 14 +++++++++++--- .../src/test/kotlin/ftl/run/TestRunnerTest.kt | 7 +------ .../src/test/kotlin/ftl/util/MatricesMocks.kt | 2 +- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/client/google/AppDetails.kt b/test_runner/src/main/kotlin/ftl/client/google/AppDetails.kt index 0a806b704e..118b74cd59 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/AppDetails.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/AppDetails.kt @@ -3,10 +3,14 @@ package ftl.client.google import com.google.testing.model.FileReference import ftl.http.executeWithRetry -fun getAndroidAppDetails(gcsAppPath: String): String = - GcTesting.get +fun getAndroidAppDetails(gcsAppPath: String): String { + // getApkDetails errors when sent non-apk files such as aab + if (gcsAppPath.trim().lowercase().endsWith(".apk").not()) return "" + + return GcTesting.get .ApplicationDetailService() .getApkDetails(FileReference().apply { gcsPath = gcsAppPath }) .apply { requestHeaders.set("X-Server-Timeout", 1800) } // 30 min .executeWithRetry() ?.apkDetail?.apkManifest?.packageName?.toString().orEmpty() +} diff --git a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt index 0aee6be347..d77b17fa5b 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/RunAndroidTests.kt @@ -13,8 +13,18 @@ import ftl.args.shardsFilePath import ftl.client.google.getAndroidAppDetails import ftl.config.FtlConstants import ftl.run.exception.FlankGeneralError -import ftl.run.model.* -import ftl.run.platform.android.* +import ftl.run.model.AndroidMatrixTestShards +import ftl.run.model.AndroidTestContext +import ftl.run.model.GameLoopContext +import ftl.run.model.InstrumentationTestContext +import ftl.run.model.RoboTestContext +import ftl.run.model.SanityRoboTestContext +import ftl.run.model.TestResult +import ftl.run.platform.android.asMatrixTestShards +import ftl.run.platform.android.createAndroidTestConfig +import ftl.run.platform.android.createAndroidTestContexts +import ftl.run.platform.android.createAndroidTestMatrixType +import ftl.run.platform.android.upload import ftl.run.platform.common.afterRunTests import ftl.run.platform.common.beforeRunMessage import ftl.run.platform.common.beforeRunTests @@ -41,9 +51,7 @@ internal suspend fun AndroidArgs.runAndroidTests(): TestResult = coroutineScope allTestShardChunks += context.shards } - if (args.appApk?.endsWith(".aab") != true) { - context.reportPackageName() - } + context.reportPackageName() } .map { createTestSetup(it) } .run { executeTestMatrixAndroid(this) } diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 0ea55ec8f6..8a2541afaa 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -35,10 +35,18 @@ import ftl.test.util.assertThrowsWithMessage import ftl.util.asFileReference import ftl.util.getMockedTestMatrix import ftl.util.mockTestMatrices -import io.mockk.* +import io.mockk.every +import io.mockk.mockkObject +import io.mockk.mockkStatic +import io.mockk.unmockkAll +import io.mockk.verify import kotlinx.coroutines.runBlocking import org.junit.After -import org.junit.Assert.* +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.Assert.fail import org.junit.Rule import org.junit.Test import org.junit.contrib.java.lang.system.SystemOutRule @@ -47,7 +55,7 @@ import picocli.CommandLine import java.io.StringReader import java.nio.file.Files import java.nio.file.Paths -import java.util.* +import java.util.UUID @Suppress("TooManyFunctions") @RunWith(FlankTestRunner::class) diff --git a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt index 5e58497146..110b90e146 100644 --- a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt @@ -2,10 +2,6 @@ package ftl.run import com.google.common.truth.Truth.assertThat import com.google.testing.Testing -import com.google.testing.model.GoogleCloudStorage -import com.google.testing.model.ResultStorage -import com.google.testing.model.TestExecution -import com.google.testing.model.TestMatrix import flank.common.isWindows import ftl.adapter.google.getFilePathToDownload import ftl.api.Artifacts.DownloadPath @@ -179,5 +175,4 @@ class TestRunnerTest { assertFalse(output.contains(matrixLink)) verify(exactly = 3) { any().executeWithRetry() } } - -} \ No newline at end of file +} diff --git a/test_runner/src/test/kotlin/ftl/util/MatricesMocks.kt b/test_runner/src/test/kotlin/ftl/util/MatricesMocks.kt index 4097c06a06..eb3935f186 100644 --- a/test_runner/src/test/kotlin/ftl/util/MatricesMocks.kt +++ b/test_runner/src/test/kotlin/ftl/util/MatricesMocks.kt @@ -30,4 +30,4 @@ fun getMockedTestMatrix() = TestMatrix().apply { } } ) -} \ No newline at end of file +} From 9a0ac14b7ee8b1a0f4792d9de75645a1184605d1 Mon Sep 17 00:00:00 2001 From: Max Oliynick Date: Tue, 21 Dec 2021 13:20:31 +0200 Subject: [PATCH 3/3] Update test --- test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 8a2541afaa..024e53b6cc 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -9,7 +9,6 @@ import ftl.args.yml.AppTestPair import ftl.args.yml.Type import ftl.client.google.GcStorage import ftl.client.google.GcStorage.exist -import ftl.client.google.getAndroidAppDetails import ftl.client.google.run.android.setupAndroidTest import ftl.config.Device import ftl.config.FtlConstants.defaultAndroidModel @@ -1739,16 +1738,11 @@ AndroidArgs getMockedTestMatrix() ) - mockkStatic("ftl.client.google.AppDetailsKt") - every { - getAndroidAppDetails(any()) - } returns "com.example.test_app" - mockkObject(Mixpanel) runBlocking { parsedYml.runAndroidTests() } - verify(exactly = 0) { Mixpanel.add(any(), any()) } + verify { Mixpanel.add(Mixpanel.APP_ID, "") } } @Test