diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt index 8164c7bf0b..6598be8af1 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateAndroidArgs.kt @@ -21,6 +21,7 @@ fun AndroidArgs.validate() = apply { assertParametersConflict() assertTestFiles() assertOtherFiles() + checkResultsDirUnique() } private fun AndroidArgs.assertDevicesSupported() = devices diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt index 7433ef0c33..4fe8784a95 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt @@ -2,6 +2,7 @@ package ftl.args import ftl.config.Device import ftl.config.FtlConstants +import ftl.gc.GcStorage import ftl.reports.FullJUnitReport import ftl.reports.JUnitReport import ftl.run.exception.FlankConfigurationError @@ -28,8 +29,8 @@ private fun List.throwIfAnyMisspelt() = private fun CommonArgs.assertProjectId() { if (project.isEmpty()) throw FlankConfigurationError( "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" + - "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" + - " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" + "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" + + " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" ) } @@ -66,3 +67,8 @@ private fun CommonArgs.assertSmartFlankGcsPath() = with(smartFlankGcsPath) { ) } } + +fun IArgs.checkResultsDirUnique() { + if (useLegacyJUnitResult && GcStorage.exist(resultsBucket, resultsDir)) + println("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled\n") +} diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt index 1eb9b3d21b..b80069f493 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateIosArgs.kt @@ -11,6 +11,7 @@ fun IosArgs.validate() = apply { assertTestTypes() assertMaxTestShards() assertTestFiles() + checkResultsDirUnique() } fun IosArgs.validateRefresh() = apply { diff --git a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt index ddfc4b7c3f..5fcea2fbf7 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcStorage.kt @@ -3,6 +3,8 @@ package ftl.gc import com.google.api.client.http.GoogleApiLogger import com.google.cloud.storage.BlobInfo import com.google.cloud.storage.Storage +import com.google.cloud.storage.Storage.BlobListOption.pageSize +import com.google.cloud.storage.Storage.BlobListOption.prefix import com.google.cloud.storage.StorageOptions import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper import com.google.common.annotations.VisibleForTesting @@ -172,4 +174,10 @@ object GcStorage { outputFile.path } } + + fun exist( + rootGcsBucket: String, + runGcsPath: String, + storage: Storage = GcStorage.storage + ) = storage.list(rootGcsBucket, pageSize(1), prefix("$runGcsPath/")).values.count() > 0 } diff --git a/test_runner/src/main/kotlin/ftl/reports/util/MatrixPath.kt b/test_runner/src/main/kotlin/ftl/reports/util/MatrixPath.kt index fb1898f69c..7665d0c452 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/MatrixPath.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/MatrixPath.kt @@ -3,9 +3,16 @@ package ftl.reports.util import java.io.File import java.nio.file.Paths -fun File.getMatrixPath(objectName: String) = getShardName()?.asObjectPath(objectName) +fun File.getMatrixPath(objectName: String) = getShardName(objectName)?.asObjectPath() -private fun File.getShardName() = shardNameRegex.find(toString())?.value?.removePrefix("/")?.removeSuffix("/") -private val shardNameRegex = "/.*(shard_|matrix_)\\d+(-rerun_\\d+)?/".toRegex() +private fun File.getShardName( + objectName: String +) = shardNameRegex(objectName) + .find(toString()) + ?.value + ?.removePrefix("/") + ?.removeSuffix("/") -private fun String.asObjectPath(objectName: String) = Paths.get(objectName, this).toString() +private fun shardNameRegex(objectName: String) = "/($objectName)/(shard_|matrix_)\\d+(-rerun_\\d+)?/".toRegex() + +private fun String.asObjectPath() = Paths.get(this).toString() diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt index 80cd1c5670..47b8e0c006 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt @@ -28,21 +28,19 @@ import kotlin.math.roundToInt object ReportManager { - private fun findXmlFiles(matrices: MatrixMap, args: IArgs): List { - val xmlFiles = mutableListOf() - val rootFolder = File(resolveLocalRunPath(matrices, args)) - - rootFolder.walk().forEach { - if (it.name.matches(Artifacts.testResultRgx)) { - xmlFiles.add(it) - } + private fun findXmlFiles( + matrices: MatrixMap, + args: IArgs + ) = File(resolveLocalRunPath(matrices, args)).walk() + .filter { it.name.matches(Artifacts.testResultRgx) } + .fold(listOf()) { xmlFiles, file -> + xmlFiles + file } - return xmlFiles - } - - private fun getWebLink(matrices: MatrixMap, xmlFile: File): String = xmlFile.getMatrixPath(matrices.runPath) - ?.findMatrixPath(matrices) ?: "".also { println("WARNING: Matrix path not found in JSON.") } + private fun getWebLink(matrices: MatrixMap, xmlFile: File): String = + xmlFile.getMatrixPath(matrices.runPath) + ?.findMatrixPath(matrices) + ?: "".also { println("WARNING: Matrix path not found in JSON.") } private val deviceStringRgx = Regex("([^-]+-[^-]+-[^-]+-[^-]+).*") @@ -201,6 +199,10 @@ object ReportManager { } } -private fun String.findMatrixPath(matrices: MatrixMap) = - matrices.map.values.firstOrNull { savedMatrix -> savedMatrix.gcsPath.endsWith(this) }?.webLink - ?: "".also { println("WARNING: Matrix path not found in JSON. $this") } +private fun String.findMatrixPath(matrices: MatrixMap) = matrices.map.values + .firstOrNull { savedMatrix -> savedMatrix.gcsPath.endsWithTextWithOptionalSlashAtTheEnd(this) } + ?.webLink + ?: "".also { println("WARNING: Matrix path not found in JSON. $this") } + +@VisibleForTesting +internal fun String.endsWithTextWithOptionalSlashAtTheEnd(text: String) = "($text)/*$".toRegex().containsMatchIn(this) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index 121388c4eb..5ed6ee527b 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -9,6 +9,8 @@ import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.config.Device import ftl.config.FtlConstants.defaultAndroidModel import ftl.config.FtlConstants.defaultAndroidVersion +import ftl.gc.GcStorage +import ftl.gc.GcStorage.exist import ftl.gc.android.setupAndroidTest import ftl.run.model.InstrumentationTestContext import ftl.run.platform.android.createAndroidTestConfig @@ -25,6 +27,7 @@ import ftl.run.exception.FlankConfigurationError import ftl.run.exception.IncompatibleTestDimensionError import ftl.util.asFileReference import io.mockk.every +import io.mockk.mockkObject import io.mockk.mockkStatic import io.mockk.unmockkAll import kotlinx.coroutines.runBlocking @@ -33,7 +36,9 @@ import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Assert.fail +import org.junit.Rule import org.junit.Test +import org.junit.contrib.java.lang.system.SystemOutRule import org.junit.runner.RunWith import picocli.CommandLine import java.io.StringReader @@ -44,6 +49,11 @@ import java.util.UUID @Suppress("TooManyFunctions") @RunWith(FlankTestRunner::class) class AndroidArgsTest { + + @Rule + @JvmField + val systemOutRule: SystemOutRule = SystemOutRule().enableLog() + private val empty = emptyList() private val appApk = "../test_projects/android/apks/app-debug.apk" private val invalidApk = "../test_projects/android/apks/invalid.apk" @@ -1596,6 +1606,7 @@ AndroidArgs """.trimIndent() AndroidArgs.load(yaml).validate() } + @Test fun `should not throw when --legacy-junit-result set and --full-junit-result not set`() { val yaml = """ @@ -1768,6 +1779,69 @@ AndroidArgs """.trimIndent() AndroidArgs.load(yaml).validate() } + + @Test + fun `should show warning message when bucket exist and legacy junit result used`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + results-dir: test + flank: + max-test-shards: -1 + legacy-junit-result: true + """.trimIndent() + mockkObject(GcStorage) { + every { exist(any(), any(), any()) } returns true + + // when + AndroidArgs.load(yaml).validate() + + // then + assertTrue(systemOutRule.log.contains("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled")) + } + } + + @Test + fun `should not print Google cloud storage warning when legacy junit results is false`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + results-dir: test + flank: + max-test-shards: -1 + legacy-junit-result: false + """.trimIndent() + + // when + AndroidArgs.load(yaml).validate() + + // then + assertFalse(systemOutRule.log.contains("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled")) + } + + @Test + fun `should not print result-dir warning when same directory does not exist`() { + val yaml = """ + gcloud: + app: $appApk + test: $testApk + results-dir: test + flank: + max-test-shards: -1 + legacy-junit-result: true + """.trimIndent() + mockkObject(GcStorage) { + every { exist(any(), any(), any()) } returns false + + // when + AndroidArgs.load(yaml).validate() + + // then + assertFalse(systemOutRule.log.contains("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled")) + } + } } private fun AndroidArgs.Companion.load(yamlData: String, cli: AndroidRunCommand? = null): AndroidArgs = diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 6525ce2986..72c803ef83 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -7,20 +7,24 @@ import ftl.config.FtlConstants import ftl.config.FtlConstants.defaultIosModel import ftl.config.FtlConstants.defaultIosVersion import ftl.config.defaultIosConfig +import ftl.gc.GcStorage +import ftl.run.exception.FlankConfigurationError import ftl.run.status.OutputStyle import ftl.test.util.FlankTestRunner import ftl.test.util.TestHelper.absolutePath import ftl.test.util.TestHelper.assert import ftl.test.util.TestHelper.getPath import ftl.test.util.assertThrowsWithMessage -import ftl.run.exception.FlankConfigurationError -import org.junit.Assert +import io.mockk.every +import io.mockk.mockkObject import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue import org.junit.Assume import org.junit.Rule import org.junit.Test import org.junit.contrib.java.lang.system.SystemErrRule +import org.junit.contrib.java.lang.system.SystemOutRule import org.junit.runner.RunWith import picocli.CommandLine import java.io.StringReader @@ -93,6 +97,10 @@ class IosArgsTest { @JvmField val systemErrRule = SystemErrRule().muteForSuccessfulTests()!! + @Rule + @JvmField + val systemOutRule: SystemOutRule = SystemOutRule().enableLog() + @Test fun `empty testTargets`() { val emptyTestTargets = """ @@ -1032,7 +1040,7 @@ IosArgs use-average-test-time-for-new-tests: true """.trimIndent() val args = IosArgs.load(yaml) - Assert.assertTrue(args.useAverageTestTimeForNewTests) + assertTrue(args.useAverageTestTimeForNewTests) } @Test @@ -1047,6 +1055,48 @@ IosArgs val args = IosArgs.load(yaml) assertFalse(args.useAverageTestTimeForNewTests) } + + @Test + fun `should show warning message when results directory exist`() { + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $testPath + results-dir: test + flank: + max-test-shards: -1 + """.trimIndent() + mockkObject(GcStorage) { + every { GcStorage.exist(any(), any(), any()) } returns true + + // when + IosArgs.load(yaml).validate() + + // then + assertTrue(systemOutRule.log.contains("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled")) + } + } + + @Test + fun `should not print result-dir warning when same directory does not exist`() { + val yaml = """ + gcloud: + test: $testPath + xctestrun-file: $testPath + results-dir: test + flank: + max-test-shards: -1 + """.trimIndent() + mockkObject(GcStorage) { + every { GcStorage.exist(any(), any(), any()) } returns false + + // when + IosArgs.load(yaml).validate() + + // then + assertFalse(systemOutRule.log.contains("WARNING: Google cloud storage result directory should be unique, otherwise results from multiple test matrices will be overwritten or intermingled")) + } + } } private fun IosArgs.Companion.load(yamlData: String, cli: IosRunCommand? = null): IosArgs = diff --git a/test_runner/src/test/kotlin/ftl/gc/GcStorageTest.kt b/test_runner/src/test/kotlin/ftl/gc/GcStorageTest.kt index 890189fa26..82a756f9b5 100644 --- a/test_runner/src/test/kotlin/ftl/gc/GcStorageTest.kt +++ b/test_runner/src/test/kotlin/ftl/gc/GcStorageTest.kt @@ -1,5 +1,6 @@ package ftl.gc +import com.google.cloud.storage.Storage import ftl.args.AndroidArgs import ftl.test.util.FlankTestRunner import io.mockk.every @@ -7,6 +8,8 @@ import io.mockk.mockk import io.mockk.unmockkAll import org.junit.After import org.junit.Assert +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue import org.junit.Test import org.junit.runner.RunWith @@ -55,4 +58,56 @@ class GcStorageTest { } ) } + + @Test + fun `should return that file exist`() { + // given + val mockedStorage: Storage = mockk { + every { + list( + "bucket", + Storage.BlobListOption.pageSize(1), + Storage.BlobListOption.prefix("path/") + ) + } returns mockk { + every { values } returns listOf(mockk()) + } + } + + // when + val actual = GcStorage.exist( + "bucket", + "path", + mockedStorage + ) + + // then + assertTrue(actual) + } + + @Test + fun `should return that file does not exist`() { + // given + val mockedStorage: Storage = mockk { + every { + list( + "bucket", + Storage.BlobListOption.pageSize(1), + Storage.BlobListOption.prefix("path/") + ) + } returns mockk { + every { values } returns listOf() + } + } + + // when + val actual = GcStorage.exist( + "bucket", + "path", + mockedStorage + ) + + // then + assertFalse(actual) + } } diff --git a/test_runner/src/test/kotlin/ftl/reports/util/EndsWithTextWithOptionalSlashAtTheEndTest.kt b/test_runner/src/test/kotlin/ftl/reports/util/EndsWithTextWithOptionalSlashAtTheEndTest.kt new file mode 100644 index 0000000000..6563431403 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/reports/util/EndsWithTextWithOptionalSlashAtTheEndTest.kt @@ -0,0 +1,21 @@ +package ftl.reports.util + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class EndsWithTextWithOptionalSlashAtTheEndTest { + + @Test + fun `should properly found end suffix matching text`() { + // given + val searchingSuffix = "test/link" + + // when + assertTrue("www.flank.com/test/link".endsWithTextWithOptionalSlashAtTheEnd(searchingSuffix)) + assertTrue("www.flank.com/test/link/".endsWithTextWithOptionalSlashAtTheEnd(searchingSuffix)) + assertFalse("www.flank.com/test/link/2".endsWithTextWithOptionalSlashAtTheEnd(searchingSuffix)) + assertFalse("www.flank.com/test/link2".endsWithTextWithOptionalSlashAtTheEnd(searchingSuffix)) + assertFalse("www.flank.com/test/2/link/".endsWithTextWithOptionalSlashAtTheEnd(searchingSuffix)) + } +} diff --git a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt index 2807a71fba..99598c8440 100644 --- a/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/utils/ReportManagerTest.kt @@ -229,8 +229,13 @@ class ReportManagerTest { fun `should get weblink from legacy path and ios path`() { val legacyPath = File("results/2020-08-06_12-08-55.641213_jGpY/matrix_0/NexusLowRes-28-en-portrait/test_result_1.xml") val iosPath = File("results/test_dir/shard_0/iphone8-12.0-en-portrait/test_result_0.xml") - val objectName = "object_name" - assertEquals("object_name/2020-08-06_12-08-55.641213_jGpY/matrix_0", legacyPath.getMatrixPath(objectName)) - assertEquals("object_name/test_dir/shard_0", iosPath.getMatrixPath(objectName)) + assertEquals("2020-08-06_12-08-55.641213_jGpY/matrix_0", legacyPath.getMatrixPath("2020-08-06_12-08-55.641213_jGpY")) + assertEquals("test_dir/shard_0", iosPath.getMatrixPath("test_dir")) + } + + @Test + fun `shouldn't contains multiple test_dir in MatrixPath`() { + val path = File("results/test_dir/test_dir/shard_0/iphone8-12.0-en-portrait/test_result_0.xml") + assertEquals("test_dir/shard_0", path.getMatrixPath("test_dir")) } }