diff --git a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt index 6db5063e27..2909c02a3a 100644 --- a/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt +++ b/test_runner/src/main/kotlin/ftl/run/platform/common/AfterRunTests.kt @@ -8,6 +8,7 @@ import ftl.json.MatrixMap import ftl.json.SavedMatrix import ftl.run.common.updateMatrixFile import ftl.util.StopWatch +import ftl.util.isInvalid import ftl.util.webLink import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.joinAll @@ -51,13 +52,18 @@ private fun saveConfigFile(matrixMap: MatrixMap, args: IArgs) { private suspend inline fun MatrixMap.printMatricesWebLinks(project: String) = coroutineScope { println("Matrices webLink") - map.values.map { launch { it.printWebLink(project) } }.joinAll() + map.values.map { + launch { + println("${FtlConstants.indent}${it.matrixId} ${getOrUpdateWebLink(it.webLink, project, it.matrixId)}") + } + }.joinAll() println() } -private suspend inline fun SavedMatrix.printWebLink(project: String) = - println("${FtlConstants.indent}$matrixId: ${getOrUpdateWebLink(webLink, project, matrixId)}") - private tailrec suspend fun getOrUpdateWebLink(link: String, project: String, matrixId: String): String = if (link.isNotBlank()) link - else getOrUpdateWebLink(GcTestMatrix.refresh(matrixId, project).webLink(), project, matrixId) + else getOrUpdateWebLink( + link = GcTestMatrix.refresh(matrixId, project).run { if (isInvalid()) "Unable to get web link" else webLink() }, + project = project, + matrixId = matrixId + ) diff --git a/test_runner/src/main/kotlin/ftl/util/MatrixState.kt b/test_runner/src/main/kotlin/ftl/util/MatrixState.kt index 28324e8b45..5eee85d82f 100644 --- a/test_runner/src/main/kotlin/ftl/util/MatrixState.kt +++ b/test_runner/src/main/kotlin/ftl/util/MatrixState.kt @@ -18,6 +18,13 @@ object MatrixState { const val CANCELLED = "CANCELLED" const val INVALID = "INVALID" + private val validStates = listOf( + VALIDATING, + PENDING, + RUNNING, + FINISHED + ) + fun inProgress(state: String): Boolean { return when (state) { VALIDATING -> true @@ -32,8 +39,18 @@ object MatrixState { fun completed(state: String): Boolean { return !inProgress(state) } + + private fun isValid(state: String): Boolean { + return validStates.contains(state) + } + + fun isInvalid(state: String): Boolean { + return !isValid(state) + } } fun TestMatrix.completed(): Boolean { return MatrixState.completed(this.state) } + +fun TestMatrix.isInvalid() = MatrixState.isInvalid(this.state) diff --git a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt index 27b80a98ec..296a53e6be 100644 --- a/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt +++ b/test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt @@ -1,18 +1,27 @@ package ftl.run +import com.google.api.services.testing.Testing +import com.google.api.services.testing.model.GoogleCloudStorage +import com.google.api.services.testing.model.ResultStorage +import com.google.api.services.testing.model.TestExecution +import com.google.api.services.testing.model.TestMatrix import com.google.common.truth.Truth.assertThat import ftl.args.AndroidArgs import ftl.args.IosArgs import ftl.config.FtlConstants.isWindows +import ftl.http.executeWithRetry import ftl.run.common.getDownloadPath import ftl.test.util.FlankTestRunner import ftl.util.ObjPath import io.mockk.every import io.mockk.mockk +import io.mockk.mockkStatic import io.mockk.unmockkAll +import io.mockk.verify import java.nio.file.Paths import kotlinx.coroutines.runBlocking import org.junit.After +import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Assume.assumeFalse import org.junit.Rule @@ -158,9 +167,45 @@ class TestRunnerTest { newTestRun(localConfig) } val matrixWebLinkHeader = "Matrices webLink" - val matrixLink = Regex("(matrix-\\d+: https://console\\.firebase\\.google\\.com/project/.*/testlab/histories/.*/matrices/.*)(/executions/.*)?") + val matrixLink = Regex("(matrix-\\d+ https://console\\.firebase\\.google\\.com/project/.*/testlab/histories/.*/matrices/.*)(/executions/.*)?") val output = systemOutRule.log assertTrue(output.contains(matrixWebLinkHeader)) assertTrue(output.contains(matrixLink)) } + + @Test + fun `flank should stop updating web link if matrix has invalid state`() { + val localConfig = AndroidArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.local.yml")) + mockkStatic("ftl.http.ExecuteWithRetryKt") + every { + any().executeWithRetry() + } returnsMany listOf( + getMockedTestMatrix().apply { state = "RUNNING" }, + getMockedTestMatrix().apply { state = "RUNNING" }, + getMockedTestMatrix() + ) + runBlocking { + newTestRun(localConfig) + } + val matrixWebLinkHeader = "Matrices webLink" + val message = "Unable to get web link" + val matrixLink = Regex("(matrix-\\d+ https://console\\.firebase\\.google\\.com/project/.*/testlab/histories/.*/matrices/.*)(/executions/.*)?") + val output = systemOutRule.log + assertTrue(output.contains(matrixWebLinkHeader)) + assertTrue(output.contains(message)) + assertFalse(output.contains(matrixLink)) + verify(exactly = 3) { any().executeWithRetry() } + } + + private fun getMockedTestMatrix() = TestMatrix().apply { + state = "INVALID" + testMatrixId = "matrix-12345" + testExecutions = listOf(TestExecution().apply { + resultStorage = ResultStorage().apply { + googleCloudStorage = GoogleCloudStorage().apply { + gcsPath = "any/Path" + } + } + }) + } }