Skip to content

Commit

Permalink
Add logic to prevent fetching from invalid test matrices (#693)
Browse files Browse the repository at this point in the history
  • Loading branch information
pawelpasterz authored Apr 1, 2020
1 parent aa6d9b8 commit f000fe4
Show file tree
Hide file tree
Showing 3 changed files with 74 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import ftl.json.MatrixMap
import ftl.json.SavedMatrix
import ftl.run.common.updateMatrixFile
import ftl.util.StopWatch
import ftl.util.isInvalid
import ftl.util.webLink
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.joinAll
Expand Down Expand Up @@ -51,13 +52,18 @@ private fun saveConfigFile(matrixMap: MatrixMap, args: IArgs) {

private suspend inline fun MatrixMap.printMatricesWebLinks(project: String) = coroutineScope {
println("Matrices webLink")
map.values.map { launch { it.printWebLink(project) } }.joinAll()
map.values.map {
launch {
println("${FtlConstants.indent}${it.matrixId} ${getOrUpdateWebLink(it.webLink, project, it.matrixId)}")
}
}.joinAll()
println()
}

private suspend inline fun SavedMatrix.printWebLink(project: String) =
println("${FtlConstants.indent}$matrixId: ${getOrUpdateWebLink(webLink, project, matrixId)}")

private tailrec suspend fun getOrUpdateWebLink(link: String, project: String, matrixId: String): String =
if (link.isNotBlank()) link
else getOrUpdateWebLink(GcTestMatrix.refresh(matrixId, project).webLink(), project, matrixId)
else getOrUpdateWebLink(
link = GcTestMatrix.refresh(matrixId, project).run { if (isInvalid()) "Unable to get web link" else webLink() },
project = project,
matrixId = matrixId
)
17 changes: 17 additions & 0 deletions test_runner/src/main/kotlin/ftl/util/MatrixState.kt
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@ object MatrixState {
const val CANCELLED = "CANCELLED"
const val INVALID = "INVALID"

private val validStates = listOf(
VALIDATING,
PENDING,
RUNNING,
FINISHED
)

fun inProgress(state: String): Boolean {
return when (state) {
VALIDATING -> true
Expand All @@ -32,8 +39,18 @@ object MatrixState {
fun completed(state: String): Boolean {
return !inProgress(state)
}

private fun isValid(state: String): Boolean {
return validStates.contains(state)
}

fun isInvalid(state: String): Boolean {
return !isValid(state)
}
}

fun TestMatrix.completed(): Boolean {
return MatrixState.completed(this.state)
}

fun TestMatrix.isInvalid() = MatrixState.isInvalid(this.state)
47 changes: 46 additions & 1 deletion test_runner/src/test/kotlin/ftl/run/TestRunnerTest.kt
Original file line number Diff line number Diff line change
@@ -1,18 +1,27 @@
package ftl.run

import com.google.api.services.testing.Testing
import com.google.api.services.testing.model.GoogleCloudStorage
import com.google.api.services.testing.model.ResultStorage
import com.google.api.services.testing.model.TestExecution
import com.google.api.services.testing.model.TestMatrix
import com.google.common.truth.Truth.assertThat
import ftl.args.AndroidArgs
import ftl.args.IosArgs
import ftl.config.FtlConstants.isWindows
import ftl.http.executeWithRetry
import ftl.run.common.getDownloadPath
import ftl.test.util.FlankTestRunner
import ftl.util.ObjPath
import io.mockk.every
import io.mockk.mockk
import io.mockk.mockkStatic
import io.mockk.unmockkAll
import io.mockk.verify
import java.nio.file.Paths
import kotlinx.coroutines.runBlocking
import org.junit.After
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Assume.assumeFalse
import org.junit.Rule
Expand Down Expand Up @@ -158,9 +167,45 @@ class TestRunnerTest {
newTestRun(localConfig)
}
val matrixWebLinkHeader = "Matrices webLink"
val matrixLink = Regex("(matrix-\\d+: https://console\\.firebase\\.google\\.com/project/.*/testlab/histories/.*/matrices/.*)(/executions/.*)?")
val matrixLink = Regex("(matrix-\\d+ https://console\\.firebase\\.google\\.com/project/.*/testlab/histories/.*/matrices/.*)(/executions/.*)?")
val output = systemOutRule.log
assertTrue(output.contains(matrixWebLinkHeader))
assertTrue(output.contains(matrixLink))
}

@Test
fun `flank should stop updating web link if matrix has invalid state`() {
val localConfig = AndroidArgs.load(Paths.get("src/test/kotlin/ftl/fixtures/flank.local.yml"))
mockkStatic("ftl.http.ExecuteWithRetryKt")
every {
any<Testing.Projects.TestMatrices.Get>().executeWithRetry()
} returnsMany listOf(
getMockedTestMatrix().apply { state = "RUNNING" },
getMockedTestMatrix().apply { state = "RUNNING" },
getMockedTestMatrix()
)
runBlocking {
newTestRun(localConfig)
}
val matrixWebLinkHeader = "Matrices webLink"
val message = "Unable to get web link"
val matrixLink = Regex("(matrix-\\d+ https://console\\.firebase\\.google\\.com/project/.*/testlab/histories/.*/matrices/.*)(/executions/.*)?")
val output = systemOutRule.log
assertTrue(output.contains(matrixWebLinkHeader))
assertTrue(output.contains(message))
assertFalse(output.contains(matrixLink))
verify(exactly = 3) { any<Testing.Projects.TestMatrices.Get>().executeWithRetry() }
}

private fun getMockedTestMatrix() = TestMatrix().apply {
state = "INVALID"
testMatrixId = "matrix-12345"
testExecutions = listOf(TestExecution().apply {
resultStorage = ResultStorage().apply {
googleCloudStorage = GoogleCloudStorage().apply {
gcsPath = "any/Path"
}
}
})
}
}

0 comments on commit f000fe4

Please sign in to comment.