From c5638715c2fa62e9c709631fb74019a674963a6c Mon Sep 17 00:00:00 2001 From: Piotr Adamczyk Date: Fri, 13 Aug 2021 23:26:08 +0200 Subject: [PATCH] refactor: Structural output refresh last run --- buildSrc/src/main/kotlin/Dependencies.kt | 1 + test_runner/build.gradle.kts | 5 +++ .../ftl/client/google/AndroidCatalog.kt | 21 +++------ .../kotlin/ftl/client/google/GcStorage.kt | 1 + .../main/kotlin/ftl/domain/RefreshLastRun.kt | 25 ++++++++--- .../main/kotlin/ftl/domain/RunTestAndroid.kt | 7 +-- .../src/main/kotlin/ftl/domain/RunTestIos.kt | 7 +-- .../main/kotlin/ftl/presentation/RunState.kt | 28 ++++++++++++ .../ftl/presentation/cli/FirebaseCommand.kt | 2 +- .../ftl/presentation/cli/MainCommand.kt | 2 +- .../test/android/AndroidRunCommand.kt | 12 ++++++ .../cli/firebase/test/ios/IosRunCommand.kt | 12 ++++++ .../test/refresh/HandleRefreshRunState.kt | 12 ++++++ .../{ => test/refresh}/RefreshCommand.kt | 19 ++++++-- .../reportmanager/HandleReportManagerState.kt | 21 +++++++++ .../kotlin/ftl/reports/MatrixResultsReport.kt | 4 +- .../ftl/reports/api/PerformanceMetrics.kt | 5 ++- .../kotlin/ftl/reports/util/ReportManager.kt | 14 +++--- .../src/main/kotlin/ftl/run/RefreshLastRun.kt | 12 +++--- .../kotlin/ftl/run/common/GetLastMatrices.kt | 5 ++- .../ftl/cli/firebase/RefreshCommandTest.kt | 2 +- .../firebase/test/ios/IosRunCommandTest.kt | 1 + .../kotlin/ftl/presentation/RunStateTest.kt | 43 +++++++++++++++++++ .../reports/outcome/BillableMinutesTest.kt | 2 - 24 files changed, 207 insertions(+), 56 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/presentation/RunState.kt create mode 100644 test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/refresh/HandleRefreshRunState.kt rename test_runner/src/main/kotlin/ftl/presentation/cli/firebase/{ => test/refresh}/RefreshCommand.kt (53%) create mode 100644 test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/reportmanager/HandleReportManagerState.kt create mode 100644 test_runner/src/test/kotlin/ftl/presentation/RunStateTest.kt diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index 52628515ef..1cd8ecb14c 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -56,6 +56,7 @@ object Dependencies { const val SYSTEM_RULES = "com.github.stefanbirkner:system-rules:${Versions.SYSTEM_RULES}" const val TRUTH = "com.google.truth:truth:${Versions.TRUTH}" const val MOCKK = "io.mockk:mockk:${Versions.MOCKK}" + const val KOTLIN_COROUTINES_TEST = "org.jetbrains.kotlinx:kotlinx-coroutines-test:${Versions.KOTLIN_COROUTINES}" //endregion const val COMMON_TEXT = "org.apache.commons:commons-text:${Versions.COMMON_TEXT}" diff --git a/test_runner/build.gradle.kts b/test_runner/build.gradle.kts index 7f77bf7a89..1be7be4835 100644 --- a/test_runner/build.gradle.kts +++ b/test_runner/build.gradle.kts @@ -217,6 +217,7 @@ dependencies { testImplementation(Dependencies.SYSTEM_RULES) testImplementation(Dependencies.TRUTH) testImplementation(Dependencies.MOCKK) + testImplementation(Dependencies.KOTLIN_COROUTINES_TEST) } buildscript { @@ -231,6 +232,10 @@ buildscript { tasks.withType { kotlinOptions.jvmTarget = "1.8" + kotlinOptions.freeCompilerArgs += listOf( + "-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi", + "-Xopt-in=kotlinx.coroutines.FlowPreview", + ) } // https://github.com/gradle/kotlin-dsl/blob/master/samples/task-dependencies/build.gradle.kts#L41 diff --git a/test_runner/src/main/kotlin/ftl/client/google/AndroidCatalog.kt b/test_runner/src/main/kotlin/ftl/client/google/AndroidCatalog.kt index 45a928d09d..9796b228b6 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/AndroidCatalog.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/AndroidCatalog.kt @@ -4,12 +4,9 @@ import com.google.testing.model.AndroidDevice import com.google.testing.model.AndroidDeviceCatalog import com.google.testing.model.AndroidModel import com.google.testing.model.Orientation -import flank.common.logLn -import ftl.api.fetchAndroidOsVersion -import ftl.environment.android.getDescription -import ftl.environment.getLocaleDescription import ftl.http.executeWithRetry -import ftl.presentation.cli.firebase.test.android.versions.toCliTable +import ftl.presentation.cli.firebase.test.reportmanager.ReportManagerState +import ftl.presentation.publish /** * Contains lists of possible Android device and version ids, as well as checks @@ -34,19 +31,9 @@ object AndroidCatalog { fun getModels(projectId: String): List = deviceCatalog(projectId).models.orEmpty() - fun supportedVersionsAsTable(projectId: String) = fetchAndroidOsVersion(projectId).toCliTable() - - fun describeSoftwareVersion(projectId: String, versionId: String) = - fetchAndroidOsVersion(projectId).getDescription(versionId) - - private fun getVersionsList(projectId: String) = deviceCatalog(projectId).versions - fun supportedOrientations(projectId: String): List = deviceCatalog(projectId).runtimeConfiguration.orientations - private fun getLocaleDescription(projectId: String, locale: String) = - getLocales(projectId).getLocaleDescription(locale) - internal fun getLocales(projectId: String) = deviceCatalog(projectId).runtimeConfiguration.locales fun androidModelIds(projectId: String) = @@ -64,7 +51,9 @@ object AndroidCatalog { val form = deviceCatalog(projectId).models .find { it.id.equals(modelId, ignoreCase = true) }?.form ?: DeviceType.PHYSICAL.name.also { - logLn("Unable to find device type for $modelId. PHYSICAL used as fallback in cost calculations") + ReportManagerState.Log( + "Unable to find device type for $modelId. PHYSICAL used as fallback in cost calculations" + ).publish() } return form.equals(DeviceType.VIRTUAL.name, ignoreCase = true) || form.equals( diff --git a/test_runner/src/main/kotlin/ftl/client/google/GcStorage.kt b/test_runner/src/main/kotlin/ftl/client/google/GcStorage.kt index e6497e3c8d..5796c87f70 100644 --- a/test_runner/src/main/kotlin/ftl/client/google/GcStorage.kt +++ b/test_runner/src/main/kotlin/ftl/client/google/GcStorage.kt @@ -106,6 +106,7 @@ object GcStorage { path: String, name: String ) { + // TODO #2136 handle messages with state runWithProgress( startMessage = "Uploading [$name] to ${GCS_STORAGE_LINK + join(bucket, path).replace(name, "")}..", action = { storage.create(BlobInfo.newBuilder(bucket, path).build(), this) }, diff --git a/test_runner/src/main/kotlin/ftl/domain/RefreshLastRun.kt b/test_runner/src/main/kotlin/ftl/domain/RefreshLastRun.kt index 8558c50f1b..fa90c70e2e 100644 --- a/test_runner/src/main/kotlin/ftl/domain/RefreshLastRun.kt +++ b/test_runner/src/main/kotlin/ftl/domain/RefreshLastRun.kt @@ -1,13 +1,26 @@ package ftl.domain import ftl.args.AndroidArgs +import ftl.presentation.Output +import ftl.presentation.RunState +import ftl.presentation.runBlockingWithObservingRunState import ftl.run.refreshLastRun -interface RefreshLastRun +interface RefreshLastRun : Output -suspend operator fun RefreshLastRun.invoke() { - refreshLastRun( - currentArgs = AndroidArgs.default(), - testShardChunks = emptyList() - ) +operator fun RefreshLastRun.invoke() { + runBlockingWithObservingRunState { + refreshLastRun( + currentArgs = AndroidArgs.default(), + testShardChunks = emptyList() + ) + } +} + +sealed interface RefreshLastRunState : RunState { + data class LoadingRun(val lastRun: String) : RefreshLastRunState + object RefreshMatricesStarted : RefreshLastRunState + data class RefreshMatrices(val matrixCount: Int) : RefreshLastRunState + data class RefreshMatrix(val matrixState: String, val matrixId: String) : RefreshLastRunState + object UpdatingMatrixFile : RefreshLastRunState } diff --git a/test_runner/src/main/kotlin/ftl/domain/RunTestAndroid.kt b/test_runner/src/main/kotlin/ftl/domain/RunTestAndroid.kt index 02d368bb0a..52bd6d9aff 100644 --- a/test_runner/src/main/kotlin/ftl/domain/RunTestAndroid.kt +++ b/test_runner/src/main/kotlin/ftl/domain/RunTestAndroid.kt @@ -10,6 +10,8 @@ import ftl.config.defaultAndroidConfig import ftl.config.loadAndroidConfig import ftl.config.plus import ftl.mock.MockServer +import ftl.presentation.Output +import ftl.presentation.runBlockingWithObservingRunState import ftl.reports.addStepTime import ftl.reports.output.configure import ftl.reports.output.log @@ -24,10 +26,9 @@ import ftl.util.TEST_TYPE import ftl.util.loadFile import ftl.util.printVersionInfo import ftl.util.setCrashReportTag -import kotlinx.coroutines.runBlocking import java.nio.file.Paths -interface RunTestAndroid { +interface RunTestAndroid : Output { val configPath: String val config: AndroidConfig val dryRun: Boolean @@ -59,7 +60,7 @@ operator fun RunTestAndroid.invoke() { ) reportConfiguration() }.validate().also { args -> - runBlocking { + runBlockingWithObservingRunState { if (dumpShards) args.dumpShards() else { diff --git a/test_runner/src/main/kotlin/ftl/domain/RunTestIos.kt b/test_runner/src/main/kotlin/ftl/domain/RunTestIos.kt index 49d65cbeec..0cb2125cc9 100644 --- a/test_runner/src/main/kotlin/ftl/domain/RunTestIos.kt +++ b/test_runner/src/main/kotlin/ftl/domain/RunTestIos.kt @@ -10,6 +10,8 @@ import ftl.config.defaultIosConfig import ftl.config.loadIosConfig import ftl.config.plus import ftl.mock.MockServer +import ftl.presentation.Output +import ftl.presentation.runBlockingWithObservingRunState import ftl.reports.addStepTime import ftl.reports.output.configure import ftl.reports.output.log @@ -24,10 +26,9 @@ import ftl.util.TEST_TYPE import ftl.util.loadFile import ftl.util.printVersionInfo import ftl.util.setCrashReportTag -import kotlinx.coroutines.runBlocking import java.nio.file.Paths -interface RunIosTest { +interface RunIosTest : Output { val configPath: String val config: IosConfig val dryRun: Boolean @@ -63,7 +64,7 @@ operator fun RunIosTest.invoke() { if (dumpShards.not()) logLn(this) }.validate().run { if (dumpShards) dumpShards() - else runBlocking { + else runBlockingWithObservingRunState { addStepTime("Preparation", prepareStopWatch.check()) newTestRun() } diff --git a/test_runner/src/main/kotlin/ftl/presentation/RunState.kt b/test_runner/src/main/kotlin/ftl/presentation/RunState.kt new file mode 100644 index 0000000000..5335abc90e --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/presentation/RunState.kt @@ -0,0 +1,28 @@ +package ftl.presentation + +import com.google.common.annotations.VisibleForTesting +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking + +interface RunState + +@VisibleForTesting +internal val runSharedFlow by lazy { + MutableSharedFlow(extraBufferCapacity = 1) +} + +internal fun Output.runBlockingWithObservingRunState(block: suspend () -> Unit) { + runBlocking { + val runStateCollectJob = launch { + runSharedFlow.collect { runState -> runState.out() } + } + block() + runStateCollectJob.cancel() + } +} + +internal fun RunState.publish() { + runSharedFlow.tryEmit(this) +} diff --git a/test_runner/src/main/kotlin/ftl/presentation/cli/FirebaseCommand.kt b/test_runner/src/main/kotlin/ftl/presentation/cli/FirebaseCommand.kt index 2471fae542..6e850f9357 100644 --- a/test_runner/src/main/kotlin/ftl/presentation/cli/FirebaseCommand.kt +++ b/test_runner/src/main/kotlin/ftl/presentation/cli/FirebaseCommand.kt @@ -1,8 +1,8 @@ package ftl.presentation.cli import ftl.presentation.cli.firebase.CancelCommand -import ftl.presentation.cli.firebase.RefreshCommand import ftl.presentation.cli.firebase.TestCommand +import ftl.presentation.cli.firebase.test.refresh.RefreshCommand import ftl.util.PrintHelpCommand import picocli.CommandLine.Command diff --git a/test_runner/src/main/kotlin/ftl/presentation/cli/MainCommand.kt b/test_runner/src/main/kotlin/ftl/presentation/cli/MainCommand.kt index 2ea07872aa..421758f42e 100644 --- a/test_runner/src/main/kotlin/ftl/presentation/cli/MainCommand.kt +++ b/test_runner/src/main/kotlin/ftl/presentation/cli/MainCommand.kt @@ -2,12 +2,12 @@ package ftl.presentation.cli import ftl.log.setDebugLogging import ftl.presentation.cli.firebase.CancelCommand -import ftl.presentation.cli.firebase.RefreshCommand import ftl.presentation.cli.firebase.test.AndroidCommand import ftl.presentation.cli.firebase.test.IPBlocksCommand import ftl.presentation.cli.firebase.test.IosCommand import ftl.presentation.cli.firebase.test.NetworkProfilesCommand import ftl.presentation.cli.firebase.test.ProvidedSoftwareCommand +import ftl.presentation.cli.firebase.test.refresh.RefreshCommand import ftl.util.PrintHelpCommand import ftl.util.printVersionInfo import picocli.CommandLine diff --git a/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/android/AndroidRunCommand.kt index e08a37b841..f66c8c5678 100644 --- a/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/android/AndroidRunCommand.kt @@ -7,6 +7,10 @@ import ftl.config.createConfiguration import ftl.domain.RunTestAndroid import ftl.domain.invoke import ftl.presentation.cli.firebase.test.CommonRunCommand +import ftl.presentation.cli.firebase.test.reportmanager.ReportManagerState +import ftl.presentation.cli.firebase.test.reportmanager.handleReportManagerState +import ftl.presentation.outputLogger +import ftl.presentation.throwUnknownType import ftl.run.ANDROID_SHARD_FILE import picocli.CommandLine import picocli.CommandLine.Command @@ -52,4 +56,12 @@ class AndroidRunCommand : } override fun run() = invoke() + + override val out = outputLogger { + // TODO add more types in #1870 + when (this) { + is ReportManagerState -> handleReportManagerState(this) + else -> throwUnknownType() + } + } } diff --git a/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/ios/IosRunCommand.kt index bab7f86199..56783bcc05 100644 --- a/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/ios/IosRunCommand.kt @@ -7,6 +7,10 @@ import ftl.config.ios.IosGcloudConfig import ftl.domain.RunIosTest import ftl.domain.invoke import ftl.presentation.cli.firebase.test.CommonRunCommand +import ftl.presentation.cli.firebase.test.reportmanager.ReportManagerState +import ftl.presentation.cli.firebase.test.reportmanager.handleReportManagerState +import ftl.presentation.outputLogger +import ftl.presentation.throwUnknownType import ftl.run.IOS_SHARD_FILE import picocli.CommandLine import picocli.CommandLine.Command @@ -52,4 +56,12 @@ class IosRunCommand : } override fun run() = invoke() + + override val out = outputLogger { + // TODO add more types in #1871 + when (this) { + is ReportManagerState -> handleReportManagerState(this) + else -> throwUnknownType() + } + } } diff --git a/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/refresh/HandleRefreshRunState.kt b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/refresh/HandleRefreshRunState.kt new file mode 100644 index 0000000000..ed570db831 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/refresh/HandleRefreshRunState.kt @@ -0,0 +1,12 @@ +package ftl.presentation.cli.firebase.test.refresh + +import ftl.config.FtlConstants +import ftl.domain.RefreshLastRunState + +internal fun handleRefreshLastRunState(state: RefreshLastRunState): String = when (state) { + is RefreshLastRunState.LoadingRun -> "Loading run ${state.lastRun}" + is RefreshLastRunState.RefreshMatrices -> "${FtlConstants.indent}Refreshing ${state.matrixCount}x matrices" + RefreshLastRunState.RefreshMatricesStarted -> "RefreshMatrices" + is RefreshLastRunState.RefreshMatrix -> "${FtlConstants.indent} ${state.matrixState} ${state.matrixId}" + RefreshLastRunState.UpdatingMatrixFile -> "${FtlConstants.indent}Updating matrix file" +} diff --git a/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/RefreshCommand.kt b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/refresh/RefreshCommand.kt similarity index 53% rename from test_runner/src/main/kotlin/ftl/presentation/cli/firebase/RefreshCommand.kt rename to test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/refresh/RefreshCommand.kt index 851eddc70b..db5378d075 100644 --- a/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/RefreshCommand.kt +++ b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/refresh/RefreshCommand.kt @@ -1,9 +1,13 @@ -package ftl.presentation.cli.firebase +package ftl.presentation.cli.firebase.test.refresh import ftl.domain.RefreshLastRun +import ftl.domain.RefreshLastRunState import ftl.domain.invoke +import ftl.presentation.cli.firebase.test.reportmanager.ReportManagerState +import ftl.presentation.cli.firebase.test.reportmanager.handleReportManagerState +import ftl.presentation.outputLogger +import ftl.presentation.throwUnknownType import ftl.util.PrintHelpCommand -import kotlinx.coroutines.runBlocking import picocli.CommandLine.Command @Command( @@ -26,5 +30,14 @@ class RefreshCommand : PrintHelpCommand(), RefreshLastRun { - override fun run() = runBlocking { invoke() } + override fun run() = invoke() + + override val out = outputLogger { + when (this) { + is RefreshLastRunState -> handleRefreshLastRunState(this) + is ReportManagerState -> handleReportManagerState(this) + // TODO #2136 handle uploading file here + else -> throwUnknownType() + } + } } diff --git a/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/reportmanager/HandleReportManagerState.kt b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/reportmanager/HandleReportManagerState.kt new file mode 100644 index 0000000000..536b3ffaa9 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/presentation/cli/firebase/test/reportmanager/HandleReportManagerState.kt @@ -0,0 +1,21 @@ +package ftl.presentation.cli.firebase.test.reportmanager + +import flank.common.newLine +import flank.common.withNewLineAtTheEnd +import ftl.presentation.RunState +import ftl.reports.util.ReportManager +import kotlin.math.roundToInt + +sealed interface ReportManagerState : RunState { + data class Log(val message: String) : ReportManagerState + data class Warning(val message: String) : ReportManagerState + data class ShardTimes(val shards: List) : ReportManagerState +} + +internal fun handleReportManagerState(state: ReportManagerState): String = when (state) { + is ReportManagerState.Log -> state.message + is ReportManagerState.Warning -> "WARNING: ${state.message}" + is ReportManagerState.ShardTimes -> "Actual shard times:$newLine" + state.shards.joinToString(newLine) { + " ${it.shard}: Expected: ${it.expectedTime.roundToInt()}s, Actual: ${it.finalTime.roundToInt()}s, Diff: ${it.timeDiff.roundToInt()}s" + }.withNewLineAtTheEnd() +} diff --git a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt index 982b2be13a..b0999379b7 100644 --- a/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt +++ b/test_runner/src/main/kotlin/ftl/reports/MatrixResultsReport.kt @@ -10,6 +10,8 @@ import ftl.config.FtlConstants.indent import ftl.json.MatrixMap import ftl.json.asPrintableTable import ftl.json.isFailed +import ftl.presentation.cli.firebase.test.reportmanager.ReportManagerState +import ftl.presentation.publish import ftl.reports.output.log import ftl.reports.output.outputReport import ftl.reports.util.IReport @@ -36,7 +38,7 @@ object MatrixResultsReport : IReport { override fun run(matrices: MatrixMap, result: JUnitTest.Result?, printToStdout: Boolean, args: IArgs) { val output = generate(matrices) - if (printToStdout) log(output) + if (printToStdout) ReportManagerState.Log(output).publish() write(matrices, output, args) ReportManager.uploadReportResult(output, args, fileName()) } diff --git a/test_runner/src/main/kotlin/ftl/reports/api/PerformanceMetrics.kt b/test_runner/src/main/kotlin/ftl/reports/api/PerformanceMetrics.kt index 9fd6ee8824..c3a5b96b27 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/PerformanceMetrics.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/PerformanceMetrics.kt @@ -1,13 +1,14 @@ package ftl.reports.api import com.google.testing.model.TestExecution -import flank.common.logLn import ftl.api.PerfMetrics import ftl.api.RemoteStorage import ftl.api.fetchPerformanceMetrics import ftl.api.uploadToRemoteStorage import ftl.args.IArgs import ftl.client.google.AndroidCatalog +import ftl.presentation.cli.firebase.test.reportmanager.ReportManagerState +import ftl.presentation.publish import ftl.run.common.prettyPrint import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async @@ -72,5 +73,5 @@ internal fun uploadPerformanceMetrics( RemoteStorage.Data("performanceMetrics.json", prettyPrint.toJson(perfMetricsSummary).toByteArray()) ) }.onFailure { - logLn("Cannot upload performance metrics ${it.message}") + ReportManagerState.Log("Cannot upload performance metrics ${it.message}").publish() }.getOrNull() diff --git a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt index 393a3cb832..fcdc3946c4 100644 --- a/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt +++ b/test_runner/src/main/kotlin/ftl/reports/util/ReportManager.kt @@ -1,7 +1,6 @@ package ftl.reports.util import com.google.common.annotations.VisibleForTesting -import flank.common.logLn import ftl.api.JUnitTest import ftl.api.RemoteStorage import ftl.api.ShardChunks @@ -20,6 +19,8 @@ import ftl.domain.junit.mergeTestTimes import ftl.domain.junit.removeStackTraces import ftl.json.MatrixMap import ftl.json.isAllSuccessful +import ftl.presentation.cli.firebase.test.reportmanager.ReportManagerState +import ftl.presentation.publish import ftl.reports.CostReport import ftl.reports.FullJUnitReport import ftl.reports.HtmlErrorReport @@ -36,7 +37,6 @@ import java.io.File import java.nio.file.Files import java.nio.file.Paths import java.util.Date -import kotlin.math.roundToInt object ReportManager { @@ -105,12 +105,12 @@ object ReportManager { private fun getWebLink(matrices: MatrixMap, xmlFile: File): String = xmlFile.getMatrixPath(matrices.runPath) ?.findMatrixPath(matrices) - ?: "".also { logLn("WARNING: Matrix path not found in JSON.") } + ?: "".also { ReportManagerState.Warning("Matrix path not found in JSON.").publish() } private fun String.findMatrixPath(matrices: MatrixMap) = matrices.map.values .firstOrNull { savedMatrix -> savedMatrix.gcsPath.endsWithTextWithOptionalSlashAtTheEnd(this) } ?.webLink - ?: "".also { logLn("WARNING: Matrix path not found in JSON. $this") } + ?: "".also { ReportManagerState.Warning("Matrix path not found in JSON. $this").publish() } @VisibleForTesting internal fun String.endsWithTextWithOptionalSlashAtTheEnd(text: String) = @@ -227,11 +227,7 @@ object ReportManager { ) { val list = createShardEfficiencyList(oldResult, newResult, args, testShardChunks) - logLn( - "Actual shard times:\n" + list.joinToString("\n") { - " ${it.shard}: Expected: ${it.expectedTime.roundToInt()}s, Actual: ${it.finalTime.roundToInt()}s, Diff: ${it.timeDiff.roundToInt()}s" - } + "\n" - ) + ReportManagerState.ShardTimes(list).publish() } private fun processJunitXml( diff --git a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt index f0deaa91d4..7a2c4517d4 100644 --- a/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt +++ b/test_runner/src/main/kotlin/ftl/run/RefreshLastRun.kt @@ -1,16 +1,16 @@ package ftl.run -import flank.common.logLn import ftl.api.ShardChunks import ftl.api.TestMatrix import ftl.api.refreshTestMatrix import ftl.args.IArgs -import ftl.config.FtlConstants +import ftl.domain.RefreshLastRunState import ftl.domain.testmatrix.needsUpdate import ftl.domain.testmatrix.updateWithMatrix import ftl.json.MatrixMap import ftl.json.updateMatrixMap import ftl.json.validate +import ftl.presentation.publish import ftl.reports.util.ReportManager import ftl.run.common.fetchAllTestRunArtifacts import ftl.run.common.getLastArgs @@ -41,7 +41,7 @@ suspend fun refreshLastRun(currentArgs: IArgs, testShardChunks: ShardChunks) { /** Refresh all in progress matrices in parallel **/ private suspend fun refreshMatrices(matrixMap: MatrixMap, args: IArgs) = coroutineScope { - logLn("RefreshMatrices") + RefreshLastRunState.RefreshMatricesStarted.publish() val jobs = arrayListOf>() val map = matrixMap.map @@ -55,14 +55,14 @@ private suspend fun refreshMatrices(matrixMap: MatrixMap, args: IArgs) = corouti } if (matrixCount != 0) { - logLn(FtlConstants.indent + "Refreshing ${matrixCount}x matrices") + RefreshLastRunState.RefreshMatrices(matrixCount).publish() } var dirty = false jobs.awaitAll().forEach { matrix -> val matrixId = matrix.matrixId - logLn(FtlConstants.indent + "${matrix.state} $matrixId") + RefreshLastRunState.RefreshMatrix(matrix.state, matrixId).publish() if (map[matrixId]?.needsUpdate(matrix) == true) { map[matrixId]?.updateWithMatrix(matrix)?.let { @@ -73,7 +73,7 @@ private suspend fun refreshMatrices(matrixMap: MatrixMap, args: IArgs) = corouti } if (dirty) { - logLn(FtlConstants.indent + "Updating matrix file") + RefreshLastRunState.UpdatingMatrixFile.publish() args.updateMatrixFile(matrixMap) } } diff --git a/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt b/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt index 462a038d42..c355317e54 100644 --- a/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt +++ b/test_runner/src/main/kotlin/ftl/run/common/GetLastMatrices.kt @@ -1,10 +1,11 @@ package ftl.run.common -import flank.common.logLn import ftl.api.TestMatrix import ftl.args.IArgs import ftl.config.FtlConstants +import ftl.domain.RefreshLastRunState import ftl.json.MatrixMap +import ftl.presentation.publish import ftl.run.exception.FlankGeneralError import java.nio.file.Paths @@ -12,7 +13,7 @@ import java.nio.file.Paths internal fun getLastMatrices(args: IArgs): MatrixMap { val lastRun = args.getLastGcsPath() ?: throw FlankGeneralError("no runs found in results/ folder") - logLn("Loading run $lastRun") + RefreshLastRunState.LoadingRun(lastRun).publish() return matrixPathToObj(lastRun, args) } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt index 3cd2d9ff99..2c62529fbf 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/RefreshCommandTest.kt @@ -2,7 +2,7 @@ package ftl.cli.firebase import com.google.common.truth.Truth.assertThat import flank.common.normalizeLineEnding -import ftl.presentation.cli.firebase.RefreshCommand +import ftl.presentation.cli.firebase.test.refresh.RefreshCommand import ftl.test.util.FlankTestRunner import org.junit.Rule import org.junit.Test diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 9a506e159e..1dac921de8 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -50,6 +50,7 @@ class IosRunCommandTest { runCmd.run() val output = systemOutRule.log + assertThat(output).contains("1 / 1 (100.00%)") } diff --git a/test_runner/src/test/kotlin/ftl/presentation/RunStateTest.kt b/test_runner/src/test/kotlin/ftl/presentation/RunStateTest.kt new file mode 100644 index 0000000000..5312207462 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/presentation/RunStateTest.kt @@ -0,0 +1,43 @@ +package ftl.presentation + +import com.google.common.truth.Truth.assertThat +import io.mockk.mockk +import kotlinx.coroutines.async +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.test.runBlockingTest +import org.junit.Test + +class RunStateTest { + + @Test + fun `should publish new run state`() = runBlockingTest { + // given + val testItem = object : RunState {} + + // when + val actual = async { runSharedFlow.first() } + testItem.publish() + + // then + assertThat(actual.await()).isEqualTo(testItem) + } + + @Test + fun `should observe run state and self cancel it`() { + // given + val testItem = mockk() + + // when + TestOutputClass().runBlockingWithObservingRunState { + testItem.publish() + } + + // then + // self cancel + } + + private class TestOutputClass : Output { + override val out: Any.() -> Unit + get() = {} + } +} diff --git a/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt b/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt index a166f1d448..f2efba4d69 100644 --- a/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/outcome/BillableMinutesTest.kt @@ -12,7 +12,6 @@ import io.mockk.mockkObject import io.mockk.unmockkAll import org.junit.After import org.junit.Assert.assertEquals -import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Rule import org.junit.Test @@ -127,7 +126,6 @@ class BillableMinutesTest { val minutes = listOf(step).calculateAndroidBillableMinutes(projectId = "anyId", timeoutValue = 1000L) verifyBilling(minutes, expectedPhysical = expectedPhysical) - assertTrue(output.log.contains("Unable to find device type for $modelId. PHYSICAL used as fallback in cost calculations")) } }