Merge branch 'master' into #1134-update-llvm-windows

buildSrc/src/main/kotlin/Versions.kt

@@ -40,10 +40,10 @@ object Versions {
     // https://github.com/googleapis/google-auth-library-java/releases
     // NOTE: https://github.com/googleapis/google-oauth-java-client is End of Life and replaced by google-auth-library-java
     // https://github.com/googleapis/google-oauth-java-client/issues/251#issuecomment-504565533
-    const val GOOGLE_AUTH = "0.24.1"
+    const val GOOGLE_AUTH = "0.25.1"
 
     // https://search.maven.org/search?q=a:google-cloud-nio%20g:com.google.cloud
-    const val GOOGLE_NIO = "0.122.10"
+    const val GOOGLE_NIO = "0.122.11"
 
     // https://search.maven.org/search?q=a:google-cloud-storage%20g:com.google.cloud
     const val GOOGLE_STORAGE = "1.113.14"
@@ -81,13 +81,13 @@ object Versions {
     const val KOTLIN_LOGGING = "2.0.6"
 
     // https://github.com/mockk/mockk
-    const val MOCKK = "1.10.6"
+    const val MOCKK = "1.11.0"
 
     // https://commons.apache.org/proper/commons-text/
     const val COMMON_TEXT = "1.9"
 
     // https://github.com/fusesource/jansi/releases
-    const val JANSI = "2.3.1"
+    const val JANSI = "2.3.2"
 
     // https://github.com/ben-manes/gradle-versions-plugin/releases
     const val BEN_MANES = "0.38.0"

docs/desktop/flank_desktop.md

@@ -0,0 +1,44 @@
+# Flank desktop prototype
+
+## Prototype Flank options
+
+The prototype will run a simple Android test on a given apk and test the apk with options to specify some flanks flags, as well
+as max tests shards. Those flags are the most popular one based on [mixpanel analytics](https://mixpanel.com/report/2288623/view/2835119/dashboards#id=879189)
+
+- Flags
+  - disable sharding
+  - disable results upload
+  - fail fast 
+  - disable usage statistics 
+  - auto Google login
+
+- Input
+  - Max tests shards 
+  - apk path 
+  - test apk path
+
+## Prototype design
+
+The mockup of the design is shown below:  
+
+![mock](flank_mock.png)
+
+### Design elements
+
+- At the top left side there are some small buttons to toggle state of boolean options
+- Below there is text field to input `maxTestsShards`
+- Below there are two text fields with buttons placed at the right to open file chooser/provide a path to files
+- At the very bottom of left column there is a button to start Flank test run.
+- On the right side there is a big window which is responsible for displaying output
+
+## Setup
+
+Please follow [getting started guide](https://github.com/JetBrains/compose-jb/tree/master/tutorials/Getting_Started) to set up new module for a prototype.
+There is also a [gradle plugin](https://github.com/JetBrains/compose-jb/tree/master/gradle-plugins) for simplifying usage of Jetbrains compose.
+More info on [project page](https://github.com/JetBrains/compose-jb/).
+
+### Additional resources
+- [IntelliJ Idea plugin](https://github.com/JetBrains/compose-jb/tree/master/examples/intelliJPlugin)
+- [Tutorial](https://developer.android.com/jetpack/compose/layout)
+- [Latest build](https://developer.android.com/jetpack/compose/layout)
+- [Template to use without gradle](https://github.com/JetBrains/compose-jb/tree/master/templates)

integration_tests/src/test/kotlin/integration/IntergrationTestsUtils.kt

@@ -86,8 +86,8 @@ enum class TestOutcome(val regex: Regex) {
 
 private val fromCommon =
     { outcome: String ->
-        if (isWindows) "\\?\\s$outcome\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex()
-        else "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex()
+        if (isWindows) "\\?\\s$outcome\\s\\?\\smatrix-[a-zA-Z0-9]*\\s\\?\\s*[a-zA-Z0-9._-]*\\s*\\?\\s*[a-zA-Z0-9-]*\\s*\\?\\s[a-zA-Z0-9\\s,-]*\\s*\\?".toRegex()
+        else "│\\s$outcome\\s│\\s${"matrix"}-[a-zA-Z0-9]*\\s│\\s*[a-zA-Z0-9._-]*\\s*│\\s*[a-zA-Z0-9-]*\\s*│\\s[a-zA-Z0-9\\s,-]*\\s*│".toRegex()
     }
 
 fun String.removeUnicode() = replace("\u001B\\[\\d{1,2}m".toRegex(), "").trimIndent()

integration_tests/src/test/resources/compare/SanityRoboIT-compare

@@ -83,11 +83,7 @@ CostReport
 
 MatrixResultsReport
   1 \/ 1 \(100\.00\%\)
-.*
-[│\?] OUTCOME [│\?]      MATRIX ID       [│\?]      TEST AXIS VALUE       [│\?] TEST DETAILS [│\?]
-.*
-[│\?] success [│\?] matrix-[a-zA-Z0-9\s]*[│\?] NexusLowRes-28-en-portrait [│\?] ---          [│\?]
-.*
+[\s\S]*
   Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*
   Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*
   Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*

test_runner/src/main/kotlin/ftl/android/AndroidCatalog.kt

@@ -81,12 +81,12 @@ object AndroidCatalog {
                 logLn("Unable to find device type for $modelId. PHYSICAL used as fallback in cost calculations")
             }
 
-        return form.equals(DeviceType.VIRTUAL.name, ignoreCase = true)
+        return form.equals(DeviceType.VIRTUAL.name, ignoreCase = true) || form.equals(DeviceType.EMULATOR.name, ignoreCase = true)
     }
 }
 
 enum class DeviceType {
-    VIRTUAL, PHYSICAL
+    VIRTUAL, PHYSICAL, EMULATOR
 }
 
 sealed class DeviceConfigCheck

test_runner/src/main/kotlin/ftl/environment/TestEnvironmentInfo.kt

@@ -28,4 +28,6 @@ const val RESOLUTION = "RESOLUTION"
 const val OS_VERSION_IDS = "OS_VERSION_IDS"
 const val TAGS = "TAGS"
 const val PHYSICAL_DEVICE = "PHYSICAL"
+const val VIRTUAL_DEVICE = "VIRTUAL"
+const val EMULATOR_DEVICE = "EMULATOR"
 const val OS_VERSION_ID = "OS_VERSION_ID"

test_runner/src/main/kotlin/ftl/environment/android/ListAndroidDevices.kt

@@ -1,6 +1,7 @@
 package ftl.environment.android
 
 import com.google.testing.model.AndroidModel
+import ftl.environment.EMULATOR_DEVICE
 import ftl.environment.FORM
 import ftl.environment.MAKE
 import ftl.environment.MODEL_ID
@@ -10,6 +11,7 @@ import ftl.environment.PHYSICAL_DEVICE
 import ftl.environment.RESOLUTION
 import ftl.environment.TAGS
 import ftl.environment.TestEnvironmentInfo
+import ftl.environment.VIRTUAL_DEVICE
 import ftl.environment.createTableColumnFor
 import ftl.environment.getOrCreateList
 import ftl.environment.orUnknown
@@ -47,5 +49,10 @@ private val AndroidModel.resolution
     get() = if (screenX == null || screenY == null) "UNKNOWN" else "$screenY x $screenX"
 
 private val formToSystemOutColorMapper: (String) -> SystemOutColor = {
-    if (it == PHYSICAL_DEVICE) SystemOutColor.YELLOW else SystemOutColor.BLUE
+    when (it) {
+        PHYSICAL_DEVICE -> SystemOutColor.YELLOW
+        VIRTUAL_DEVICE -> SystemOutColor.BLUE
+        EMULATOR_DEVICE -> SystemOutColor.GREEN
+        else -> SystemOutColor.DEFAULT
+    }
 }

test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt

@@ -1,11 +1,14 @@
 package ftl.json
 
+import com.google.testing.model.FileReference
 import com.google.testing.model.TestMatrix
+import ftl.analytics.toJSONObject
 import ftl.environment.orUnknown
 import ftl.reports.outcome.BillableMinutes
 import ftl.reports.outcome.TestOutcome
 import ftl.reports.outcome.createMatrixOutcomeSummary
 import ftl.reports.outcome.fetchTestOutcomeContext
+import ftl.run.common.prettyPrint
 import ftl.util.MatrixState.FINISHED
 import ftl.util.MatrixState.INVALID
 import ftl.util.StepOutcome
@@ -32,11 +35,14 @@ data class SavedMatrix(
     val gcsPathWithoutRootBucket: String = "",
     val gcsRootBucket: String = "",
     val webLinkWithoutExecutionDetails: String? = "",
-    val testAxises: List<TestOutcome> = emptyList()
+    val testAxises: List<TestOutcome> = emptyList(),
+    val appFileName: String = fallbackAppName
 ) {
     val outcome = testAxises.maxByOrNull { StepOutcome.order.indexOf(it.outcome) }?.outcome.orEmpty()
 }
 
+private const val fallbackAppName = "N/A"
+
 fun createSavedMatrix(testMatrix: TestMatrix) = SavedMatrix().updateWithMatrix(testMatrix)
 
 fun SavedMatrix.canceledByUser() = testAxises.any { it.details == ABORTED_BY_USER_MESSAGE }
@@ -97,9 +103,24 @@ private fun SavedMatrix.updateProperties(newMatrix: TestMatrix) = copy(
     clientDetails = newMatrix.getClientDetails(),
     gcsPathWithoutRootBucket = newMatrix.getGcsPathWithoutRootBucket(),
     gcsRootBucket = newMatrix.getGcsRootBucket(),
-    webLinkWithoutExecutionDetails = newMatrix.webLinkWithoutExecutionDetails()
+    webLinkWithoutExecutionDetails = newMatrix.webLinkWithoutExecutionDetails(),
+    appFileName = newMatrix.extractAppFileName() ?: fallbackAppName
 )
 
+private fun TestMatrix.extractAppFileName() = testSpecification?.run {
+    listOf(
+        androidInstrumentationTest,
+        androidTestLoop,
+        androidRoboTest,
+        iosXcTest,
+        iosTestLoop
+    )
+        .firstOrNull { it != null }
+        ?.toJSONObject()
+        ?.let { prettyPrint.fromJson(it.toString(), AppPath::class.java).gcsPath }
+        ?.substringAfterLast('/')
+}
+
 private fun SavedMatrix.updateBillableMinutes(billableMinutes: BillableMinutes) = copy(
     billablePhysicalMinutes = billableMinutes.physical,
     billableVirtualMinutes = billableMinutes.virtual,
@@ -113,3 +134,12 @@ private fun TestMatrix.invalidTestOutcome() = TestOutcome(
     outcome = INVALID,
     details = invalidMatrixDetails.orUnknown()
 )
+
+private data class AppPath(
+    private val appApk: FileReference?,
+    private val testsZip: FileReference?,
+    private val appIpa: FileReference?
+) {
+    val gcsPath: String?
+        get() = (appApk ?: testsZip ?: appIpa)?.gcsPath
+}

test_runner/src/main/kotlin/ftl/json/SavedMatrixTableUtil.kt

@@ -20,6 +20,10 @@ fun List<SavedMatrix>.asPrintableTable(): String = buildTable(
         header = MATRIX_ID_COLUMN_HEADER,
         data = flatMapTestAxis { matrix -> matrix.matrixId }
     ),
+    TableColumn(
+        header = APP_NAME_COLUMN_HEADER,
+        data = flatMapTestAxis { matrix -> matrix.appFileName }
+    ),
     TableColumn(
         header = TEST_AXIS_VALUE_HEADER,
         data = flatMapTestAxis { device }
@@ -43,5 +47,6 @@ private val TestOutcome.outcomeColor
 
 private const val OUTCOME_COLUMN_HEADER = "OUTCOME"
 private const val MATRIX_ID_COLUMN_HEADER = "MATRIX ID"
+private const val APP_NAME_COLUMN_HEADER = "APP NAME"
 private const val TEST_AXIS_VALUE_HEADER = "TEST AXIS VALUE"
 private const val OUTCOME_DETAILS_COLUMN_HEADER = "TEST DETAILS"

test_runner/src/main/kotlin/ftl/reports/output/OutputReportLoggers.kt

@@ -17,7 +17,10 @@ internal fun OutputReport.log(matrices: Collection<SavedMatrix>) {
     add(
         "test_results",
         matrices.map {
-            it.matrixId to it.testAxises
+            it.matrixId to mapOf(
+                "app" to it.appFileName,
+                "test-axises" to it.testAxises
+            )
         }.toMap()
     )
 }

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -208,7 +208,7 @@ class AndroidArgsTest {
           app: $appApk
           test: $testApk
           device:
-          - model: shamu
+          - model: walleye
             version: 18
       """
             ).validate()
@@ -836,7 +836,7 @@ AndroidArgs
     @Test
     fun `cli device`() {
         val cli = AndroidRunCommand()
-        CommandLine(cli).parseArgs("--device=model=shamu,version=22,locale=zh_CN,orientation=default")
+        CommandLine(cli).parseArgs("--device=model=walleye,version=27,locale=zh_CN,orientation=default")
 
         val yaml = """
         gcloud:
@@ -849,7 +849,7 @@ AndroidArgs
         assertThat(defaultDevices.size).isEqualTo(1)
 
         val androidArgs = AndroidArgs.load(yaml, cli).validate()
-        val expectedDevice = Device("shamu", "22", "zh_CN", "default", isVirtual = false)
+        val expectedDevice = Device("walleye", "27", "zh_CN", "default", isVirtual = false)
         val actualDevices = androidArgs.devices
         assertThat(actualDevices.first()).isEqualTo(expectedDevice)
         assertThat(actualDevices.size).isEqualTo(1)
@@ -858,7 +858,7 @@ AndroidArgs
     @Test
     fun `cli device repeat`() {
         val cli = AndroidRunCommand()
-        val deviceCmd = "--device=model=shamu,version=22,locale=zh_CN,orientation=default"
+        val deviceCmd = "--device=model=walleye,version=27,locale=zh_CN,orientation=default"
         CommandLine(cli).parseArgs(deviceCmd, deviceCmd)
 
         val yaml = """
@@ -867,7 +867,7 @@ AndroidArgs
           test: $testApk
       """
         val androidArgs = AndroidArgs.load(yaml, cli).validate()
-        val expectedDevice = Device("shamu", "22", "zh_CN", "default", false)
+        val expectedDevice = Device("walleye", "27", "zh_CN", "default", false)
         val actualDevices = androidArgs.devices
         assertThat(actualDevices.size).isEqualTo(2)
         assertThat(actualDevices[0]).isEqualTo(expectedDevice)