Refactor

Flank · Jun 25, 2020 · e80131f · e80131f
1 parent 14b649b
commit e80131f
Show file tree

Hide file tree

Showing 12 changed files with 118 additions and 116 deletions.
diff --git a/test_runner/docs/ascii/flank.jar_-android-run.adoc b/test_runner/docs/ascii/flank.jar_-android-run.adoc
@@ -13,12 +13,13 @@ flank.jar
 
 *flank.jar
  android run* [*-h*] [*--async*] [*--auto-google-login*]
-                       [*--disable-sharding*] [*--dry*] [*--dump-shards*]
-                       [*--full-junit-result*] [*--ignore-failed-tests*]
-                       [*--keep-file-path*] [*--legacy-junit-result*]
-                       [*--no-auto-google-login*] [*--no-performance-metrics*]
-                       [*--no-record-video*] [*--no-use-orchestrator*]
-                       [*--obfuscate*] [*--performance-metrics*] [*--record-video*]
+                       [*--disable-results-upload*] [*--disable-sharding*] [*--dry*]
+                       [*--dump-shards*] [*--full-junit-result*]
+                       [*--ignore-failed-tests*] [*--keep-file-path*]
+                       [*--legacy-junit-result*] [*--no-auto-google-login*]
+                       [*--no-performance-metrics*] [*--no-record-video*]
+                       [*--no-use-orchestrator*] [*--obfuscate*]
+                       [*--performance-metrics*] [*--record-video*]
                        [*--smart-flank-disable-upload*] [*--use-orchestrator*]
                        [*--app*=_<app>_] [*-c*=_<configPath>_]
                        [*--local-result-dir*=_<localResultsDir>_]
@@ -159,12 +160,12 @@ Configuration is read from flank.yml
 *--output-style*=_<outputStyle>_::
   Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`.
 
-*--app*=_<app>_::
-  The path to the application binary file. The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
-
 *--disable-results-upload*::
   Disables flank results upload on gcloud storage.
 
+*--app*=_<app>_::
+  The path to the application binary file. The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
+
 *--test*=_<test>_::
   The path to the binary file containing instrumentation tests. The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://.
 
@@ -177,9 +178,6 @@ Configuration is read from flank.yml
 *--no-auto-google-login*::
   Google account not logged in (default behavior). Use --auto-google-login to enable
 
-*--app*=_<app>_::
-  The path to the application binary file. The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
-
 *--use-orchestrator*::
   Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more information about Android Test Orchestrator.
 

diff --git a/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc b/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc
@@ -172,6 +172,9 @@ Configuration is read from flank.yml
 *--output-style*=_<outputStyle>_::
   Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`.
 
+*--disable-results-upload*::
+  Disables flank results upload on gcloud storage.
+
 *--app*=_<app>_::
   The path to the application binary file. The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
 
@@ -187,9 +190,6 @@ Configuration is read from flank.yml
 *--no-auto-google-login*::
   Google account not logged in (default behavior). Use --auto-google-login to enable
 
-*--app*=_<app>_::
-  The path to the application binary file. The path may be in the local filesystem or in Google Cloud Storage using gs:// notation.
-
 *--use-orchestrator*::
   Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator (default: Orchestrator is used. To disable, use --no-use-orchestrator). Orchestrator is only compatible with AndroidJUnitRunner v1.0 or higher. See https://developer.android.com/training/testing/junit-runner.html#using-android-test-orchestrator for more information about Android Test Orchestrator.
 

diff --git a/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc b/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc
@@ -12,11 +12,11 @@ flank.jar
 == Synopsis
 
 *flank.jar
- firebase test ios run* [*-h*] [*--async*] [*--disable-sharding*] [*--dry*]
-                                 [*--dump-shards*] [*--full-junit-result*]
-                                 [*--ignore-failed-tests*] [*--keep-file-path*]
-                                 [*--no-record-video*] [*--obfuscate*]
-                                 [*--record-video*]
+ firebase test ios run* [*-h*] [*--async*] [*--disable-results-upload*]
+                                 [*--disable-sharding*] [*--dry*] [*--dump-shards*]
+                                 [*--full-junit-result*] [*--ignore-failed-tests*]
+                                 [*--keep-file-path*] [*--no-record-video*]
+                                 [*--obfuscate*] [*--record-video*]
                                  [*--smart-flank-disable-upload*]
                                  [*-c*=_<configPath>_]
                                  [*--local-result-dir*=_<localResultsDir>_]
@@ -151,6 +151,9 @@ Configuration is read from flank.yml
 *--output-style*=_<outputStyle>_::
   Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`.
 
+*--disable-results-upload*::
+  Disables flank results upload on gcloud storage.
+
 *--test*=_<test>_::
   The path to the test package (a zip file containing the iOS app and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file is specified.
 

diff --git a/test_runner/docs/ascii/flank.jar_-ios-run.adoc b/test_runner/docs/ascii/flank.jar_-ios-run.adoc
@@ -12,7 +12,8 @@ flank.jar
 == Synopsis
 
 *flank.jar
- ios run* [*-h*] [*--async*] [*--disable-sharding*] [*--dry*] [*--dump-shards*]
+ ios run* [*-h*] [*--async*] [*--disable-results-upload*]
+                   [*--disable-sharding*] [*--dry*] [*--dump-shards*]
                    [*--full-junit-result*] [*--ignore-failed-tests*]
                    [*--keep-file-path*] [*--no-record-video*] [*--obfuscate*]
                    [*--record-video*] [*--smart-flank-disable-upload*]
@@ -142,6 +143,9 @@ Configuration is read from flank.yml
 *--output-style*=_<outputStyle>_::
   Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`.
 
+*--disable-results-upload*::
+  Disables flank results upload on gcloud storage.
+
 *--test*=_<test>_::
   The path to the test package (a zip file containing the iOS app and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file is specified.
 

diff --git a/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt b/test_runner/src/main/kotlin/ftl/json/SavedMatrix.kt
@@ -5,19 +5,19 @@ import com.google.api.services.testing.model.TestMatrix
 import com.google.api.services.toolresults.model.Outcome
 import ftl.android.AndroidCatalog
 import ftl.gc.GcToolResults
-import ftl.util.Billing
 import ftl.util.MatrixState.FINISHED
 import ftl.util.StepOutcome.failure
 import ftl.util.StepOutcome.flaky
 import ftl.util.StepOutcome.inconclusive
 import ftl.util.StepOutcome.skipped
 import ftl.util.StepOutcome.success
 import ftl.util.StepOutcome.unset
+import ftl.util.billableMinutes
 import ftl.util.timeoutToSeconds
 import ftl.util.webLink
 
 // execution gcs paths aren't API accessible.
-class SavedMatrix(matrix: TestMatrix, testTimeout: String = "0s") {
+class SavedMatrix(matrix: TestMatrix, testTimeout: String) {
     private val testTimeoutSeconds = timeoutToSeconds(testTimeout)
     val matrixId: String = matrix.testMatrixId
     var state: String = matrix.state
@@ -92,7 +92,7 @@ class SavedMatrix(matrix: TestMatrix, testTimeout: String = "0s") {
             val stepResult = GcToolResults.getStepResult(it.toolResultsStep)
             val testTimeSeconds = stepResult.testExecutionStep?.testTiming?.testProcessDuration?.seconds ?: return
 
-            val billableMinutes = Billing.billableMinutes(testTimeSeconds, testTimeoutSeconds)
+            val billableMinutes = billableMinutes(testTimeSeconds, testTimeoutSeconds)
 
             if (AndroidCatalog.isVirtualDevice(it.environment?.androidDevice, matrix.projectId ?: "")) {
                 billableVirtualMinutes += billableMinutes

diff --git a/test_runner/src/main/kotlin/ftl/reports/CostReport.kt b/test_runner/src/main/kotlin/ftl/reports/CostReport.kt
@@ -6,7 +6,7 @@ import ftl.gc.GcStorage
 import ftl.json.MatrixMap
 import ftl.reports.util.IReport
 import ftl.reports.xml.model.JUnitTestResult
-import ftl.util.Billing
+import ftl.util.estimateCosts
 import ftl.util.println
 import java.io.StringWriter
 
@@ -24,7 +24,7 @@ object CostReport : IReport {
             totalBillablePhysicalMinutes += it.billablePhysicalSecondsMinutes
         }
 
-        return Billing.estimateCosts(totalBillableVirtualMinutes, totalBillablePhysicalMinutes)
+        return estimateCosts(totalBillableVirtualMinutes, totalBillablePhysicalMinutes)
     }
 
     private fun generate(matrices: MatrixMap): String {

diff --git a/test_runner/src/main/kotlin/ftl/util/Billing.kt b/test_runner/src/main/kotlin/ftl/util/Billing.kt
@@ -3,91 +3,79 @@ package ftl.util
 import java.math.BigDecimal
 import java.math.RoundingMode
 import java.util.concurrent.TimeUnit
+import kotlin.math.min
 
-object Billing {
+private val physicalCostPerMinute = divBy60(5) // $5/hr
+private val virtualCostPerMinute = divBy60(1) // $1/hr
 
-    private fun divBy60(value: Long): BigDecimal {
-        return BigDecimal(value).divide(BigDecimal(60), 10, RoundingMode.HALF_UP)
-    }
-
-    private fun divBy60(value: BigDecimal): BigDecimal {
-        return value.divide(BigDecimal(60), 10, RoundingMode.HALF_UP)
-    }
-
-    private var PHYSICAL_COST_PER_MINUTE = divBy60(5) // $5/hr
-    private var VIRTUAL_COST_PER_MINUTE = divBy60(1) // $1/hr
+private fun divBy60(value: Long) = BigDecimal(value).divide(BigDecimal(60), 10, RoundingMode.HALF_UP)
 
-    fun billableMinutes(testDurationSeconds: Long, testTimeoutSeconds: Long): Long =
-        if (testTimeoutSeconds in 1 until testDurationSeconds) billableMinutes(BigDecimal(testTimeoutSeconds))
-        else billableMinutes(BigDecimal(testDurationSeconds))
+private fun divBy60(value: BigDecimal) = value.divide(BigDecimal(60), 10, RoundingMode.HALF_UP)
 
-    private fun billableMinutes(testDurationSeconds: BigDecimal): Long {
-        val billableMinutes = divBy60(checkForZero(testDurationSeconds))
-        // round decimals up.  0.01 minutes is billable at 1 minute.
-        return billableMinutes.setScale(0, RoundingMode.UP).longValueExact()
-    }
+fun billableMinutes(testDurationSeconds: Long, testTimeoutSeconds: Long): Long =
+    billableMinutes(min(testDurationSeconds, testTimeoutSeconds))
 
-    private fun checkForZero(testDurationSeconds: BigDecimal): BigDecimal {
-        // 0s duration => 1s
-        if (testDurationSeconds.compareTo(BigDecimal(0)) == 0) {
-            return BigDecimal(1)
-        }
+// round decimals up. 0.01 minutes is billable at 1 minute.
+private fun billableMinutes(testDurationSeconds: Long) = divBy60(checkForZero(BigDecimal(testDurationSeconds)))
+    .setScale(0, RoundingMode.UP)
+    .longValueExact()
 
-        return testDurationSeconds
-    }
+// 0s duration => 1s
+private fun checkForZero(testDurationSeconds: BigDecimal) =
+    if (testDurationSeconds == BigDecimal.ZERO) BigDecimal.ONE
+    else testDurationSeconds
 
-    fun estimateCosts(billableVirtualMinutes: Long, billablePhysicalMinutes: Long): String {
-        return estimateCosts(BigDecimal(billableVirtualMinutes), BigDecimal(billablePhysicalMinutes))
-    }
+fun estimateCosts(billableVirtualMinutes: Long, billablePhysicalMinutes: Long): String {
+    return estimateCosts(BigDecimal(billableVirtualMinutes), BigDecimal(billablePhysicalMinutes))
+}
 
-    private fun estimateCosts(billableVirtualMinutes: BigDecimal, billablePhysicalMinutes: BigDecimal): String {
-        val virtualCost = billableVirtualMinutes.multiply(VIRTUAL_COST_PER_MINUTE).setScale(2, RoundingMode.HALF_UP)
-        val physicalCost = billablePhysicalMinutes.multiply(PHYSICAL_COST_PER_MINUTE).setScale(2, RoundingMode.HALF_UP)
-        val totalCost = (virtualCost + physicalCost).setScale(2, RoundingMode.HALF_UP)
+private fun estimateCosts(billableVirtualMinutes: BigDecimal, billablePhysicalMinutes: BigDecimal): String {
+    val virtualCost = billableVirtualMinutes.multiply(virtualCostPerMinute).setScale(2, RoundingMode.HALF_UP)
+    val physicalCost = billablePhysicalMinutes.multiply(physicalCostPerMinute).setScale(2, RoundingMode.HALF_UP)
+    val totalCost = (virtualCost + physicalCost).setScale(2, RoundingMode.HALF_UP)
 
-        val billableVirtualTime = prettyTime(billableVirtualMinutes)
-        val billablePhysicalTime = prettyTime(billablePhysicalMinutes)
-        val totalTime = prettyTime(billableVirtualMinutes + billablePhysicalMinutes)
+    val billableVirtualTime = prettyTime(billableVirtualMinutes)
+    val billablePhysicalTime = prettyTime(billablePhysicalMinutes)
+    val totalTime = prettyTime(billableVirtualMinutes + billablePhysicalMinutes)
 
-        val displayPhysical = billablePhysicalMinutes.signum() == 1
-        val displayVirtual = billableVirtualMinutes.signum() == 1 // 1 = positive number > 0
-        val displayTotal = displayPhysical && displayVirtual
-        var result = ""
+    val displayPhysical = billablePhysicalMinutes.signum() == 1
+    val displayVirtual = billableVirtualMinutes.signum() == 1 // 1 = positive number > 0
+    val displayTotal = displayPhysical && displayVirtual
+    var result = ""
 
-        if (!displayPhysical && !displayVirtual) {
-            result = "No cost. 0m"
-        }
+    if (!displayPhysical && !displayVirtual) {
+        result = "No cost. 0m"
+    }
 
-        if (displayPhysical) {
-            result += """
+    if (displayPhysical) {
+        result += """
 Physical devices
   $$physicalCost for $billablePhysicalTime
 """
-        }
+    }
 
-        if (displayVirtual) {
-            result += """
+    if (displayVirtual) {
+        result += """
 Virtual devices
   $$virtualCost for $billableVirtualTime
 """
-        }
+    }
 
-        if (displayTotal) {
-            result += """
+    if (displayTotal) {
+        result += """
 Total
   $$totalCost for $totalTime
 """
-        }
-
-        return result.trim()
     }
 
-    private fun prettyTime(billableMinutes: BigDecimal): String {
-        val remainder = billableMinutes.toLong()
-        val hours = TimeUnit.MINUTES.toHours(remainder)
-        val minutes = remainder % 60
+    return result.trim()
+}
 
-        return if (hours > 0) "${hours}h ${minutes}m"
-        else "${minutes}m"
-    }
+private fun prettyTime(billableMinutes: BigDecimal): String {
+    val remainder = billableMinutes.toLong()
+    val hours = TimeUnit.MINUTES.toHours(remainder)
+    val minutes = remainder % 60
+
+    return if (hours > 0) "${hours}h ${minutes}m"
+    else "${minutes}m"
 }
diff --git a/test_runner/src/test/kotlin/ftl/json/MatrixMapTest.kt b/test_runner/src/test/kotlin/ftl/json/MatrixMapTest.kt
@@ -5,6 +5,7 @@ import com.google.common.truth.Truth.assertThat
 import ftl.json.SavedMatrixTest.Companion.createResultsStorage
 import ftl.json.SavedMatrixTest.Companion.createStepExecution
 import ftl.test.util.FlankTestRunner
+import ftl.test.util.defaultTestTimeout
 import ftl.util.MatrixState
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -25,15 +26,16 @@ class MatrixMapTest {
 
     private fun matrixForExecution(executionId: Int): SavedMatrix {
         return SavedMatrix(
-            TestMatrix()
+            matrix = TestMatrix()
                 .setResultStorage(createResultsStorage())
                 .setState(MatrixState.FINISHED)
                 .setTestMatrixId("123")
                 .setTestExecutions(
                     listOf(
                         createStepExecution(executionId, "")
                     )
-                )
+                ),
+            testTimeout = defaultTestTimeout
         )
     }
 

diff --git a/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt b/test_runner/src/test/kotlin/ftl/json/SavedMatrixTest.kt
@@ -10,6 +10,7 @@ import com.google.common.truth.Truth.assertThat
 import ftl.config.Device
 import ftl.gc.GcAndroidDevice
 import ftl.test.util.FlankTestRunner
+import ftl.test.util.defaultTestTimeout
 import ftl.util.MatrixState.FINISHED
 import ftl.util.MatrixState.PENDING
 import ftl.util.webLink
@@ -68,7 +69,7 @@ class SavedMatrixTest {
         testMatrix.resultStorage = createResultsStorage()
         testMatrix.testExecutions = testExecutions
 
-        val savedMatrix = SavedMatrix(testMatrix)
+        val savedMatrix = SavedMatrix(testMatrix, defaultTestTimeout)
         assertThat(savedMatrix.outcome).isEqualTo("failure")
 
         // assert other properties
@@ -100,7 +101,7 @@ class SavedMatrixTest {
         testMatrix.resultStorage = createResultsStorage()
         testMatrix.testExecutions = testExecutions
 
-        val savedMatrix = SavedMatrix(testMatrix)
+        val savedMatrix = SavedMatrix(testMatrix, defaultTestTimeout)
         assertThat(savedMatrix.outcome).isEqualTo("skipped")
 
         // assert other properties
@@ -130,7 +131,7 @@ class SavedMatrixTest {
         testMatrix.resultStorage = createResultsStorage()
         testMatrix.testExecutions = testExecutions
 
-        val savedMatrix = SavedMatrix(testMatrix)
+        val savedMatrix = SavedMatrix(testMatrix, defaultTestTimeout)
         savedMatrix.update(testMatrix)
         testMatrix.state = FINISHED
         testMatrix.webLink()

diff --git a/test_runner/src/test/kotlin/ftl/test/util/Constants.kt b/test_runner/src/test/kotlin/ftl/test/util/Constants.kt
@@ -1,6 +1,8 @@
 package ftl.test.util
 
+import ftl.args.AndroidArgs
 import ftl.test.util.TestHelper.getPath
 
-val mixedConfigYaml = getPath("src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml")
-val ios2ConfigYaml = getPath("src/test/kotlin/ftl/fixtures/flank2.ios.yml")
+internal val mixedConfigYaml = getPath("src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-mixed.yml")
+internal val ios2ConfigYaml = getPath("src/test/kotlin/ftl/fixtures/flank2.ios.yml")
+internal val defaultTestTimeout = AndroidArgs.default().testTimeout