Add default flank settings to optimize performance (#644)

Add default flank settings to optimize performance

README.md

@@ -67,7 +67,7 @@ gcloud:
   ## (default: a timestamp with a random suffix).
   # results-dir: tmp
 
-  ## Enable video recording during the test. Enabled by default, use --no-record-video to disable.
+  ## Enable video recording during the test. Disabled by default. Use --record-video to enable.
   # record-video: true
 
   ## The max time this test execution can run before it is cancelled (default: 15m).
@@ -189,7 +189,7 @@ gcloud:
   ## (default: a timestamp with a random suffix).
   # results-dir: tmp
 
-  ## Enable video recording during the test. Enabled by default, use --no-record-video to disable.
+  ## Enable video recording during the test. Disabled by default. Use --record-video to enable.
   # record-video: true
 
   ## The max time this test execution can run before it is cancelled (default: 15m).
@@ -222,7 +222,7 @@ gcloud:
   test: ../test_app/apks/app-debug-androidTest.apk
 
   ## Automatically log into the test device using a preconfigured Google account before beginning the test.
-  ## Enabled by default, use --no-auto-google-login to disable.
+  ## Disabled by default. Use --auto-google-login to enable.
   # auto-google-login: true
 
   ## Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator
@@ -244,10 +244,10 @@ gcloud:
   #   - /sdcard/
 
   ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only).
-  ## Enabled by default, use --no-performance-metrics to disable.
+  ## Disabled by default. Use --performance-metrics to enable.
   # performance-metrics: true
-  
-  ## The fully-qualified Java class name of the instrumentation test runner 
+
+  ## The fully-qualified Java class name of the instrumentation test runner
   ## (default: the last name extracted from the APK manifest).
   # test-runner-class: com.foo.TestRunner
 
@@ -315,7 +315,7 @@ flank:
   ## Keeps the full path of downloaded files. Required when file names are not unique.
   ## Default: false  
   # keep-file-path: false
-    
+
   ## Include additional app/test apk pairs in the run. Apks are unique by just filename and not by path!
   ## If app is omitted, then the top level app is used for that pair.
   # additional-app-test-apks:
@@ -346,7 +346,7 @@ android {
       testCoverageEnabled true
     }
   }
- 
+
   // https://google.github.io/android-gradle-dsl/current/com.android.build.gradle.internal.dsl.TestOptions.html#com.android.build.gradle.internal.dsl.TestOptions:animationsDisabled
   testOptions {
         execution 'ANDROIDX_TEST_ORCHESTRATOR'
@@ -411,7 +411,7 @@ import static android.Manifest.permission.READ_EXTERNAL_STORAGE;
 import static android.Manifest.permission.WRITE_EXTERNAL_STORAGE;
 
 class MyEspressoTest {
-  
+
   @Rule
   GrantPermissionRule grantPermissionRule = GrantPermissionRule.grant(
           READ_EXTERNAL_STORAGE, WRITE_EXTERNAL_STORAGE);
@@ -427,11 +427,11 @@ Here's an example flank.yml. Note that `coverage` and `coverageFilePath` must be
 gcloud:
   app: ./app/build/outputs/apk/debug/app-debug.apk
   test: ./app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk
-  environment-variables: 
+  environment-variables:
     coverage: true
     coverageFilePath: /sdcard/
     clearPackageData: true
-  directories-to-pull: 
+  directories-to-pull:
     - /sdcard/
   # use a named results dir that's used by the gradle task
   results-dir: coverage_ec

test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt

@@ -3,6 +3,7 @@ package ftl.args.yml
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 import ftl.config.Device
+import ftl.config.FlankDefaults
 import ftl.config.FtlConstants.defaultAndroidModel
 import ftl.config.FtlConstants.defaultAndroidVersion
 
@@ -17,7 +18,7 @@ class AndroidGcloudYmlParams(
     val test: String? = null,
 
     @field:JsonProperty("auto-google-login")
-    val autoGoogleLogin: Boolean = true,
+    val autoGoogleLogin: Boolean = FlankDefaults.DISABLE_AUTO_LOGIN,
 
     @field:JsonProperty("use-orchestrator")
     val useOrchestrator: Boolean = true,
@@ -29,7 +30,7 @@ class AndroidGcloudYmlParams(
     val directoriesToPull: List<String> = emptyList(),
 
     @field:JsonProperty("performance-metrics")
-    val performanceMetrics: Boolean = true,
+    val performanceMetrics: Boolean = FlankDefaults.DISABLE_PERFORMANCE_METRICS,
 
     @field:JsonProperty("test-runner-class")
     val testRunnerClass: String? = null,

test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt

@@ -2,6 +2,7 @@ package ftl.args.yml
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
+import ftl.config.FlankDefaults
 
 /**
  * Common Gcloud parameters shared between iOS and Android
@@ -18,7 +19,7 @@ class GcloudYmlParams(
     var resultsDir: String? = null,
 
     @field:JsonProperty("record-video")
-    val recordVideo: Boolean = true,
+    val recordVideo: Boolean = FlankDefaults.DISABLE_VIDEO_RECORDING,
 
     val timeout: String = "15m",
 

test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt

@@ -94,13 +94,13 @@ class AndroidRunCommand : Runnable {
     @Option(
         names = ["--auto-google-login"],
         description = ["Automatically log into the test device using a preconfigured " +
-                "Google account before beginning the test. Enabled by default, use --no-auto-google-login to disable."]
+                "Google account before beginning the test. Disabled by default."]
     )
     var autoGoogleLogin: Boolean? = null
 
     @Option(
         names = ["--no-auto-google-login"],
-        description = ["Google account not logged in. See --auto-google-login."]
+        description = ["Google account not logged in (default behavior). Use --auto-google-login to enable"]
     )
     var noAutoGoogleLogin: Boolean? = null
 
@@ -145,13 +145,13 @@ class AndroidRunCommand : Runnable {
     @Option(
         names = ["--performance-metrics"],
         description = ["Monitor and record performance metrics: CPU, memory, " +
-                "network usage, and FPS (game-loop only). Enabled by default, use --no-performance-metrics to disable."]
+                "network usage, and FPS (game-loop only). Disabled by default."]
     )
     var performanceMetrics: Boolean? = null
 
     @Option(
         names = ["--no-performance-metrics"],
-        description = ["Disables performance metrics. See --performance-metrics"]
+        description = ["Disables performance metrics (default behavior). Use --performance-metrics to enable."]
     )
     var noPerformanceMetrics: Boolean? = null
 
@@ -221,13 +221,13 @@ class AndroidRunCommand : Runnable {
     @Option(
         names = ["--record-video"],
         description = ["Enable video recording during the test. " +
-                "Enabled by default, use --no-record-video to disable."]
+                "Disabled by default."]
     )
     var recordVideo: Boolean? = null
 
     @Option(
         names = ["--no-record-video"],
-        description = ["Disable video recording during the test. See --record-video to enable."]
+        description = ["Disable video recording during the test (default behavior). Use --record-video to enable."]
     )
     var noRecordVideo: Boolean? = null
 

test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt

@@ -144,13 +144,13 @@ class IosRunCommand : Runnable {
     @Option(
         names = ["--record-video"],
         description = ["Enable video recording during the test. " +
-            "Enabled by default, use --no-record-video to disable."]
+            "Disabled by default."]
     )
     var recordVideo: Boolean? = null
 
     @Option(
         names = ["--no-record-video"],
-        description = ["Disable video recording during the test. See --record-video to enable."]
+        description = ["Disable video recording during the test (default behavior). Use --record-video to enable."]
     )
     var noRecordVideo: Boolean? = null
 

test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt

@@ -0,0 +1,12 @@
+package ftl.config
+
+/**
+ * Flank specific default settings.
+ * Values should assure best (quickest) test performance.
+ * Each of value can be overwritten by config *.yml file
+ */
+object FlankDefaults {
+    const val DISABLE_VIDEO_RECORDING = false
+    const val DISABLE_AUTO_LOGIN = false
+    const val DISABLE_PERFORMANCE_METRICS = false
+}

test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt

@@ -85,9 +85,6 @@ class AndroidArgsFileTest {
                     Device("shamu", "22", "zh_CN", "default")
                 )
             )
-        }
-
-        with(args) {
             assert(maxTestShards, 1)
             assert(repeatTests, 1)
         }

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -271,19 +271,19 @@ AndroidArgs
         with(androidArgs) {
             // GcloudYml
             assert(resultsBucket, "mockBucket")
-            assert(recordVideo, true)
+            assert(recordVideo, false)
             assert(testTimeout, "15m")
             assert(async, false)
             assert(project, "mockProjectId")
 
             // AndroidGcloudYml
             assert(appApk, appApkAbsolutePath)
             assert(testApk, testApkAbsolutePath)
-            assert(autoGoogleLogin, true)
+            assert(autoGoogleLogin, false)
             assert(useOrchestrator, true)
             assert(environmentVariables, emptyMap<String, String>())
             assert(directoriesToPull, empty)
-            assert(performanceMetrics, true)
+            assert(performanceMetrics, false)
             assert(testRunnerClass, null)
             assert(testTargets, empty)
             assert(devices, listOf(Device("NexusLowRes", "28")))

test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt

@@ -14,6 +14,8 @@ import ftl.config.FtlConstants.defaultIosVersion
 import ftl.test.util.FlankTestRunner
 import ftl.test.util.TestHelper.absolutePath
 import ftl.test.util.TestHelper.assert
+import ftl.test.util.TestHelper.getPath
+import org.junit.Assert.assertFalse
 import org.junit.Assume
 import org.junit.Rule
 import org.junit.Test
@@ -26,6 +28,7 @@ import picocli.CommandLine
 @RunWith(FlankTestRunner::class)
 class IosArgsTest {
     private val empty = emptyList<String>()
+    private val simpleFlankPath = getPath("src/test/kotlin/ftl/fixtures/simple-ios-flank.yml")
     private val testPath = "./src/test/kotlin/ftl/fixtures/tmp/ios_earlgrey2.zip"
     private val xctestrunFile =
         "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun"
@@ -216,7 +219,7 @@ IosArgs
         with(args) {
             // GcloudYml
             assert(resultsBucket, "mockBucket")
-            assert(recordVideo, true)
+            assert(recordVideo, false)
             assert(testTimeout, "15m")
             assert(async, false)
             assert(project, "mockProjectId")
@@ -749,4 +752,10 @@ IosArgs
 
         assertThat(actual).containsExactlyElementsIn(expected)
     }
+
+    @Test
+    fun `verify flank default settings for ios`() {
+        val args = IosArgs.load(simpleFlankPath)
+        assertFalse(args.recordVideo)
+    }
 }

test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt

@@ -50,7 +50,7 @@ class CancelCommandTest {
     fun cancelCommandRuns() {
         exit.expectSystemExit()
         val runCmd = AndroidRunCommand()
-        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/android.yml"
+        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml"
         runCmd.run()
         CancelCommand().run()
         val output = systemOutRule.log

test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt

@@ -40,7 +40,7 @@ class AndroidRunCommandTest {
     fun androidRunCommandRuns() {
         exit.expectSystemExit()
         val runCmd = AndroidRunCommand()
-        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/android.yml"
+        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml"
         runCmd.run()
         val output = systemOutRule.log
         assertThat(output).contains("1 / 1 (100.00%)")

test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml

@@ -0,0 +1,3 @@
+gcloud:
+  test: ./src/test/kotlin/ftl/fixtures/tmp/ios_earlgrey2.zip
+  xctestrun-file: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun