From 25ac2b8d1d4b0a310e1be62b8c6db6ac77883a4b Mon Sep 17 00:00:00 2001
From: pawelpasterz <32893017+pawelpasterz@users.noreply.github.com>
Date: Fri, 28 Feb 2020 21:54:27 +0100
Subject: [PATCH] Add default flank settings to optimize performance (#644)

Add default flank settings to optimize performance
---
 README.md                                     | 22 +++++++++----------
 .../kotlin/ftl/args/yml/AndroidGcloudYml.kt   |  5 +++--
 .../src/main/kotlin/ftl/args/yml/GcloudYml.kt |  3 ++-
 .../test/android/AndroidRunCommand.kt         | 12 +++++-----
 .../cli/firebase/test/ios/IosRunCommand.kt    |  4 ++--
 .../main/kotlin/ftl/config/FlankDefaults.kt   | 12 ++++++++++
 .../kotlin/ftl/args/AndroidArgsFileTest.kt    |  3 ---
 .../test/kotlin/ftl/args/AndroidArgsTest.kt   |  6 ++---
 .../src/test/kotlin/ftl/args/IosArgsTest.kt   | 11 +++++++++-
 .../ftl/cli/firebase/CancelCommandTest.kt     |  2 +-
 .../test/android/AndroidRunCommandTest.kt     |  2 +-
 .../{android.yml => simple-android-flank.yml} |  0
 .../kotlin/ftl/fixtures/simple-ios-flank.yml  |  3 +++
 13 files changed, 54 insertions(+), 31 deletions(-)
 create mode 100644 test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt
 rename test_runner/src/test/kotlin/ftl/fixtures/{android.yml => simple-android-flank.yml} (100%)
 create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml

diff --git a/README.md b/README.md
index 941991ab92..39a065240d 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,7 @@ gcloud:
   ## (default: a timestamp with a random suffix).
   # results-dir: tmp
 
-  ## Enable video recording during the test. Enabled by default, use --no-record-video to disable.
+  ## Enable video recording during the test. Disabled by default. Use --record-video to enable.
   # record-video: true
 
   ## The max time this test execution can run before it is cancelled (default: 15m).
@@ -189,7 +189,7 @@ gcloud:
   ## (default: a timestamp with a random suffix).
   # results-dir: tmp
 
-  ## Enable video recording during the test. Enabled by default, use --no-record-video to disable.
+  ## Enable video recording during the test. Disabled by default. Use --record-video to enable.
   # record-video: true
 
   ## The max time this test execution can run before it is cancelled (default: 15m).
@@ -222,7 +222,7 @@ gcloud:
   test: ../test_app/apks/app-debug-androidTest.apk
 
   ## Automatically log into the test device using a preconfigured Google account before beginning the test.
-  ## Enabled by default, use --no-auto-google-login to disable.
+  ## Disabled by default. Use --auto-google-login to enable.
   # auto-google-login: true
 
   ## Whether each test runs in its own Instrumentation instance with the Android Test Orchestrator
@@ -244,10 +244,10 @@ gcloud:
   #   - /sdcard/
 
   ## Monitor and record performance metrics: CPU, memory, network usage, and FPS (game-loop only).
-  ## Enabled by default, use --no-performance-metrics to disable.
+  ## Disabled by default. Use --performance-metrics to enable.
   # performance-metrics: true
-  
-  ## The fully-qualified Java class name of the instrumentation test runner 
+
+  ## The fully-qualified Java class name of the instrumentation test runner
   ## (default: the last name extracted from the APK manifest).
   # test-runner-class: com.foo.TestRunner
 
@@ -315,7 +315,7 @@ flank:
   ## Keeps the full path of downloaded files. Required when file names are not unique.
   ## Default: false  
   # keep-file-path: false
-    
+
   ## Include additional app/test apk pairs in the run. Apks are unique by just filename and not by path!
   ## If app is omitted, then the top level app is used for that pair.
   # additional-app-test-apks:
@@ -346,7 +346,7 @@ android {
       testCoverageEnabled true
     }
   }
- 
+
   // https://google.github.io/android-gradle-dsl/current/com.android.build.gradle.internal.dsl.TestOptions.html#com.android.build.gradle.internal.dsl.TestOptions:animationsDisabled
   testOptions {
         execution 'ANDROIDX_TEST_ORCHESTRATOR'
@@ -411,7 +411,7 @@ import static android.Manifest.permission.READ_EXTERNAL_STORAGE;
 import static android.Manifest.permission.WRITE_EXTERNAL_STORAGE;
 
 class MyEspressoTest {
-  
+
   @Rule
   GrantPermissionRule grantPermissionRule = GrantPermissionRule.grant(
           READ_EXTERNAL_STORAGE, WRITE_EXTERNAL_STORAGE);
@@ -427,11 +427,11 @@ Here's an example flank.yml. Note that `coverage` and `coverageFilePath` must be
 gcloud:
   app: ./app/build/outputs/apk/debug/app-debug.apk
   test: ./app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk
-  environment-variables: 
+  environment-variables:
     coverage: true
     coverageFilePath: /sdcard/
     clearPackageData: true
-  directories-to-pull: 
+  directories-to-pull:
     - /sdcard/
   # use a named results dir that's used by the gradle task
   results-dir: coverage_ec
diff --git a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt
index 046555592c..69ba19dbd8 100644
--- a/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt
+++ b/test_runner/src/main/kotlin/ftl/args/yml/AndroidGcloudYml.kt
@@ -3,6 +3,7 @@ package ftl.args.yml
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
 import ftl.config.Device
+import ftl.config.FlankDefaults
 import ftl.config.FtlConstants.defaultAndroidModel
 import ftl.config.FtlConstants.defaultAndroidVersion
 
@@ -17,7 +18,7 @@ class AndroidGcloudYmlParams(
     val test: String? = null,
 
     @field:JsonProperty("auto-google-login")
-    val autoGoogleLogin: Boolean = true,
+    val autoGoogleLogin: Boolean = FlankDefaults.DISABLE_AUTO_LOGIN,
 
     @field:JsonProperty("use-orchestrator")
     val useOrchestrator: Boolean = true,
@@ -29,7 +30,7 @@ class AndroidGcloudYmlParams(
     val directoriesToPull: List<String> = emptyList(),
 
     @field:JsonProperty("performance-metrics")
-    val performanceMetrics: Boolean = true,
+    val performanceMetrics: Boolean = FlankDefaults.DISABLE_PERFORMANCE_METRICS,
 
     @field:JsonProperty("test-runner-class")
     val testRunnerClass: String? = null,
diff --git a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt
index d1ab29136a..4a219bb1c0 100644
--- a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt
+++ b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt
@@ -2,6 +2,7 @@ package ftl.args.yml
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
 import com.fasterxml.jackson.annotation.JsonProperty
+import ftl.config.FlankDefaults
 
 /**
  * Common Gcloud parameters shared between iOS and Android
@@ -18,7 +19,7 @@ class GcloudYmlParams(
     var resultsDir: String? = null,
 
     @field:JsonProperty("record-video")
-    val recordVideo: Boolean = true,
+    val recordVideo: Boolean = FlankDefaults.DISABLE_VIDEO_RECORDING,
 
     val timeout: String = "15m",
 
diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt
index 3ec8ebe391..21464d9d8f 100644
--- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt
+++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt
@@ -94,13 +94,13 @@ class AndroidRunCommand : Runnable {
     @Option(
         names = ["--auto-google-login"],
         description = ["Automatically log into the test device using a preconfigured " +
-                "Google account before beginning the test. Enabled by default, use --no-auto-google-login to disable."]
+                "Google account before beginning the test. Disabled by default."]
     )
     var autoGoogleLogin: Boolean? = null
 
     @Option(
         names = ["--no-auto-google-login"],
-        description = ["Google account not logged in. See --auto-google-login."]
+        description = ["Google account not logged in (default behavior). Use --auto-google-login to enable"]
     )
     var noAutoGoogleLogin: Boolean? = null
 
@@ -145,13 +145,13 @@ class AndroidRunCommand : Runnable {
     @Option(
         names = ["--performance-metrics"],
         description = ["Monitor and record performance metrics: CPU, memory, " +
-                "network usage, and FPS (game-loop only). Enabled by default, use --no-performance-metrics to disable."]
+                "network usage, and FPS (game-loop only). Disabled by default."]
     )
     var performanceMetrics: Boolean? = null
 
     @Option(
         names = ["--no-performance-metrics"],
-        description = ["Disables performance metrics. See --performance-metrics"]
+        description = ["Disables performance metrics (default behavior). Use --performance-metrics to enable."]
     )
     var noPerformanceMetrics: Boolean? = null
 
@@ -221,13 +221,13 @@ class AndroidRunCommand : Runnable {
     @Option(
         names = ["--record-video"],
         description = ["Enable video recording during the test. " +
-                "Enabled by default, use --no-record-video to disable."]
+                "Disabled by default."]
     )
     var recordVideo: Boolean? = null
 
     @Option(
         names = ["--no-record-video"],
-        description = ["Disable video recording during the test. See --record-video to enable."]
+        description = ["Disable video recording during the test (default behavior). Use --record-video to enable."]
     )
     var noRecordVideo: Boolean? = null
 
diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt
index 402cc5bcae..f770bc06f3 100644
--- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt
+++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt
@@ -144,13 +144,13 @@ class IosRunCommand : Runnable {
     @Option(
         names = ["--record-video"],
         description = ["Enable video recording during the test. " +
-            "Enabled by default, use --no-record-video to disable."]
+            "Disabled by default."]
     )
     var recordVideo: Boolean? = null
 
     @Option(
         names = ["--no-record-video"],
-        description = ["Disable video recording during the test. See --record-video to enable."]
+        description = ["Disable video recording during the test (default behavior). Use --record-video to enable."]
     )
     var noRecordVideo: Boolean? = null
 
diff --git a/test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt b/test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt
new file mode 100644
index 0000000000..645c4849e6
--- /dev/null
+++ b/test_runner/src/main/kotlin/ftl/config/FlankDefaults.kt
@@ -0,0 +1,12 @@
+package ftl.config
+
+/**
+ * Flank specific default settings.
+ * Values should assure best (quickest) test performance.
+ * Each of value can be overwritten by config *.yml file
+ */
+object FlankDefaults {
+    const val DISABLE_VIDEO_RECORDING = false
+    const val DISABLE_AUTO_LOGIN = false
+    const val DISABLE_PERFORMANCE_METRICS = false
+}
diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt
index 146af2619b..c2ab26d1a5 100644
--- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt
+++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsFileTest.kt
@@ -85,9 +85,6 @@ class AndroidArgsFileTest {
                     Device("shamu", "22", "zh_CN", "default")
                 )
             )
-        }
-
-        with(args) {
             assert(maxTestShards, 1)
             assert(repeatTests, 1)
         }
diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt
index 36373b2819..2529357ca1 100644
--- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt
+++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt
@@ -271,7 +271,7 @@ AndroidArgs
         with(androidArgs) {
             // GcloudYml
             assert(resultsBucket, "mockBucket")
-            assert(recordVideo, true)
+            assert(recordVideo, false)
             assert(testTimeout, "15m")
             assert(async, false)
             assert(project, "mockProjectId")
@@ -279,11 +279,11 @@ AndroidArgs
             // AndroidGcloudYml
             assert(appApk, appApkAbsolutePath)
             assert(testApk, testApkAbsolutePath)
-            assert(autoGoogleLogin, true)
+            assert(autoGoogleLogin, false)
             assert(useOrchestrator, true)
             assert(environmentVariables, emptyMap<String, String>())
             assert(directoriesToPull, empty)
-            assert(performanceMetrics, true)
+            assert(performanceMetrics, false)
             assert(testRunnerClass, null)
             assert(testTargets, empty)
             assert(devices, listOf(Device("NexusLowRes", "28")))
diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt
index 3149acd277..9897d0e9a5 100644
--- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt
+++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt
@@ -14,6 +14,8 @@ import ftl.config.FtlConstants.defaultIosVersion
 import ftl.test.util.FlankTestRunner
 import ftl.test.util.TestHelper.absolutePath
 import ftl.test.util.TestHelper.assert
+import ftl.test.util.TestHelper.getPath
+import org.junit.Assert.assertFalse
 import org.junit.Assume
 import org.junit.Rule
 import org.junit.Test
@@ -26,6 +28,7 @@ import picocli.CommandLine
 @RunWith(FlankTestRunner::class)
 class IosArgsTest {
     private val empty = emptyList<String>()
+    private val simpleFlankPath = getPath("src/test/kotlin/ftl/fixtures/simple-ios-flank.yml")
     private val testPath = "./src/test/kotlin/ftl/fixtures/tmp/ios_earlgrey2.zip"
     private val xctestrunFile =
         "./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun"
@@ -216,7 +219,7 @@ IosArgs
         with(args) {
             // GcloudYml
             assert(resultsBucket, "mockBucket")
-            assert(recordVideo, true)
+            assert(recordVideo, false)
             assert(testTimeout, "15m")
             assert(async, false)
             assert(project, "mockProjectId")
@@ -749,4 +752,10 @@ IosArgs
 
         assertThat(actual).containsExactlyElementsIn(expected)
     }
+
+    @Test
+    fun `verify flank default settings for ios`() {
+        val args = IosArgs.load(simpleFlankPath)
+        assertFalse(args.recordVideo)
+    }
 }
diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt
index bd1b0f1142..beaf66bb81 100644
--- a/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt
+++ b/test_runner/src/test/kotlin/ftl/cli/firebase/CancelCommandTest.kt
@@ -50,7 +50,7 @@ class CancelCommandTest {
     fun cancelCommandRuns() {
         exit.expectSystemExit()
         val runCmd = AndroidRunCommand()
-        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/android.yml"
+        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml"
         runCmd.run()
         CancelCommand().run()
         val output = systemOutRule.log
diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt
index 9d9c386311..c1327d4c42 100644
--- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt
+++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt
@@ -40,7 +40,7 @@ class AndroidRunCommandTest {
     fun androidRunCommandRuns() {
         exit.expectSystemExit()
         val runCmd = AndroidRunCommand()
-        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/android.yml"
+        runCmd.configPath = "./src/test/kotlin/ftl/fixtures/simple-android-flank.yml"
         runCmd.run()
         val output = systemOutRule.log
         assertThat(output).contains("1 / 1 (100.00%)")
diff --git a/test_runner/src/test/kotlin/ftl/fixtures/android.yml b/test_runner/src/test/kotlin/ftl/fixtures/simple-android-flank.yml
similarity index 100%
rename from test_runner/src/test/kotlin/ftl/fixtures/android.yml
rename to test_runner/src/test/kotlin/ftl/fixtures/simple-android-flank.yml
diff --git a/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml b/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml
new file mode 100644
index 0000000000..42cea075c8
--- /dev/null
+++ b/test_runner/src/test/kotlin/ftl/fixtures/simple-ios-flank.yml
@@ -0,0 +1,3 @@
+gcloud:
+  test: ./src/test/kotlin/ftl/fixtures/tmp/ios_earlgrey2.zip
+  xctestrun-file: ./src/test/kotlin/ftl/fixtures/tmp/EarlGreyExampleSwiftTests_iphoneos12.1-arm64e.xctestrun