Add --flaky-test-attempts (#454)

* Add flakyTestAttempts support for Android and iOS * Fix crash * Update docs * Update tests * Update release_notes.md * Fix lint issue
Flank · Jan 11, 2019 · 6b12a99 · 6b12a99
1 parent 513054d
commit 6b12a99
Show file tree

Hide file tree

Showing 20 changed files with 109 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -62,6 +62,8 @@ gcloud:
       version: 11.2
       locale: en
       orientation: portrait
+  # The number of times to retry failed tests. Default is 0. Max is 10.
+  flaky-test-attempts: 0
 
 flank:
   # test shards - the amount of groups to split the test suite into
@@ -118,6 +120,8 @@ gcloud:
   device:
     - model: NexusLowRes
       version: 28
+  # The number of times to retry failed tests. Default is 0. Max is 10.
+  flaky-test-attempts: 0
 
 flank:
   # test shards - the amount of groups to split the test suite into

diff --git a/release_notes.md b/release_notes.md
@@ -3,6 +3,7 @@
 - Fix create Gcs bucket [#444](https://github.com/TestArmada/flank/pull/444)
 - Add `files-to-download` to Android and iOS. Specify a list of regular expressions to download files from the Google Cloud Storage bucket. [#441](https://github.com/TestArmada/flank/pull/441)
 - Add `flank auth login` to authorize with a user account instead of a service account. [#446](https://github.com/TestArmada/flank/pull/436)
+- Add `flaky-test-attempts` support on Android and iOS which automatically retries failed tests. [#454](https://github.com/TestArmada/flank/pull/454)
 
 ## v4.1.1
 

diff --git a/test_app/apks/flaky-androidTest.apk b/test_app/apks/flaky-androidTest.apk
diff --git a/test_app/app/src/androidTest/java/com/example/app/ExampleUiTest.java b/test_app/app/src/androidTest/java/com/example/app/ExampleUiTest.java
@@ -1,6 +1,7 @@
 package com.example.app;
 
 import android.Manifest;
+import android.os.SystemClock;
 import android.support.test.rule.GrantPermissionRule;
 import android.support.test.runner.AndroidJUnit4;
 import com.instructure.espresso.ScreenshotTestRule;
@@ -9,6 +10,7 @@
 import org.junit.runner.RunWith;
 
 import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 @RunWith(AndroidJUnit4.class)
 public class ExampleUiTest {
@@ -30,4 +32,11 @@ public void testPasses() {
     public void testFails() {
         assertEquals(true, false);
     }
+
+    @Test
+    public void testIsFlaky() {
+        if ((SystemClock.uptimeMillis() %2) == 0) {
+            fail();
+        }
+    }
 }
diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml
@@ -17,6 +17,8 @@ gcloud:
       version: 12.0
       locale: en
       orientation: portrait
+  # The number of times to retry failed tests. Default is 0. Max is 10.
+  flaky-test-attempts: 0
 
 flank:
   # # Google cloud storage path to store the JUnit XML results from the last run.

diff --git a/test_runner/flank.yml b/test_runner/flank.yml
@@ -25,6 +25,7 @@ gcloud:
   device:
     - model: NexusLowRes
       version: 28
+  flaky-test-attempts: 1 # default is 0
 
 flank:
   # # Google cloud storage path to store the JUnit XML results from the last run.

diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt
@@ -48,6 +48,7 @@ class AndroidArgs(
     override val async = cli?.async ?: gcloud.async
     override val project = cli?.project ?: gcloud.project
     override val resultsHistoryName = cli?.resultsHistoryName ?: gcloud.resultsHistoryName
+    override val flakyTestAttempts = cli?.flakyTestAttempts ?: gcloud.flakyTestAttempts
 
     private val androidGcloud = androidGcloudYml.gcloud
     var appApk = cli?.app ?: androidGcloud.app
@@ -155,6 +156,7 @@ ${listToString(directoriesToPull)}
 ${listToString(testTargets)}
       device:
 ${devicesToString(devices)}
+      flaky-test-attempts: $flakyTestAttempts
 
     flank:
       testShards: $testShards

diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt
@@ -12,6 +12,7 @@ interface IArgs {
     val async: Boolean
     val project: String
     val resultsHistoryName: String?
+    val flakyTestAttempts: Int
 
     // FlankYml
     val testShards: Int

diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt
@@ -40,6 +40,7 @@ class IosArgs(
     override val async = cli?.async ?: gcloud.async
     override val project = cli?.project ?: gcloud.project
     override val resultsHistoryName = cli?.resultsHistoryName ?: gcloud.resultsHistoryName
+    override val flakyTestAttempts = cli?.flakyTestAttempts ?: gcloud.flakyTestAttempts
 
     private val iosGcloud = iosGcloudYml.gcloud
     var xctestrunZip = cli?.test ?: iosGcloud.test
@@ -117,6 +118,7 @@ IosArgs
       xcode-version: $xcodeVersion
       device:
 ${devicesToString(devices)}
+      flaky-test-attempts: $flakyTestAttempts
 
     flank:
       testShards: $testShards

diff --git a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt
@@ -31,11 +31,14 @@ class GcloudYmlParams(
     val project: String = getDefaultProjectId() ?: "",
 
     @field:JsonProperty("results-history-name")
-    val resultsHistoryName: String? = null
+    val resultsHistoryName: String? = null,
+
+    @field:JsonProperty("flaky-test-attempts")
+    val flakyTestAttempts: Int = 0
 ) {
     companion object : IYmlKeys {
         override val keys =
-            listOf("results-bucket", "record-video", "timeout", "async", "project", "results-history-name", "results-dir")
+            listOf("results-bucket", "results-dir", "record-video", "timeout", "async", "project", "results-history-name", "flaky-test-attempts")
     }
 
     init {

diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt
@@ -254,4 +254,11 @@ class AndroidRunCommand : Runnable {
                 "intermingled."]
     )
     var resultsDir: String? = null
+
+    @Option(
+        names = ["--flaky-test-attempts"],
+        description = ["The number of times a TestExecution should be re-attempted if one or more of its test cases " +
+                "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."]
+    )
+    var flakyTestAttempts: Int? = null
 }
diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt
@@ -195,4 +195,11 @@ class IosRunCommand : Runnable {
                 "intermingled."]
     )
     var resultsDir: String? = null
+
+    @Option(
+        names = ["--flaky-test-attempts"],
+        description = ["The number of times a TestExecution should be re-attempted if one or more of its test cases " +
+                "fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns."]
+    )
+    var flakyTestAttempts: Int? = null
 }
diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt
@@ -95,6 +95,7 @@ object GcAndroidTestMatrix {
             .setTestSpecification(testSpecification)
             .setResultStorage(resultsStorage)
             .setEnvironmentMatrix(environmentMatrix)
+            .setFlakyTestAttempts(args.flakyTestAttempts)
         try {
             return GcTesting.get.projects().testMatrices().create(args.project, testMatrix)
         } catch (e: Exception) {

diff --git a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt
@@ -78,6 +78,7 @@ object GcIosTestMatrix {
             .setTestSpecification(testSpecification)
             .setEnvironmentMatrix(environmentMatrix)
             .setResultStorage(resultStorage)
+            .setFlakyTestAttempts(args.flakyTestAttempts)
 
         try {
             return GcTesting.get.projects().testMatrices().create(args.project, testMatrix)

diff --git a/test_runner/src/main/kotlin/ftl/run/TestRunner.kt b/test_runner/src/main/kotlin/ftl/run/TestRunner.kt
@@ -293,6 +293,11 @@ object TestRunner {
                 break
             }
 
+            // flaky-test-attempts restarts progress array at size 1
+            if (lastProgressLen > progress.size) {
+                lastProgressLen = 0
+            }
+
             // Progress contains all messages. only print new updates
             for (msg in progress.listIterator(lastProgressLen)) {
                 puts(msg)

diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt
@@ -55,6 +55,7 @@ class AndroidArgsTest {
             version: 24
             locale: en
             orientation: portrait
+          flaky-test-attempts: 3
 
         flank:
           testShards: 7
@@ -153,6 +154,7 @@ class AndroidArgsTest {
                     Device("NexusLowRes", "24", "en", "portrait")
                 )
             )
+            assert(flakyTestAttempts, 3)
 
             // FlankYml
             assert(testShards, 7)
@@ -205,6 +207,7 @@ AndroidArgs
           version: 24
           locale: en
           orientation: portrait
+      flaky-test-attempts: 3
 
     flank:
       testShards: 7
@@ -248,6 +251,7 @@ AndroidArgs
             assert(performanceMetrics, true)
             assert(testTargets, empty)
             assert(devices, listOf(Device("NexusLowRes", "28")))
+            assert(flakyTestAttempts, 0)
 
             // FlankYml
             assert(testShards, 1)
@@ -710,4 +714,21 @@ AndroidArgs
         assertThat(AndroidArgs.load(yaml).resultsDir).isEqualTo("a")
         assertThat(AndroidArgs.load(yaml, cli).resultsDir).isEqualTo("b")
     }
+
+    @Test
+    fun cli_flakyTestAttempts() {
+        val cli = AndroidRunCommand()
+        CommandLine(cli).parse("--flaky-test-attempts=3")
+
+        val yaml = """
+        gcloud:
+          app: $appApk
+          test: $testApk
+          results-dir: a
+      """
+        assertThat(AndroidArgs.load(yaml).flakyTestAttempts).isEqualTo(0)
+
+        val androidArgs = AndroidArgs.load(yaml, cli)
+        assertThat(androidArgs.flakyTestAttempts).isEqualTo(3)
+    }
 }
diff --git a/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt b/test_runner/src/test/kotlin/ftl/args/ArgsHelperTest.kt
@@ -39,7 +39,7 @@ class ArgsHelperTest {
     fun mergeYmlMaps_succeeds() {
         val merged = mergeYmlMaps(GcloudYml, IosGcloudYml)
         assertThat(merged.keys.size).isEqualTo(1)
-        assertThat(merged["gcloud"]?.size).isEqualTo(11)
+        assertThat(merged["gcloud"]?.size).isEqualTo(12)
     }
 
     @Test

diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt
@@ -49,6 +49,7 @@ class IosArgsTest {
             version: 11.2
             locale: c
             orientation: d
+          flaky-test-attempts: 4
 
         flank:
           testShards: 7
@@ -123,6 +124,8 @@ class IosArgsTest {
 
             // IosFlankYml
             assert(testTargets, listOf("b/testBasicSelection", "b/testBasicSelection2"))
+
+            assert(flakyTestAttempts, 4)
         }
     }
 
@@ -153,6 +156,7 @@ IosArgs
           version: 11.2
           locale: c
           orientation: d
+      flaky-test-attempts: 4
 
     flank:
       testShards: 7
@@ -193,6 +197,7 @@ IosArgs
             assert(xctestrunZip, testAbsolutePath)
             assert(xctestrunFile, xctestrunFileAbsolutePath)
             assert(devices, listOf(Device("iphone8", "12.0")))
+            assert(flakyTestAttempts, 0)
 
             // FlankYml
             assert(testShards, 1)
@@ -528,4 +533,20 @@ IosArgs
         val androidArgs = IosArgs.load(yaml, cli)
         assertThat(androidArgs.filesToDownload).isEqualTo(listOf("a", "b"))
     }
+
+    @Test
+    fun cli_flakyTestAttempts() {
+        val cli = IosRunCommand()
+        CommandLine(cli).parse("--flaky-test-attempts=3")
+
+        val yaml = """
+        gcloud:
+          test: $testPath
+          xctestrun-file: $testPath
+      """
+        assertThat(IosArgs.load(yaml).flakyTestAttempts).isEqualTo(0)
+
+        val androidArgs = IosArgs.load(yaml, cli)
+        assertThat(androidArgs.flakyTestAttempts).isEqualTo(3)
+    }
 }
diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt
@@ -82,6 +82,7 @@ class AndroidRunCommandTest {
         assertThat(cmd.testTargetsAlwaysRun).isNull()
         assertThat(cmd.filesToDownload).isNull()
         assertThat(cmd.resultsDir).isNull()
+        assertThat(cmd.flakyTestAttempts).isNull()
     }
 
     @Test
@@ -282,4 +283,12 @@ class AndroidRunCommandTest {
 
         assertThat(cmd.filesToDownload).isEqualTo(arrayListOf("a", "b"))
     }
+
+    @Test
+    fun flakyTestAttempts_parse() {
+        val cmd = AndroidRunCommand()
+        CommandLine(cmd).parse("--flaky-test-attempts=10")
+
+        assertThat(cmd.flakyTestAttempts).isEqualTo(10)
+    }
 }
diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt
@@ -75,6 +75,7 @@ class IosRunCommandTest {
         assertThat(cmd.xcodeVersion).isNull()
         assertThat(cmd.device).isNull()
         assertThat(cmd.resultsDir).isNull()
+        assertThat(cmd.flakyTestAttempts).isNull()
     }
 
     @Test
@@ -216,4 +217,12 @@ class IosRunCommandTest {
 
         assertThat(cmd.filesToDownload).isEqualTo(arrayListOf("a", "b"))
     }
+
+    @Test
+    fun flakyTestAttempts_parse() {
+        val cmd = IosRunCommand()
+        CommandLine(cmd).parse("--flaky-test-attempts=10")
+
+        assertThat(cmd.flakyTestAttempts).isEqualTo(10)
+    }
 }