Add support for network-profiles list command & --network-profile option

release_notes.md

@@ -1,4 +1,5 @@
 ## next (unreleased)
+- [#692](https://github.com/Flank/flank/pull/692) Add support for network-profiles list command & --network-profile option. ([jan-gogo](https://github.com/jan-gogo))
 - [#689](https://github.com/Flank/flank/pull/689) Add support for client-details option. ([jan-gogo](https://github.com/jan-gogo))
 - [#687](https://github.com/Flank/flank/pull/687) Debug message printed after every command. ([pawelpasterz](https://github.com/pawelpasterz))
 - [#684](https://github.com/Flank/flank/pull/684) Add overhead time to junit test case report. ([jan-gogo](https://github.com/jan-gogo))

test_runner/flank.ios.yml

@@ -30,6 +30,12 @@ gcloud:
   #   key1: value1
   #   key2: value2
 
+  ## The name of the network traffic profile, for example LTE, HSPA, etc,
+  ## which consists of a set of parameters to emulate network conditions when running the test
+  ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command).
+  ## This feature only works on physical devices.
+  # network-profile: LTE
+
   ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest).
   ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list.
   # results-history-name: android-history

test_runner/flank.yml

@@ -30,6 +30,12 @@ gcloud:
   #   key1: value1
   #   key2: value2
 
+  ## The name of the network traffic profile, for example LTE, HSPA, etc,
+  ## which consists of a set of parameters to emulate network conditions when running the test
+  ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command).
+  ## This feature only works on physical devices.
+  # network-profile: LTE
+
   ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest).
   ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list.
   # results-history-name: android-history

test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt

@@ -76,6 +76,7 @@ class AndroidArgs(
     override val runTimeout = cli?.runTimeout ?: flank.runTimeout
     override val useLegacyJUnitResult = cli?.useLegacyJUnitResult ?: flank.useLegacyJUnitResult
     override val clientDetails = cli?.clientDetails ?: gcloud.clientDetails
+    override val networkProfile = cli?.networkProfile ?: gcloud.networkProfile
 
     private val androidFlank = androidFlankYml.flank
     val additionalAppTestApks = cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks
@@ -125,6 +126,7 @@ AndroidArgs
       timeout: $testTimeout
       async: $async
       client-details: ${mapToString(clientDetails)}
+      network-profile: $networkProfile
       results-history-name: $resultsHistoryName
       # Android gcloud
       app: $appApk

test_runner/src/main/kotlin/ftl/args/IArgs.kt

@@ -15,6 +15,7 @@ interface IArgs {
     val testTimeout: String
     val async: Boolean
     val clientDetails: Map<String, String>?
+    val networkProfile: String?
     val project: String
     val resultsHistoryName: String?
     val flakyTestAttempts: Int

test_runner/src/main/kotlin/ftl/args/IosArgs.kt

@@ -64,6 +64,7 @@ class IosArgs(
     override val localResultDir = cli?.localResultsDir ?: flank.localResultsDir
     override val runTimeout = cli?.runTimeout ?: flank.runTimeout
     override val clientDetails = cli?.clientDetails ?: gcloud.clientDetails
+    override val networkProfile = cli?.networkProfile ?: gcloud.networkProfile
 
     private val iosFlank = iosFlankYml.flank
     val testTargets = cli?.testTargets ?: iosFlank.testTargets.filterNotNull()
@@ -120,6 +121,7 @@ IosArgs
       timeout: $testTimeout
       async: $async
       client-details: ${mapToString(clientDetails)}
+      network-profile: $networkProfile
       results-history-name: $resultsHistoryName
       # iOS gcloud
       test: $xctestrunZip

test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt

@@ -28,6 +28,9 @@ class GcloudYmlParams(
     @field:JsonProperty("client-details")
     val clientDetails: Map<String, String>? = null,
 
+    @field:JsonProperty("network-profile")
+    val networkProfile: String? = null,
+
     @field:JsonProperty("results-history-name")
     val resultsHistoryName: String? = null,
 

test_runner/src/main/kotlin/ftl/cli/firebase/TestCommand.kt

@@ -2,6 +2,7 @@ package ftl.cli.firebase
 
 import ftl.cli.firebase.test.AndroidCommand
 import ftl.cli.firebase.test.IosCommand
+import ftl.cli.firebase.test.NetworkProfilesCommand
 import picocli.CommandLine
 import picocli.CommandLine.Command
 
@@ -10,7 +11,8 @@ import picocli.CommandLine.Command
     synopsisHeading = "",
     subcommands = [
         AndroidCommand::class,
-        IosCommand::class
+        IosCommand::class,
+        NetworkProfilesCommand::class
     ],
     usageHelpAutoWidth = true
 )

test_runner/src/main/kotlin/ftl/cli/firebase/test/CommonRunCommand.kt

@@ -84,6 +84,17 @@ abstract class CommonRunCommand {
     )
     var clientDetails: Map<String, String>? = null
 
+    @CommandLine.Option(
+        names = ["--network-profile"],
+        description = [
+            "The name of the network traffic profile, for example --network-profile=LTE, ",
+            "which consists of a set of parameters to emulate network conditions when running the test ",
+            "(default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). ",
+            "This feature only works on physical devices. "
+        ]
+    )
+    var networkProfile: String? = null
+
     @CommandLine.Option(
         names = ["--results-history-name"],
         description = ["The history name for your test results " +

test_runner/src/main/kotlin/ftl/cli/firebase/test/NetworkProfilesCommand.kt

@@ -0,0 +1,18 @@
+package ftl.cli.firebase.test
+
+import ftl.cli.firebase.test.networkprofiles.NetworkProfilesListCommand
+import picocli.CommandLine
+
+@CommandLine.Command(
+    name = "network-profiles",
+    synopsisHeading = "",
+    subcommands = [
+        NetworkProfilesListCommand::class
+    ],
+    usageHelpAutoWidth = true
+)
+class NetworkProfilesCommand : Runnable {
+    override fun run() {
+        CommandLine.usage(NetworkProfilesCommand(), System.out)
+    }
+}

test_runner/src/main/kotlin/ftl/cli/firebase/test/networkprofiles/NetworkProfilesListCommand.kt

@@ -0,0 +1,31 @@
+package ftl.cli.firebase.test.networkprofiles
+
+import ftl.gc.GcTesting
+import ftl.http.executeWithRetry
+import ftl.run.common.prettyPrint
+import picocli.CommandLine
+
+@CommandLine.Command(
+    name = "list",
+    sortOptions = false,
+    headerHeading = "",
+    synopsisHeading = "%n",
+    descriptionHeading = "%n@|bold,underline Description:|@%n%n",
+    parameterListHeading = "%n@|bold,underline Parameters:|@%n",
+    optionListHeading = "%n@|bold,underline Options:|@%n",
+    header = ["List all network profiles available for testing "],
+    usageHelpAutoWidth = true
+)
+class NetworkProfilesListCommand : Runnable {
+    override fun run() {
+        println("fetching available network profiles...")
+        val configurations = GcTesting.get.testEnvironmentCatalog()
+            .get("NETWORK_CONFIGURATION")
+            .executeWithRetry()
+            ?.networkConfigurationCatalog
+            ?.configurations
+            ?: emptyList()
+        println()
+        println(prettyPrint.toJson(configurations))
+    }
+}

test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt

@@ -79,6 +79,7 @@ object GcAndroidTestMatrix {
 
         val testSetup = TestSetup()
             .setAccount(account)
+            .setNetworkProfile(args.networkProfile)
             .setDirectoriesToPull(args.directoriesToPull)
 
         if (args.environmentVariables.isNotEmpty()) {

test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt

@@ -57,7 +57,7 @@ object GcIosTestMatrix {
             .setXcodeVersion(args.xcodeVersion)
 
         val iOSTestSetup = IosTestSetup()
-            .setNetworkProfile(null)
+            .setNetworkProfile(args.networkProfile)
 
         val testTimeoutSeconds = timeoutToSeconds(args.testTimeout)
 

test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt

@@ -40,6 +40,7 @@ class AndroidArgsTest {
           client-details:
             key1: value1
             key2: value2
+          network-profile: LTE
           project: projectFoo
           results-history-name: android-history
 
@@ -171,6 +172,7 @@ class AndroidArgsTest {
                     "key2" to "value2"
                 )
             )
+            assert(networkProfile, "LTE")
             assert(project, "projectFoo")
             assert(resultsHistoryName ?: "", "android-history")
 
@@ -229,6 +231,7 @@ AndroidArgs
       client-details: 
         key1: value1
         key2: value2
+      network-profile: LTE
       results-history-name: android-history
       # Android gcloud
       app: $appApkAbsolutePath
@@ -292,6 +295,7 @@ AndroidArgs
       timeout: 15m
       async: false
       client-details: 
+      network-profile: null
       results-history-name: null
       # Android gcloud
       app: $appApkAbsolutePath
@@ -345,6 +349,8 @@ AndroidArgs
             assert(testTimeout, "15m")
             assert(async, false)
             assert(project, "mockProjectId")
+            assert(clientDetails, null)
+            assert(networkProfile, null)
 
             // AndroidGcloudYml
             assert(appApk, appApkAbsolutePath)

test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt

@@ -45,6 +45,7 @@ class IosArgsTest {
           client-details:
             key1: value1
             key2: value2
+          network-profile: LTE
           project: projectFoo
           results-history-name: ios-history
 
@@ -144,6 +145,7 @@ flank:
                     "key2" to "value2"
                 )
             )
+            assert(networkProfile, "LTE")
             assert(project, "projectFoo")
             assert(resultsHistoryName ?: "", "ios-history")
 
@@ -184,6 +186,7 @@ IosArgs
       client-details: 
         key1: value1
         key2: value2
+      network-profile: LTE
       results-history-name: ios-history
       # iOS gcloud
       test: $testAbsolutePath
@@ -235,6 +238,7 @@ IosArgs
       timeout: 15m
       async: false
       client-details: 
+      network-profile: null
       results-history-name: null
       # iOS gcloud
       test: $testAbsolutePath
@@ -281,6 +285,8 @@ IosArgs
             assert(testTimeout, "15m")
             assert(async, false)
             assert(project, "mockProjectId")
+            assert(clientDetails, null)
+            assert(networkProfile, null)
 
             // IosGcloudYml
             assert(xctestrunZip, testAbsolutePath)

test_runner/src/test/kotlin/ftl/cli/firebase/test/NetworkProfilesCommandTest.kt

@@ -0,0 +1,30 @@
+package ftl.cli.firebase.test
+
+import ftl.test.util.TestHelper.normalizeLineEnding
+import org.junit.Assert.assertEquals
+import org.junit.Rule
+import org.junit.Test
+import org.junit.contrib.java.lang.system.SystemOutRule
+
+class NetworkProfilesCommandTest {
+
+    @Rule
+    @JvmField
+    val systemOutRule: SystemOutRule = SystemOutRule().enableLog().muteForSuccessfulTests()
+
+    @Test
+    fun printHelp() {
+        NetworkProfilesCommand().run()
+
+        val expected = listOf(
+            "network-profiles [COMMAND]",
+            "Commands:",
+            "  list  List all network profiles available for testing",
+            ""
+        ).joinToString("\n")
+
+        val actual = systemOutRule.log.normalizeLineEnding()
+
+        assertEquals(expected, actual)
+    }
+}

test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt

@@ -82,6 +82,7 @@ class AndroidRunCommandTest {
         assertThat(cmd.timeout).isNull()
         assertThat(cmd.async).isNull()
         assertThat(cmd.clientDetails).isNull()
+        assertThat(cmd.networkProfile).isNull()
         assertThat(cmd.project).isNull()
         assertThat(cmd.resultsHistoryName).isNull()
         assertThat(cmd.maxTestShards).isNull()
@@ -262,6 +263,14 @@ class AndroidRunCommandTest {
         )
     }
 
+    @Test
+    fun `networkProfile parse`() {
+        val cmd = AndroidRunCommand()
+        CommandLine(cmd).parseArgs("--network-profile=a")
+
+        assertThat(cmd.networkProfile).isEqualTo("a")
+    }
+
     @Test
     fun `project parse`() {
         val cmd = AndroidRunCommand()

test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt

@@ -72,6 +72,8 @@ class IosRunCommandTest {
         assertThat(cmd.noRecordVideo).isNull()
         assertThat(cmd.timeout).isNull()
         assertThat(cmd.async).isNull()
+        assertThat(cmd.clientDetails).isNull()
+        assertThat(cmd.networkProfile).isNull()
         assertThat(cmd.project).isNull()
         assertThat(cmd.resultsHistoryName).isNull()
         assertThat(cmd.maxTestShards).isNull()
@@ -146,6 +148,14 @@ class IosRunCommandTest {
         )
     }
 
+    @Test
+    fun `networkProfile parse`() {
+        val cmd = AndroidRunCommand()
+        CommandLine(cmd).parseArgs("--network-profile=a")
+
+        assertThat(cmd.networkProfile).isEqualTo("a")
+    }
+
     @Test
     fun `project parse`() {
         val cmd = IosRunCommand()

test_runner/src/test/kotlin/ftl/cli/firebase/test/networkprofiles/NetworkProfilesListCommandTest.kt

@@ -0,0 +1,51 @@
+package ftl.cli.firebase.test.networkprofiles
+
+import com.google.api.services.testing.Testing
+import com.google.api.services.testing.model.NetworkConfiguration
+import com.google.api.services.testing.model.NetworkConfigurationCatalog
+import com.google.api.services.testing.model.TestEnvironmentCatalog
+import ftl.http.executeWithRetry
+import ftl.run.common.prettyPrint
+import io.mockk.every
+import io.mockk.mockkStatic
+import io.mockk.spyk
+import io.mockk.unmockkAll
+import io.mockk.verify
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+import picocli.CommandLine
+
+class NetworkProfilesListCommandTest {
+
+    @Before
+    fun setUp() {
+        mockkStatic(
+            "ftl.http.ExecuteWithRetryKt",
+            "ftl.run.common.PrettyPrintKt"
+        )
+    }
+
+    @After
+    fun tearDown() {
+        unmockkAll()
+    }
+
+    @Test
+    fun run() {
+        val configurationsMock = emptyList<NetworkConfiguration>()
+        val prettyPrintSpy = spyk(prettyPrint)
+        every { prettyPrint } returns prettyPrintSpy
+        every {
+            any<Testing.TestEnvironmentCatalog.Get>().executeWithRetry()
+        } returns TestEnvironmentCatalog().apply {
+            networkConfigurationCatalog = NetworkConfigurationCatalog().apply {
+                configurations = configurationsMock
+            }
+        }
+
+        CommandLine(NetworkProfilesListCommand()).execute()
+
+        verify { prettyPrintSpy.toJson(configurationsMock) }
+    }
+}