diff --git a/release_notes.md b/release_notes.md index f2e4b1082c..fc4a99d66f 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,4 +1,5 @@ ## next (unreleased) +- [#692](https://github.com/Flank/flank/pull/692) Add support for network-profiles list command & --network-profile option. ([jan-gogo](https://github.com/jan-gogo)) - [#689](https://github.com/Flank/flank/pull/689) Add support for client-details option. ([jan-gogo](https://github.com/jan-gogo)) - [#687](https://github.com/Flank/flank/pull/687) Debug message printed after every command. ([pawelpasterz](https://github.com/pawelpasterz)) - [#684](https://github.com/Flank/flank/pull/684) Add overhead time to junit test case report. ([jan-gogo](https://github.com/jan-gogo)) diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 590e0cfd07..6150d3c018 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -30,6 +30,12 @@ gcloud: # key1: value1 # key2: value2 + ## The name of the network traffic profile, for example LTE, HSPA, etc, + ## which consists of a set of parameters to emulate network conditions when running the test + ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). + ## This feature only works on physical devices. + # network-profile: LTE + ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history diff --git a/test_runner/flank.yml b/test_runner/flank.yml index bd596642b3..d98452cd1a 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -30,6 +30,12 @@ gcloud: # key1: value1 # key2: value2 + ## The name of the network traffic profile, for example LTE, HSPA, etc, + ## which consists of a set of parameters to emulate network conditions when running the test + ## (default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). + ## This feature only works on physical devices. + # network-profile: LTE + ## The history name for your test results (an arbitrary string label; default: the application's label from the APK manifest). ## All tests which use the same history name will have their results grouped together in the Firebase console in a time-ordered test history list. # results-history-name: android-history diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 7a42699777..1fa869b9e5 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -76,6 +76,7 @@ class AndroidArgs( override val runTimeout = cli?.runTimeout ?: flank.runTimeout override val useLegacyJUnitResult = cli?.useLegacyJUnitResult ?: flank.useLegacyJUnitResult override val clientDetails = cli?.clientDetails ?: gcloud.clientDetails + override val networkProfile = cli?.networkProfile ?: gcloud.networkProfile private val androidFlank = androidFlankYml.flank val additionalAppTestApks = cli?.additionalAppTestApks ?: androidFlank.additionalAppTestApks @@ -125,6 +126,7 @@ AndroidArgs timeout: $testTimeout async: $async client-details: ${mapToString(clientDetails)} + network-profile: $networkProfile results-history-name: $resultsHistoryName # Android gcloud app: $appApk diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 83bceea14c..4b3a299c12 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -15,6 +15,7 @@ interface IArgs { val testTimeout: String val async: Boolean val clientDetails: Map? + val networkProfile: String? val project: String val resultsHistoryName: String? val flakyTestAttempts: Int diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index dfbb4b24a6..40252c6bb3 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -64,6 +64,7 @@ class IosArgs( override val localResultDir = cli?.localResultsDir ?: flank.localResultsDir override val runTimeout = cli?.runTimeout ?: flank.runTimeout override val clientDetails = cli?.clientDetails ?: gcloud.clientDetails + override val networkProfile = cli?.networkProfile ?: gcloud.networkProfile private val iosFlank = iosFlankYml.flank val testTargets = cli?.testTargets ?: iosFlank.testTargets.filterNotNull() @@ -120,6 +121,7 @@ IosArgs timeout: $testTimeout async: $async client-details: ${mapToString(clientDetails)} + network-profile: $networkProfile results-history-name: $resultsHistoryName # iOS gcloud test: $xctestrunZip diff --git a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt index 7f6da6ddf7..a73cf7807d 100644 --- a/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt +++ b/test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt @@ -28,6 +28,9 @@ class GcloudYmlParams( @field:JsonProperty("client-details") val clientDetails: Map? = null, + @field:JsonProperty("network-profile") + val networkProfile: String? = null, + @field:JsonProperty("results-history-name") val resultsHistoryName: String? = null, diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/TestCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/TestCommand.kt index c69499f2b6..f05d7ad011 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/TestCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/TestCommand.kt @@ -2,6 +2,7 @@ package ftl.cli.firebase import ftl.cli.firebase.test.AndroidCommand import ftl.cli.firebase.test.IosCommand +import ftl.cli.firebase.test.NetworkProfilesCommand import picocli.CommandLine import picocli.CommandLine.Command @@ -10,7 +11,8 @@ import picocli.CommandLine.Command synopsisHeading = "", subcommands = [ AndroidCommand::class, - IosCommand::class + IosCommand::class, + NetworkProfilesCommand::class ], usageHelpAutoWidth = true ) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/CommonRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/CommonRunCommand.kt index 1b7648d8f6..5c84d68e28 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/CommonRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/CommonRunCommand.kt @@ -84,6 +84,17 @@ abstract class CommonRunCommand { ) var clientDetails: Map? = null + @CommandLine.Option( + names = ["--network-profile"], + description = [ + "The name of the network traffic profile, for example --network-profile=LTE, ", + "which consists of a set of parameters to emulate network conditions when running the test ", + "(default: no network shaping; see available profiles listed by the `flank test network-profiles list` command). ", + "This feature only works on physical devices. " + ] + ) + var networkProfile: String? = null + @CommandLine.Option( names = ["--results-history-name"], description = ["The history name for your test results " + diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/NetworkProfilesCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/NetworkProfilesCommand.kt new file mode 100644 index 0000000000..755635c474 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/NetworkProfilesCommand.kt @@ -0,0 +1,18 @@ +package ftl.cli.firebase.test + +import ftl.cli.firebase.test.networkprofiles.NetworkProfilesListCommand +import picocli.CommandLine + +@CommandLine.Command( + name = "network-profiles", + synopsisHeading = "", + subcommands = [ + NetworkProfilesListCommand::class + ], + usageHelpAutoWidth = true +) +class NetworkProfilesCommand : Runnable { + override fun run() { + CommandLine.usage(NetworkProfilesCommand(), System.out) + } +} diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/networkprofiles/NetworkProfilesListCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/networkprofiles/NetworkProfilesListCommand.kt new file mode 100644 index 0000000000..3e359b11fb --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/networkprofiles/NetworkProfilesListCommand.kt @@ -0,0 +1,31 @@ +package ftl.cli.firebase.test.networkprofiles + +import ftl.gc.GcTesting +import ftl.http.executeWithRetry +import ftl.run.common.prettyPrint +import picocli.CommandLine + +@CommandLine.Command( + name = "list", + sortOptions = false, + headerHeading = "", + synopsisHeading = "%n", + descriptionHeading = "%n@|bold,underline Description:|@%n%n", + parameterListHeading = "%n@|bold,underline Parameters:|@%n", + optionListHeading = "%n@|bold,underline Options:|@%n", + header = ["List all network profiles available for testing "], + usageHelpAutoWidth = true +) +class NetworkProfilesListCommand : Runnable { + override fun run() { + println("fetching available network profiles...") + val configurations = GcTesting.get.testEnvironmentCatalog() + .get("NETWORK_CONFIGURATION") + .executeWithRetry() + ?.networkConfigurationCatalog + ?.configurations + ?: emptyList() + println() + println(prettyPrint.toJson(configurations)) + } +} diff --git a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt index 46485c869c..a00684537d 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcAndroidTestMatrix.kt @@ -79,6 +79,7 @@ object GcAndroidTestMatrix { val testSetup = TestSetup() .setAccount(account) + .setNetworkProfile(args.networkProfile) .setDirectoriesToPull(args.directoriesToPull) if (args.environmentVariables.isNotEmpty()) { diff --git a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt index c0539bdb36..8b355376fb 100644 --- a/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt +++ b/test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt @@ -57,7 +57,7 @@ object GcIosTestMatrix { .setXcodeVersion(args.xcodeVersion) val iOSTestSetup = IosTestSetup() - .setNetworkProfile(null) + .setNetworkProfile(args.networkProfile) val testTimeoutSeconds = timeoutToSeconds(args.testTimeout) diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index a0603e5b80..10ac9c1f8f 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -40,6 +40,7 @@ class AndroidArgsTest { client-details: key1: value1 key2: value2 + network-profile: LTE project: projectFoo results-history-name: android-history @@ -171,6 +172,7 @@ class AndroidArgsTest { "key2" to "value2" ) ) + assert(networkProfile, "LTE") assert(project, "projectFoo") assert(resultsHistoryName ?: "", "android-history") @@ -229,6 +231,7 @@ AndroidArgs client-details: key1: value1 key2: value2 + network-profile: LTE results-history-name: android-history # Android gcloud app: $appApkAbsolutePath @@ -292,6 +295,7 @@ AndroidArgs timeout: 15m async: false client-details: + network-profile: null results-history-name: null # Android gcloud app: $appApkAbsolutePath @@ -345,6 +349,8 @@ AndroidArgs assert(testTimeout, "15m") assert(async, false) assert(project, "mockProjectId") + assert(clientDetails, null) + assert(networkProfile, null) // AndroidGcloudYml assert(appApk, appApkAbsolutePath) diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 3ec82ed7e1..c2d89abc0c 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -45,6 +45,7 @@ class IosArgsTest { client-details: key1: value1 key2: value2 + network-profile: LTE project: projectFoo results-history-name: ios-history @@ -144,6 +145,7 @@ flank: "key2" to "value2" ) ) + assert(networkProfile, "LTE") assert(project, "projectFoo") assert(resultsHistoryName ?: "", "ios-history") @@ -184,6 +186,7 @@ IosArgs client-details: key1: value1 key2: value2 + network-profile: LTE results-history-name: ios-history # iOS gcloud test: $testAbsolutePath @@ -235,6 +238,7 @@ IosArgs timeout: 15m async: false client-details: + network-profile: null results-history-name: null # iOS gcloud test: $testAbsolutePath @@ -281,6 +285,8 @@ IosArgs assert(testTimeout, "15m") assert(async, false) assert(project, "mockProjectId") + assert(clientDetails, null) + assert(networkProfile, null) // IosGcloudYml assert(xctestrunZip, testAbsolutePath) diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/NetworkProfilesCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/NetworkProfilesCommandTest.kt new file mode 100644 index 0000000000..578c8d6464 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/NetworkProfilesCommandTest.kt @@ -0,0 +1,30 @@ +package ftl.cli.firebase.test + +import ftl.test.util.TestHelper.normalizeLineEnding +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test +import org.junit.contrib.java.lang.system.SystemOutRule + +class NetworkProfilesCommandTest { + + @Rule + @JvmField + val systemOutRule: SystemOutRule = SystemOutRule().enableLog().muteForSuccessfulTests() + + @Test + fun printHelp() { + NetworkProfilesCommand().run() + + val expected = listOf( + "network-profiles [COMMAND]", + "Commands:", + " list List all network profiles available for testing", + "" + ).joinToString("\n") + + val actual = systemOutRule.log.normalizeLineEnding() + + assertEquals(expected, actual) + } +} diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index e3763e1f3b..acab94068f 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -82,6 +82,7 @@ class AndroidRunCommandTest { assertThat(cmd.timeout).isNull() assertThat(cmd.async).isNull() assertThat(cmd.clientDetails).isNull() + assertThat(cmd.networkProfile).isNull() assertThat(cmd.project).isNull() assertThat(cmd.resultsHistoryName).isNull() assertThat(cmd.maxTestShards).isNull() @@ -262,6 +263,14 @@ class AndroidRunCommandTest { ) } + @Test + fun `networkProfile parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--network-profile=a") + + assertThat(cmd.networkProfile).isEqualTo("a") + } + @Test fun `project parse`() { val cmd = AndroidRunCommand() diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt index 7779c2b4af..fa8c01f058 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/ios/IosRunCommandTest.kt @@ -72,6 +72,8 @@ class IosRunCommandTest { assertThat(cmd.noRecordVideo).isNull() assertThat(cmd.timeout).isNull() assertThat(cmd.async).isNull() + assertThat(cmd.clientDetails).isNull() + assertThat(cmd.networkProfile).isNull() assertThat(cmd.project).isNull() assertThat(cmd.resultsHistoryName).isNull() assertThat(cmd.maxTestShards).isNull() @@ -146,6 +148,14 @@ class IosRunCommandTest { ) } + @Test + fun `networkProfile parse`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--network-profile=a") + + assertThat(cmd.networkProfile).isEqualTo("a") + } + @Test fun `project parse`() { val cmd = IosRunCommand() diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/networkprofiles/NetworkProfilesListCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/networkprofiles/NetworkProfilesListCommandTest.kt new file mode 100644 index 0000000000..7514b542b0 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/networkprofiles/NetworkProfilesListCommandTest.kt @@ -0,0 +1,51 @@ +package ftl.cli.firebase.test.networkprofiles + +import com.google.api.services.testing.Testing +import com.google.api.services.testing.model.NetworkConfiguration +import com.google.api.services.testing.model.NetworkConfigurationCatalog +import com.google.api.services.testing.model.TestEnvironmentCatalog +import ftl.http.executeWithRetry +import ftl.run.common.prettyPrint +import io.mockk.every +import io.mockk.mockkStatic +import io.mockk.spyk +import io.mockk.unmockkAll +import io.mockk.verify +import org.junit.After +import org.junit.Before +import org.junit.Test +import picocli.CommandLine + +class NetworkProfilesListCommandTest { + + @Before + fun setUp() { + mockkStatic( + "ftl.http.ExecuteWithRetryKt", + "ftl.run.common.PrettyPrintKt" + ) + } + + @After + fun tearDown() { + unmockkAll() + } + + @Test + fun run() { + val configurationsMock = emptyList() + val prettyPrintSpy = spyk(prettyPrint) + every { prettyPrint } returns prettyPrintSpy + every { + any().executeWithRetry() + } returns TestEnvironmentCatalog().apply { + networkConfigurationCatalog = NetworkConfigurationCatalog().apply { + configurations = configurationsMock + } + } + + CommandLine(NetworkProfilesListCommand()).execute() + + verify { prettyPrintSpy.toJson(configurationsMock) } + } +} diff --git a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt index 65987bd5f1..f003770ae0 100644 --- a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt +++ b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt @@ -1,13 +1,13 @@ package ftl.test.util -import com.google.common.truth.Truth +import org.junit.Assert import java.nio.file.Path import java.nio.file.Paths object TestHelper { fun assert(actual: Any?, expected: Any?) = - Truth.assertThat(actual).isEqualTo(expected) + Assert.assertEquals(expected, actual) fun getPath(path: String): Path = Paths.get(path).toAbsolutePath().normalize()