From cbb00aebce509deeab87e7d66564b4211a23d91c Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Fri, 25 Sep 2020 09:40:09 +0200 Subject: [PATCH 01/12] Implement sanityRobo --- .../flank/gradle/FladlePluginDelegate.kt | 2 + .../flank/gradle/FlankGradleExtension.kt | 3 + .../com/osacky/flank/gradle/YamlWriter.kt | 27 +- .../gradle/integration/SanityRoboCheck.kt | 328 ++++++++++++++++++ .../flank/gradle/integration/TestFixtures.kt | 19 + 5 files changed, 368 insertions(+), 11 deletions(-) create mode 100644 buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt index 15415501..5fa1894c 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt @@ -40,6 +40,8 @@ class FladlePluginDelegate { // Must be done afterEvaluate otherwise extension values will not be set. project.dependencies.add(FLADLE_CONFIG, "${base.flankCoordinates.get()}:${base.flankVersion.get()}") + if (project.hasProperty("sanityRobo")) base.sanityRobo.set(true) + // Only use automatic apk path detection for 'com.android.application' projects. project.pluginManager.withPlugin("com.android.application") { if (!base.debugApk.isPresent || !base.instrumentationApk.isPresent) { diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt index 0196ab75..f9e1c415 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt @@ -19,6 +19,9 @@ open class FlankGradleExtension @Inject constructor(objects: ObjectFactory) : Fl @get:Input val flankCoordinates: Property = objects.property(String::class.java).convention("com.github.flank:flank") + @get:Input + val sanityRobo: Property = objects.property().convention(false) + @get:Input val flankVersion: Property = objects.property(String::class.java).convention("20.09.3") // Project id is automatically discovered by default. Use this to override the project id. diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt index dcd183d2..93a13c31 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt @@ -12,7 +12,7 @@ internal class YamlWriter { check(base.serviceAccountCredentials.isPresent) { "ServiceAccountCredentials in fladle extension not set. https://github.com/runningcode/fladle#serviceaccountcredentials" } } check(base.debugApk.isPresent) { "debugApk must be specified" } - check(base.instrumentationApk.isPresent xor !base.roboScript.orNull.isNullOrBlank()) { + check(base.sanityRobo.get() == true || (base.instrumentationApk.isPresent xor !base.roboScript.orNull.isNullOrBlank())) { val prefix = if (base.instrumentationApk.isPresent && !base.roboScript.orNull.isNullOrBlank()) { "Both instrumentationApk file and roboScript file were specified, but only one is expected." } else { @@ -25,13 +25,15 @@ internal class YamlWriter { """.trimIndent() } - val additionalProperties = writeAdditionalProperties(config) - val flankProperties = writeFlankProperties(config) + val shouldPrintTestAndRobo = base.sanityRobo.get().not() + val additionalProperties = writeAdditionalProperties(config, shouldPrintTestAndRobo) + val flankProperties = writeFlankProperties(config, shouldPrintTestAndRobo) return buildString { appendln("gcloud:") appendln(" app: ${base.debugApk.get()}") - if (base.instrumentationApk.isPresent) { + // We don't want to print instrumentation apks if sanityRobo == true + if (shouldPrintTestAndRobo && base.instrumentationApk.isPresent) { appendln(" test: ${base.instrumentationApk.get()}") } if (config.devices.isPresentAndNotEmpty) appendln(createDeviceString(config.devices.get())) @@ -40,7 +42,7 @@ internal class YamlWriter { } } - internal fun writeFlankProperties(config: FladleConfig): String = buildString { + internal fun writeFlankProperties(config: FladleConfig, printApk: Boolean = true): String = buildString { appendln("flank:") appendProperty(config.testShards, name = "max-test-shards") @@ -50,7 +52,8 @@ internal class YamlWriter { appendProperty(config.projectId, name = "project") appendProperty(config.keepFilePath, name = "keep-file-path") appendListProperty(config.filesToDownload, name = "files-to-download") { appendln(" - $it") } - appendListProperty(config.additionalTestApks, name = "additional-app-test-apks") { appendln(" $it") } + if (printApk) + appendListProperty(config.additionalTestApks, name = "additional-app-test-apks") { appendln(" $it") } appendProperty(config.runTimeout, name = "run-timeout") appendProperty(config.ignoreFailedTests, name = "ignore-failed-tests") appendProperty(config.disableSharding, name = "disable-sharding") @@ -62,7 +65,7 @@ internal class YamlWriter { appendProperty(config.outputStyle, name = "output-style") } - internal fun writeAdditionalProperties(config: FladleConfig): String = buildString { + internal fun writeAdditionalProperties(config: FladleConfig, printRobo: Boolean = true): String = buildString { appendProperty(config.useOrchestrator, name = "use-orchestrator") appendProperty(config.autoGoogleLogin, name = "auto-google-login") appendProperty(config.recordVideo, name = "record-video") @@ -82,10 +85,12 @@ internal class YamlWriter { appendMapProperty(config.clientDetails, name = "client-details") { appendln(" ${it.key}: ${it.value}") } appendMapProperty(config.otherFiles, name = "other-files") { appendln(" ${it.key}: ${it.value}") } appendProperty(config.networkProfile, name = "network-profile") - appendProperty(config.roboScript, name = "robo-script") - appendListProperty(config.roboDirectives, name = "robo-directives") { - val value = it.getOrElse(2) { "" }.let { stringValue -> if (stringValue.isBlank()) "\"\"" else stringValue } - appendln(" ${it[0]}:${it[1]}: $value") + if (printRobo) { + appendProperty(config.roboScript, name = "robo-script") + appendListProperty(config.roboDirectives, name = "robo-directives") { + val value = it.getOrElse(2) { "" }.let { stringValue -> if (stringValue.isBlank()) "\"\"" else stringValue } + appendln(" ${it[0]}:${it[1]}: $value") + } } } diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt new file mode 100644 index 00000000..1b69f9ee --- /dev/null +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt @@ -0,0 +1,328 @@ +package com.osacky.flank.gradle.integration + +import com.google.common.truth.Truth.assertThat +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +class SanityRoboCheck { + @get:Rule + var testProjectRoot = TemporaryFolder() + + @Before + fun setUp() = testProjectRoot.newFile("flank-gradle-service.json").writeText("{}") + + @Test + fun checkSanityRoboRunSimpleCase() { + makeGradleFile(where = testProjectRoot) { + """ + |plugins { + | id "com.osacky.fladle" + |} + | + |fladle { + | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") + | debugApk = "foo.apk" + |} + """ + } + + val result = gradleRun { + arguments = listOf("writeConfigProps", "-PsanityRobo") + projectDir = testProjectRoot.root + } + + assertThat(result.output).contains("SUCCESS") + + "build/fladle/flank.yml" readAndCompareWith { + """ + |gcloud: + | app: foo.apk + | device: + | - model: NexusLowRes + | version: 28 + | + | use-orchestrator: false + | auto-google-login: false + | record-video: true + | performance-metrics: true + | timeout: 15m + | num-flaky-test-attempts: 0 + | + |flank: + | keep-file-path: false + | ignore-failed-tests: false + | disable-sharding: false + | smart-flank-disable-upload: false + | legacy-junit-result: false + | full-junit-result: false + | output-style: single + """ + } + } + + @Test + fun checkSanityRoboRunWithApksAdded() { + makeGradleFile(where = testProjectRoot) { + """ + |plugins { + | id "com.osacky.fladle" + |} + | + |fladle { + | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") + | debugApk = "foo.apk" + | instrumentationApk = "test.apk" + | additionalTestApks = [ + | "- app: debug2.apk", + | " test: test2.apk", + | "- test: test3.apk" + | ] + |} + """ + } + + val result = gradleRun { + arguments = listOf("writeConfigProps", "-PsanityRobo") + projectDir = testProjectRoot.root + } + + assertThat(result.output).contains("SUCCESS") + + "build/fladle/flank.yml" readAndCompareWith { + """ + |gcloud: + | app: foo.apk + | device: + | - model: NexusLowRes + | version: 28 + | + | use-orchestrator: false + | auto-google-login: false + | record-video: true + | performance-metrics: true + | timeout: 15m + | num-flaky-test-attempts: 0 + | + |flank: + | keep-file-path: false + | ignore-failed-tests: false + | disable-sharding: false + | smart-flank-disable-upload: false + | legacy-junit-result: false + | full-junit-result: false + | output-style: single + """ + } + } + + @Test + fun checkSanityRoboRunMultipleConfigs() { + makeGradleFile(where = testProjectRoot) { + """ + |plugins { + | id "com.osacky.fladle" + |} + | + |fladle { + | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") + | debugApk = "foo.apk" + | instrumentationApk = "test.apk" + | additionalTestApks = [ + | "- app: debug2.apk", + | " test: test2.apk", + | "- test: test3.apk" + | ] + | configs { + | orange { + | testTargets = ['override'] + | localResultsDir.set('overrideDir') + | } + | } + |} + """ + } + + val result = gradleRun { + arguments = listOf("writeConfigProps", "-PsanityRobo") + projectDir = testProjectRoot.root + } + + assertThat(result.output).contains("SUCCESS") + + "build/fladle/flank.yml" readAndCompareWith { + """ + |gcloud: + | app: foo.apk + | device: + | - model: NexusLowRes + | version: 28 + | + | use-orchestrator: false + | auto-google-login: false + | record-video: true + | performance-metrics: true + | timeout: 15m + | num-flaky-test-attempts: 0 + | + |flank: + | keep-file-path: false + | ignore-failed-tests: false + | disable-sharding: false + | smart-flank-disable-upload: false + | legacy-junit-result: false + | full-junit-result: false + | output-style: single + """ + } + + val resultOrange = gradleRun { + arguments = listOf("writeConfigPropsOrange", "-PsanityRobo") + projectDir = testProjectRoot.root + } + + assertThat(resultOrange.output).contains("SUCCESS") + + "build/fladle/orange/flank.yml" readAndCompareWith { + """ + |gcloud: + | app: foo.apk + | device: + | - model: NexusLowRes + | version: 28 + | + | use-orchestrator: false + | auto-google-login: false + | record-video: true + | performance-metrics: true + | timeout: 15m + | test-targets: + | - override + | num-flaky-test-attempts: 0 + | + |flank: + | keep-file-path: false + | ignore-failed-tests: false + | disable-sharding: false + | smart-flank-disable-upload: false + | local-result-dir: overrideDir + | legacy-junit-result: false + | full-junit-result: false + | output-style: single + """ + } + } + + @Test + fun checkSanityRoboRunRoboScript() { + makeGradleFile(where = testProjectRoot) { + """ + |plugins { + | id "com.osacky.fladle" + |} + | + |fladle { + | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") + | debugApk = "foo.apk" + | roboScript = "some/path/script.json" + |} + """ + } + + val result = gradleRun { + arguments = listOf("writeConfigProps", "-PsanityRobo") + projectDir = testProjectRoot.root + } + + assertThat(result.output).contains("SUCCESS") + + "build/fladle/flank.yml" readAndCompareWith { + """ + |gcloud: + | app: foo.apk + | device: + | - model: NexusLowRes + | version: 28 + | + | use-orchestrator: false + | auto-google-login: false + | record-video: true + | performance-metrics: true + | timeout: 15m + | num-flaky-test-attempts: 0 + | + |flank: + | keep-file-path: false + | ignore-failed-tests: false + | disable-sharding: false + | smart-flank-disable-upload: false + | legacy-junit-result: false + | full-junit-result: false + | output-style: single + """ + } + } + + @Test + fun checkSanityRoboRunRoboDirectives() { + makeGradleFile(where = testProjectRoot) { + """ + |plugins { + | id "com.osacky.fladle" + |} + | + |fladle { + | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") + | debugApk = "foo.apk" + | roboDirectives = [ + | ["click", "button1", ""], + | ["ignore", "button2"], + | ["text", "field1", "my text"], + | ] + |} + """ + } + + val result = gradleRun { + arguments = listOf("writeConfigProps", "-PsanityRobo") + projectDir = testProjectRoot.root + } + + assertThat(result.output).contains("SUCCESS") + + "build/fladle/flank.yml" readAndCompareWith { + """ + |gcloud: + | app: foo.apk + | device: + | - model: NexusLowRes + | version: 28 + | + | use-orchestrator: false + | auto-google-login: false + | record-video: true + | performance-metrics: true + | timeout: 15m + | num-flaky-test-attempts: 0 + | + |flank: + | keep-file-path: false + | ignore-failed-tests: false + | disable-sharding: false + | smart-flank-disable-upload: false + | legacy-junit-result: false + | full-junit-result: false + | output-style: single + """ + } + } + + private infix fun String.readAndCompareWith(block: () -> String) = testProjectRoot + .root + .resolve(this) + .readText() + .run { + assertThat(this).contains(block().trimMargin()) + } +} diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt index 1d65a757..c2c6e865 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt @@ -1,8 +1,27 @@ package com.osacky.flank.gradle.integration +import org.gradle.testkit.runner.GradleRunner import org.junit.rules.TemporaryFolder import java.io.File fun TemporaryFolder.setupFixture(fixtureName: String) { File(this::class.java.classLoader.getResource(fixtureName)!!.file).copyRecursively(newFile(fixtureName), true) } + +internal fun makeGradleFile(where: TemporaryFolder, stringProvider: () -> String) = where + .newFile("build.gradle") + .writeText(stringProvider().trimMargin()) + +internal fun gradleRun(block: GradleRunFladle.() -> Unit) = GradleRunFladle().apply(block).run { + GradleRunner.create() + .withPluginClasspath() + .withArguments(arguments) + .forwardOutput() + .withProjectDir(projectDir) + .build() +} + +internal data class GradleRunFladle( + var arguments: List = emptyList(), + var projectDir: File? = null +) From abed4781b27cab9b47999b5be9f6591fbe695e62 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Fri, 25 Sep 2020 10:10:15 +0200 Subject: [PATCH 02/12] Update doc --- docs/configuration.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index b40457ab..2722e33b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -743,3 +743,9 @@ Enables creating an additional local junit result on local storage with failure ``` kotlin fullJunitResult.set(false) ``` + +## Sanity robo test +Just add `-PsanityRobo` -- this will launch simple robo test without robo scripts and/or robo directives. (`roboScript`, `roboDirectives`, `test`, `additionalTestApks` are skipped) +``` +./gradlew runFlank -PsanityRobo +``` From 3095ac138e531f45b8d01de36e72a0eeb51b3ada Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Fri, 25 Sep 2020 11:44:40 +0200 Subject: [PATCH 03/12] PR review changes --- .../com/osacky/flank/gradle/YamlWriter.kt | 4 +- .../gradle/integration/SanityRoboCheck.kt | 78 +++++++++++-------- .../flank/gradle/integration/TestFixtures.kt | 12 +-- 3 files changed, 50 insertions(+), 44 deletions(-) diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt index 93a13c31..54440945 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt @@ -12,7 +12,8 @@ internal class YamlWriter { check(base.serviceAccountCredentials.isPresent) { "ServiceAccountCredentials in fladle extension not set. https://github.com/runningcode/fladle#serviceaccountcredentials" } } check(base.debugApk.isPresent) { "debugApk must be specified" } - check(base.sanityRobo.get() == true || (base.instrumentationApk.isPresent xor !base.roboScript.orNull.isNullOrBlank())) { + if (base.sanityRobo.get() == false) { + check(base.instrumentationApk.isPresent xor !base.roboScript.orNull.isNullOrBlank()) { val prefix = if (base.instrumentationApk.isPresent && !base.roboScript.orNull.isNullOrBlank()) { "Both instrumentationApk file and roboScript file were specified, but only one is expected." } else { @@ -23,6 +24,7 @@ internal class YamlWriter { instrumentationApk=${base.instrumentationApk.orNull} roboScript=${base.roboScript.orNull} """.trimIndent() + } } val shouldPrintTestAndRobo = base.sanityRobo.get().not() diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt index 1b69f9ee..0f5f4c1c 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt @@ -14,9 +14,11 @@ class SanityRoboCheck { fun setUp() = testProjectRoot.newFile("flank-gradle-service.json").writeText("{}") @Test - fun checkSanityRoboRunSimpleCase() { - makeGradleFile(where = testProjectRoot) { - """ + fun checkSanityRoboRunWithProjectProperty() { + makeGradleFile( + where = testProjectRoot, + buildScript = + """ |plugins { | id "com.osacky.fladle" |} @@ -26,12 +28,12 @@ class SanityRoboCheck { | debugApk = "foo.apk" |} """ - } + ) - val result = gradleRun { - arguments = listOf("writeConfigProps", "-PsanityRobo") + val result = gradleRun( + arguments = listOf("writeConfigProps", "-PsanityRobo"), projectDir = testProjectRoot.root - } + ) assertThat(result.output).contains("SUCCESS") @@ -64,8 +66,10 @@ class SanityRoboCheck { @Test fun checkSanityRoboRunWithApksAdded() { - makeGradleFile(where = testProjectRoot) { - """ + makeGradleFile( + where = testProjectRoot, + buildScript = + """ |plugins { | id "com.osacky.fladle" |} @@ -81,12 +85,12 @@ class SanityRoboCheck { | ] |} """ - } + ) - val result = gradleRun { - arguments = listOf("writeConfigProps", "-PsanityRobo") + val result = gradleRun( + arguments = listOf("writeConfigProps", "-PsanityRobo"), projectDir = testProjectRoot.root - } + ) assertThat(result.output).contains("SUCCESS") @@ -119,8 +123,10 @@ class SanityRoboCheck { @Test fun checkSanityRoboRunMultipleConfigs() { - makeGradleFile(where = testProjectRoot) { - """ + makeGradleFile( + where = testProjectRoot, + buildScript = + """ |plugins { | id "com.osacky.fladle" |} @@ -142,12 +148,12 @@ class SanityRoboCheck { | } |} """ - } + ) - val result = gradleRun { - arguments = listOf("writeConfigProps", "-PsanityRobo") + val result = gradleRun( + arguments = listOf("writeConfigProps", "-PsanityRobo"), projectDir = testProjectRoot.root - } + ) assertThat(result.output).contains("SUCCESS") @@ -177,10 +183,10 @@ class SanityRoboCheck { """ } - val resultOrange = gradleRun { - arguments = listOf("writeConfigPropsOrange", "-PsanityRobo") + val resultOrange = gradleRun( + arguments = listOf("writeConfigPropsOrange", "-PsanityRobo"), projectDir = testProjectRoot.root - } + ) assertThat(resultOrange.output).contains("SUCCESS") @@ -216,8 +222,10 @@ class SanityRoboCheck { @Test fun checkSanityRoboRunRoboScript() { - makeGradleFile(where = testProjectRoot) { - """ + makeGradleFile( + where = testProjectRoot, + buildScript = + """ |plugins { | id "com.osacky.fladle" |} @@ -228,12 +236,12 @@ class SanityRoboCheck { | roboScript = "some/path/script.json" |} """ - } + ) - val result = gradleRun { - arguments = listOf("writeConfigProps", "-PsanityRobo") + val result = gradleRun( + arguments = listOf("writeConfigProps", "-PsanityRobo"), projectDir = testProjectRoot.root - } + ) assertThat(result.output).contains("SUCCESS") @@ -266,8 +274,10 @@ class SanityRoboCheck { @Test fun checkSanityRoboRunRoboDirectives() { - makeGradleFile(where = testProjectRoot) { - """ + makeGradleFile( + where = testProjectRoot, + buildScript = + """ |plugins { | id "com.osacky.fladle" |} @@ -282,12 +292,12 @@ class SanityRoboCheck { | ] |} """ - } + ) - val result = gradleRun { - arguments = listOf("writeConfigProps", "-PsanityRobo") + val result = gradleRun( + arguments = listOf("writeConfigProps", "-PsanityRobo"), projectDir = testProjectRoot.root - } + ) assertThat(result.output).contains("SUCCESS") diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt index c2c6e865..37c3ee3f 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt @@ -8,20 +8,14 @@ fun TemporaryFolder.setupFixture(fixtureName: String) { File(this::class.java.classLoader.getResource(fixtureName)!!.file).copyRecursively(newFile(fixtureName), true) } -internal fun makeGradleFile(where: TemporaryFolder, stringProvider: () -> String) = where +internal fun makeGradleFile(where: TemporaryFolder, buildScript: String) = where .newFile("build.gradle") - .writeText(stringProvider().trimMargin()) + .writeText(buildScript.trimMargin()) -internal fun gradleRun(block: GradleRunFladle.() -> Unit) = GradleRunFladle().apply(block).run { +internal fun gradleRun(projectDir: File, arguments: List = emptyList()) = GradleRunner.create() .withPluginClasspath() .withArguments(arguments) .forwardOutput() .withProjectDir(projectDir) .build() -} - -internal data class GradleRunFladle( - var arguments: List = emptyList(), - var projectDir: File? = null -) From 79b0e1134a24cffa52f2efa5793b3c4fe37dabaf Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Fri, 25 Sep 2020 12:00:09 +0200 Subject: [PATCH 04/12] Refactor sanity check test --- .../gradle/integration/SanityRoboCheck.kt | 62 +++++++------------ 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt index 0f5f4c1c..6de4c1b9 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt @@ -31,13 +31,12 @@ class SanityRoboCheck { ) val result = gradleRun( - arguments = listOf("writeConfigProps", "-PsanityRobo"), + arguments = listOf("printYml", "-PsanityRobo"), projectDir = testProjectRoot.root ) assertThat(result.output).contains("SUCCESS") - - "build/fladle/flank.yml" readAndCompareWith { + assertThat(result.output).contains( """ |gcloud: | app: foo.apk @@ -60,8 +59,8 @@ class SanityRoboCheck { | legacy-junit-result: false | full-junit-result: false | output-style: single - """ - } + """.trimMargin() + ) } @Test @@ -88,13 +87,12 @@ class SanityRoboCheck { ) val result = gradleRun( - arguments = listOf("writeConfigProps", "-PsanityRobo"), + arguments = listOf("printYml", "-PsanityRobo"), projectDir = testProjectRoot.root ) assertThat(result.output).contains("SUCCESS") - - "build/fladle/flank.yml" readAndCompareWith { + assertThat(result.output).contains( """ |gcloud: | app: foo.apk @@ -117,8 +115,8 @@ class SanityRoboCheck { | legacy-junit-result: false | full-junit-result: false | output-style: single - """ - } + """.trimMargin() + ) } @Test @@ -151,13 +149,12 @@ class SanityRoboCheck { ) val result = gradleRun( - arguments = listOf("writeConfigProps", "-PsanityRobo"), + arguments = listOf("printYml", "-PsanityRobo"), projectDir = testProjectRoot.root ) assertThat(result.output).contains("SUCCESS") - - "build/fladle/flank.yml" readAndCompareWith { + assertThat(result.output).contains( """ |gcloud: | app: foo.apk @@ -180,17 +177,16 @@ class SanityRoboCheck { | legacy-junit-result: false | full-junit-result: false | output-style: single - """ - } + """.trimMargin() + ) val resultOrange = gradleRun( - arguments = listOf("writeConfigPropsOrange", "-PsanityRobo"), + arguments = listOf("printYmlOrange", "-PsanityRobo"), projectDir = testProjectRoot.root ) assertThat(resultOrange.output).contains("SUCCESS") - - "build/fladle/orange/flank.yml" readAndCompareWith { + assertThat(resultOrange.output).contains( """ |gcloud: | app: foo.apk @@ -216,8 +212,8 @@ class SanityRoboCheck { | legacy-junit-result: false | full-junit-result: false | output-style: single - """ - } + """.trimMargin() + ) } @Test @@ -239,13 +235,12 @@ class SanityRoboCheck { ) val result = gradleRun( - arguments = listOf("writeConfigProps", "-PsanityRobo"), + arguments = listOf("printYml", "-PsanityRobo"), projectDir = testProjectRoot.root ) assertThat(result.output).contains("SUCCESS") - - "build/fladle/flank.yml" readAndCompareWith { + assertThat(result.output).contains( """ |gcloud: | app: foo.apk @@ -268,8 +263,8 @@ class SanityRoboCheck { | legacy-junit-result: false | full-junit-result: false | output-style: single - """ - } + """.trimMargin() + ) } @Test @@ -295,13 +290,12 @@ class SanityRoboCheck { ) val result = gradleRun( - arguments = listOf("writeConfigProps", "-PsanityRobo"), + arguments = listOf("printYml", "-PsanityRobo"), projectDir = testProjectRoot.root ) assertThat(result.output).contains("SUCCESS") - - "build/fladle/flank.yml" readAndCompareWith { + assertThat(result.output).contains( """ |gcloud: | app: foo.apk @@ -324,15 +318,7 @@ class SanityRoboCheck { | legacy-junit-result: false | full-junit-result: false | output-style: single - """ - } + """.trimMargin() + ) } - - private infix fun String.readAndCompareWith(block: () -> String) = testProjectRoot - .root - .resolve(this) - .readText() - .run { - assertThat(this).contains(block().trimMargin()) - } } From e4dd6ee8a99a8499989a1559e0796e36990c98ed Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Fri, 25 Sep 2020 12:45:23 +0200 Subject: [PATCH 05/12] Remove gradle properties --- .../flank/gradle/FladlePluginDelegate.kt | 2 - .../gradle/integration/SanityRoboCheck.kt | 56 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt index 5fa1894c..15415501 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt @@ -40,8 +40,6 @@ class FladlePluginDelegate { // Must be done afterEvaluate otherwise extension values will not be set. project.dependencies.add(FLADLE_CONFIG, "${base.flankCoordinates.get()}:${base.flankVersion.get()}") - if (project.hasProperty("sanityRobo")) base.sanityRobo.set(true) - // Only use automatic apk path detection for 'com.android.application' projects. project.pluginManager.withPlugin("com.android.application") { if (!base.debugApk.isPresent || !base.instrumentationApk.isPresent) { diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt index 6de4c1b9..eff6e8c1 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt @@ -24,6 +24,58 @@ class SanityRoboCheck { |} | |fladle { + | sanityRobo = project.hasProperty('sanityRobo') + | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") + | debugApk = "foo.apk" + |} + """ + ) + + val result = gradleRun( + arguments = listOf("printYml", "-PsanityRobo"), + projectDir = testProjectRoot.root + ) + + assertThat(result.output).contains("SUCCESS") + assertThat(result.output).contains( + """ + |gcloud: + | app: foo.apk + | device: + | - model: NexusLowRes + | version: 28 + | + | use-orchestrator: false + | auto-google-login: false + | record-video: true + | performance-metrics: true + | timeout: 15m + | num-flaky-test-attempts: 0 + | + |flank: + | keep-file-path: false + | ignore-failed-tests: false + | disable-sharding: false + | smart-flank-disable-upload: false + | legacy-junit-result: false + | full-junit-result: false + | output-style: single + """.trimMargin() + ) + } + + @Test + fun checkSanityRoboRunWithProjectPropertySetAsExtensionProperty() { + makeGradleFile( + where = testProjectRoot, + buildScript = + """ + |plugins { + | id "com.osacky.fladle" + |} + | + |fladle { + | sanityRobo = true | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") | debugApk = "foo.apk" |} @@ -74,6 +126,7 @@ class SanityRoboCheck { |} | |fladle { + | sanityRobo = project.hasProperty('sanityRobo') | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") | debugApk = "foo.apk" | instrumentationApk = "test.apk" @@ -130,6 +183,7 @@ class SanityRoboCheck { |} | |fladle { + | sanityRobo = project.hasProperty('sanityRobo') | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") | debugApk = "foo.apk" | instrumentationApk = "test.apk" @@ -227,6 +281,7 @@ class SanityRoboCheck { |} | |fladle { + | sanityRobo = project.hasProperty('sanityRobo') | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") | debugApk = "foo.apk" | roboScript = "some/path/script.json" @@ -278,6 +333,7 @@ class SanityRoboCheck { |} | |fladle { + | sanityRobo = project.hasProperty('sanityRobo') | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") | debugApk = "foo.apk" | roboDirectives = [ From 750ae1b7714919566c5daedfa287632a73a76150 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Mon, 28 Sep 2020 14:32:51 +0200 Subject: [PATCH 06/12] After rebase - resolve problems --- .../java/com/osacky/flank/gradle/YamlWriter.kt | 14 +++++++------- .../flank/gradle/integration/SanityRoboCheck.kt | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt index 54440945..2f49ad61 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt @@ -13,13 +13,13 @@ internal class YamlWriter { } check(base.debugApk.isPresent) { "debugApk must be specified" } if (base.sanityRobo.get() == false) { - check(base.instrumentationApk.isPresent xor !base.roboScript.orNull.isNullOrBlank()) { - val prefix = if (base.instrumentationApk.isPresent && !base.roboScript.orNull.isNullOrBlank()) { - "Both instrumentationApk file and roboScript file were specified, but only one is expected." - } else { - "Must specify either a instrumentationApk file or a roboScript file." - } - """ + check(base.instrumentationApk.isPresent xor !base.roboScript.orNull.isNullOrBlank()) { + val prefix = if (base.instrumentationApk.isPresent && !base.roboScript.orNull.isNullOrBlank()) { + "Both instrumentationApk file and roboScript file were specified, but only one is expected." + } else { + "Must specify either a instrumentationApk file or a roboScript file." + } + """ $prefix instrumentationApk=${base.instrumentationApk.orNull} roboScript=${base.roboScript.orNull} diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt index eff6e8c1..b941a639 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt @@ -194,7 +194,7 @@ class SanityRoboCheck { | ] | configs { | orange { - | testTargets = ['override'] + | testTargets.set(project.provider { ['override'] }) | localResultsDir.set('overrideDir') | } | } From f08f456531af4ff6ecc1c2c6b96b67a209b915e8 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Tue, 29 Sep 2020 14:58:27 +0200 Subject: [PATCH 07/12] Add validation --- .../com/osacky/flank/gradle/FladleConfig.kt | 13 + .../osacky/flank/gradle/FladleConfigImpl.kt | 13 +- .../flank/gradle/FladlePluginDelegate.kt | 3 + .../flank/gradle/FlankGradleExtension.kt | 57 +--- .../flank/gradle/SanityConfigValidation.kt | 45 +++ .../com/osacky/flank/gradle/YamlWriter.kt | 18 +- .../gradle/integration/SanityRoboCheck.kt | 270 ++++++++++-------- .../flank/gradle/integration/TestFixtures.kt | 7 +- docs/configuration.md | 6 - 9 files changed, 242 insertions(+), 190 deletions(-) create mode 100644 buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfig.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfig.kt index 1edbdb67..d845b5df 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfig.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfig.kt @@ -18,6 +18,19 @@ interface FladleConfig { @get:Optional val serviceAccountCredentials: RegularFileProperty + /** + * debugApk and instrumentationApk are [Property] and not [RegularFileProperty] because we support wildcard characters. + */ + @get:Input + @get:Optional + val debugApk: Property + @get:Input + @get:Optional + val instrumentationApk: Property + + @get:Input + val sanityRobo: Property + @get:Input val useOrchestrator: Property diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt index 61a86922..05e2a0c5 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt @@ -9,6 +9,9 @@ data class FladleConfigImpl( internal val name: String, override val projectId: Property, override val serviceAccountCredentials: RegularFileProperty, + override val debugApk: Property, + override val instrumentationApk: Property, + override val sanityRobo: Property, override val useOrchestrator: Property, override val autoGoogleLogin: Property, override val devices: ListProperty>, @@ -45,4 +48,12 @@ data class FladleConfigImpl( override val outputStyle: Property, override val legacyJunitResult: Property, override val fullJunitResult: Property -) : FladleConfig +) : FladleConfig { + fun sanityRoboRun() { + sanityRobo.set(true) + additionalTestApks.empty() + instrumentationApk.set("") + roboDirectives.empty() + roboScript.set("") + } +} diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt index 15415501..b5f5ce9c 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FladlePluginDelegate.kt @@ -40,6 +40,9 @@ class FladlePluginDelegate { // Must be done afterEvaluate otherwise extension values will not be set. project.dependencies.add(FLADLE_CONFIG, "${base.flankCoordinates.get()}:${base.flankVersion.get()}") + checkIfSanityAndValidateConfigs(base) + base.configs.forEach(::checkIfSanityAndValidateConfigs) + // Only use automatic apk path detection for 'com.android.application' projects. project.pluginManager.withPlugin("com.android.application") { if (!base.debugApk.isPresent || !base.instrumentationApk.isPresent) { diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt index f9e1c415..3529a91e 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt @@ -20,96 +20,58 @@ open class FlankGradleExtension @Inject constructor(objects: ObjectFactory) : Fl val flankCoordinates: Property = objects.property(String::class.java).convention("com.github.flank:flank") @get:Input - val sanityRobo: Property = objects.property().convention(false) + val flankVersion: Property = objects.property(String::class.java).convention("20.09.3") + // Variant to use for configuring output APK. @get:Input - val flankVersion: Property = objects.property(String::class.java).convention("20.09.3") + @get:Optional + val variant: Property = objects.property() + // Project id is automatically discovered by default. Use this to override the project id. override val projectId: Property = objects.property() override val serviceAccountCredentials: RegularFileProperty = objects.fileProperty() override val useOrchestrator: Property = objects.property().convention(false) override val autoGoogleLogin: Property = objects.property().convention(false) override val devices: ListProperty> = objects.listProperty>().convention(listOf(mapOf("model" to "NexusLowRes", "version" to "28"))) + override val sanityRobo: Property = objects.property().convention(false) // https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run override val testTargets: ListProperty = objects.listProperty() - override val testShards: Property = objects.property() override val shardTime: Property = objects.property() override val repeatTests: Property = objects.property() // Shard Android tests by time using historical run data. The amount of shards used is set by `testShards`. override val smartFlankGcsPath: Property = objects.property() - override val resultsHistoryName: Property = objects.property() - override val flakyTestAttempts: Property = objects.property().convention(0) - - // Variant to use for configuring output APK. - @get:Input - @get:Optional - val variant: Property = objects.property() - - /** - * debugApk and instrumentationApk are [Property] and not [RegularFileProperty] because we support wildcard characters. - */ - @get:Input - @get:Optional - val debugApk: Property = objects.property() - @get:Input - @get:Optional - val instrumentationApk: Property = objects.property() - + override val debugApk: Property = objects.property() + override val instrumentationApk: Property = objects.property() override val directoriesToPull: ListProperty = objects.listProperty() - override val filesToDownload: ListProperty = objects.listProperty() - override val environmentVariables: MapProperty = objects.mapProperty() - override val recordVideo: Property = objects.property().convention(true) - override val performanceMetrics: Property = objects.property().convention(true) - override val resultsBucket: Property = objects.property() - override val keepFilePath: Property = objects.property().convention(false) - override val resultsDir: Property = objects.property() - override val additionalTestApks: ListProperty = objects.listProperty() - override val runTimeout: Property = objects.property() - override val ignoreFailedTests: Property = objects.property().convention(false) - override val disableSharding: Property = objects.property().convention(false) - override val smartFlankDisableUpload: Property = objects.property().convention(false) - override val testRunnerClass: Property = objects.property() - override val localResultsDir: Property = objects.property() - override val numUniformShards: Property = objects.property() - override val clientDetails: MapProperty = objects.mapProperty() - override val testTargetsAlwaysRun: ListProperty = objects.listProperty() - override val otherFiles: MapProperty = objects.mapProperty() - override val networkProfile: Property = objects.property() - override val roboScript: Property = objects.property() - override val roboDirectives: ListProperty> = objects.listProperty() - override val testTimeout: Property = objects.property().convention("15m") - override val outputStyle: Property = objects.property().convention("single") - override val legacyJunitResult: Property = objects.property().convention(false) - override val fullJunitResult: Property = objects.property().convention(false) @Internal @@ -118,6 +80,9 @@ open class FlankGradleExtension @Inject constructor(objects: ObjectFactory) : Fl name = it, projectId = objects.property().convention(projectId), serviceAccountCredentials = objects.fileProperty().convention(serviceAccountCredentials), + debugApk = objects.property().convention(debugApk), + instrumentationApk = objects.property().convention(instrumentationApk), + sanityRobo = objects.property().convention(false), useOrchestrator = objects.property().convention(useOrchestrator), autoGoogleLogin = objects.property().convention(autoGoogleLogin), devices = objects.listProperty>().convention(devices), diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt new file mode 100644 index 00000000..0588ce13 --- /dev/null +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt @@ -0,0 +1,45 @@ +package com.osacky.flank.gradle + +import org.gradle.api.GradleException +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property + +@Throws(GradleException::class) +fun checkIfSanityAndValidateConfigs(config: FladleConfig) = when (config) { + is FlankGradleExtension -> config.checkAndValidateExtension() + is FladleConfigImpl -> config.checkAndValidateConfig() + else -> throw GradleException("Unable to check for sanity, check config type") +} + +private fun FlankGradleExtension.checkAndValidateExtension() { + if (sanityRobo.getOrElse(false)) when { + instrumentationApk.isNotPresentOrBlank -> throwBaseConfigError("instrumentationApk") + additionalTestApks.isNotPresentOrEmpty -> throwBaseConfigError("additionalTestApks") + roboDirectives.isNotPresentOrEmpty -> throwBaseConfigError("roboDirectives") + roboScript.isNotPresentOrBlank -> throwBaseConfigError("roboScript") + } +} + +private fun FladleConfigImpl.checkAndValidateConfig() { + if (sanityRobo.getOrElse(false)) when { + roboDirectives.isNotPresentOrEmpty -> throwAdditionalConfigError("roboDirectives", name) + roboScript.isNotPresentOrBlank -> throwAdditionalConfigError("roboScript", name) + instrumentationApk.isNotPresentOrBlank -> throwAdditionalConfigError("instrumentationApk", name) + additionalTestApks.isNotPresentOrEmpty -> throwAdditionalConfigError("additionalTestApks", name) + } +} + +private fun throwBaseConfigError(option: String): Nothing = + throw GradleException("Incorrect [base] configuration. [$option] can't be used together with sanityRobo.") + +private fun throwAdditionalConfigError(option: String, name: String): Nothing = + throw GradleException( + "Incorrect [$name] configuration. [$option] can't be used together with sanityRobo. " + + "If you want to launch robo test run without robo script place only sanityRoboRun() into [$name] configuration" + ) + +private val Property.isNotPresentOrBlank + get() = orNull.isNullOrBlank().not() + +private val ListProperty.isNotPresentOrEmpty + get() = getOrElse(emptyList()).isEmpty().not() diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt index 2f49ad61..74fa98a8 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt @@ -12,31 +12,31 @@ internal class YamlWriter { check(base.serviceAccountCredentials.isPresent) { "ServiceAccountCredentials in fladle extension not set. https://github.com/runningcode/fladle#serviceaccountcredentials" } } check(base.debugApk.isPresent) { "debugApk must be specified" } - if (base.sanityRobo.get() == false) { - check(base.instrumentationApk.isPresent xor !base.roboScript.orNull.isNullOrBlank()) { - val prefix = if (base.instrumentationApk.isPresent && !base.roboScript.orNull.isNullOrBlank()) { + if (config.sanityRobo.get() == false) { + check(config.instrumentationApk.isPresent xor !config.roboScript.orNull.isNullOrBlank()) { + val prefix = if (base.instrumentationApk.isPresent && !config.roboScript.orNull.isNullOrBlank()) { "Both instrumentationApk file and roboScript file were specified, but only one is expected." } else { "Must specify either a instrumentationApk file or a roboScript file." } """ $prefix - instrumentationApk=${base.instrumentationApk.orNull} - roboScript=${base.roboScript.orNull} + instrumentationApk=${config.instrumentationApk.orNull} + roboScript=${config.roboScript.orNull} """.trimIndent() } } - val shouldPrintTestAndRobo = base.sanityRobo.get().not() + val shouldPrintTestAndRobo = config.sanityRobo.get().not() val additionalProperties = writeAdditionalProperties(config, shouldPrintTestAndRobo) val flankProperties = writeFlankProperties(config, shouldPrintTestAndRobo) return buildString { appendln("gcloud:") - appendln(" app: ${base.debugApk.get()}") + appendln(" app: ${config.debugApk.get()}") // We don't want to print instrumentation apks if sanityRobo == true - if (shouldPrintTestAndRobo && base.instrumentationApk.isPresent) { - appendln(" test: ${base.instrumentationApk.get()}") + if (shouldPrintTestAndRobo && config.instrumentationApk.isPresent) { + appendln(" test: ${config.instrumentationApk.get()}") } if (config.devices.isPresentAndNotEmpty) appendln(createDeviceString(config.devices.get())) appendln(additionalProperties) diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt index b941a639..e38de32f 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt @@ -6,6 +6,9 @@ import org.junit.Rule import org.junit.Test import org.junit.rules.TemporaryFolder +private fun baseConfigMessage(option: String) = "Incorrect [base] configuration. [$option] can't be used together with sanityRobo." +private fun additionalConfigMessage(option: String, name: String) = "Incorrect [$name] configuration. [$option] can't be used together with sanityRobo. If you want to launch robo test run without robo script place only sanityRoboRun() into [$name] configuration" + class SanityRoboCheck { @get:Rule var testProjectRoot = TemporaryFolder() @@ -14,7 +17,7 @@ class SanityRoboCheck { fun setUp() = testProjectRoot.newFile("flank-gradle-service.json").writeText("{}") @Test - fun checkSanityRoboRunWithProjectProperty() { + fun `sanityRobo - should throw an error if instrumentationApk set`() { makeGradleFile( where = testProjectRoot, buildScript = @@ -24,48 +27,39 @@ class SanityRoboCheck { |} | |fladle { - | sanityRobo = project.hasProperty('sanityRobo') + | debugApk = "debug.apk" + | sanityRobo = true | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") - | debugApk = "foo.apk" + | instrumentationApk = "test.apk" + | configs { + | sanity { + | sanityRobo.set(true) + | makeSanityRun() + | } + | } |} """ ) - val result = gradleRun( - arguments = listOf("printYml", "-PsanityRobo"), + val result = failedGradleRun( + arguments = listOf("printYml"), projectDir = testProjectRoot.root ) - assertThat(result.output).contains("SUCCESS") - assertThat(result.output).contains( - """ - |gcloud: - | app: foo.apk - | device: - | - model: NexusLowRes - | version: 28 - | - | use-orchestrator: false - | auto-google-login: false - | record-video: true - | performance-metrics: true - | timeout: 15m - | num-flaky-test-attempts: 0 - | - |flank: - | keep-file-path: false - | ignore-failed-tests: false - | disable-sharding: false - | smart-flank-disable-upload: false - | legacy-junit-result: false - | full-junit-result: false - | output-style: single - """.trimMargin() + assertThat(result.output).contains("FAILED") + assertThat(result.output).contains(baseConfigMessage("instrumentationApk")) + + val resultSanity = failedGradleRun( + arguments = listOf("printYmlSanity"), + projectDir = testProjectRoot.root ) + + assertThat(resultSanity.output).contains("FAILED") + assertThat(result.output).contains(baseConfigMessage("instrumentationApk")) } @Test - fun checkSanityRoboRunWithProjectPropertySetAsExtensionProperty() { + fun `sanityRobo - should throw an error if roboScript set`() { makeGradleFile( where = testProjectRoot, buildScript = @@ -75,48 +69,89 @@ class SanityRoboCheck { |} | |fladle { + | debugApk = "debug.apk" | sanityRobo = true | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") - | debugApk = "foo.apk" + | roboScript = "some/path/script.json" |} """ ) - val result = gradleRun( - arguments = listOf("printYml", "-PsanityRobo"), + val result = failedGradleRun( + arguments = listOf("printYml"), projectDir = testProjectRoot.root ) - assertThat(result.output).contains("SUCCESS") - assertThat(result.output).contains( - """ - |gcloud: - | app: foo.apk - | device: - | - model: NexusLowRes - | version: 28 + assertThat(result.output).contains("FAILED") + assertThat(result.output).contains(baseConfigMessage("roboScript")) + } + + @Test + fun `sanityRobo - should throw an error if roboDirectives set`() { + makeGradleFile( + where = testProjectRoot, + buildScript = + """ + |plugins { + | id "com.osacky.fladle" + |} | - | use-orchestrator: false - | auto-google-login: false - | record-video: true - | performance-metrics: true - | timeout: 15m - | num-flaky-test-attempts: 0 + |fladle { + | debugApk = "debug.apk" + | sanityRobo = true + | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") + | roboDirectives = [ + | ["click", "button1", ""], + | ["ignore", "button2"], + | ["text", "field1", "my text"], + | ] + |} + """ + ) + + val result = failedGradleRun( + arguments = listOf("printYml"), + projectDir = testProjectRoot.root + ) + + assertThat(result.output).contains("FAILED") + assertThat(result.output).contains(baseConfigMessage("roboDirectives")) + } + + @Test + fun `sanityRobo - should throw an error if additionalTestApks set`() { + makeGradleFile( + where = testProjectRoot, + buildScript = + """ + |plugins { + | id "com.osacky.fladle" + |} | - |flank: - | keep-file-path: false - | ignore-failed-tests: false - | disable-sharding: false - | smart-flank-disable-upload: false - | legacy-junit-result: false - | full-junit-result: false - | output-style: single - """.trimMargin() + |fladle { + | debugApk = "debug.apk" + | sanityRobo = true + | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") + | additionalTestApks = [ + | "- app: debug2.apk", + | " test: test2.apk", + | "- test: test3.apk" + | ] + |} + """ ) + + val result = failedGradleRun( + arguments = listOf("printYml"), + projectDir = testProjectRoot.root + ) + + assertThat(result.output).contains("FAILED") + assertThat(result.output).contains(baseConfigMessage("additionalTestApks")) } @Test - fun checkSanityRoboRunWithApksAdded() { + fun `sanityRobo - should throw an error if roboScript set (multiple config)`() { makeGradleFile( where = testProjectRoot, buildScript = @@ -126,7 +161,6 @@ class SanityRoboCheck { |} | |fladle { - | sanityRobo = project.hasProperty('sanityRobo') | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") | debugApk = "foo.apk" | instrumentationApk = "test.apk" @@ -135,45 +169,37 @@ class SanityRoboCheck { | " test: test2.apk", | "- test: test3.apk" | ] + | configs { + | sanity { + | sanityRobo.set(true) + | roboScript.set("path/to/script.json") + | } + | } |} """ ) - val result = gradleRun( - arguments = listOf("printYml", "-PsanityRobo"), + val expectedMessage = additionalConfigMessage("roboScript", "sanity") + + val result = failedGradleRun( + arguments = listOf("printYml"), projectDir = testProjectRoot.root ) - assertThat(result.output).contains("SUCCESS") - assertThat(result.output).contains( - """ - |gcloud: - | app: foo.apk - | device: - | - model: NexusLowRes - | version: 28 - | - | use-orchestrator: false - | auto-google-login: false - | record-video: true - | performance-metrics: true - | timeout: 15m - | num-flaky-test-attempts: 0 - | - |flank: - | keep-file-path: false - | ignore-failed-tests: false - | disable-sharding: false - | smart-flank-disable-upload: false - | legacy-junit-result: false - | full-junit-result: false - | output-style: single - """.trimMargin() + assertThat(result.output).contains("FAILED") + assertThat(result.output).contains(expectedMessage) + + val resultOrange = failedGradleRun( + arguments = listOf("printYmlSanity"), + projectDir = testProjectRoot.root ) + + assertThat(resultOrange.output).contains("FAILED") + assertThat(resultOrange.output).contains(expectedMessage) } @Test - fun checkSanityRoboRunMultipleConfigs() { + fun `sanityRobo - should print correct config yamls (inner config is sanity run)`() { makeGradleFile( where = testProjectRoot, buildScript = @@ -183,7 +209,6 @@ class SanityRoboCheck { |} | |fladle { - | sanityRobo = project.hasProperty('sanityRobo') | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") | debugApk = "foo.apk" | instrumentationApk = "test.apk" @@ -194,8 +219,7 @@ class SanityRoboCheck { | ] | configs { | orange { - | testTargets.set(project.provider { ['override'] }) - | localResultsDir.set('overrideDir') + | sanityRoboRun() | } | } |} @@ -203,7 +227,7 @@ class SanityRoboCheck { ) val result = gradleRun( - arguments = listOf("printYml", "-PsanityRobo"), + arguments = listOf("printYml"), projectDir = testProjectRoot.root ) @@ -212,6 +236,7 @@ class SanityRoboCheck { """ |gcloud: | app: foo.apk + | test: test.apk | device: | - model: NexusLowRes | version: 28 @@ -225,6 +250,10 @@ class SanityRoboCheck { | |flank: | keep-file-path: false + | additional-app-test-apks: + | - app: debug2.apk + | test: test2.apk + | - test: test3.apk | ignore-failed-tests: false | disable-sharding: false | smart-flank-disable-upload: false @@ -235,7 +264,7 @@ class SanityRoboCheck { ) val resultOrange = gradleRun( - arguments = listOf("printYmlOrange", "-PsanityRobo"), + arguments = listOf("printYmlOrange"), projectDir = testProjectRoot.root ) @@ -253,8 +282,6 @@ class SanityRoboCheck { | record-video: true | performance-metrics: true | timeout: 15m - | test-targets: - | - override | num-flaky-test-attempts: 0 | |flank: @@ -262,7 +289,6 @@ class SanityRoboCheck { | ignore-failed-tests: false | disable-sharding: false | smart-flank-disable-upload: false - | local-result-dir: overrideDir | legacy-junit-result: false | full-junit-result: false | output-style: single @@ -271,7 +297,7 @@ class SanityRoboCheck { } @Test - fun checkSanityRoboRunRoboScript() { + fun `sanityRobo - should print correct config yamls (base config is sanity run)`() { makeGradleFile( where = testProjectRoot, buildScript = @@ -281,16 +307,25 @@ class SanityRoboCheck { |} | |fladle { - | sanityRobo = project.hasProperty('sanityRobo') | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") | debugApk = "foo.apk" - | roboScript = "some/path/script.json" + | sanityRobo = true + | configs { + | orange { + | instrumentationApk.set("test.apk") + | additionalTestApks.set(project.provider { [ + | "- app: debug2.apk", + | " test: test2.apk", + | "- test: test3.apk" + | ] }) + | } + | } |} """ ) val result = gradleRun( - arguments = listOf("printYml", "-PsanityRobo"), + arguments = listOf("printYml"), projectDir = testProjectRoot.root ) @@ -320,41 +355,18 @@ class SanityRoboCheck { | output-style: single """.trimMargin() ) - } - - @Test - fun checkSanityRoboRunRoboDirectives() { - makeGradleFile( - where = testProjectRoot, - buildScript = - """ - |plugins { - | id "com.osacky.fladle" - |} - | - |fladle { - | sanityRobo = project.hasProperty('sanityRobo') - | serviceAccountCredentials = layout.projectDirectory.file("flank-gradle-service.json") - | debugApk = "foo.apk" - | roboDirectives = [ - | ["click", "button1", ""], - | ["ignore", "button2"], - | ["text", "field1", "my text"], - | ] - |} - """ - ) - val result = gradleRun( - arguments = listOf("printYml", "-PsanityRobo"), + val resultOrange = gradleRun( + arguments = listOf("printYmlOrange"), projectDir = testProjectRoot.root ) - assertThat(result.output).contains("SUCCESS") - assertThat(result.output).contains( + assertThat(resultOrange.output).contains("SUCCESS") + assertThat(resultOrange.output).contains( """ |gcloud: | app: foo.apk + | test: test.apk | device: | - model: NexusLowRes | version: 28 @@ -368,6 +380,10 @@ class SanityRoboCheck { | |flank: | keep-file-path: false + | additional-app-test-apks: + | - app: debug2.apk + | test: test2.apk + | - test: test3.apk | ignore-failed-tests: false | disable-sharding: false | smart-flank-disable-upload: false diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt index 37c3ee3f..7af64458 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt @@ -13,9 +13,14 @@ internal fun makeGradleFile(where: TemporaryFolder, buildScript: String) = where .writeText(buildScript.trimMargin()) internal fun gradleRun(projectDir: File, arguments: List = emptyList()) = + commonGradleRunConfig(projectDir, arguments).build() + +internal fun failedGradleRun(projectDir: File, arguments: List = emptyList()) = + commonGradleRunConfig(projectDir, arguments).buildAndFail() + +private fun commonGradleRunConfig(projectDir: File, arguments: List) = GradleRunner.create() .withPluginClasspath() .withArguments(arguments) .forwardOutput() .withProjectDir(projectDir) - .build() diff --git a/docs/configuration.md b/docs/configuration.md index 2722e33b..b40457ab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -743,9 +743,3 @@ Enables creating an additional local junit result on local storage with failure ``` kotlin fullJunitResult.set(false) ``` - -## Sanity robo test -Just add `-PsanityRobo` -- this will launch simple robo test without robo scripts and/or robo directives. (`roboScript`, `roboDirectives`, `test`, `additionalTestApks` are skipped) -``` -./gradlew runFlank -PsanityRobo -``` From 1a6c8b6c664e4ca013fc956a5491679d8b6adbc9 Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Wed, 30 Sep 2020 07:21:01 +0200 Subject: [PATCH 08/12] Refactor --- buildSrc/build.gradle.kts | 1 + .../flank/gradle/FlankGradleExtension.kt | 48 ++++++++++++++++--- .../{SanityRoboCheck.kt => SanityRoboTest.kt} | 14 +++--- .../flank/gradle/integration/TestFixtures.kt | 2 +- 4 files changed, 50 insertions(+), 15 deletions(-) rename buildSrc/src/test/java/com/osacky/flank/gradle/integration/{SanityRoboCheck.kt => SanityRoboTest.kt} (98%) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 93e65370..264a06b5 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -153,6 +153,7 @@ fun org.gradle.api.publish.maven.MavenPom.configureForFladle(pluginName: String) tasks.withType(Test::class.java).configureEach { // Test fixtures are stored in here so we should re-run tests if the test projects change. inputs.dir("src/test/resources") + maxParallelForks = Runtime.getRuntime().availableProcessors() testLogging { events = setOf(TestLogEvent.SKIPPED, TestLogEvent.FAILED, TestLogEvent.PASSED) } diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt index 3529a91e..7cc93eee 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FlankGradleExtension.kt @@ -19,59 +19,93 @@ open class FlankGradleExtension @Inject constructor(objects: ObjectFactory) : Fl @get:Input val flankCoordinates: Property = objects.property(String::class.java).convention("com.github.flank:flank") - @get:Input - val flankVersion: Property = objects.property(String::class.java).convention("20.09.3") + override val sanityRobo: Property = objects.property().convention(false) - // Variant to use for configuring output APK. @get:Input - @get:Optional - val variant: Property = objects.property() - + val flankVersion: Property = objects.property(String::class.java).convention("20.09.3") // Project id is automatically discovered by default. Use this to override the project id. override val projectId: Property = objects.property() override val serviceAccountCredentials: RegularFileProperty = objects.fileProperty() override val useOrchestrator: Property = objects.property().convention(false) override val autoGoogleLogin: Property = objects.property().convention(false) override val devices: ListProperty> = objects.listProperty>().convention(listOf(mapOf("model" to "NexusLowRes", "version" to "28"))) - override val sanityRobo: Property = objects.property().convention(false) // https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run override val testTargets: ListProperty = objects.listProperty() + override val testShards: Property = objects.property() override val shardTime: Property = objects.property() override val repeatTests: Property = objects.property() // Shard Android tests by time using historical run data. The amount of shards used is set by `testShards`. override val smartFlankGcsPath: Property = objects.property() + override val resultsHistoryName: Property = objects.property() + override val flakyTestAttempts: Property = objects.property().convention(0) + + // Variant to use for configuring output APK. + @get:Input + @get:Optional + val variant: Property = objects.property() + + /** + * debugApk and instrumentationApk are [Property] and not [RegularFileProperty] because we support wildcard characters. + */ override val debugApk: Property = objects.property() + override val instrumentationApk: Property = objects.property() + override val directoriesToPull: ListProperty = objects.listProperty() + override val filesToDownload: ListProperty = objects.listProperty() + override val environmentVariables: MapProperty = objects.mapProperty() + override val recordVideo: Property = objects.property().convention(true) + override val performanceMetrics: Property = objects.property().convention(true) + override val resultsBucket: Property = objects.property() + override val keepFilePath: Property = objects.property().convention(false) + override val resultsDir: Property = objects.property() + override val additionalTestApks: ListProperty = objects.listProperty() + override val runTimeout: Property = objects.property() + override val ignoreFailedTests: Property = objects.property().convention(false) + override val disableSharding: Property = objects.property().convention(false) + override val smartFlankDisableUpload: Property = objects.property().convention(false) + override val testRunnerClass: Property = objects.property() + override val localResultsDir: Property = objects.property() + override val numUniformShards: Property = objects.property() + override val clientDetails: MapProperty = objects.mapProperty() + override val testTargetsAlwaysRun: ListProperty = objects.listProperty() + override val otherFiles: MapProperty = objects.mapProperty() + override val networkProfile: Property = objects.property() + override val roboScript: Property = objects.property() + override val roboDirectives: ListProperty> = objects.listProperty() + override val testTimeout: Property = objects.property().convention("15m") + override val outputStyle: Property = objects.property().convention("single") + override val legacyJunitResult: Property = objects.property().convention(false) + override val fullJunitResult: Property = objects.property().convention(false) @Internal diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt similarity index 98% rename from buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt rename to buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt index e38de32f..93dac6f0 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboCheck.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt @@ -18,7 +18,7 @@ class SanityRoboCheck { @Test fun `sanityRobo - should throw an error if instrumentationApk set`() { - makeGradleFile( + makeBuildDotGradle( where = testProjectRoot, buildScript = """ @@ -60,7 +60,7 @@ class SanityRoboCheck { @Test fun `sanityRobo - should throw an error if roboScript set`() { - makeGradleFile( + makeBuildDotGradle( where = testProjectRoot, buildScript = """ @@ -88,7 +88,7 @@ class SanityRoboCheck { @Test fun `sanityRobo - should throw an error if roboDirectives set`() { - makeGradleFile( + makeBuildDotGradle( where = testProjectRoot, buildScript = """ @@ -120,7 +120,7 @@ class SanityRoboCheck { @Test fun `sanityRobo - should throw an error if additionalTestApks set`() { - makeGradleFile( + makeBuildDotGradle( where = testProjectRoot, buildScript = """ @@ -152,7 +152,7 @@ class SanityRoboCheck { @Test fun `sanityRobo - should throw an error if roboScript set (multiple config)`() { - makeGradleFile( + makeBuildDotGradle( where = testProjectRoot, buildScript = """ @@ -200,7 +200,7 @@ class SanityRoboCheck { @Test fun `sanityRobo - should print correct config yamls (inner config is sanity run)`() { - makeGradleFile( + makeBuildDotGradle( where = testProjectRoot, buildScript = """ @@ -298,7 +298,7 @@ class SanityRoboCheck { @Test fun `sanityRobo - should print correct config yamls (base config is sanity run)`() { - makeGradleFile( + makeBuildDotGradle( where = testProjectRoot, buildScript = """ diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt index 7af64458..175d5531 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/TestFixtures.kt @@ -8,7 +8,7 @@ fun TemporaryFolder.setupFixture(fixtureName: String) { File(this::class.java.classLoader.getResource(fixtureName)!!.file).copyRecursively(newFile(fixtureName), true) } -internal fun makeGradleFile(where: TemporaryFolder, buildScript: String) = where +internal fun makeBuildDotGradle(where: TemporaryFolder, buildScript: String) = where .newFile("build.gradle") .writeText(buildScript.trimMargin()) From e4b1e1fa742c9aa698b75b945e75ce97ad3e520c Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Wed, 30 Sep 2020 12:05:24 +0200 Subject: [PATCH 09/12] Add test for autoconfig sanity run --- .../gradle/integration/SanityRoboTest.kt | 2 +- .../SanityWithAutoConfigureTest.kt | 274 ++++++++++++++++++ 2 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt index 93dac6f0..a905ed6a 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt @@ -9,7 +9,7 @@ import org.junit.rules.TemporaryFolder private fun baseConfigMessage(option: String) = "Incorrect [base] configuration. [$option] can't be used together with sanityRobo." private fun additionalConfigMessage(option: String, name: String) = "Incorrect [$name] configuration. [$option] can't be used together with sanityRobo. If you want to launch robo test run without robo script place only sanityRoboRun() into [$name] configuration" -class SanityRoboCheck { +class SanityRoboTest { @get:Rule var testProjectRoot = TemporaryFolder() diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt new file mode 100644 index 00000000..a810c161 --- /dev/null +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt @@ -0,0 +1,274 @@ +package com.osacky.flank.gradle.integration + +import com.google.common.truth.Truth.assertThat +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +private const val commonScriptPart = """ + plugins { + id 'com.android.application' + id 'com.osacky.fladle' + } + + android { + compileSdkVersion 29 + defaultConfig { + applicationId "com.osacky.flank.gradle.sample" + minSdkVersion 23 + targetSdkVersion 29 + versionCode 1 + versionName "1.0" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + testOptions { + execution 'ANDROIDX_TEST_ORCHESTRATOR' + } + } +""" + +class SanityWithAutoConfigureTest { + + @get:Rule + var testProjectRoot = TemporaryFolder() + + @Before + fun setUp() { + testProjectRoot.newFile("flank-gradle-service.json").writeText("{}") + testProjectRoot.newFile("local.properties").writeText("sdk.dir=${androidHome()}\n") + testProjectRoot.newFile("gradle.properties").writeText("android.useAndroidX=true") + } + + @Test + fun `test auto configuration with sanityRobo set (inner config)`() { + makeBuildDotGradle( + where = testProjectRoot, + buildScript = + """ + $commonScriptPart + + fladle { + serviceAccountCredentials = project.layout.projectDirectory.file("flank-gradle-service.json") + useOrchestrator = true + environmentVariables = [ + "clearPackageData": "true" + ] + testTargets = [ + "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#seeView" + ] + devices = [ + [ "model": "Pixel2", "version": "26" ], + [ "model": "Nexus5", "version": "23" ] + ] + smartFlankGcsPath = "gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml" + configs { + sanity { + sanityRoboRun() + useOrchestrator.set(false) + testTargets.set(project.provider { [ + "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail" + ] }) + flakyTestAttempts.set(3) + } + } + } + """ + ) + + val baseResult = gradleRun( + projectDir = testProjectRoot.root, + arguments = listOf("printYml") + ) + + assertThat(baseResult.output).contains("BUILD SUCCESSFUL") + assertThat(baseResult.output).containsMatch( + """ + gcloud: + app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk + test: [0-9a-zA-Z\/_]*/build/outputs/apk/androidTest/debug/[0-9a-zA-Z\/_]*-debug-androidTest.apk + device: + - model: Pixel2 + version: 26 + - model: Nexus5 + version: 23 + + use-orchestrator: true + auto-google-login: false + record-video: true + performance-metrics: true + timeout: 15m + environment-variables: + clearPackageData: true + test-targets: + - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#seeView + num-flaky-test-attempts: 0 + + flank: + smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml + keep-file-path: false + ignore-failed-tests: false + disable-sharding: false + smart-flank-disable-upload: false + legacy-junit-result: false + full-junit-result: false + output-style: single + """.trimIndent() + ) + + val sanityResult = gradleRun( + projectDir = testProjectRoot.root, + arguments = listOf("printYmlSanity") + ) + + assertThat(sanityResult.output).contains("BUILD SUCCESSFUL") + assertThat(sanityResult.output).containsMatch( + """ + gcloud: + app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk + device: + - model: Pixel2 + version: 26 + - model: Nexus5 + version: 23 + + use-orchestrator: false + auto-google-login: false + record-video: true + performance-metrics: true + timeout: 15m + environment-variables: + clearPackageData: true + test-targets: + - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail + num-flaky-test-attempts: 3 + + flank: + smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml + keep-file-path: false + ignore-failed-tests: false + disable-sharding: false + smart-flank-disable-upload: false + legacy-junit-result: false + full-junit-result: false + output-style: single + """.trimIndent() + ) + } + + @Test + fun `test auto configuration with sanityRobo set (base config)`() { + makeBuildDotGradle( + where = testProjectRoot, + buildScript = + """ + $commonScriptPart + + fladle { + sanityRobo = true + serviceAccountCredentials = project.layout.projectDirectory.file("flank-gradle-service.json") + useOrchestrator = true + environmentVariables = [ + "clearPackageData": "true" + ] + testTargets = [ + "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#seeView" + ] + devices = [ + [ "model": "Pixel2", "version": "26" ], + [ "model": "Nexus5", "version": "23" ] + ] + smartFlankGcsPath = "gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml" + configs { + oranges { + useOrchestrator.set(false) + testTargets.set(project.provider { [ + "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail" + ] }) + flakyTestAttempts.set(3) + } + } + } + """ + ) + + val baseResult = gradleRun( + projectDir = testProjectRoot.root, + arguments = listOf("printYml") + ) + + assertThat(baseResult.output).contains("BUILD SUCCESSFUL") + assertThat(baseResult.output).containsMatch( + """ + gcloud: + app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk + device: + - model: Pixel2 + version: 26 + - model: Nexus5 + version: 23 + + use-orchestrator: true + auto-google-login: false + record-video: true + performance-metrics: true + timeout: 15m + environment-variables: + clearPackageData: true + test-targets: + - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#seeView + num-flaky-test-attempts: 0 + + flank: + smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml + keep-file-path: false + ignore-failed-tests: false + disable-sharding: false + smart-flank-disable-upload: false + legacy-junit-result: false + full-junit-result: false + output-style: single + """.trimIndent() + ) + + val orangesResult = gradleRun( + projectDir = testProjectRoot.root, + arguments = listOf("printYmlOranges") + ) + + assertThat(orangesResult.output).contains("BUILD SUCCESSFUL") + assertThat(orangesResult.output).containsMatch( + """ + gcloud: + app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk + test: [0-9a-zA-Z\/_]*/build/outputs/apk/androidTest/debug/[0-9a-zA-Z\/_]*-debug-androidTest.apk + device: + - model: Pixel2 + version: 26 + - model: Nexus5 + version: 23 + + use-orchestrator: false + auto-google-login: false + record-video: true + performance-metrics: true + timeout: 15m + environment-variables: + clearPackageData: true + test-targets: + - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail + num-flaky-test-attempts: 3 + + flank: + smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml + keep-file-path: false + ignore-failed-tests: false + disable-sharding: false + smart-flank-disable-upload: false + legacy-junit-result: false + full-junit-result: false + output-style: single + """.trimIndent() + ) + } +} From cd22c9b90ebeb60e1a1b7cc09364b02c3c9cb8fd Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Wed, 30 Sep 2020 12:08:08 +0200 Subject: [PATCH 10/12] Update test --- .../SanityWithAutoConfigureTest.kt | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt index a810c161..ee30b0fd 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt @@ -71,6 +71,13 @@ class SanityWithAutoConfigureTest { ] }) flakyTestAttempts.set(3) } + oranges { + useOrchestrator.set(false) + testTargets.set(project.provider { [ + "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail" + ] }) + flakyTestAttempts.set(6) + } } } """ @@ -154,6 +161,46 @@ class SanityWithAutoConfigureTest { output-style: single """.trimIndent() ) + + val orangesResult = gradleRun( + projectDir = testProjectRoot.root, + arguments = listOf("printYmlOranges") + ) + + assertThat(orangesResult.output).contains("BUILD SUCCESSFUL") + assertThat(orangesResult.output).containsMatch( + """ + gcloud: + app: [0-9a-zA-Z\/_]*/build/outputs/apk/debug/[0-9a-zA-Z\/_]*-debug.apk + test: [0-9a-zA-Z\/_]*/build/outputs/apk/androidTest/debug/[0-9a-zA-Z\/_]*-debug-androidTest.apk + device: + - model: Pixel2 + version: 26 + - model: Nexus5 + version: 23 + + use-orchestrator: false + auto-google-login: false + record-video: true + performance-metrics: true + timeout: 15m + environment-variables: + clearPackageData: true + test-targets: + - class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail + num-flaky-test-attempts: 6 + + flank: + smart-flank-gcs-path: gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml + keep-file-path: false + ignore-failed-tests: false + disable-sharding: false + smart-flank-disable-upload: false + legacy-junit-result: false + full-junit-result: false + output-style: single + """.trimIndent() + ) } @Test From 65213bd2c260e620fddb64cc1c7442cb19ffda8b Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Thu, 1 Oct 2020 08:21:22 +0200 Subject: [PATCH 11/12] Another round of refactor --- buildSrc/build.gradle.kts | 1 - .../osacky/flank/gradle/FladleConfigImpl.kt | 9 ++++- .../flank/gradle/SanityConfigValidation.kt | 37 ++++++------------- .../com/osacky/flank/gradle/YamlWriter.kt | 12 +++--- .../gradle/integration/SanityRoboTest.kt | 7 ++-- .../SanityWithAutoConfigureTest.kt | 2 +- 6 files changed, 30 insertions(+), 38 deletions(-) diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 264a06b5..93e65370 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -153,7 +153,6 @@ fun org.gradle.api.publish.maven.MavenPom.configureForFladle(pluginName: String) tasks.withType(Test::class.java).configureEach { // Test fixtures are stored in here so we should re-run tests if the test projects change. inputs.dir("src/test/resources") - maxParallelForks = Runtime.getRuntime().availableProcessors() testLogging { events = setOf(TestLogEvent.SKIPPED, TestLogEvent.FAILED, TestLogEvent.PASSED) } diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt index 05e2a0c5..b8c21d3f 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/FladleConfigImpl.kt @@ -49,7 +49,14 @@ data class FladleConfigImpl( override val legacyJunitResult: Property, override val fullJunitResult: Property ) : FladleConfig { - fun sanityRoboRun() { + /** + * Prepare config to run sanity robo. + * + * Sets [sanityRobo] property as `true`. + * + * Cleans [instrumentationApk], [additionalTestApks], [roboDirectives], [roboScript] properties. + */ + fun clearPropertiesForSanityRobo() { sanityRobo.set(true) additionalTestApks.empty() instrumentationApk.set("") diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt index 0588ce13..5539bbb7 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/SanityConfigValidation.kt @@ -6,38 +6,25 @@ import org.gradle.api.provider.Property @Throws(GradleException::class) fun checkIfSanityAndValidateConfigs(config: FladleConfig) = when (config) { - is FlankGradleExtension -> config.checkAndValidateExtension() - is FladleConfigImpl -> config.checkAndValidateConfig() - else -> throw GradleException("Unable to check for sanity, check config type") -} - -private fun FlankGradleExtension.checkAndValidateExtension() { - if (sanityRobo.getOrElse(false)) when { - instrumentationApk.isNotPresentOrBlank -> throwBaseConfigError("instrumentationApk") - additionalTestApks.isNotPresentOrEmpty -> throwBaseConfigError("additionalTestApks") - roboDirectives.isNotPresentOrEmpty -> throwBaseConfigError("roboDirectives") - roboScript.isNotPresentOrBlank -> throwBaseConfigError("roboScript") + is FlankGradleExtension -> config.checkAndValidateConfig() { option, _ -> + "Incorrect [base] configuration. [$option] can't be used together with sanityRobo." + } + is FladleConfigImpl -> config.checkAndValidateConfig(config.name) { option, name -> + "Incorrect [$name] configuration. [$option] can't be used together with sanityRobo. " + + "If you want to launch robo test run without robo script place only clearPropertiesForSanityRobo() into [$name] configuration" } + else -> throw GradleException("Unable to check for sanity, check config type") } -private fun FladleConfigImpl.checkAndValidateConfig() { +private fun FladleConfig.checkAndValidateConfig(name: String = "base", message: (String, String) -> String) { if (sanityRobo.getOrElse(false)) when { - roboDirectives.isNotPresentOrEmpty -> throwAdditionalConfigError("roboDirectives", name) - roboScript.isNotPresentOrBlank -> throwAdditionalConfigError("roboScript", name) - instrumentationApk.isNotPresentOrBlank -> throwAdditionalConfigError("instrumentationApk", name) - additionalTestApks.isNotPresentOrEmpty -> throwAdditionalConfigError("additionalTestApks", name) + roboDirectives.isNotPresentOrEmpty -> throw GradleException(message("roboDirectives", name)) + roboScript.isNotPresentOrBlank -> throw GradleException(message("roboScript", name)) + instrumentationApk.isNotPresentOrBlank -> throw GradleException(message("instrumentationApk", name)) + additionalTestApks.isNotPresentOrEmpty -> throw GradleException(message("additionalTestApks", name)) } } -private fun throwBaseConfigError(option: String): Nothing = - throw GradleException("Incorrect [base] configuration. [$option] can't be used together with sanityRobo.") - -private fun throwAdditionalConfigError(option: String, name: String): Nothing = - throw GradleException( - "Incorrect [$name] configuration. [$option] can't be used together with sanityRobo. " + - "If you want to launch robo test run without robo script place only sanityRoboRun() into [$name] configuration" - ) - private val Property.isNotPresentOrBlank get() = orNull.isNullOrBlank().not() diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt index 74fa98a8..1a2b7d5c 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt @@ -28,8 +28,8 @@ internal class YamlWriter { } val shouldPrintTestAndRobo = config.sanityRobo.get().not() - val additionalProperties = writeAdditionalProperties(config, shouldPrintTestAndRobo) - val flankProperties = writeFlankProperties(config, shouldPrintTestAndRobo) + val additionalProperties = writeAdditionalProperties(config) + val flankProperties = writeFlankProperties(config) return buildString { appendln("gcloud:") @@ -44,7 +44,7 @@ internal class YamlWriter { } } - internal fun writeFlankProperties(config: FladleConfig, printApk: Boolean = true): String = buildString { + internal fun writeFlankProperties(config: FladleConfig): String = buildString { appendln("flank:") appendProperty(config.testShards, name = "max-test-shards") @@ -54,7 +54,7 @@ internal class YamlWriter { appendProperty(config.projectId, name = "project") appendProperty(config.keepFilePath, name = "keep-file-path") appendListProperty(config.filesToDownload, name = "files-to-download") { appendln(" - $it") } - if (printApk) + if (!config.sanityRobo.get()) appendListProperty(config.additionalTestApks, name = "additional-app-test-apks") { appendln(" $it") } appendProperty(config.runTimeout, name = "run-timeout") appendProperty(config.ignoreFailedTests, name = "ignore-failed-tests") @@ -67,7 +67,7 @@ internal class YamlWriter { appendProperty(config.outputStyle, name = "output-style") } - internal fun writeAdditionalProperties(config: FladleConfig, printRobo: Boolean = true): String = buildString { + internal fun writeAdditionalProperties(config: FladleConfig): String = buildString { appendProperty(config.useOrchestrator, name = "use-orchestrator") appendProperty(config.autoGoogleLogin, name = "auto-google-login") appendProperty(config.recordVideo, name = "record-video") @@ -87,7 +87,7 @@ internal class YamlWriter { appendMapProperty(config.clientDetails, name = "client-details") { appendln(" ${it.key}: ${it.value}") } appendMapProperty(config.otherFiles, name = "other-files") { appendln(" ${it.key}: ${it.value}") } appendProperty(config.networkProfile, name = "network-profile") - if (printRobo) { + if (!config.sanityRobo.get()) { appendProperty(config.roboScript, name = "robo-script") appendListProperty(config.roboDirectives, name = "robo-directives") { val value = it.getOrElse(2) { "" }.let { stringValue -> if (stringValue.isBlank()) "\"\"" else stringValue } diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt index a905ed6a..e1127cff 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityRoboTest.kt @@ -7,7 +7,7 @@ import org.junit.Test import org.junit.rules.TemporaryFolder private fun baseConfigMessage(option: String) = "Incorrect [base] configuration. [$option] can't be used together with sanityRobo." -private fun additionalConfigMessage(option: String, name: String) = "Incorrect [$name] configuration. [$option] can't be used together with sanityRobo. If you want to launch robo test run without robo script place only sanityRoboRun() into [$name] configuration" +private fun additionalConfigMessage(option: String, name: String) = "Incorrect [$name] configuration. [$option] can't be used together with sanityRobo. If you want to launch robo test run without robo script place only clearPropertiesForSanityRobo() into [$name] configuration" class SanityRoboTest { @get:Rule @@ -33,7 +33,6 @@ class SanityRoboTest { | instrumentationApk = "test.apk" | configs { | sanity { - | sanityRobo.set(true) | makeSanityRun() | } | } @@ -171,7 +170,7 @@ class SanityRoboTest { | ] | configs { | sanity { - | sanityRobo.set(true) + | clearPropertiesForSanityRobo() | roboScript.set("path/to/script.json") | } | } @@ -219,7 +218,7 @@ class SanityRoboTest { | ] | configs { | orange { - | sanityRoboRun() + | clearPropertiesForSanityRobo() | } | } |} diff --git a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt index ee30b0fd..f9edfb2a 100644 --- a/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt +++ b/buildSrc/src/test/java/com/osacky/flank/gradle/integration/SanityWithAutoConfigureTest.kt @@ -64,7 +64,7 @@ class SanityWithAutoConfigureTest { smartFlankGcsPath = "gs://test-lab-yr9w6qsdvy45q-iurp80dm95h8a/flank/test_app_android.xml" configs { sanity { - sanityRoboRun() + clearPropertiesForSanityRobo() useOrchestrator.set(false) testTargets.set(project.provider { [ "class com.osacky.flank.gradle.sample.ExampleInstrumentedTest#runAndFail" From 82cd138182f6742edc7396f5df2f042068d323cc Mon Sep 17 00:00:00 2001 From: Pawel Pasterz Date: Thu, 1 Oct 2020 08:33:33 +0200 Subject: [PATCH 12/12] Linting --- buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt index 1a2b7d5c..dd3f27d4 100644 --- a/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt +++ b/buildSrc/src/main/java/com/osacky/flank/gradle/YamlWriter.kt @@ -23,7 +23,7 @@ internal class YamlWriter { $prefix instrumentationApk=${config.instrumentationApk.orNull} roboScript=${config.roboScript.orNull} - """.trimIndent() + """.trimIndent() } }