diff --git a/docs/bugs/smart_flank_upload_results_rules.md b/docs/bugs/smart_flank_upload_results_rules.md new file mode 100644 index 0000000000..41cc0e25fd --- /dev/null +++ b/docs/bugs/smart_flank_upload_results_rules.md @@ -0,0 +1,25 @@ +# Smart Flank rules of validation result types + +Flank trying to avoid override smart-flank-gcs-path by different JUnit report type. That's means: + +1. If user select in ```smart-flank-gcs-path``` command FulJUnitResult.xml and flag ```--full-junit-result``` is not set Flank fail with the message + + ```txt + + smart-flank-gcs-path is set with FullJUnitReport.xml but in this run --full-junit-result is disabled, please set --full-junit-result flag + + + ``` + +2. If user set in ```smart-flank-gcs-path``` command JUnitResult.xml and flag ```--full-junit-result``` is set Flank fails with message + + ```txt + + smart-flank-gcs-path is set with JUnitReport.xml but in this run --full-junit-result enabled, please turn off --full-junit-result flag + + + ``` + +3. If ```smart-flank-gcs-path``` is set to a different name than ```JUnitReport.xml``` and ```FullJUnitReport.xml``` flank not validating report type + +4. Flank not validating report type if flag ```--smart-flank-disable-upload``` set diff --git a/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt index 2af477f600..7433ef0c33 100644 --- a/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/ValidateCommonArgs.kt @@ -2,6 +2,8 @@ package ftl.args import ftl.config.Device import ftl.config.FtlConstants +import ftl.reports.FullJUnitReport +import ftl.reports.JUnitReport import ftl.run.exception.FlankConfigurationError import ftl.run.exception.FlankGeneralError @@ -26,8 +28,8 @@ private fun List.throwIfAnyMisspelt() = private fun CommonArgs.assertProjectId() { if (project.isEmpty()) throw FlankConfigurationError( "The project is not set. Define GOOGLE_CLOUD_PROJECT, set project in flank.yml\n" + - "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" + - " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" + "or save service account credential to ${FtlConstants.defaultCredentialPath}\n" + + " See https://github.com/GoogleCloudPlatform/google-cloud-java#specifying-a-project-id" ) } @@ -54,5 +56,13 @@ private fun CommonArgs.assertSmartFlankGcsPath() = with(smartFlankGcsPath) { count { it == '/' } <= 2 || endsWith(".xml").not() -> throw FlankConfigurationError( "smart-flank-gcs-path must be in the format gs://bucket/foo.xml" ) + + smartFlankDisableUpload.not() && contains("/${FullJUnitReport.fileName()}") && fullJUnitResult.not() -> throw FlankConfigurationError( + "smart-flank-gcs-path is set with ${FullJUnitReport.fileName()} but in this run --full-junit-result is disabled, please set --full-junit-result flag" + ) + + smartFlankDisableUpload.not() && contains("/${JUnitReport.fileName()}") && fullJUnitResult -> throw FlankConfigurationError( + "smart-flank-gcs-path is set with ${JUnitReport.fileName()} but in this run --full-junit-result enabled, please turn off --full-junit-result flag" + ) } } diff --git a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt index ae3e4d59c7..a716577726 100644 --- a/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt +++ b/test_runner/src/test/kotlin/ftl/cli/firebase/test/android/AndroidRunCommandTest.kt @@ -4,6 +4,7 @@ import com.google.common.truth.Truth.assertThat import ftl.args.yml.AppTestPair import ftl.config.Device import ftl.config.FtlConstants +import ftl.run.exception.FlankConfigurationError import ftl.test.util.FlankTestRunner import org.junit.Assert.assertEquals import org.junit.Rule @@ -483,4 +484,43 @@ class AndroidRunCommandTest { assertThat(cmd.config.common.flank.useAverageTestTimeForNewTests).isTrue() } + + @Test(expected = FlankConfigurationError::class) + fun `should throw if --full-junit-result and JUnitResult xml used`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--full-junit-result", "--smart-flank-gcs-path=gs://test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-08-26_15-20-23.850738_rtGt/JUnitReport.xml") + cmd.run() + } + + @Test(expected = FlankConfigurationError::class) + fun `should throw if FullJUnitResult xml used and --full-junit-result not set`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--smart-flank-gcs-path=gs://test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-08-26_15-20-23.850738_rtGt/FullJUnitReport.xml") + cmd.run() + } + + @Test + fun `should not throw if --full-junit-result and smart flank path different than JUnitReport xml`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--full-junit-result", "--smart-flank-gcs-path=gs://test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-08-26_15-20-23.850738_rtGt/JUnitReportTest.xml") + cmd.run() + } + + @Test + fun `should not throw if --full-junit-result not set and smart flank path different than JUnitReport xml`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs("--smart-flank-gcs-path=gs://test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-08-26_15-20-23.850738_rtGt/JUnitReportTest.xml") + cmd.run() + } + + @Test + fun `should not validate if smart-flank-disable-upload set`() { + val cmd = AndroidRunCommand() + CommandLine(cmd).parseArgs( + "--smart-flank-disable-upload", + "--full-junit-result", + "--smart-flank-gcs-path=gs://test-lab-v9cn46bb990nx-kz69ymd4nm9aq/2020-08-26_15-20-23.850738_rtGt/JUnitReport.xml" + ) + cmd.run() + } }