From 16e3b5be91583ce4565587cb6de826c70198cdb2 Mon Sep 17 00:00:00 2001 From: Mansfield Mark Date: Tue, 23 Jun 2020 17:23:37 -0700 Subject: [PATCH] Add weblink to all test results, not just failures --- release_notes.md | 5 +---- .../docs/ascii/flank.jar_-android-run.adoc | 12 ++++++------ .../flank.jar_-firebase-test-android-run.adoc | 12 ++++++------ .../flank.jar_-firebase-test-ios-run.adoc | 12 ++++++------ .../docs/ascii/flank.jar_-ios-run.adoc | 12 ++++++------ .../ftl/reports/api/CreateJUnitTestCase.kt | 4 +--- .../reports/api/CreateJUnitTestCaseKtTest.kt | 19 ++++++++++++++----- 7 files changed, 40 insertions(+), 36 deletions(-) diff --git a/release_notes.md b/release_notes.md index 06569383ee..a11a66ff85 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,14 +1,11 @@ ## next (unreleased) - [#857](https://github.com/Flank/flank/pull/857) Added multimodule setup for test app. ([piotradamczyk5](https://github.com/piotradamczyk5)) - [#837](https://github.com/Flank/flank/pull/837) Added obfuscate option to dump shards. ([piotradamczyk5](https://github.com/piotradamczyk5)) -- -- +- [#868](https://github.com/Flank/flank/pull/868) Restored weblinks to all test results, not just failures. ([rainnapper](https://github.com/rainnapper)) ## v20.06.2 - [#853](https://github.com/Flank/flank/pull/853) Store @Ignore tests in the JUnit XML without sending ignored tests to FTL. ([piotradamczyk5](https://github.com/piotradamczyk5), [adamfilipow92](https://github.com/adamfilipow92)) - [#853](https://github.com/Flank/flank/pull/858) Handle duplicated apk names. ([jan-gogo](https://github.com/jan-gogo)) -- -- ## v20.06.1 - [#840](https://github.com/Flank/flank/pull/840) Fix parametrized tests. ([jan-gogo](https://github.com/jan-gogo), [adamfilipow92](https://github.com/adamfilipow92), [pawelpasterz](https://github.com/pawelpasterz), [piotradamczyk5](https://github.com/piotradamczyk5)) diff --git a/test_runner/docs/ascii/flank.jar_-android-run.adoc b/test_runner/docs/ascii/flank.jar_-android-run.adoc index 8f028a2e51..16a49e292d 100644 --- a/test_runner/docs/ascii/flank.jar_-android-run.adoc +++ b/test_runner/docs/ascii/flank.jar_-android-run.adoc @@ -114,12 +114,12 @@ Configuration is read from flank.yml *--num-flaky-test-attempts*=__:: The number of times a TestExecution should be re-attempted if one or more of its test cases fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. -*--shard-time*=__:: - The max amount of seconds each shard should run. - *--max-test-shards*=__:: The amount of matrices to split the tests across. +*--shard-time*=__:: + The max amount of seconds each shard should run. + *--num-test-runs*=__:: The amount of times to run the test executions. @@ -224,12 +224,12 @@ You can guide the Robo test to perform specific actions by recording a Robo Scri + Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. -*--legacy-junit-result*:: - Fallback for legacy xml junit results parsing. - *--additional-app-test-apks*=__[,__...]:: A list of app & test apks to include in the run. Useful for running multiple module tests within a single Flank run. +*--legacy-junit-result*:: + Fallback for legacy xml junit results parsing. + *--dump-shards*:: Measures test shards from given test apks and writes them into android_shards.json file instead of executing. diff --git a/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc b/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc index c235735ee2..f84ce3bbab 100644 --- a/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc +++ b/test_runner/docs/ascii/flank.jar_-firebase-test-android-run.adoc @@ -126,12 +126,12 @@ Configuration is read from flank.yml *--num-flaky-test-attempts*=__:: The number of times a TestExecution should be re-attempted if one or more of its test cases fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. -*--shard-time*=__:: - The max amount of seconds each shard should run. - *--max-test-shards*=__:: The amount of matrices to split the tests across. +*--shard-time*=__:: + The max amount of seconds each shard should run. + *--num-test-runs*=__:: The amount of times to run the test executions. @@ -236,12 +236,12 @@ You can guide the Robo test to perform specific actions by recording a Robo Scri + Learn more at https://firebase.google.com/docs/test-lab/robo-ux-test#scripting. -*--legacy-junit-result*:: - Fallback for legacy xml junit results parsing. - *--additional-app-test-apks*=__[,__...]:: A list of app & test apks to include in the run. Useful for running multiple module tests within a single Flank run. +*--legacy-junit-result*:: + Fallback for legacy xml junit results parsing. + *--dump-shards*:: Measures test shards from given test apks and writes them into android_shards.json file instead of executing. diff --git a/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc b/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc index d602db4dbb..1d81707c47 100644 --- a/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc +++ b/test_runner/docs/ascii/flank.jar_-firebase-test-ios-run.adoc @@ -106,12 +106,12 @@ Configuration is read from flank.yml *--num-flaky-test-attempts*=__:: The number of times a TestExecution should be re-attempted if one or more of its test cases fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. -*--shard-time*=__:: - The max amount of seconds each shard should run. - *--max-test-shards*=__:: The amount of matrices to split the tests across. +*--shard-time*=__:: + The max amount of seconds each shard should run. + *--num-test-runs*=__:: The amount of times to run the test executions. @@ -151,15 +151,15 @@ Configuration is read from flank.yml *--output-style*=__:: Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`. +*--test*=__:: + The path to the test package (a zip file containing the iOS app and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file is specified. + *--xctestrun-file*=__:: The path to an .xctestrun file that will override any .xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. *--xcode-version*=__:: The version of Xcode that should be used to run an XCTest. Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by all iOS versions selected in the test matrix. -*--test*=__:: - The path to the test package (a zip file containing the iOS app and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file is specified. - *--test-targets*=__[,__...]:: A list of one or more test method names to run (default: run all test targets). diff --git a/test_runner/docs/ascii/flank.jar_-ios-run.adoc b/test_runner/docs/ascii/flank.jar_-ios-run.adoc index 1ce7ddb96f..675025df34 100644 --- a/test_runner/docs/ascii/flank.jar_-ios-run.adoc +++ b/test_runner/docs/ascii/flank.jar_-ios-run.adoc @@ -97,12 +97,12 @@ Configuration is read from flank.yml *--num-flaky-test-attempts*=__:: The number of times a TestExecution should be re-attempted if one or more of its test cases fail for any reason. The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. -*--shard-time*=__:: - The max amount of seconds each shard should run. - *--max-test-shards*=__:: The amount of matrices to split the tests across. +*--shard-time*=__:: + The max amount of seconds each shard should run. + *--num-test-runs*=__:: The amount of times to run the test executions. @@ -142,15 +142,15 @@ Configuration is read from flank.yml *--output-style*=__:: Output style of execution status. May be one of [verbose, multi, single]. For runs with only one test execution the default value is 'verbose', in other cases 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles which don't support ansi codes, to avoid corrupted output use `single` or `verbose`. +*--test*=__:: + The path to the test package (a zip file containing the iOS app and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file is specified. + *--xctestrun-file*=__:: The path to an .xctestrun file that will override any .xctestrun file contained in the --test package. Because the .xctestrun file contains environment variables along with test methods to run and/or ignore, this can be useful for customizing or sharding test suites. The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. *--xcode-version*=__:: The version of Xcode that should be used to run an XCTest. Defaults to the latest Xcode version supported in Firebase Test Lab. This Xcode version must be supported by all iOS versions selected in the test matrix. -*--test*=__:: - The path to the test package (a zip file containing the iOS app and XCTest files). The given path may be in the local filesystem or in Google Cloud Storage using a URL beginning with gs://. Note: any .xctestrun file in this zip file will be ignored if --xctestrun-file is specified. - *--test-targets*=__[,__...]:: A list of one or more test method names to run (default: run all test targets). diff --git a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt b/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt index 7b89c705d7..c0e06e00a7 100644 --- a/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt +++ b/test_runner/src/main/kotlin/ftl/reports/api/CreateJUnitTestCase.kt @@ -34,9 +34,7 @@ private fun createJUnitTestCase( // skipped = true is represented by null. skipped = false is "absent" skipped = if (testCase.status == "skipped") null else "absent" ).apply { - if (errors != null || failures != null) { - webLink = getWebLink(toolResultsStep, testCase.testCaseId) - } + webLink = getWebLink(toolResultsStep, testCase.testCaseId) if (testCase.flaky) { flaky = true } diff --git a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt index df58eb811e..11cb8eea1f 100644 --- a/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt +++ b/test_runner/src/test/kotlin/ftl/reports/api/CreateJUnitTestCaseKtTest.kt @@ -49,20 +49,27 @@ class CreateJUnitTestCaseKtTest { name = "test1", classname = "TestClassName", time = "2.200" - ), + ).apply { + webLink = + "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test1" + }, JUnitTestCase( name = "test2", classname = "TestClassName", time = "2.200", skipped = null - ), + ).apply { + webLink = + "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test2" + }, JUnitTestCase( name = "test3", classname = "TestClassName", time = "2.200", errors = listOf("exception") ).apply { - webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test3" + webLink = + "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test3" }, JUnitTestCase( name = "test4", @@ -70,7 +77,8 @@ class CreateJUnitTestCaseKtTest { time = "2.200", failures = listOf("exception") ).apply { - webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test4" + webLink = + "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test4" }, JUnitTestCase( name = "test5", @@ -78,7 +86,8 @@ class CreateJUnitTestCaseKtTest { time = "2.200", failures = listOf("exception") ).apply { - webLink = "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test5" + webLink = + "https://console.firebase.google.com/project/projectId/testlab/histories/historyId/matrices/executionId/executions/stepId/testcases/test5" flaky = true } )