From 9eb491d616963ce2409b4b1cd93901c149c8089f Mon Sep 17 00:00:00 2001 From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com> Date: Fri, 15 May 2020 16:04:49 +0200 Subject: [PATCH] Improve error messages (#780) * Improve error messages * No state no class, fix wrong auto formating * Add support for get full chain with properties contains '-' in name --- release_notes.md | 2 + .../src/main/kotlin/ftl/args/AndroidArgs.kt | 7 +- .../src/main/kotlin/ftl/args/ArgsHelper.kt | 10 ++- .../yml/MissingParameterErrorConverter.kt | 15 ++++ .../kotlin/ftl/args/yml/YamlObjectMapper.kt | 18 ++++ .../ConfigurationErrorMessageBuilder.kt | 50 +++++++++++ .../yml/errors/ConfigurationErrorModel.kt | 3 + .../yml/errors/ConfigurationErrorParser.kt | 30 +++++++ .../ftl/args/yml/errors/ErrorNodeResolver.kt | 23 +++++ .../main/kotlin/ftl/util/FlankException.kt | 9 ++ test_runner/src/main/kotlin/ftl/util/Utils.kt | 12 ++- test_runner/src/test/kotlin/Debug.kt | 2 +- .../kotlin/ftl/args/yml/ErrorParserTest.kt | 90 +++++++++++++++++++ .../flank-no-device-version.yml | 31 +++++++ .../flank-no-model-name.yml | 10 +++ .../flank-no-model-node.yml | 9 ++ .../flank-multiple-parse-error.yml | 32 +++++++ .../ftl/test/util/FlankTestExceptions.kt | 3 + .../test/kotlin/ftl/test/util/TestHelper.kt | 7 ++ 19 files changed, 353 insertions(+), 10 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/args/yml/MissingParameterErrorConverter.kt create mode 100644 test_runner/src/main/kotlin/ftl/args/yml/YamlObjectMapper.kt create mode 100644 test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorMessageBuilder.kt create mode 100644 test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorModel.kt create mode 100644 test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorParser.kt create mode 100644 test_runner/src/main/kotlin/ftl/args/yml/errors/ErrorNodeResolver.kt create mode 100644 test_runner/src/test/kotlin/ftl/args/yml/ErrorParserTest.kt create mode 100644 test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-device-version.yml create mode 100644 test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-name.yml create mode 100644 test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-node.yml create mode 100644 test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-parse-error.yml create mode 100644 test_runner/src/test/kotlin/ftl/test/util/FlankTestExceptions.kt diff --git a/release_notes.md b/release_notes.md index b26630fd44..7915796415 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,4 +1,6 @@ ## next (unreleased) + +- [#656](https://github.com/Flank/flank/issues/656) Improve error message reporting. ([adamfilipow92](https://github.com/adamfilipow92)) - [#783](https://github.com/Flank/flank/pull/783) Use legacy results for iOS by default. ([pawelpasterz](https://github.com/pawelpasterz)) ## v20.05.1 diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 27452176aa..0c599bea18 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -20,10 +20,10 @@ import ftl.args.ArgsToString.mapToString import ftl.args.ArgsToString.objectsToString import ftl.args.yml.AndroidFlankYml import ftl.args.yml.AndroidGcloudYml -import ftl.args.yml.AndroidGcloudYmlParams -import ftl.args.yml.AppTestPair import ftl.args.yml.FlankYml import ftl.args.yml.GcloudYml +import ftl.args.yml.AppTestPair +import ftl.args.yml.AndroidGcloudYmlParams import ftl.args.yml.YamlDeprecated import ftl.cli.firebase.test.android.AndroidRunCommand import ftl.config.Device @@ -43,6 +43,7 @@ class AndroidArgs( override val data: String, val cli: AndroidRunCommand? = null ) : IArgs { + private val gcloud = gcloudYml.gcloud override val resultsBucket: String override val resultsDir = (cli?.resultsDir ?: gcloud.resultsDir)?.also { assertFileExists(it, "from results-dir") } @@ -193,8 +194,8 @@ AndroidArgs @VisibleForTesting internal fun load(yamlReader: Reader, cli: AndroidRunCommand? = null): AndroidArgs { - val data = YamlDeprecated.modifyAndThrow(yamlReader, android = true) + val data = YamlDeprecated.modifyAndThrow(yamlReader, android = true) val flankYml = yamlMapper.readValue(data, FlankYml::class.java) val gcloudYml = yamlMapper.readValue(data, GcloudYml::class.java) val androidGcloudYml = yamlMapper.readValue(data, AndroidGcloudYml::class.java) diff --git a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt index 05f1abe837..7ef73ea3e8 100644 --- a/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt +++ b/test_runner/src/main/kotlin/ftl/args/ArgsHelper.kt @@ -1,8 +1,7 @@ package ftl.args import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule +import com.fasterxml.jackson.module.kotlin.registerKotlinModule import com.google.api.client.json.GenericJson import com.google.api.client.json.JsonObjectParser import com.google.api.client.util.Charsets @@ -12,6 +11,7 @@ import com.google.cloud.storage.Storage import com.google.cloud.storage.StorageClass import com.google.cloud.storage.StorageOptions import ftl.args.yml.IYmlMap +import ftl.args.yml.YamlObjectMapper import ftl.config.FtlConstants import ftl.config.FtlConstants.GCS_PREFIX import ftl.config.FtlConstants.JSON_FACTORY @@ -23,9 +23,9 @@ import ftl.reports.xml.model.JUnitTestResult import ftl.shard.Shard import ftl.shard.StringShards import ftl.shard.stringShards +import ftl.util.FlankFatalError import ftl.util.FlankTestMethod import ftl.util.assertNotEmpty -import ftl.util.FlankFatalError import java.io.File import java.net.URI import java.nio.file.Files @@ -35,7 +35,9 @@ import java.util.regex.Pattern object ArgsHelper { - val yamlMapper: ObjectMapper by lazy { ObjectMapper(YAMLFactory()).registerModule(KotlinModule()) } + val yamlMapper: ObjectMapper by lazy { + YamlObjectMapper().registerKotlinModule() + } fun mergeYmlMaps(vararg ymlMaps: IYmlMap): Map> { val result = mutableMapOf>() diff --git a/test_runner/src/main/kotlin/ftl/args/yml/MissingParameterErrorConverter.kt b/test_runner/src/main/kotlin/ftl/args/yml/MissingParameterErrorConverter.kt new file mode 100644 index 0000000000..6cd76efb89 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/yml/MissingParameterErrorConverter.kt @@ -0,0 +1,15 @@ +package ftl.args.yml + +import com.fasterxml.jackson.databind.JsonNode +import ftl.args.yml.errors.ConfigurationErrorMessageBuilder +import ftl.util.FlankConfigurationException + +fun convertConfigurationErrorExceptions(missingParameterError: Exception, yaml: JsonNode): Throwable { + val errorMessageBuilder = ConfigurationErrorMessageBuilder + val errorMessage = missingParameterError.message + return if (errorMessage != null) { + FlankConfigurationException(errorMessageBuilder(errorMessage, yaml)) + } else { + missingParameterError + } +} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/YamlObjectMapper.kt b/test_runner/src/main/kotlin/ftl/args/yml/YamlObjectMapper.kt new file mode 100644 index 0000000000..4eb5849021 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/yml/YamlObjectMapper.kt @@ -0,0 +1,18 @@ +package ftl.args.yml + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.exc.MismatchedInputException +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.module.kotlin.MissingKotlinParameterException + +class YamlObjectMapper : ObjectMapper(YAMLFactory()) { + override fun readValue(content: String?, valueType: Class?): T { + try { + return readValue(content, _typeFactory.constructType(valueType)) + } catch (missingParameterError: MissingKotlinParameterException) { + throw convertConfigurationErrorExceptions(missingParameterError, readTree(content)) + } catch (mismatchedInputException: MismatchedInputException) { + throw convertConfigurationErrorExceptions(mismatchedInputException, readTree(content)) + } + } +} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorMessageBuilder.kt b/test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorMessageBuilder.kt new file mode 100644 index 0000000000..1020ea04d0 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorMessageBuilder.kt @@ -0,0 +1,50 @@ +package ftl.args.yml.errors + +import com.fasterxml.jackson.databind.JsonNode +import java.lang.Exception + +object ConfigurationErrorMessageBuilder { + + private val parseMessage = ConfigurationErrorParser + private val resolveErrorNode = ErrorNodeResolver + + //region error message elements + private const val messageHeader = "Error on parse config: " + private const val missingElementMessage = "Missing element or value for: '%s'" + private const val atMessage = "At line: %s, column: %s" + private const val errorNodeMessage = "Error node: %s" + //endregion + + private const val exceptionTemplate = "Parse message error: %s" + + operator fun invoke(errorMessage: String, yamlTreeNode: JsonNode? = null) = + try { + val errorModel = parseMessage(errorMessage) + val errorMessageBuilder = StringBuilder(messageHeader) + errorMessageBuilder.appendln(createReferenceChain(errorModel.referenceChain)) + if (errorModel.propertyName != "") { + errorMessageBuilder.appendln(missingElementMessage.format(errorModel.propertyName)) + } + errorMessageBuilder.appendln(atMessage.format(errorModel.line, errorModel.column)) + yamlTreeNode?.let { + errorMessageBuilder.appendln(errorNodeMessage.format(resolveErrorNode(yamlTreeNode, errorModel))) + } + errorMessageBuilder.toString().trim() + } catch (error: Exception) { + exceptionTemplate.format(errorMessage) + } + + private fun createReferenceChain(referenceChain: List): String { + val chainBuilder = StringBuilder() + referenceChain.forEachIndexed { index, chainPart -> + chainBuilder.append(appendChainElement(chainPart, index > 0)) + } + return chainBuilder.toString() + } + + private fun appendChainElement(chainPart: String, withSeparator: Boolean): String = when { + chainPart.toIntOrNull() != null -> "[$chainPart]" + withSeparator -> "->$chainPart" + else -> chainPart + } +} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorModel.kt b/test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorModel.kt new file mode 100644 index 0000000000..47106b1f8a --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorModel.kt @@ -0,0 +1,3 @@ +package ftl.args.yml.errors + +internal data class ConfigurationErrorModel(val propertyName: String, val line: Int, val column: Int, val referenceChain: List) diff --git a/test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorParser.kt b/test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorParser.kt new file mode 100644 index 0000000000..1f037a1f9a --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/yml/errors/ConfigurationErrorParser.kt @@ -0,0 +1,30 @@ +package ftl.args.yml.errors + +internal object ConfigurationErrorParser { + + //region regex patterns + private val propertyNameRegex = "(?<=property\\s)[a-z]*".toRegex() + private val referenceChainRegex = "(?<=chain:\\s).*(?=[)])".toRegex() + private val referenceChainCleanUpRegex = "(?<=[\\[])\"?[\\w]*\"?(?=])".toRegex() + private val lineAndColumnRegex = "((?<=line:\\s)\\d*), column:\\s(\\d*)".toRegex() + //endregion + + operator fun invoke(errorMessage: String): ConfigurationErrorModel { + val (line, column) = parseErrorPositionLine(errorMessage) + return ConfigurationErrorModel( + parsePropertyName(errorMessage), + line.toInt(), + column.toInt(), + parseReferenceChain(errorMessage) + ) + } + + private fun parsePropertyName(errorMessage: String) = propertyNameRegex.find(errorMessage)?.value ?: "" + private fun parseErrorPositionLine(errorMessage: String) = lineAndColumnRegex.find(errorMessage)!!.destructured + + private fun parseReferenceChain(errorMessage: String) = + cleanUpReferenceChain(referenceChainRegex.find(errorMessage)!!.value) + + private fun cleanUpReferenceChain(referenceChain: String): List = + referenceChainCleanUpRegex.findAll(referenceChain).map { it.value.replace("\"", "") }.toList() +} diff --git a/test_runner/src/main/kotlin/ftl/args/yml/errors/ErrorNodeResolver.kt b/test_runner/src/main/kotlin/ftl/args/yml/errors/ErrorNodeResolver.kt new file mode 100644 index 0000000000..8967c5c5c9 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/args/yml/errors/ErrorNodeResolver.kt @@ -0,0 +1,23 @@ +package ftl.args.yml.errors + +import com.fasterxml.jackson.databind.JsonNode + +internal object ErrorNodeResolver { + operator fun invoke(treeNode: JsonNode, errorModel: ConfigurationErrorModel): String { + var currentNode: JsonNode = treeNode + + val lastNode = errorModel.referenceChain.last() + for (chainNode in errorModel.referenceChain) { + if (chainNode == lastNode) { + break + } + val nodeAsIntValue = chainNode.toIntOrNull() + currentNode = if (nodeAsIntValue != null) { + currentNode[nodeAsIntValue] + } else { + currentNode[chainNode] + } + } + return currentNode.toPrettyString() + } +} diff --git a/test_runner/src/main/kotlin/ftl/util/FlankException.kt b/test_runner/src/main/kotlin/ftl/util/FlankException.kt index 2b7386c251..04643fa14f 100644 --- a/test_runner/src/main/kotlin/ftl/util/FlankException.kt +++ b/test_runner/src/main/kotlin/ftl/util/FlankException.kt @@ -60,3 +60,12 @@ class FlankFatalError(message: String) : FlankException(message) * @param message [String] message to be printed to [System.err] */ class FlankCommonException(message: String) : FlankException(message) + +/** + * Exception throws when required parameters is missing + * + * Exit code: 1 + * + * @param message [String] message to be printed to [System.err] + */ +class FlankConfigurationException(message: String) : FlankException(message) diff --git a/test_runner/src/main/kotlin/ftl/util/Utils.kt b/test_runner/src/main/kotlin/ftl/util/Utils.kt index c607c400c7..e3adc55220 100644 --- a/test_runner/src/main/kotlin/ftl/util/Utils.kt +++ b/test_runner/src/main/kotlin/ftl/util/Utils.kt @@ -151,6 +151,10 @@ fun withGlobalExceptionHandling(block: () -> Int) { } exitProcess(1) } + is FlankConfigurationException -> { + System.err.println(t) + exitProcess(1) + } is FTLError -> { t.matrix.logError("not finished") exitProcess(3) @@ -159,6 +163,7 @@ fun withGlobalExceptionHandling(block: () -> Int) { System.err.println(t.message) exitProcess(2) } + // We need to cover the case where some component in the call stack starts a non-daemon // thread, and then throws an Error that kills the main thread. This is extra safe implementation else -> { @@ -183,6 +188,9 @@ fun , T> mutableMapProperty( defaultValue: () -> T ) = object : ReadWriteProperty { @Suppress("UNCHECKED_CAST") - override fun getValue(thisRef: R, property: KProperty<*>): T = thisRef.getOrElse(name ?: property.name, defaultValue) as T - override fun setValue(thisRef: R, property: KProperty<*>, value: T) = thisRef.set(name ?: property.name, value as Any) + override fun getValue(thisRef: R, property: KProperty<*>): T = + thisRef.getOrElse(name ?: property.name, defaultValue) as T + + override fun setValue(thisRef: R, property: KProperty<*>, value: T) = + thisRef.set(name ?: property.name, value as Any) } diff --git a/test_runner/src/test/kotlin/Debug.kt b/test_runner/src/test/kotlin/Debug.kt index 3b1e592633..c0c8288396 100644 --- a/test_runner/src/test/kotlin/Debug.kt +++ b/test_runner/src/test/kotlin/Debug.kt @@ -11,7 +11,7 @@ fun main() { val projectId = System.getenv("GOOGLE_CLOUD_PROJECT") ?: "YOUR PROJECT ID" val quantity = "multiple" - val type = "error" + val type = "parse-error" // Bugsnag keeps the process alive so we must call exitProcess // https://github.com/bugsnag/bugsnag-java/issues/151 diff --git a/test_runner/src/test/kotlin/ftl/args/yml/ErrorParserTest.kt b/test_runner/src/test/kotlin/ftl/args/yml/ErrorParserTest.kt new file mode 100644 index 0000000000..b177264ef9 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/args/yml/ErrorParserTest.kt @@ -0,0 +1,90 @@ +package ftl.args.yml + +import ftl.args.AndroidArgs +import ftl.args.yml.errors.ConfigurationErrorMessageBuilder +import ftl.test.util.TestHelper +import ftl.test.util.TestHelper.getThrowable +import ftl.util.FlankConfigurationException +import org.junit.Assert +import org.junit.Test + +class ErrorParserTest { + private val yamlWithoutDeviceVersion = + TestHelper.getPath("src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-device-version.yml") + private val yamlNoModelName = + TestHelper.getPath("src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-name.yml") + private val yamlNoModelNode = + TestHelper.getPath("src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-node.yml") + + @Test + fun `parse json mapping error`() { + val instantionError = + "Instantiation of [simple type, class ftl.config.Device] value failed for JSON property version due to missing (therefore NULL) value for creator parameter version which is a non-nullable type\n" + + " at [Source: (StringReader); line: 23, column: 3] (through reference chain: ftl.args.yml.AndroidGcloudYml[\"gcloud\"]->ftl.args.yml.AndroidGcloudYmlParams[\"device\"]->java.util.ArrayList[4]->ftl.config.Device[\"version\"])" + + val expected = """ +Error on parse config: gcloud->device[4]->version +Missing element or value for: 'version' +At line: 23, column: 3 +""".trimIndent() + val buildErrorMessage = ConfigurationErrorMessageBuilder + Assert.assertEquals(expected, buildErrorMessage(instantionError)) + } + + @Test + fun `return exception with inner message on parse error`() { + val instantionError = + "Instantiation oflParams[\"device\"]->java.util.A" + val expected = "Parse message error: Instantiation oflParams[\"device\"]->java.util.A".trimIndent() + val buildErrorMessage = ConfigurationErrorMessageBuilder + + Assert.assertEquals(expected, buildErrorMessage(instantionError)) + } + + @Test(expected = FlankConfigurationException::class) + fun `should throw FlankConfigException without device version`() { + AndroidArgs.load(yamlWithoutDeviceVersion) + } + + @Test + fun `without model name should have message`() { + val actualMessage = getThrowable { AndroidArgs.load(yamlNoModelName) }.message + val exceptedMessage = """ + Error on parse config: gcloud->device[0]->model + Missing element or value for: 'model' + At line: 8, column: 1 + Error node: { + "model" : null, + "version" : "test" + } + """.trimIndent() + Assert.assertEquals(exceptedMessage, actualMessage) + } + + @Test + fun `without model node should have message`() { + val actualMessage = getThrowable { AndroidArgs.load(yamlNoModelNode) }.message + val exceptedMessage = """ +Error on parse config: gcloud->device +At line: 6, column: 5 +Error node: { + "app" : "./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk", + "test" : "./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk", + "device" : { + "version" : "test" + } +} + """.trimIndent() + Assert.assertEquals(exceptedMessage, actualMessage) + } + + @Test(expected = FlankConfigurationException::class) + fun `should throw FlankConfigException without model name`() { + AndroidArgs.load(yamlNoModelName) + } + + @Test(expected = FlankConfigurationException::class) + fun `should throw FlankConfigException without model node`() { + AndroidArgs.load(yamlNoModelNode) + } +} diff --git a/test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-device-version.yml b/test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-device-version.yml new file mode 100644 index 0000000000..9bf6952f77 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-device-version.yml @@ -0,0 +1,31 @@ +gcloud: + app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + device: + - model: NexusLowRes + version: 23 + - model: NexusLowRes + version: 23 + orientation: landscape + - model: shamu + version: 22 + locale: zh_CN + orientation: default + # Google Pixel 3 + - model: blueline + version: 28 + locale: en + orientation: portrait + # Samsung Galaxy S9 SM-G9600 + - model: starqltechn + versxion: 28 + locale: en + orientation: portrait + # LG Nexus 5 + - model: hammerhead + version: 21 + locale: en + orientation: portrait +flank: + disable-sharding: true + diff --git a/test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-name.yml b/test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-name.yml new file mode 100644 index 0000000000..4122b1e1c8 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-name.yml @@ -0,0 +1,10 @@ +gcloud: + app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + device: + - model: + version: test + +flank: + disable-sharding: true + diff --git a/test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-node.yml b/test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-node.yml new file mode 100644 index 0000000000..665b5ffeaf --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/args/yml/test_error_yaml_cases/flank-no-model-node.yml @@ -0,0 +1,9 @@ +gcloud: + app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + device: + version: test + +flank: + disable-sharding: true + diff --git a/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-parse-error.yml b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-parse-error.yml new file mode 100644 index 0000000000..0fa875262f --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/fixtures/test_app_cases/flank-multiple-parse-error.yml @@ -0,0 +1,32 @@ +gcloud: + app: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-debug.apk + test: ./src/test/kotlin/ftl/fixtures/tmp/apk/app-single-success-debug-androidTest.apk + environment-variables: "123, 12312, 12312" + device: + - model: NexusLowRes + orientation: landscape + - model: NexusLowRes + version: 23 + orientation: landscape + - model: shamu + version: 22 + locale: zh_CN + orientation: default + # Google Pixel 3 + - model: blueline + version: 28 + locale: en + orientation: portrait + # Samsung Galaxy S9 SM-G9600 + - model: starqltechn + version: 28 + locale: en + orientation: portrait + # LG Nexus 5 + - model: hammerhead + version: 21 + locale: en + orientation: portrait +flank: + disable-sharding: true + diff --git a/test_runner/src/test/kotlin/ftl/test/util/FlankTestExceptions.kt b/test_runner/src/test/kotlin/ftl/test/util/FlankTestExceptions.kt new file mode 100644 index 0000000000..28466e0d51 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/test/util/FlankTestExceptions.kt @@ -0,0 +1,3 @@ +package ftl.test.util + +class FlankTestNotFoundException(message: String) : Exception(message) diff --git a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt index f003770ae0..e580490fff 100644 --- a/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt +++ b/test_runner/src/test/kotlin/ftl/test/util/TestHelper.kt @@ -21,4 +21,11 @@ object TestHelper { // required for tests to pass on Windows return this.replace("\r\n", "\n") } + + fun getThrowable(action: () -> Unit): Throwable = try { + action() + throw FlankTestNotFoundException("Action not throwing exception") + } catch (exception: Throwable) { + exception + } }