Skip to content

Commit

Permalink
Adjust HtmlErrorReport
Browse files Browse the repository at this point in the history
  • Loading branch information
jan-goral committed Apr 30, 2020
1 parent 6a059b6 commit 721e9c2
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 154 deletions.
146 changes: 51 additions & 95 deletions test_runner/src/main/kotlin/ftl/reports/HtmlErrorReport.kt
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import ftl.json.MatrixMap
import ftl.reports.util.IReport
import ftl.reports.xml.model.JUnitTestCase
import ftl.reports.xml.model.JUnitTestResult
import ftl.reports.xml.model.JUnitTestSuite
import ftl.util.readTextResource
import java.nio.file.Files
import java.nio.file.Paths
Expand All @@ -14,109 +15,64 @@ import java.nio.file.Paths
* Outputs HTML report for Bitrise based on JUnit XML. Only run on failures.
* */
object HtmlErrorReport : IReport {
override val extension = ".html"

data class Group(val key: String, val name: String, val startIndex: Int, val count: Int)
data class Item(val key: String, val name: String, val link: String)

private val gson = Gson()

fun groupItemList(testSuites: JUnitTestResult): Pair<List<HtmlErrorReport.Group>, List<HtmlErrorReport.Item>>? {
val groupList = mutableListOf<Group>()
val itemList = mutableListOf<Item>()

var groupId = 0
var itemId = 0

val failures = mutableMapOf<String, MutableList<JUnitTestCase>>()
testSuites.testsuites?.forEach { suite ->
suite.testcases?.forEach testCase@{ testCase ->
if (!testCase.failed()) return@testCase
val key = "${suite.name} ${testCase.classname}#${testCase.name}".trim()

if (failures[key] == null) {
failures[key] = mutableListOf(testCase)
} else {
failures[key]?.add(testCase)
}
}
}

if (failures.isEmpty()) return null

failures.forEach { (testName, testResults) ->
groupList.add(
Group(
"group-$groupId",
testName,
groupId,
testResults.size
)
)
groupId += 1

testResults.forEach { failure ->
itemList.add(
Item(
"item-$itemId",
failure.stackTrace().split("\n").firstOrNull() ?: "",
failure.webLink ?: ""
)
)
itemId += 1
}
}

return groupList to itemList
}

private fun reactJson(testSuites: JUnitTestResult): Pair<String, String>? {
val groupItemList = groupItemList(testSuites) ?: return null

val groupJson = gson.toJson(groupItemList.first)
val itemJson = gson.toJson(groupItemList.second)
return groupJson to itemJson
}

override fun run(matrices: MatrixMap, testSuite: JUnitTestResult?, printToStdout: Boolean, args: IArgs) {
if (testSuite == null) return
val reactJson = reactJson(testSuite) ?: return
val newGroupJson = reactJson.first
val newItemsJson = reactJson.second

var templateData = readTextResource("inline.html")

templateData = replaceRange(templateData, findGroupRange(templateData), newGroupJson)
templateData = replaceRange(templateData, findItemRange(templateData), newItemsJson)
override val extension = ".html"

val writePath = Paths.get(reportPath(matrices, args))
// Print out full path so you can click into the report from the terminal
println("${this.javaClass.simpleName} written to ${writePath.toAbsolutePath()}")
Files.write(writePath, templateData.toByteArray())
internal data class Group(val label: String, val items: List<Item>)
internal data class Item(val label: String, val url: String)

override fun run(
matrices: MatrixMap,
result: JUnitTestResult?,
printToStdout: Boolean,
args: IArgs
) {
val suites = result?.testsuites?.process()?.takeIf { it.isNotEmpty() } ?: return
Files.write(
Paths.get(reportPath(matrices, args)),
suites.createHtmlReport().toByteArray()
)
}
}

private fun replaceRange(data: String, deleteRange: IntRange, insert: String): String {
val before = data.substring(0, deleteRange.first)
val after = data.substring(deleteRange.last)
internal fun List<JUnitTestSuite>.process(): List<HtmlErrorReport.Group> = this
.getFailures()
.groupByLabel()
.createGroups()

return before + insert + after
private fun List<JUnitTestSuite>.getFailures(): List<Pair<String, List<JUnitTestCase>>> =
mapNotNull { suite ->
suite.testcases?.let { testCases ->
suite.name to testCases.filter { it.failed() }
}
}

private fun findItemRange(data: String): IntRange {
return findJsonBounds(data, startPattern = "=[{key:\"item-0\",")
private fun List<Pair<String, List<JUnitTestCase>>>.groupByLabel(): Map<String, List<JUnitTestCase>> = this
.map { (suiteName, testCases) ->
testCases.map { testCase ->
"$suiteName ${testCase.classname}#${testCase.name}".trim() to testCase
}
}

private fun findGroupRange(data: String): IntRange {
return findJsonBounds(data, startPattern = "=[{key:\"group-0\",")
.flatten()
.groupBy({ (label: String, _) -> label }) { (_, useCase) -> useCase }

private fun Map<String, List<JUnitTestCase>>.createGroups(): List<HtmlErrorReport.Group> =
map { (label, testCases: List<JUnitTestCase>) ->
HtmlErrorReport.Group(
label = label,
items = testCases.createItems()
)
}

// return start/stop index for the matched JSON object
private fun findJsonBounds(data: String, startPattern: String): IntRange {
val startIndex = data.indexOf(startPattern) + 1
if (startIndex == -1) throw RuntimeException("failed to find $startPattern")

val endIndex = data.indexOf(']', startIndex) + 1

return IntRange(startIndex, endIndex)
}
private fun List<JUnitTestCase>.createItems(): List<HtmlErrorReport.Item> = map { testCase ->
HtmlErrorReport.Item(
label = testCase.stackTrace().split("\n").firstOrNull() ?: "",
url = testCase.webLink ?: ""
)
}

private fun List<HtmlErrorReport.Group>.createHtmlReport(): String =
readTextResource("inline.html").replace(
oldValue = "\"INJECT-DATA-HERE\"",
newValue = "`${Gson().toJson(this)}`"
)
95 changes: 36 additions & 59 deletions test_runner/src/test/kotlin/ftl/reports/HtmlErrorReportTest.kt
Original file line number Diff line number Diff line change
@@ -1,102 +1,79 @@
package ftl.reports

import com.google.common.truth.Truth.assertThat
import ftl.reports.xml.JUnitXmlTest
import ftl.reports.xml.model.JUnitTestResult
import ftl.reports.xml.parseAllSuitesXml
import ftl.reports.xml.parseOneSuiteXml
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test

class HtmlErrorReportTest {

@Test
fun `reactJson androidPassXml`() {
val results = HtmlErrorReport.groupItemList(parseOneSuiteXml(JUnitXmlTest.androidPassXml))
assertThat(results).isNull()
val results = parseOneSuiteXml(JUnitXmlTest.androidPassXml).testsuites!!.process()
assertTrue(results.isEmpty())
}

@Test
fun `reactJson androidFailXml`() {
val results = HtmlErrorReport.groupItemList(parseOneSuiteXml(JUnitXmlTest.androidFailXml))
?: throw RuntimeException("null")
val results: List<HtmlErrorReport.Group> = parseOneSuiteXml(JUnitXmlTest.androidFailXml).testsuites!!.process()

val group = results.first
assertThat(group.size).isEqualTo(1)
val group = results.first()
assertEquals(1, results.size)

with(group.first()) {
assertThat(key).isEqualTo("group-0")
assertThat(name).isEqualTo("com.example.app.ExampleUiTest#testFails")
assertThat(startIndex).isEqualTo(0)
assertThat(count).isEqualTo(1)
}
assertEquals("com.example.app.ExampleUiTest#testFails", group.label)
assertEquals(1, group.items.size)

val item = results.second
assertThat(item.size).isEqualTo(1)
with(item.first()) {
assertThat(key).isEqualTo("item-0")
assertThat(name).isEqualTo("junit.framework.AssertionFailedError: expected:<true> but was:<false>")
assertThat(link).isEqualTo("")
}
val item = group.items.first()
assertEquals("junit.framework.AssertionFailedError: expected:<true> but was:<false>", item.label)
assertEquals("", item.url)
}

@Test
fun `reactJson androidFailXml merged`() {
// 4 tests - 2 pass, 2 fail. we should have 2 failures in the report
val mergedXml = parseOneSuiteXml(JUnitXmlTest.androidFailXml)
val mergedXml: JUnitTestResult = parseOneSuiteXml(JUnitXmlTest.androidFailXml)
mergedXml.merge(mergedXml)

assertThat(mergedXml.testsuites?.first()?.testcases?.size).isEqualTo(4)
assertEquals(4, mergedXml.testsuites?.first()?.testcases?.size)

val results = HtmlErrorReport.groupItemList(mergedXml)
?: throw RuntimeException("null")
val results: List<HtmlErrorReport.Group> = mergedXml.testsuites!!.process()

val group = results.first
assertThat(group.size).isEqualTo(1)
val group = results.first()
assertEquals(1, results.size)

with(group.first()) {
assertThat(key).isEqualTo("group-0")
assertThat(name).isEqualTo("com.example.app.ExampleUiTest#testFails")
assertThat(startIndex).isEqualTo(0)
assertThat(count).isEqualTo(2)
}
assertEquals("com.example.app.ExampleUiTest#testFails", group.label)
assertEquals(2, group.items.size)

val items = results.second
assertThat(items.size).isEqualTo(2)
items.forEachIndexed { index, item ->
with(item) {
assertThat(key).isEqualTo("item-$index")
assertThat(name).isEqualTo("junit.framework.AssertionFailedError: expected:<true> but was:<false>")
assertThat(link).isEqualTo("")
}
group.items.forEach { item ->
assertEquals("junit.framework.AssertionFailedError: expected:<true> but was:<false>", item.label)
assertEquals("", item.url)
}
}

@Test
fun `reactJson iosPassXml`() {
val results = HtmlErrorReport.groupItemList(parseAllSuitesXml(JUnitXmlTest.iosPassXml))
assertThat(results).isNull()
val results = parseAllSuitesXml(JUnitXmlTest.iosPassXml).testsuites!!.process()
assertTrue(results.isEmpty())
}

@Test
fun `reactJson iosFailXml`() {
val results =
HtmlErrorReport.groupItemList(parseAllSuitesXml(JUnitXmlTest.iosFailXml)) ?: throw RuntimeException("null")
val results = parseAllSuitesXml(JUnitXmlTest.iosFailXml).testsuites!!.process()

val group = results.first
assertThat(group.size).isEqualTo(1)
val group = results.first()
assertEquals(1, results.size)

with(group.first()) {
assertThat(key).isEqualTo("group-0")
assertThat(name).isEqualTo("EarlGreyExampleSwiftTests EarlGreyExampleSwiftTests#testBasicSelectionAndAction()")
assertThat(startIndex).isEqualTo(0)
assertThat(count).isEqualTo(1)
}
assertEquals("EarlGreyExampleSwiftTests EarlGreyExampleSwiftTests#testBasicSelectionAndAction()", group.label)
assertEquals(1, group.items.size)

val item = results.second
assertThat(item.size).isEqualTo(1)
with(item.first()) {
assertThat(key).isEqualTo("item-0")
assertThat(name).isEqualTo("Exception: NoMatchingElementException, failed: caught \"EarlGreyInternalTestInterruptException\", \"Immediately halt execution of testcase\"null")
assertThat(link).isEqualTo("")
}
val item = group.items.first()
assertEquals(
"Exception: NoMatchingElementException, failed: caught \"EarlGreyInternalTestInterruptException\", \"Immediately halt execution of testcase\"null",
item.label
)
assertEquals("", item.url)
}
}

0 comments on commit 721e9c2

Please sign in to comment.