Skip to content

Commit

Permalink
Remove tests for class TestsetsReporter (trilinos/Trilinos#3887)
Browse files Browse the repository at this point in the history 
This is tested indrectly through the tests for the clas
IssueTrackerTestsStatusReporter.  We don't want to maintain large complex
redundant tests.
  • Loading branch information
@bartlettroscoe
bartlettroscoe committed Jun 7, 2020
1 parent 7dc05a7 commit 6699e4c
Showing 1 changed file with 26 additions and 198 deletions.
224 changes: 26 additions & 198 deletions test/ci_support/CDashQueryAnalyzeReport_UnitTests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3880,202 +3880,6 @@ def test_all(self):
)


#############################################################################
#
# Test CDashQueryAnalyzeReport.TestsetsReporter
#
#############################################################################


def makeTestPassing(testDict):
testDict['status'] = u'Passed'
testDict['status_color'] = cdashColorPassed()
testDict['details'] = u'Completed (Passed)\n'


def makeTestMissing(testDict):
setTestDictAsMissing(testDict)


cdash_analyze_and_report_dir = g_testBaseDir+'/cdash_analyze_and_report'


g_twoif_10_twoinr2_twif_8_twinr_1_test_data_out = \
eval(
open(cdash_analyze_and_report_dir+'/twoif_10_twoinr2_twif_8_twinr_1/test_data.json',
'r').read())


class test_TestsetsReporter(unittest.TestCase):


def test_empty(self):
cdashReportData = CDashReportData()
testsetsReporter = TestsetsReporter(cdashReportData, verbose=False)
testsLOD = []
testsetsReporter.reportTestsets(testsLOD)
self.assertEqual(cdashReportData.summaryLineDataNumbersList, [])
self.assertEqual(cdashReportData.htmlEmailBodyTop, "")
self.assertEqual(cdashReportData.htmlEmailBodyBottom, "")


def test_twif_8_twinr_1(self):
allTestsLOD = copy.deepcopy(g_twoif_10_twoinr2_twif_8_twinr_1_test_data_out)
cdashReportData = CDashReportData()
testsetsReporter = TestsetsReporter(cdashReportData,
htmlStyle="", # GitHub Markdown does not like the default HTML style
verbose=False)
testsLOD = allTestsLOD
#g_pp.pprint(testsLOD)
testsetsReporter.reportTestsets(testsLOD)
#print("\ncdashReportData.summaryLineDataNumbersList = "+\
# str(cdashReportData.summaryLineDataNumbersList))
#print("\ncdashReportData.htmlEmailBodyTop:\n"+\
# cdashReportData.htmlEmailBodyTop)
summaryLineDataNumbersList_expected = \
['twif=8', 'twinr=1']
self.assertEqual(cdashReportData.summaryLineDataNumbersList,
summaryLineDataNumbersList_expected)
htmlEmailBodyTop_expected = \
'<font color="red">Tests with issue trackers Failed: twif=8</font><br>\n'+\
'<font color="orange">Tests with issue trackers Not Run: twinr=1</font><br>\n'
self.assertEqual(cdashReportData.htmlEmailBodyTop,
htmlEmailBodyTop_expected)
#print("\ncdashReportData.htmlEmailBodyBottom:\n"+\
# cdashReportData.htmlEmailBodyBottom)
assertListOfRegexsFoundInLinstOfStrs(self,
regexList=[
"<h3><font color=.red.>Tests with issue trackers Failed: twif=8</font></h3>",
"<td align=\"left\">cee-rhel6</td>",
"<td align=\"left\"><a href=\"https://something.com/cdash/testDetails.php[?]test=57816429&build=4107319\">MueLu_&shy;UnitTestsBlockedEpetra_&shy;MPI_&shy;1</a></td>",
"<td align=\"left\"><a href=\"https://something.com/cdash/testDetails.php[?]test=57816429&build=4107319\"><font color=\"red\">Failed</font></a></td>",
"<td align=\"right\"><a href=\"https://github.com/trilinos/Trilinos/issues/3640\">#3640</a></td>",
"<h3><font color=\"orange\">Tests with issue trackers Not Run: twinr=1</font></h3>",
"<td align=\"left\">cee-rhel6</td>",
"<td align=\"left\"><a href=\"https://something.com/cdash/testDetails.php[?]test=57816373&build=4107331\">Teko_&shy;ModALPreconditioner_&shy;MPI_&shy;1</a></td>",
"<td align=\"left\"><a href=\"https://something.com/cdash/testDetails.php[?]test=57816373&build=4107331\"><font color=\"orange\">Not Run</font></a></td>",
"<td align=\"left\">Required Files Missing</td>",
"<td align=\"right\"><a href=\"https://github.com/trilinos/Trilinos/issues/3638\">#3638</a></td>",
],
stringsList=cdashReportData.htmlEmailBodyBottom.split('\n'),
stringsListName="cdashReportData.htmlEmailBodyBottom",
debugPrint=False
)
testsHtmlReportStr = testsetsReporter.getTestsHtmlReportStr(
"Status of tests for issue #1234")
# TODO: REMOVE THIS FILE WRITE!!!
#with open("testsHtmlReport.html", 'w') as testsHtmlReportFile:
# testsHtmlReportFile.write(testsHtmlReportStr)
assertListOfRegexsFoundInLinstOfStrs(self,
regexList=[
'<h2>Status of tests for issue #1234</h2>',
'<font color="red">Tests with issue trackers Failed: twif=8</font><br>',
'<font color="orange">Tests with issue trackers Not Run: twinr=1</font><br>',
'<h3><font color="red">Tests with issue trackers Failed: twif=8</font></h3>',
'<h3><font color="orange">Tests with issue trackers Not Run: twinr=1</font></h3>',
],
stringsList=testsHtmlReportStr.split('\n'),
stringsListName="testsHtmlReportStr",
debugPrint=False
)


def test_twip_1_twif_5_twim_2_twinr_1(self):
allTestsLOD = copy.deepcopy(g_twoif_10_twoinr2_twif_8_twinr_1_test_data_out)
# Change a test from failing to passing
testIdx = getIdxOfTestInTestLOD(allTestsLOD,
'cee-rhel6', 'Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial',
'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2')
makeTestPassing(allTestsLOD[testIdx])
# Change a from from faling to missing
testIdx = getIdxOfTestInTestLOD(allTestsLOD,
'cee-rhel6', 'Trilinos-atdm-cee-rhel6-intel-opt-serial',
'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2')
makeTestMissing(allTestsLOD[testIdx])
# Change another a test from failing to missing
testIdx = getIdxOfTestInTestLOD(allTestsLOD,
'cee-rhel6', 'Trilinos-atdm-cee-rhel6-intel-opt-serial',
'PanzerAdaptersIOSS_tIOSSConnManager3_MPI_3')
makeTestMissing(allTestsLOD[testIdx])
# Run the reporter
cdashReportData = CDashReportData()
testsetsReporter = TestsetsReporter(cdashReportData,
htmlStyle="",
verbose=False)
testsLOD = allTestsLOD
#g_pp.pprint(testsLOD)
testsetsReporter.reportTestsets(testsLOD)
#print("\ncdashReportData.summaryLineDataNumbersList = "+\
# str(cdashReportData.summaryLineDataNumbersList))
#print("\ncdashReportData.htmlEmailBodyTop:\n"+\
# cdashReportData.htmlEmailBodyTop)
summaryLineDataNumbersList_expected = \
['twip=1', 'twim=2', 'twif=5', 'twinr=1']
self.assertEqual(cdashReportData.summaryLineDataNumbersList,
summaryLineDataNumbersList_expected)
htmlEmailBodyTop_expected = \
'<font color="green">Tests with issue trackers Passed: twip=1</font><br>\n'+\
'<font color="gray">Tests with issue trackers Missing: twim=2</font><br>\n'+\
'<font color="red">Tests with issue trackers Failed: twif=5</font><br>\n'+\
'<font color="orange">Tests with issue trackers Not Run: twinr=1</font><br>\n'
self.assertEqual(cdashReportData.htmlEmailBodyTop,
htmlEmailBodyTop_expected)
#print("\ncdashReportData.htmlEmailBodyBottom:\n"+\
# cdashReportData.htmlEmailBodyBottom)
assertListOfRegexsFoundInLinstOfStrs(self,
regexList=[
'<h3><font color="green">Tests with issue trackers Passed: twip=1</font></h3>',
'<td align="left"><a href=".*">Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial</a></td>',
'<td align="left"><a href=".*">PanzerAdaptersIOSS_&shy;tIOSSConnManager2_&shy;MPI_&shy;2</a></td>',
'<td align="left"><a href=".*"><font color="green">Passed</font></a></td>',
'<td align="left">Completed [(]Passed[)]</td>',

'<h3><font color="gray">Tests with issue trackers Missing: twim=2</font></h3>',
'<td align="left"><a href=".*">Trilinos-atdm-cee-rhel6-intel-opt-serial</a></td>',
'<td align="left"><a href=".*">PanzerAdaptersIOSS_&shy;tIOSSConnManager2_&shy;MPI_&shy;2</a></td>',
'<td align="left"><a href=".*"><font color="gray">Missing</font></a></td>',
'<td align="left">Missing</td>',

'<h3><font color="red">Tests with issue trackers Failed: twif=5</font></h3>',
'<td align="left"><a href=".*">Trilinos-atdm-cee-rhel6-clang-opt-serial</a></td>',
'<td align="left"><a href=".*">MueLu_&shy;UnitTestsBlockedEpetra_&shy;MPI_&shy;1</a></td>',
'<td align="left"><a href=".*"><font color="red">Failed</font></a></td>',
'<td align="left">Completed [(]Failed[)]</td>',

'<h3><font color="orange">Tests with issue trackers Not Run: twinr=1</font></h3>',
'<td align="left"><a href=".*">Trilinos-atdm-cee-rhel6-clang-opt-serial</a></td>',
'<td align="left"><a href=".*">Teko_&shy;ModALPreconditioner_&shy;MPI_&shy;1</a></td>',
'<td align="left"><a href=".*"><font color="orange">Not Run</font></a></td>',
'<td align="left">Required Files Missing</td>',
],
stringsList=cdashReportData.htmlEmailBodyBottom.split('\n'),
stringsListName="cdashReportData.htmlEmailBodyBottom",
debugPrint=False
)
# Get the summary report
testsHtmlReportStr = testsetsReporter.getTestsHtmlReportStr(
"Status of tests for issue #1236")
# TODO: REMOVE THIS FILE WRITE!!!
#with open("testsHtmlReport.html", 'w') as testsHtmlReportFile:
# testsHtmlReportFile.write(testsHtmlReportStr)
assertListOfRegexsFoundInLinstOfStrs(self,
regexList=[
'<h2>Status of tests for issue #1236</h2>',
'<font color="green">Tests with issue trackers Passed: twip=1</font><br>',
'<font color="gray">Tests with issue trackers Missing: twim=2</font><br>',
'<font color="red">Tests with issue trackers Failed: twif=5</font><br>',
'<font color="orange">Tests with issue trackers Not Run: twinr=1</font><br>',
'<h3><font color="green">Tests with issue trackers Passed: twip=1</font></h3>',
'<h3><font color="gray">Tests with issue trackers Missing: twim=2</font></h3>',
'<h3><font color="red">Tests with issue trackers Failed: twif=5</font></h3>',
'<h3><font color="orange">Tests with issue trackers Not Run: twinr=1</font></h3>',
],
stringsList=testsHtmlReportStr.split('\n'),
stringsListName="testsHtmlReportStr",
debugPrint=False
)


#############################################################################
#
# Test CDashQueryAnalyzeReport.getIssueTrackerAndAssertAllSame()
Expand Down Expand Up @@ -4200,11 +4004,28 @@ def test_inconsistent_issue_tracker_field(self):
#
# Test CDashQueryAnalyzeReport.IssueTrackerTestsStatusReporter
#
# NOTE: This also tests the class CDashQueryAnalyzeReport.TestsetsReporter
#
#############################################################################


cdash_analyze_and_report_dir = g_testBaseDir+'/cdash_analyze_and_report'


g_twoif_10_twoinr2_twif_8_twinr_1_test_data_out = \
eval(
open(cdash_analyze_and_report_dir+'/twoif_10_twoinr2_twif_8_twinr_1/test_data.json',
'r').read())


def makeTestPassing(testDict):
testDict['status'] = u'Passed'
testDict['status_color'] = cdashColorPassed()
testDict['details'] = u'Completed (Passed)\n'


def makeTestMissing(testDict):
setTestDictAsMissing(testDict)


def setIssueTrackerFields(testsLOD, issue_tracker, issue_tracker_url):
for testDict in testsLOD:
testDict['issue_tracker'] = issue_tracker
Expand Down Expand Up @@ -4387,6 +4208,13 @@ def test_twip_1_twif_5_twim_2_twinr_1(self):
debugPrint=False
)

# NOTE: The above tests for the class
# CDashQueryAnalyzeReport.IssueTrackerTestsStatusReporter also tests the
# classes CDashQueryAnalyzeReport.SingleTestsetReporter and
# CDashQueryAnalyzeReport.TestsetsReporter. I just did not want to
# duplicate all of those large and complex tests for little added value.



#
# Run the unit tests!
Expand Down

0 comments on commit 6699e4c

Please sign in to comment.