WIP: Add 'test_twip_2_twim_2' that 'FAILED' and make it 'PASSED'

While working on trilinos/Trilinos#3887, I noticed faulty logic in determining
global pass/fail related to 'twip' and 'twim' tests.  The existance of these
test categories should not result in a global failure.  The assumption with
'twim' is that the tests were disabled (on purpose hopefully).

To make this pass, I added the field TestsetInfo.existanceTriggersGlobalFail
that defaults to 'True' but is set to 'False' for the 'twip' and 'twim'
test-sets.

test/ci_support/cdash_analyze_and_report_UnitTests.py

@@ -1472,6 +1472,309 @@ def test_bm_1_twoif_12_twip_2_twim_2_twif_5(self):
       '2018-10-28T06:10:33 UTC')
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  # Test to check that when only 'twip' and 'twim' exists, then we consider
+  # this to be global PASSED.
+  #
+  # We use test rseults that exist in the directory twoif_12_twif_9/ to create
+  # these cases.  We copy out and modify the test dicts that we want, modify
+  # them to be passing and missing tests, then write then back to the file
+  # fullCDashNonpassingTests.json
+  #
+  def test_twip_2_twim_2(self):
+
+    testCaseName = "twip_2_twim_2"
+
+    # Copy the raw files to get started
+    testOutputDir = cdash_analyze_and_report_setup_test_dir(testCaseName)
+
+    # Remove all of the failing tests
+    testListFilePath = \
+      testOutputDir+"/ProjectName_Nightly_Builds_fullCDashNonpassingTests.json"
+    CDQAR.pprintPythonDataToFile( {'builds':[]}, testListFilePath )
+
+    # Write a new tests with issue trackers file just for the passing and missing tests
+    with open(testOutputDir+"/testsWithIssueTrackers.csv", 'w') as testsWithIssueTrackerFile:
+      testsWithIssueTrackerFile.write('''site, buildName, testname, issue_tracker_url, issue_tracker
+cee-rhel6, Trilinos-atdm-cee-rhel6-clang-opt-serial, MueLu_UnitTestsBlockedEpetra_MPI_1, https://github.com/trilinos/Trilinos/issues/3640, #3640
+cee-rhel6, Trilinos-atdm-cee-rhel6-intel-opt-serial, PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2, https://github.com/trilinos/Trilinos/issues/3632, #3632
+cee-rhel6, Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial, PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2, https://github.com/trilinos/Trilinos/issues/3632, #3632
+cee-rhel6, Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial, PanzerAdaptersIOSS_tIOSSConnManager3_MPI_3, https://github.com/trilinos/Trilinos/issues/3632, #3632
+'''
+        )
+
+    # Change the history for some tests so that they show up correctly as
+    # Passing and Missing
+
+    daysOfHistory = 30
+
+    # Make a test passed
+    testHistoryFileName = CDQAR.getTestHistoryCacheFileName( "2018-10-28",
+      'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-clang-opt-serial',
+      'MueLu_UnitTestsBlockedEpetra_MPI_1',
+      daysOfHistory)
+    testHistoryLOD = getTestHistoryDictListFromCDashJsonFile(
+      testOutputDir, testHistoryFileName)
+    testHistoryLOD.sort(reverse=True, key=CDQAR.DictSortFunctor(['buildstarttime']))
+    testHistoryLOD[0]['status'] = u'Passed'
+    testHistoryLOD[0]['details'] = u'Completed (Passed)'
+    writeTestHistoryDictListFromCDashJsonFile(testHistoryLOD, testOutputDir,
+      testHistoryFileName)
+
+    # Make a test passed
+    testHistoryFileName = CDQAR.getTestHistoryCacheFileName( "2018-10-28",
+      'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-intel-opt-serial',
+      'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2',
+      daysOfHistory)
+    testHistoryLOD = getTestHistoryDictListFromCDashJsonFile(
+      testOutputDir, testHistoryFileName)
+    testHistoryLOD.sort(reverse=True, key=CDQAR.DictSortFunctor(['buildstarttime']))
+    testHistoryLOD[0]['status'] = u'Passed'
+    testHistoryLOD[0]['details'] = u'Completed (Passed)'
+    writeTestHistoryDictListFromCDashJsonFile(testHistoryLOD, testOutputDir,
+      testHistoryFileName)
+
+    # Make a test missing (here, we need to move the date back)
+    testHistoryFileName = CDQAR.getTestHistoryCacheFileName( "2018-10-28",
+      'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial',
+      'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2',
+      daysOfHistory)
+    testHistoryLOD = getTestHistoryDictListFromCDashJsonFile(
+      testOutputDir, testHistoryFileName)
+    testHistoryLOD.sort(reverse=True, key=CDQAR.DictSortFunctor(['buildstarttime']))
+    # There is just one day in history so we need to duplicate it
+    testHistoryLOD.append(copy.deepcopy(testHistoryLOD[0]))
+    # Make first non-missing day failed
+    testHistoryLOD[0]['buildstarttime'] = "2018-10-26T12:00:00 UTC"
+    testHistoryLOD[0]['status'] = u'Failed'
+    testHistoryLOD[0]['details'] = u'Completed (Failed)'
+    # Make second non-missing day passed
+    testHistoryLOD[1]['buildstarttime'] = "2018-10-25T12:00:00 UTC"
+    testHistoryLOD[1]['status'] = u'Passed'
+    testHistoryLOD[1]['details'] = u'Completed (Passed)'
+    writeTestHistoryDictListFromCDashJsonFile(testHistoryLOD, testOutputDir,
+      testHistoryFileName)
+
+    # Make a test missing (no days of test history)
+    testHistoryFileName = CDQAR.getTestHistoryCacheFileName( "2018-10-28",
+      'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial',
+      'PanzerAdaptersIOSS_tIOSSConnManager3_MPI_3',
+      daysOfHistory)
+    testHistoryLOD = []
+    writeTestHistoryDictListFromCDashJsonFile(testHistoryLOD, testOutputDir,
+      testHistoryFileName)
+
+    # ToDo: Remove history for missing tests to test different numbers of
+    # missing days.  (This will need to be done once we tablulate "Consecutive
+    # Pass Days" and "Consecutive Missing Days".)
+
+    # Run the script and make sure it outputs the right stuff
+    cdash_analyze_and_report_run_case(
+      self,
+      testCaseName,
+      [ "--limit-test-history-days=30",  # Test that you can set this as int
+        "--write-test-data-to-file=test_data.json",
+        ],
+      0,
+      "PASSED (twip=2, twim=2):"+\
+        " ProjectName Nightly Builds on 2018-10-28",
+      [
+        "Num expected builds = 6",
+        "Num tests with issue trackers = 4",
+        "Num builds = 6",
+        "Num nonpassing tests direct from CDash query = 0",
+        "Num nonpassing tests after removing duplicate tests = 0",
+        "Num nonpassing tests without issue trackers = 0",
+        "Num nonpassing tests with issue trackers = 0",
+        "Num nonpassing tests without issue trackers Failed = 0",
+        "Num nonpassing tests without issue trackers Not Run = 0",
+        "Num nonpassing tests with issue trackers Failed = 0",
+        "Num nonpassing tests with issue trackers Not Run = 0",
+        "Num tests with issue trackers gross passing or missing = 4",
+
+        "Builds Missing: bm=0",
+        "Builds with Configure Failures: cf=0",
+        "Builds with Build Failures: bf=0",
+
+        "Num tests with issue trackers passing or missing matching posted builds = 4",
+
+        "Tests with issue trackers missing that match missing expected builds: num=0",
+
+        "Getting test history for tests with issue trackers passing or missing: num=4",
+        "Getting 30 days of history for MueLu_UnitTestsBlockedEpetra_MPI_1 in the build Trilinos-atdm-cee-rhel6-clang-opt-serial on cee-rhel6 from cache file",
+        "Getting 30 days of history for PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2 in the build Trilinos-atdm-cee-rhel6-intel-opt-serial on cee-rhel6 from cache file",
+        "Getting 30 days of history for PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2 in the build Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial on cee-rhel6 from cache file",
+        "Getting 30 days of history for PanzerAdaptersIOSS_tIOSSConnManager3_MPI_3 in the build Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial on cee-rhel6 from cache file",
+
+        "Num tests with issue trackers Passed = 2",
+        "Num tests with issue trackers Missing = 2",
+
+        "Tests without issue trackers Failed: twoif=0",
+
+        "Tests with issue trackers Passed: twip=2",
+
+        "Tests with issue trackers Missing: twim=2",
+
+        "Tests with issue trackers Failed: twif=0",
+
+        "Tests with issue trackers Not Run: twinr=0",
+
+        ],
+      [
+
+        "<h3><font color=\"green\">Tests with issue trackers Passed: twip=2</font></h3>",
+        # Pin down table headers
+        "<th>Site</th>",
+        "<th>Build Name</th>",
+        "<th>Test Name</th>",
+        "<th>Status</th>",
+        "<th>Details</th>",
+        "<th>Consec&shy;utive Pass Days</th>",
+        "<th>Non-pass Last 30 Days</th>",
+        "<th>Pass Last 30 Days</th>",
+        "<th>Issue Tracker</th>",
+        # Pin down first row
+        "<td align=\"left\">cee-rhel6</td>",
+        "<td align=\"left\"><a href=\".+\">Trilinos-atdm-cee-rhel6-clang-opt-serial</a></td>",
+        "<td align=\"left\"><a href=\".+\">MueLu_&shy;UnitTestsBlockedEpetra_&shy;MPI_&shy;1</a></td>",
+        "<td align=\"left\"><a href=\".+\"><font color=\"green\">Passed</font></a></td>",
+        "<td align=\"left\">Completed [(]Passed[)]</td>",
+        "<td align=\"right\"><a href=\".+\"><font color=\"green\">1</font></a></td>",
+        "<td align=\"right\"><a href=\".+\"><font color=\"red\">14</font></a></td>",
+        "<td align=\"right\"><a href=\".+\"><font color=\"green\">1</font></a></td>",
+        "<td align=\"right\"><a href=\".+\">#3640</a></td>",
+
+        "<h3>Tests with issue trackers Missing: twim=2</h3>",
+        # Pin down table headers
+        "<th>Site</th>",
+        "<th>Build Name</th>",
+        "<th>Test Name</th>",
+        "<th>Status</th>",
+        "<th>Details</th>",
+        "<th>Consec&shy;utive Missing Days</th>",
+        "<th>Non-pass Last 30 Days</th>",
+        "<th>Pass Last 30 Days</th>",
+        "<th>Issue Tracker</th>",
+        # Pin down first row
+        "<td align=\"left\">cee-rhel6</td>",
+        "<td align=\"left\"><a href=\".+\">Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial</a></td>",
+        "<td align=\"left\">PanzerAdaptersIOSS_&shy;tIOSSConnManager2_&shy;MPI_&shy;2</td>",
+        "<td align=\"left\"><font color=\"gray\">Missing</font></td>",
+        "<td align=\"left\">Missing</td>",
+        "<td align=\"right\"><a href=\".+\"><font color=\"gray\">2</font></a></td>",
+        "<td align=\"right\"><a href=\".+\"><font color=\"red\">1</font></a></td>",
+        "<td align=\"right\"><a href=\".+\"><font color=\"green\">1</font></a></td>",
+        "<td align=\"right\"><a href=\".+\">#3632</a></td>",
+
+        ],
+      #verbose=True,
+      #debugPrint=True,
+      )
+
+    # Read the written file 'test_data.json' and verify that it is correct
+    testDataLOD = eval(open(testOutputDir+"/test_data.json", 'r').read())
+    self.assertEqual(len(testDataLOD), 4)
+    # Make sure an entry from 'twip' exists!
+    testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-clang-opt-serial',
+      'MueLu_UnitTestsBlockedEpetra_MPI_1')
+    testDict = testDataLOD[testIdx]
+    self.assertEqual(testDict['site'], 'cee-rhel6')
+    self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-clang-opt-serial')
+    self.assertEqual(testDict['testname'], 'MueLu_UnitTestsBlockedEpetra_MPI_1')
+    self.assertEqual(testDict['status'], 'Passed')
+    self.assertEqual(testDict['details'], 'Completed (Passed)')
+    self.assertEqual(testDict['cdash_testing_day'], u'2018-10-28')
+    self.assertEqual(testDict['test_history_list'][0]['buildstarttime'],
+      '2018-10-28T06:10:33 UTC')
+    # Make sure an entry from 'twim' exists!
+    testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial',
+      'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2')
+    testDict = testDataLOD[testIdx]
+    self.assertEqual(testDict['site'], 'cee-rhel6')
+    self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial')
+    self.assertEqual(testDict['testname'], 'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2')
+    self.assertEqual(testDict['status'], 'Missing')
+    self.assertEqual(testDict['details'], 'Missing')
+    self.assertEqual(testDict['cdash_testing_day'], u'2018-10-28')
+    self.assertEqual(testDict['test_history_list'][0]['buildstarttime'],
+      '2018-10-26T12:00:00 UTC')
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 #
 # Run the unit tests!
 #

tribits/ci_support/CDashQueryAnalyzeReport.py

@@ -317,11 +317,14 @@ def getStandardTestsetAcroList():
 #
 class TestsetInfo(object):
 
-  def __init__(self, testsetAcro, testsetDescr, testsetTableType, testsetColor):
+  def __init__(self, testsetAcro, testsetDescr, testsetTableType, testsetColor,
+      existanceTriggersGlobalFail=True,
+    ):
     self.testsetAcro = testsetAcro
     self.testsetDescr = testsetDescr
     self.testsetTableType = testsetTableType
     self.testsetColor = testsetColor
+    self.existanceTriggersGlobalFail = existanceTriggersGlobalFail
 
 
 # Return the TestsetInfo object for the standard types of test sets that get
@@ -342,10 +345,10 @@ def getStandardTestsetInfo(testsetAcro, testsetColor=None):
       cdashColorNotRun())
   elif testsetAcro == "twip":
     tsi = TestsetInfo(testsetAcro, "Tests with issue trackers Passed", 'pass',
-      cdashColorPassed())
+      cdashColorPassed(), existanceTriggersGlobalFail=False)
   elif testsetAcro == "twim":
     tsi = TestsetInfo(testsetAcro, "Tests with issue trackers Missing", 'missing',
-      cdashColorMissing())
+      cdashColorMissing(), existanceTriggersGlobalFail=False)
   elif testsetAcro == "twif":
     tsi = TestsetInfo(testsetAcro, "Tests with issue trackers Failed", 'nopass',
       cdashColorFailed())
@@ -2608,7 +2611,7 @@ def __init__(self, cdashReportData,
   #   will be written, along with formatting.
   #
   def reportSingleTestset(self, testsetInfo, testsetTotalSize, testsetLOD,
-      testsetNonzeroSizeTriggerGlobalFail=True, sortTests=True,
+      sortTests=True,
       limitTableRows=None,   # Change to 'int' > 0 to limit table rows
       getTestHistory=False,
     ):
@@ -2623,7 +2626,7 @@ def reportSingleTestset(self, testsetInfo, testsetTotalSize, testsetLOD,
 
     if testsetTotalSize > 0:
 
-      if testsetNonzeroSizeTriggerGlobalFail:
+      if testsetInfo.existanceTriggersGlobalFail:
         self.cdashReportData.globalPass = False
 
       self.cdashReportData.summaryLineDataNumbersList.append(