Merge branch '304-write-test-data-to-file' (#304)

Needed as part of trilinos/Trilinos#3887

Build/Test Cases Summary
Enabled Packages:
Enabled all Packages
0) MPI_DEBUG => passed: passed=361,notpassed=0 (1.40 min)
1) SERIAL_RELEASE => passed: passed=361,notpassed=0 (1.22 min)
Other local commits for this build/test group: d9134b4

test/ci_support/cdash_analyze_and_report_UnitTests.py

@@ -559,7 +559,8 @@ def test_twoif_10_twoinr2_twif_8_twinr_1(self):
       testCaseName,
       [
         "--limit-test-history-days=30",   # Test that you can set this as int
-        "--write-failing-tests-without-issue-trackers-to-file=twoif.csv"
+        "--write-failing-tests-without-issue-trackers-to-file=twoif.csv",
+        "--write-test-data-to-file=test_data.json"
         ],
       1,
       "FAILED (twoif=10, twoinr=2, twif=8, twinr=1): ProjectName Nightly Builds on 2018-10-28",
@@ -709,6 +710,34 @@ def test_twoif_10_twoinr2_twif_8_twinr_1(self):
     # right number of tests are being written and the first and last are
     # correct.
 
+    # Read the written file 'test_data.json' and verify that it is correct
+    testDataLOD = eval(open(testOutputDir+"/test_data.json", 'r').read())
+    self.assertEqual(len(testDataLOD), 9)
+    # Make sure an entry from 'twif' exists!
+    testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-clang-opt-serial',
+      'MueLu_UnitTestsBlockedEpetra_MPI_1')
+    testDict = testDataLOD[testIdx]
+    self.assertEqual(testDict['site'], 'cee-rhel6')
+    self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-clang-opt-serial')
+    self.assertEqual(testDict['testname'], 'MueLu_UnitTestsBlockedEpetra_MPI_1')
+    self.assertEqual(testDict['status'], 'Failed')
+    self.assertEqual(testDict['details'], 'Completed (Failed)\n')
+    self.assertEqual(testDict['test_history_list'][0]['buildstarttime'],
+      '2018-10-28T06:10:33 UTC')
+    # Make sure an entry from 'twinr' exists!
+    testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-clang-opt-serial',
+      'Teko_ModALPreconditioner_MPI_1')
+    testDict = testDataLOD[testIdx]
+    self.assertEqual(testDict['site'], 'cee-rhel6')
+    self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-clang-opt-serial')
+    self.assertEqual(testDict['testname'], 'Teko_ModALPreconditioner_MPI_1')
+    self.assertEqual(testDict['status'], 'Not Run')
+    self.assertEqual(testDict['details'], 'Required Files Missing')
+    self.assertEqual(testDict['test_history_list'][0]['buildstarttime'],
+      '2018-10-28T06:10:33 UTC')
+
 
   # Test with some duplicate tests from CDash query (this happens in real life
   # sometimes!)
@@ -1355,7 +1384,9 @@ def test_twip_twim(self):
     cdash_analyze_and_report_run_case(
       self,
       testCaseName,
-      ["--limit-test-history-days=30"], # Test that you can set this as int
+      [ "--limit-test-history-days=30",  # Test that you can set this as int
+        "--write-test-data-to-file=test_data.json",
+        ],
       1,
       "FAILED (bm=1, twoif=12, twip=2, twim=2, twif=5):"+\
         " ProjectName Nightly Builds on 2018-10-28",
@@ -1473,6 +1504,47 @@ def test_twip_twim(self):
       #debugPrint=True,
       )
 
+    # Read the written file 'test_data.json' and verify that it is correct
+    testDataLOD = eval(open(testOutputDir+"/test_data.json", 'r').read())
+    self.assertEqual(len(testDataLOD), 9)
+    # Make sure an entry from 'twip' exists!
+    testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-clang-opt-serial',
+      'MueLu_UnitTestsBlockedEpetra_MPI_1')
+    testDict = testDataLOD[testIdx]
+    self.assertEqual(testDict['site'], 'cee-rhel6')
+    self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-clang-opt-serial')
+    self.assertEqual(testDict['testname'], 'MueLu_UnitTestsBlockedEpetra_MPI_1')
+    self.assertEqual(testDict['status'], 'Passed')
+    self.assertEqual(testDict['details'], 'Completed (Passed)')
+    self.assertEqual(testDict['test_history_list'][0]['buildstarttime'],
+      '2018-10-28T06:10:33 UTC')
+    # Make sure an entry from 'twim' exists!
+    testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial',
+      'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2')
+    testDict = testDataLOD[testIdx]
+    self.assertEqual(testDict['site'], 'cee-rhel6')
+    self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial')
+    self.assertEqual(testDict['testname'], 'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2')
+    self.assertEqual(testDict['status'], 'Missing')
+    self.assertEqual(testDict['details'], 'Missing')
+    self.assertEqual(testDict['test_history_list'][0]['buildstarttime'],
+      '2018-10-26T12:00:00 UTC')
+    # Make sure an entry from 'twif' exists!
+    testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6',
+      'Trilinos-atdm-cee-rhel6-clang-opt-serial',
+      'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2')
+    testDict = testDataLOD[testIdx]
+    self.assertEqual(testDict['site'], 'cee-rhel6')
+    self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-clang-opt-serial')
+    self.assertEqual(testDict['testname'], 'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2')
+    self.assertEqual(testDict['status'], 'Failed')
+    self.assertEqual(testDict['details'], 'Completed (Failed)\n')
+    self.assertEqual(testDict['test_history_list'][0]['buildstarttime'],
+      '2018-10-28T06:10:33 UTC')
+
+
 #
 # Run the unit tests!
 #

tribits/ci_support/cdash_analyze_and_report.py

@@ -193,15 +193,15 @@ def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""):
     "--require-test-history-match-nonpassing-tests",
     "requireTestHistoryMatchNonpassingTestsStr",
     ("on", "off"), 0,
-    "Require that the status for each tracked test listed in the tests with issue"+\
-    " trackers CSV file match the status of that test returned from the test history"+\
-    " returned from CDash.  In general, these should match up but these may not if extra"+\
-    " filter criteria has been added to the list on nonpassing tests in the"+\
-    " --cdash-nonpassed-tests-filters=<filters> set of filters (such as to filter out"+\
-    " a large number of random system failures).  In this case, an error will be returned"+\
-    " by default and the script will crash.  But this can be relaxed by setting this to 'off'"+\
-    " which will result in these tracked tests being listed in the 'twim' table but typically"+\
-    " with status 'Failed'.",
+    "Require that the status for each tracked test listed in the tests with issue"\
+    +" trackers CSV file match the status of that test returned from the test history"\
+    +" returned from CDash.  In general, these should match up but these may not if extra"\
+    +" filter criteria has been added to the list on nonpassing tests in the"\
+    +" --cdash-nonpassed-tests-filters=<filters> set of filters (such as to filter out"\
+    +" a large number of random system failures).  In this case, an error will be"\
+    +" returned by default and the script will crash.  But this can be relaxed by"\
+    +" setting this to 'off' which will result in these tracked tests being listed in"\
+    +" the 'twim' table but typically with status 'Failed'.",
     clp )
 
   addOptionParserChoiceOption(
@@ -214,10 +214,19 @@ def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""):
   clp.add_option(
     "--write-failing-tests-without-issue-trackers-to-file",
     dest="writeFailingTestsWithoutIssueTrackersToFile", type="string", default="",
-    help="Write CSV file with a list of tets with issue trackers failed 'twif'." \
+    help="Write a CSV file with a list of tets with issue trackers failed 'twif'." \
     +"  This is to make it easy to add new entires to the file read by" \
     +" the option --tests-with-issue-trackers-file=<file>. [default = '']" )
 
+  clp.add_option(
+    "--write-test-data-to-file",
+    dest="writeTestDataToFile", type="string", default="",
+    help="Write pretty-printed Python list of dictionaries for tests with" \
+    +" with issue trackers.  This includes the history of the tests for" \
+    +" --limit-test-history-days=<days> of history.  This contains all of the" \
+    +" information that appears in the generated summary tables for tests with" \
+    +" associated issue trackers.  [default = '']" )
+
   clp.add_option(
     "--write-email-to-file", dest="writeEmailToFile", type="string", default="",
     help="Write the body of the HTML email to this file. [default = '']" )
@@ -298,6 +307,7 @@ def fwdCmndLineOptions(inOptions, lt=""):
     "  --require-test-history-match-nonpassing-tests='"+io.requireTestHistoryMatchNonpassingTestsStr+"'"+lt+\
     "  --print-details='"+io.printDetailsStr+"'"+lt+\
     "  --write-failing-tests-without-issue-trackers-to-file='"+io.writeFailingTestsWithoutIssueTrackersToFile+"'"+lt+\
+    "  --write-test-data-to-file='"+io.writeTestDataToFile+"'"+lt+\
     "  --write-email-to-file='"+io.writeEmailToFile+"'"+lt+\
     "  --email-from-address='"+io.emailFromAddress+"'"+lt+\
     "  --send-email-to='"+io.sendEmailTo+"'"+lt
@@ -959,13 +969,21 @@ def testSetGetDataAnalyzeReport( self,
     #
 
     if inOptions.writeFailingTestsWithoutIssueTrackersToFile:
-
       twoifCsvFileName = inOptions.writeFailingTestsWithoutIssueTrackersToFile
-
       print("\nWriting list of 'twiof' to file "+twoifCsvFileName+" ...")
-
       CDQAR.writeTestsLODToCsvFile(twoifLOD, twoifCsvFileName)
 
+    #
+    # D.7) Write out test data to CSV file
+    #
+
+    if inOptions.writeTestDataToFile:
+      testDataFileName = inOptions.writeTestDataToFile
+      print("\nWriting out gathered test data to file "+testDataFileName+" ...")
+      testDataLOD = twipLOD + twimLOD + twifLOD + twinrLOD
+      # ToDo: Add the first inOptions.limitTableRows elements of twiofLOD and twoinrLOD ...
+      CDQAR.pprintPythonDataToFile(testDataLOD, testDataFileName)
+
   except Exception:
     # Traceback!
     print("")