From d9134b4b45b0b34a331ce5179f5aa589db636e42 Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 27 Feb 2020 10:24:42 -0700 Subject: [PATCH] cdash_analyze_and_report.py: Add option --write-test-data-to-file (#304) --- .../cdash_analyze_and_report_UnitTests.py | 76 ++++++++++++++++++- .../ci_support/cdash_analyze_and_report.py | 44 +++++++---- 2 files changed, 105 insertions(+), 15 deletions(-) diff --git a/test/ci_support/cdash_analyze_and_report_UnitTests.py b/test/ci_support/cdash_analyze_and_report_UnitTests.py index 1052fe887..6ebfa0d6e 100644 --- a/test/ci_support/cdash_analyze_and_report_UnitTests.py +++ b/test/ci_support/cdash_analyze_and_report_UnitTests.py @@ -559,7 +559,8 @@ def test_twoif_10_twoinr2_twif_8_twinr_1(self): testCaseName, [ "--limit-test-history-days=30", # Test that you can set this as int - "--write-failing-tests-without-issue-trackers-to-file=twoif.csv" + "--write-failing-tests-without-issue-trackers-to-file=twoif.csv", + "--write-test-data-to-file=test_data.json" ], 1, "FAILED (twoif=10, twoinr=2, twif=8, twinr=1): ProjectName Nightly Builds on 2018-10-28", @@ -709,6 +710,34 @@ def test_twoif_10_twoinr2_twif_8_twinr_1(self): # right number of tests are being written and the first and last are # correct. + # Read the written file 'test_data.json' and verify that it is correct + testDataLOD = eval(open(testOutputDir+"/test_data.json", 'r').read()) + self.assertEqual(len(testDataLOD), 9) + # Make sure an entry from 'twif' exists! + testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6', + 'Trilinos-atdm-cee-rhel6-clang-opt-serial', + 'MueLu_UnitTestsBlockedEpetra_MPI_1') + testDict = testDataLOD[testIdx] + self.assertEqual(testDict['site'], 'cee-rhel6') + self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-clang-opt-serial') + self.assertEqual(testDict['testname'], 'MueLu_UnitTestsBlockedEpetra_MPI_1') + self.assertEqual(testDict['status'], 'Failed') + self.assertEqual(testDict['details'], 'Completed (Failed)\n') + self.assertEqual(testDict['test_history_list'][0]['buildstarttime'], + '2018-10-28T06:10:33 UTC') + # Make sure an entry from 'twinr' exists! + testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6', + 'Trilinos-atdm-cee-rhel6-clang-opt-serial', + 'Teko_ModALPreconditioner_MPI_1') + testDict = testDataLOD[testIdx] + self.assertEqual(testDict['site'], 'cee-rhel6') + self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-clang-opt-serial') + self.assertEqual(testDict['testname'], 'Teko_ModALPreconditioner_MPI_1') + self.assertEqual(testDict['status'], 'Not Run') + self.assertEqual(testDict['details'], 'Required Files Missing') + self.assertEqual(testDict['test_history_list'][0]['buildstarttime'], + '2018-10-28T06:10:33 UTC') + # Test with some duplicate tests from CDash query (this happens in real life # sometimes!) @@ -1355,7 +1384,9 @@ def test_twip_twim(self): cdash_analyze_and_report_run_case( self, testCaseName, - ["--limit-test-history-days=30"], # Test that you can set this as int + [ "--limit-test-history-days=30", # Test that you can set this as int + "--write-test-data-to-file=test_data.json", + ], 1, "FAILED (bm=1, twoif=12, twip=2, twim=2, twif=5):"+\ " ProjectName Nightly Builds on 2018-10-28", @@ -1473,6 +1504,47 @@ def test_twip_twim(self): #debugPrint=True, ) + # Read the written file 'test_data.json' and verify that it is correct + testDataLOD = eval(open(testOutputDir+"/test_data.json", 'r').read()) + self.assertEqual(len(testDataLOD), 9) + # Make sure an entry from 'twip' exists! + testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6', + 'Trilinos-atdm-cee-rhel6-clang-opt-serial', + 'MueLu_UnitTestsBlockedEpetra_MPI_1') + testDict = testDataLOD[testIdx] + self.assertEqual(testDict['site'], 'cee-rhel6') + self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-clang-opt-serial') + self.assertEqual(testDict['testname'], 'MueLu_UnitTestsBlockedEpetra_MPI_1') + self.assertEqual(testDict['status'], 'Passed') + self.assertEqual(testDict['details'], 'Completed (Passed)') + self.assertEqual(testDict['test_history_list'][0]['buildstarttime'], + '2018-10-28T06:10:33 UTC') + # Make sure an entry from 'twim' exists! + testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6', + 'Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial', + 'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2') + testDict = testDataLOD[testIdx] + self.assertEqual(testDict['site'], 'cee-rhel6') + self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-gnu-4.9.3-opt-serial') + self.assertEqual(testDict['testname'], 'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2') + self.assertEqual(testDict['status'], 'Missing') + self.assertEqual(testDict['details'], 'Missing') + self.assertEqual(testDict['test_history_list'][0]['buildstarttime'], + '2018-10-26T12:00:00 UTC') + # Make sure an entry from 'twif' exists! + testIdx = getIdxOfTestInTestLOD(testDataLOD, 'cee-rhel6', + 'Trilinos-atdm-cee-rhel6-clang-opt-serial', + 'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2') + testDict = testDataLOD[testIdx] + self.assertEqual(testDict['site'], 'cee-rhel6') + self.assertEqual(testDict['buildName'], 'Trilinos-atdm-cee-rhel6-clang-opt-serial') + self.assertEqual(testDict['testname'], 'PanzerAdaptersIOSS_tIOSSConnManager2_MPI_2') + self.assertEqual(testDict['status'], 'Failed') + self.assertEqual(testDict['details'], 'Completed (Failed)\n') + self.assertEqual(testDict['test_history_list'][0]['buildstarttime'], + '2018-10-28T06:10:33 UTC') + + # # Run the unit tests! # diff --git a/tribits/ci_support/cdash_analyze_and_report.py b/tribits/ci_support/cdash_analyze_and_report.py index 00a74ba44..cf2fd75da 100755 --- a/tribits/ci_support/cdash_analyze_and_report.py +++ b/tribits/ci_support/cdash_analyze_and_report.py @@ -193,15 +193,15 @@ def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""): "--require-test-history-match-nonpassing-tests", "requireTestHistoryMatchNonpassingTestsStr", ("on", "off"), 0, - "Require that the status for each tracked test listed in the tests with issue"+\ - " trackers CSV file match the status of that test returned from the test history"+\ - " returned from CDash. In general, these should match up but these may not if extra"+\ - " filter criteria has been added to the list on nonpassing tests in the"+\ - " --cdash-nonpassed-tests-filters= set of filters (such as to filter out"+\ - " a large number of random system failures). In this case, an error will be returned"+\ - " by default and the script will crash. But this can be relaxed by setting this to 'off'"+\ - " which will result in these tracked tests being listed in the 'twim' table but typically"+\ - " with status 'Failed'.", + "Require that the status for each tracked test listed in the tests with issue"\ + +" trackers CSV file match the status of that test returned from the test history"\ + +" returned from CDash. In general, these should match up but these may not if extra"\ + +" filter criteria has been added to the list on nonpassing tests in the"\ + +" --cdash-nonpassed-tests-filters= set of filters (such as to filter out"\ + +" a large number of random system failures). In this case, an error will be"\ + +" returned by default and the script will crash. But this can be relaxed by"\ + +" setting this to 'off' which will result in these tracked tests being listed in"\ + +" the 'twim' table but typically with status 'Failed'.", clp ) addOptionParserChoiceOption( @@ -214,10 +214,19 @@ def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""): clp.add_option( "--write-failing-tests-without-issue-trackers-to-file", dest="writeFailingTestsWithoutIssueTrackersToFile", type="string", default="", - help="Write CSV file with a list of tets with issue trackers failed 'twif'." \ + help="Write a CSV file with a list of tets with issue trackers failed 'twif'." \ +" This is to make it easy to add new entires to the file read by" \ +" the option --tests-with-issue-trackers-file=. [default = '']" ) + clp.add_option( + "--write-test-data-to-file", + dest="writeTestDataToFile", type="string", default="", + help="Write pretty-printed Python list of dictionaries for tests with" \ + +" with issue trackers. This includes the history of the tests for" \ + +" --limit-test-history-days= of history. This contains all of the" \ + +" information that appears in the generated summary tables for tests with" \ + +" associated issue trackers. [default = '']" ) + clp.add_option( "--write-email-to-file", dest="writeEmailToFile", type="string", default="", help="Write the body of the HTML email to this file. [default = '']" ) @@ -298,6 +307,7 @@ def fwdCmndLineOptions(inOptions, lt=""): " --require-test-history-match-nonpassing-tests='"+io.requireTestHistoryMatchNonpassingTestsStr+"'"+lt+\ " --print-details='"+io.printDetailsStr+"'"+lt+\ " --write-failing-tests-without-issue-trackers-to-file='"+io.writeFailingTestsWithoutIssueTrackersToFile+"'"+lt+\ + " --write-test-data-to-file='"+io.writeTestDataToFile+"'"+lt+\ " --write-email-to-file='"+io.writeEmailToFile+"'"+lt+\ " --email-from-address='"+io.emailFromAddress+"'"+lt+\ " --send-email-to='"+io.sendEmailTo+"'"+lt @@ -959,13 +969,21 @@ def testSetGetDataAnalyzeReport( self, # if inOptions.writeFailingTestsWithoutIssueTrackersToFile: - twoifCsvFileName = inOptions.writeFailingTestsWithoutIssueTrackersToFile - print("\nWriting list of 'twiof' to file "+twoifCsvFileName+" ...") - CDQAR.writeTestsLODToCsvFile(twoifLOD, twoifCsvFileName) + # + # D.7) Write out test data to CSV file + # + + if inOptions.writeTestDataToFile: + testDataFileName = inOptions.writeTestDataToFile + print("\nWriting out gathered test data to file "+testDataFileName+" ...") + testDataLOD = twipLOD + twimLOD + twifLOD + twinrLOD + # ToDo: Add the first inOptions.limitTableRows elements of twiofLOD and twoinrLOD ... + CDQAR.pprintPythonDataToFile(testDataLOD, testDataFileName) + except Exception: # Traceback! print("")