From 848be17ba2eacb195b7bb2a829021dce9c05f446 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Wed, 19 Oct 2022 11:40:43 +0200 Subject: [PATCH 1/9] run nano comparison if requested and IB supports it --- pr_testing/run-pr-comparisons | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/pr_testing/run-pr-comparisons b/pr_testing/run-pr-comparisons index a59eec01fe29..bb3b5875a7a5 100755 --- a/pr_testing/run-pr-comparisons +++ b/pr_testing/run-pr-comparisons @@ -1,4 +1,38 @@ #!/bin/sh -ex +function process_nano(){ + mkdir -p $1 + pushd $1 + for r in $(ls -d $2/*/step2.root 2>/dev/null); do + WF_DIR=$(dirname $r) + [ -e ${WF_DIR}/JobReport2.xml ] || continue + while [ $(jobs -p | wc -l) -ge ${NUM_PROC} ] ; do sleep 0.1 ; done + WF=$(basename ${WF_DIR} | cut -d_ -f1) + ${NANO_TEST_DIR}/inspectNanoFile.py $r -j ${WF}-size.json & + while [ $(jobs -p | wc -l) -ge ${NUM_PROC} ] ; do sleep 0.1 ; done + ${NANO_TEST_DIR}/get_timing_from_jobreport.py ${WF_DIR}/JobReport2.xml ${WF}-timing.json & + done + popd +} + +function nano_summary(){ + pushd nano + WFS="" + rm -f NANO_report.md + for wf in $(ls -d *-size.json | sed 's|-size.json||') ; do + ok_wf=true + for f in ${wf}-timing.json ref/${wf}-size.json ref/${wf}-size.json ; do + if [ ! -e $f ] ; then echo "- Nano ERROR: Missing $f" >> NANO_report.md ; ok_wf=false ; fi + done + $ok_wf && WFS="${WFS} ${wf}" + done + if [ "${WFS}" != "" ] ; then + ${NANO_TEST_DIR}/compare_sizes_json.py -H -f md --base "{}-size.json,{}-timing.json" --ref "./ref" ${WFS} >> NANO_report.md + else + echo "- No valid nano workflows found" >> NANO_report.md + fi + popd +} + source $(dirname $0)/setup-pr-test-env.sh TEST_FLAVOR_STR="" UC_TEST_FLAVOR=$(echo ${TEST_FLAVOR} | tr '[a-z]' '[A-Z]') @@ -54,12 +88,21 @@ if $DQM_COMPARISON_TEST ; then fi fi +NANO_TEST=false +if [ "${UC_TEST_FLAVOR}" = "nano" ] ; then + NANO_TEST_DIR=${CMSSW_BASE}/src/PhysicsTools/NanoAOD/test + [ -e ${NANO_TEST_DIR} ] || NANO_TEST_DIR=${CMSSW_RELEASE_BASE}/src/PhysicsTools/NanoAOD/test + [ -x ${NANO_TEST_DIR}/compare_sizes_json.py ] && NANO_TEST=true +fi + cd $WORKSPACE/results echo "Downloading Ref: `date`" get_jenkins_artifacts ${BASELINE_DIR}/ $WORKSPACE/data/$COMPARISON_RELEASE/ || true echo "Downloading PR: `date`" +${NANO_TEST} && process_nano nano/ref $WORKSPACE/data/$COMPARISON_RELEASE get_jenkins_artifacts ${PR_BASELINE_DIR}/ $WORKSPACE/data/PR-${PR_NUM}/ echo "Done Downloading `date`" +${NANO_TEST} && process_nano nano $WORKSPACE/data/PR-${PR_NUM} cat $WORKSPACE/data/$COMPARISON_RELEASE/wf_mapping.*txt | sort | uniq > $WORKSPACE/$MAPPING_FILE cat $WORKSPACE/data/$COMPARISON_RELEASE/wf_errors.*txt | sort | uniq > $WORKSPACE/$ERRORS_FILE @@ -337,6 +380,9 @@ if [ "X$RUN_TR_COMP" = Xtrue ]; then fi #----------------------------------- +#Nano Workflows summary +${NANO_TEST} && nano_summary + jobs wait || true @@ -393,6 +439,9 @@ for x in $(find results/JR-comparison -maxdepth 2 -name 'command*' -o -name '*.l mv results/JR-comparison/$f $d/ done +#nano +[ -d $WORKSPACE/results/nano ] && mv $WORKSPACE/results/nano upload/ + #files if [ -e results/files ] ; then mkdir -p upload/files From 5fe53b1f77623faa5cb5f8f33daabbbaf0a378d8 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Wed, 19 Oct 2022 11:43:16 +0200 Subject: [PATCH 2/9] added few more nano relval --- cmssw-pr-test-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmssw-pr-test-config b/cmssw-pr-test-config index 69e88555f721..75f8f8f84692 100644 --- a/cmssw-pr-test-config +++ b/cmssw-pr-test-config @@ -2,4 +2,4 @@ PR_TEST_MATRIX_EXTRAS=1306.0,101.0,9.0,25202.0,10224.0,250202.181 PR_TEST_MATRIX_EXTRAS_GPU=11634.506,11634.512,11634.522 PR_TEST_MATRIX_EXTRAS_PROFILING=39634.21,21034.21,11834.21,136.889 PR_TEST_MATRIX_EXTRAS_HIGH_STATS=35034.0 -PR_TEST_MATRIX_EXTRAS_NANO=2500.0 +PR_TEST_MATRIX_EXTRAS_NANO=2500.0,2500.001,2500.1,2500.101,2500.11 From e8e73e98b180cf324197eec6a831c1cd20d6d527 Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Wed, 19 Oct 2022 18:16:37 +0200 Subject: [PATCH 3/9] typo --- pr_testing/run-pr-comparisons | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_testing/run-pr-comparisons b/pr_testing/run-pr-comparisons index bb3b5875a7a5..55adb83e4817 100755 --- a/pr_testing/run-pr-comparisons +++ b/pr_testing/run-pr-comparisons @@ -89,7 +89,7 @@ if $DQM_COMPARISON_TEST ; then fi NANO_TEST=false -if [ "${UC_TEST_FLAVOR}" = "nano" ] ; then +if [ "${UC_TEST_FLAVOR}" = "NANO" ] ; then NANO_TEST_DIR=${CMSSW_BASE}/src/PhysicsTools/NanoAOD/test [ -e ${NANO_TEST_DIR} ] || NANO_TEST_DIR=${CMSSW_RELEASE_BASE}/src/PhysicsTools/NanoAOD/test [ -x ${NANO_TEST_DIR}/compare_sizes_json.py ] && NANO_TEST=true From 6fa35e1636c6adad8fa47586c5f55d366e02f945 Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Wed, 19 Oct 2022 18:56:22 +0200 Subject: [PATCH 4/9] Update run-pr-comparisons --- pr_testing/run-pr-comparisons | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_testing/run-pr-comparisons b/pr_testing/run-pr-comparisons index 55adb83e4817..b55599b9811e 100755 --- a/pr_testing/run-pr-comparisons +++ b/pr_testing/run-pr-comparisons @@ -15,7 +15,7 @@ function process_nano(){ } function nano_summary(){ - pushd nano + pushd $WORKSPACE/results/nano WFS="" rm -f NANO_report.md for wf in $(ls -d *-size.json | sed 's|-size.json||') ; do From 430f27cbf1104da65c61642b6112eb4b90662871 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Wed, 19 Oct 2022 20:18:15 +0200 Subject: [PATCH 5/9] fix nano comparison url; add size comparison --- pr_testing/run-pr-comparisons | 4 ++++ templates/js/renderPRTests.js | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/pr_testing/run-pr-comparisons b/pr_testing/run-pr-comparisons index b55599b9811e..68329e1a76fe 100755 --- a/pr_testing/run-pr-comparisons +++ b/pr_testing/run-pr-comparisons @@ -469,6 +469,10 @@ ${CMS_BOT_DIR}/report-pull-request-results "COMPARISON_READY" --report-url ${PR_ if [ "${TEST_FLAVOR}" != "" ] ; then sed -i -e "s|## Comparison Summary|## ${UC_TEST_FLAVOR} Comparison Summary|" $WORKSPACE/testsResults/20-comparison-report.res mv $WORKSPACE/testsResults/20-comparison-report.res $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res + if [ -e $WORKSPACE/upload/NANO_report.md ] ; then + echo "- Nano Size comparison results" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res + cat $WORKSPACE/upload/NANO_report.md >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res + fi fi if [ "$DRY_RUN" = "" ] ; then diff --git a/templates/js/renderPRTests.js b/templates/js/renderPRTests.js index 407d65efa795..d743062b502e 100644 --- a/templates/js/renderPRTests.js +++ b/templates/js/renderPRTests.js @@ -45,6 +45,11 @@ getResultRow = function( resultsDict , resultsKey ){ column2Label = 'See High Stats Comparison Results' linkURL = BASE_COMPARISONS_HIGH_STATS_URL + resultsDict[ BASE_IB_KEY ] + '+' + resultsDict[ PR_NUMBER_KEY ] + '/' + testResult + '/' + }else if( resultsKey == COMPARISON_NANO ){ + + column2Label = 'See Nano Comparison Results' + linkURL = BASE_COMPARISONS_NANO_URL + resultsDict[ BASE_IB_KEY ] + '+' + resultsDict[ PR_NUMBER_KEY ] + '/' + testResult + '/' + } @@ -212,6 +217,7 @@ PR_NUMBERS_KEY = 'PR_NUMBERS'; COMPARISON_KEY = 'COMPARISON'; COMPARISON_GPU_KEY = 'COMPARISON_GPU'; COMPARISON_HIGH_STATS = 'COMPARISON_HIGH_STATS'; +COMPARISON_NANO = 'COMPARISON_NANO'; BASE_IB_URL = '/SDT/html/showIB.html'; COMPARISON_IB_KEY = "COMPARISON_IB" IB_PAGE_V2 = '/SDT/html/cmssdt-ib/#/ib/'; @@ -224,4 +230,5 @@ LOCATIONS = {}; BASE_COMPARISONS_URL = '/SDT/@JENKINS_PREFIX@-artifacts/baseLineComparisons/'; BASE_COMPARISONS_GPU_URL = '/SDT/@JENKINS_PREFIX@-artifacts/baseLineComparisonsGPU/'; BASE_COMPARISONS_HIGH_STATS_URL = '/SDT/@JENKINS_PREFIX@-artifacts/baseLineComparisonsHIGH_STATS/' +BASE_COMPARISONS_NANO_URL = '/SDT/@JENKINS_PREFIX@-artifacts/baseLineComparisonsNANO/' From 61d21574de36262b25d6c78f0e3e0b845de8f4b4 Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Wed, 19 Oct 2022 23:52:28 +0200 Subject: [PATCH 6/9] proper formatting for nano results --- pr_testing/run-pr-comparisons | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pr_testing/run-pr-comparisons b/pr_testing/run-pr-comparisons index 68329e1a76fe..f5e456256e8c 100755 --- a/pr_testing/run-pr-comparisons +++ b/pr_testing/run-pr-comparisons @@ -21,14 +21,14 @@ function nano_summary(){ for wf in $(ls -d *-size.json | sed 's|-size.json||') ; do ok_wf=true for f in ${wf}-timing.json ref/${wf}-size.json ref/${wf}-size.json ; do - if [ ! -e $f ] ; then echo "- Nano ERROR: Missing $f" >> NANO_report.md ; ok_wf=false ; fi + if [ ! -e $f ] ; then echo " - Nano ERROR: Missing $f" >> NANO_report.md ; ok_wf=false ; fi done $ok_wf && WFS="${WFS} ${wf}" done if [ "${WFS}" != "" ] ; then ${NANO_TEST_DIR}/compare_sizes_json.py -H -f md --base "{}-size.json,{}-timing.json" --ref "./ref" ${WFS} >> NANO_report.md else - echo "- No valid nano workflows found" >> NANO_report.md + echo " - No valid nano workflows found" >> NANO_report.md fi popd } @@ -469,9 +469,11 @@ ${CMS_BOT_DIR}/report-pull-request-results "COMPARISON_READY" --report-url ${PR_ if [ "${TEST_FLAVOR}" != "" ] ; then sed -i -e "s|## Comparison Summary|## ${UC_TEST_FLAVOR} Comparison Summary|" $WORKSPACE/testsResults/20-comparison-report.res mv $WORKSPACE/testsResults/20-comparison-report.res $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res - if [ -e $WORKSPACE/upload/NANO_report.md ] ; then + if [ -e $WORKSPACE/upload/nano/NANO_report.md ] ; then echo "- Nano Size comparison results" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res - cat $WORKSPACE/upload/NANO_report.md >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res + echo "" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res + cat $WORKSPACE/upload/nano/NANO_report.md >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res + echo "" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res fi fi From 98119095b5514f66a04109349cd18658689f47ad Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Thu, 20 Oct 2022 00:06:39 +0200 Subject: [PATCH 7/9] for summary format --- pr_testing/run-pr-comparisons | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pr_testing/run-pr-comparisons b/pr_testing/run-pr-comparisons index f5e456256e8c..4e397ca249f7 100755 --- a/pr_testing/run-pr-comparisons +++ b/pr_testing/run-pr-comparisons @@ -21,14 +21,14 @@ function nano_summary(){ for wf in $(ls -d *-size.json | sed 's|-size.json||') ; do ok_wf=true for f in ${wf}-timing.json ref/${wf}-size.json ref/${wf}-size.json ; do - if [ ! -e $f ] ; then echo " - Nano ERROR: Missing $f" >> NANO_report.md ; ok_wf=false ; fi + if [ ! -e $f ] ; then echo " - Nano ERROR: Missing $f" >> NANO_report.md ; ok_wf=false ; fi done $ok_wf && WFS="${WFS} ${wf}" done if [ "${WFS}" != "" ] ; then ${NANO_TEST_DIR}/compare_sizes_json.py -H -f md --base "{}-size.json,{}-timing.json" --ref "./ref" ${WFS} >> NANO_report.md else - echo " - No valid nano workflows found" >> NANO_report.md + echo " - No valid nano workflows found" >> NANO_report.md fi popd } @@ -470,7 +470,7 @@ if [ "${TEST_FLAVOR}" != "" ] ; then sed -i -e "s|## Comparison Summary|## ${UC_TEST_FLAVOR} Comparison Summary|" $WORKSPACE/testsResults/20-comparison-report.res mv $WORKSPACE/testsResults/20-comparison-report.res $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res if [ -e $WORKSPACE/upload/nano/NANO_report.md ] ; then - echo "- Nano Size comparison results" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res + echo " - Nano Size comparison results" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res echo "" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res cat $WORKSPACE/upload/nano/NANO_report.md >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res echo "" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res From 1057b6a3ab715b53cf844715b2feb24d46375036 Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Thu, 20 Oct 2022 06:51:59 +0200 Subject: [PATCH 8/9] Update run-pr-comparisons --- pr_testing/run-pr-comparisons | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pr_testing/run-pr-comparisons b/pr_testing/run-pr-comparisons index 4e397ca249f7..600320612d47 100755 --- a/pr_testing/run-pr-comparisons +++ b/pr_testing/run-pr-comparisons @@ -21,14 +21,14 @@ function nano_summary(){ for wf in $(ls -d *-size.json | sed 's|-size.json||') ; do ok_wf=true for f in ${wf}-timing.json ref/${wf}-size.json ref/${wf}-size.json ; do - if [ ! -e $f ] ; then echo " - Nano ERROR: Missing $f" >> NANO_report.md ; ok_wf=false ; fi + if [ ! -e $f ] ; then echo " - Nano ERROR: Missing $f" >> NANO_report.md ; ok_wf=false ; fi done $ok_wf && WFS="${WFS} ${wf}" done if [ "${WFS}" != "" ] ; then ${NANO_TEST_DIR}/compare_sizes_json.py -H -f md --base "{}-size.json,{}-timing.json" --ref "./ref" ${WFS} >> NANO_report.md else - echo " - No valid nano workflows found" >> NANO_report.md + echo " - No valid nano workflows found" >> NANO_report.md fi popd } @@ -470,10 +470,10 @@ if [ "${TEST_FLAVOR}" != "" ] ; then sed -i -e "s|## Comparison Summary|## ${UC_TEST_FLAVOR} Comparison Summary|" $WORKSPACE/testsResults/20-comparison-report.res mv $WORKSPACE/testsResults/20-comparison-report.res $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res if [ -e $WORKSPACE/upload/nano/NANO_report.md ] ; then - echo " - Nano Size comparison results" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res - echo "" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res - cat $WORKSPACE/upload/nano/NANO_report.md >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res - echo "" >> $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res + echo "**Nano size comparison Summary**:" >> $WORKSPACE/testsResults/21-${TEST_FLAVOR}-comparison-report.res + echo "" >> $WORKSPACE/testsResults/21-${TEST_FLAVOR}-comparison-report.res + cat $WORKSPACE/upload/nano/NANO_report.md >> $WORKSPACE/testsResults/21-${TEST_FLAVOR}-comparison-report.res + echo "" >> $WORKSPACE/testsResults/21-${TEST_FLAVOR}-comparison-report.res fi fi From 7559370ee9f182cf205d1a3a214d099770f25df5 Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Thu, 20 Oct 2022 16:38:58 +0200 Subject: [PATCH 9/9] add all nano workflows --- cmssw-pr-test-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmssw-pr-test-config b/cmssw-pr-test-config index 75f8f8f84692..c4ae7f62fbe3 100644 --- a/cmssw-pr-test-config +++ b/cmssw-pr-test-config @@ -2,4 +2,4 @@ PR_TEST_MATRIX_EXTRAS=1306.0,101.0,9.0,25202.0,10224.0,250202.181 PR_TEST_MATRIX_EXTRAS_GPU=11634.506,11634.512,11634.522 PR_TEST_MATRIX_EXTRAS_PROFILING=39634.21,21034.21,11834.21,136.889 PR_TEST_MATRIX_EXTRAS_HIGH_STATS=35034.0 -PR_TEST_MATRIX_EXTRAS_NANO=2500.0,2500.001,2500.1,2500.101,2500.11 +PR_TEST_MATRIX_EXTRAS_NANO=2500.0,2500.001,2500.1,2500.101,2500.11,2500.12,2500.2,2500.21,2500.3,2500.301,2500.302,2500.31,2500.311,2500.312,2500.32,2500.321,2500.322,2500.33,2500.331,2500.332,2500.4,2500.401,2500.5,2500.501,2500.51,2500.511,2500.6,2500.601