Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added nano relval comparison #1867

Merged
merged 9 commits into from
Oct 23, 2022
Merged

Added nano relval comparison #1867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
848be17
run nano comparison if requested and IB supports it
smuzaffar Oct 19, 2022
5fe53b1
added few more nano relval
smuzaffar Oct 19, 2022
e8e73e9
typo
smuzaffar Oct 19, 2022
6fa35e1
Update run-pr-comparisons
smuzaffar Oct 19, 2022
430f27c
fix nano comparison url; add size comparison
smuzaffar Oct 19, 2022
61d2157
proper formatting for nano results
smuzaffar Oct 19, 2022
9811909
for summary format
smuzaffar Oct 19, 2022
1057b6a
Update run-pr-comparisons
smuzaffar Oct 20, 2022
7559370
add all nano workflows
smuzaffar Oct 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmssw-pr-test-config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ PR_TEST_MATRIX_EXTRAS=1306.0,101.0,9.0,25202.0,10224.0,250202.181
PR_TEST_MATRIX_EXTRAS_GPU=11634.506,11634.512,11634.522
PR_TEST_MATRIX_EXTRAS_PROFILING=39634.21,21034.21,11834.21,136.889
PR_TEST_MATRIX_EXTRAS_HIGH_STATS=35034.0
PR_TEST_MATRIX_EXTRAS_NANO=2500.0
PR_TEST_MATRIX_EXTRAS_NANO=2500.0,2500.001,2500.1,2500.101,2500.11,2500.12,2500.2,2500.21,2500.3,2500.301,2500.302,2500.31,2500.311,2500.312,2500.32,2500.321,2500.322,2500.33,2500.331,2500.332,2500.4,2500.401,2500.5,2500.501,2500.51,2500.511,2500.6,2500.601
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no way to have this being a wild-card 2500.* ?
the nano workflows are done dynamically ; maybe I need to make them static

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no wild card does not work. It think static wf are better which can easily match entires in matrix_RE.txt otherwise we have to keep changing matrix_RE.txt for every change in workflow number

55 changes: 55 additions & 0 deletions pr_testing/run-pr-comparisons
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,38 @@
#!/bin/sh -ex
function process_nano(){
mkdir -p $1
pushd $1
for r in $(ls -d $2/*/step2.root 2>/dev/null); do
WF_DIR=$(dirname $r)
[ -e ${WF_DIR}/JobReport2.xml ] || continue
while [ $(jobs -p | wc -l) -ge ${NUM_PROC} ] ; do sleep 0.1 ; done
WF=$(basename ${WF_DIR} | cut -d_ -f1)
${NANO_TEST_DIR}/inspectNanoFile.py $r -j ${WF}-size.json &
while [ $(jobs -p | wc -l) -ge ${NUM_PROC} ] ; do sleep 0.1 ; done
${NANO_TEST_DIR}/get_timing_from_jobreport.py ${WF_DIR}/JobReport2.xml ${WF}-timing.json &
done
popd
}

function nano_summary(){
pushd $WORKSPACE/results/nano
WFS=""
rm -f NANO_report.md
for wf in $(ls -d *-size.json | sed 's|-size.json||') ; do
ok_wf=true
for f in ${wf}-timing.json ref/${wf}-size.json ref/${wf}-size.json ; do
if [ ! -e $f ] ; then echo " - Nano ERROR: Missing $f" >> NANO_report.md ; ok_wf=false ; fi
done
$ok_wf && WFS="${WFS} ${wf}"
done
if [ "${WFS}" != "" ] ; then
${NANO_TEST_DIR}/compare_sizes_json.py -H -f md --base "{}-size.json,{}-timing.json" --ref "./ref" ${WFS} >> NANO_report.md
else
echo " - No valid nano workflows found" >> NANO_report.md
fi
popd
}

source $(dirname $0)/setup-pr-test-env.sh
TEST_FLAVOR_STR=""
UC_TEST_FLAVOR=$(echo ${TEST_FLAVOR} | tr '[a-z]' '[A-Z]')
Expand Down Expand Up @@ -54,12 +88,21 @@ if $DQM_COMPARISON_TEST ; then
fi
fi

NANO_TEST=false
if [ "${UC_TEST_FLAVOR}" = "NANO" ] ; then
NANO_TEST_DIR=${CMSSW_BASE}/src/PhysicsTools/NanoAOD/test
[ -e ${NANO_TEST_DIR} ] || NANO_TEST_DIR=${CMSSW_RELEASE_BASE}/src/PhysicsTools/NanoAOD/test
[ -x ${NANO_TEST_DIR}/compare_sizes_json.py ] && NANO_TEST=true
fi

cd $WORKSPACE/results
echo "Downloading Ref: `date`"
get_jenkins_artifacts ${BASELINE_DIR}/ $WORKSPACE/data/$COMPARISON_RELEASE/ || true
echo "Downloading PR: `date`"
${NANO_TEST} && process_nano nano/ref $WORKSPACE/data/$COMPARISON_RELEASE
get_jenkins_artifacts ${PR_BASELINE_DIR}/ $WORKSPACE/data/PR-${PR_NUM}/
echo "Done Downloading `date`"
${NANO_TEST} && process_nano nano $WORKSPACE/data/PR-${PR_NUM}

cat $WORKSPACE/data/$COMPARISON_RELEASE/wf_mapping.*txt | sort | uniq > $WORKSPACE/$MAPPING_FILE
cat $WORKSPACE/data/$COMPARISON_RELEASE/wf_errors.*txt | sort | uniq > $WORKSPACE/$ERRORS_FILE
Expand Down Expand Up @@ -337,6 +380,9 @@ if [ "X$RUN_TR_COMP" = Xtrue ]; then
fi
#-----------------------------------

#Nano Workflows summary
${NANO_TEST} && nano_summary

jobs
wait || true

Expand Down Expand Up @@ -393,6 +439,9 @@ for x in $(find results/JR-comparison -maxdepth 2 -name 'command*' -o -name '*.l
mv results/JR-comparison/$f $d/
done

#nano
[ -d $WORKSPACE/results/nano ] && mv $WORKSPACE/results/nano upload/

#files
if [ -e results/files ] ; then
mkdir -p upload/files
Expand Down Expand Up @@ -420,6 +469,12 @@ ${CMS_BOT_DIR}/report-pull-request-results "COMPARISON_READY" --report-url ${PR_
if [ "${TEST_FLAVOR}" != "" ] ; then
sed -i -e "s|## Comparison Summary|## ${UC_TEST_FLAVOR} Comparison Summary|" $WORKSPACE/testsResults/20-comparison-report.res
mv $WORKSPACE/testsResults/20-comparison-report.res $WORKSPACE/testsResults/20-${TEST_FLAVOR}-comparison-report.res
if [ -e $WORKSPACE/upload/nano/NANO_report.md ] ; then
echo "**Nano size comparison Summary**:" >> $WORKSPACE/testsResults/21-${TEST_FLAVOR}-comparison-report.res
echo "" >> $WORKSPACE/testsResults/21-${TEST_FLAVOR}-comparison-report.res
cat $WORKSPACE/upload/nano/NANO_report.md >> $WORKSPACE/testsResults/21-${TEST_FLAVOR}-comparison-report.res
echo "" >> $WORKSPACE/testsResults/21-${TEST_FLAVOR}-comparison-report.res
fi
fi

if [ "$DRY_RUN" = "" ] ; then
Expand Down
7 changes: 7 additions & 0 deletions templates/js/renderPRTests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ getResultRow = function( resultsDict , resultsKey ){
column2Label = 'See High Stats Comparison Results'
linkURL = BASE_COMPARISONS_HIGH_STATS_URL + resultsDict[ BASE_IB_KEY ] + '+' + resultsDict[ PR_NUMBER_KEY ] + '/' + testResult + '/'

}else if( resultsKey == COMPARISON_NANO ){

column2Label = 'See Nano Comparison Results'
linkURL = BASE_COMPARISONS_NANO_URL + resultsDict[ BASE_IB_KEY ] + '+' + resultsDict[ PR_NUMBER_KEY ] + '/' + testResult + '/'

}


Expand Down Expand Up @@ -212,6 +217,7 @@ PR_NUMBERS_KEY = 'PR_NUMBERS';
COMPARISON_KEY = 'COMPARISON';
COMPARISON_GPU_KEY = 'COMPARISON_GPU';
COMPARISON_HIGH_STATS = 'COMPARISON_HIGH_STATS';
COMPARISON_NANO = 'COMPARISON_NANO';
BASE_IB_URL = '/SDT/html/showIB.html';
COMPARISON_IB_KEY = "COMPARISON_IB"
IB_PAGE_V2 = '/SDT/html/cmssdt-ib/#/ib/';
Expand All @@ -224,4 +230,5 @@ LOCATIONS = {};
BASE_COMPARISONS_URL = '/SDT/@JENKINS_PREFIX@-artifacts/baseLineComparisons/';
BASE_COMPARISONS_GPU_URL = '/SDT/@JENKINS_PREFIX@-artifacts/baseLineComparisonsGPU/';
BASE_COMPARISONS_HIGH_STATS_URL = '/SDT/@JENKINS_PREFIX@-artifacts/baseLineComparisonsHIGH_STATS/'
BASE_COMPARISONS_NANO_URL = '/SDT/@JENKINS_PREFIX@-artifacts/baseLineComparisonsNANO/'