-
Notifications
You must be signed in to change notification settings - Fork 246
/
test_multiple_prs.sh
executable file
·1512 lines (1408 loc) · 73.5 KB
/
test_multiple_prs.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash -ex
# This script will be called by Jenkins job 'ib-run-pr-tests'
# and
# 1) will merge multiple PRs for multiple repos
# 2) run tests and post result on github
# ---
#common function
function order_workflow_list(){
echo ${1} | tr ' ' '\n' | tr ',' '\n' | grep '^[0-9]\|^all$' | sort -n | uniq | tr '\n' ',' | sed 's|,*$||'
}
function get_pr_baseline_worklflow() {
order_workflow_list $($CMS_BOT_DIR/cmssw-pr-test-config $1),${2}
}
function get_compilation_warnings() {
grep -E ': warning:|: warning #[0-9]+-D:' $1 | grep -E "/$CMSSW_IB/src/|^\s*src/" | sed -E "s|: warning #[0-9]+-D:|: warning:|;s|.*/$CMSSW_IB/||"
}
function get_warnings_files(){
local logFile="$1"
local changedFiles="$2"
for i in $(cat $logFile | sed 's|^src/||;s|:.*||;s| ||g;s|[(].*||' | sort -u) ; do
[ $(grep "$i" "$changedFiles" | wc -l) -gt 0 ] && echo $i
done
}
function get_pr_relval_args() {
local WF_ARGS
local WF_LIST
local WF_LIST2
if $1 ; then
WF_LIST=$(get_pr_baseline_worklflow "$2")
[ "$WF_LIST" = "" ] || WF_LIST="-l $WF_LIST"
WF_LIST2=$(order_workflow_list "$(eval echo \${MATRIX_EXTRAS$2})")
if [ "${WF_LIST2}" = "" ] ; then
WF_ARGS="${WF_LIST}"
else
WF_ARGS="${WF_LIST};-l ${WF_LIST2} $(eval echo \${EXTRA_MATRIX_ARGS$2})"
fi
else
WF_LIST=$(get_pr_baseline_worklflow "$2" "$(eval echo \${MATRIX_EXTRAS$2})")
[ "$WF_LIST" = "" ] || WF_LIST="-l $WF_LIST"
WF_ARGS="${WF_LIST} $(eval echo \${EXTRA_MATRIX_ARGS$2})"
fi
echo "${WF_ARGS}"
}
# Function to extract filenames by headername and append to indirectly-changed-files.txt
function extract_filenames() {
local headername="$1"
local input_file="./etc/dependencies/usedby.out"
local output_file="$2"
# Extract lines starting with headername, split them, and append each filename to the temp file
grep "^$headername" "$input_file" | while read -r line; do
# Split the line into an array
IFS=' ' read -r -a array <<< "$line"
# Loop through each element after the first (which is the headername)
for filename in "${array[@]:1}"; do
echo "$filename" >> "$output_file"
done
done
}
# Function to get indirectly changed files
function process_changed_files() {
local directlyChangedFiles="$1"
local allChangedFiles="$2"
cat </dev/null >"$WORKSPACE/indirectly-changed-files.txt"
# Iterate over each line in $WORKSPACE/changed-files.txt
while IFS= read -r headername; do
# Call the function to extract files that use $headername and append them to $WORKSPACE/indirectly-changed-files
extract_filenames "$headername" "$WORKSPACE/indirectly-changed-files.txt"
done < "$directlyChangedFiles"
# Merge lists
sort -u "$directlyChangedFiles" $WORKSPACE/indirectly-changed-files.txt > "$allChangedFiles"
}
# Constants
echo LD_LIBRARY_PATH=${LD_LIBRARY_PATH} || true
ls ${LD_LIBRARY_PATH} || true
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" # Absolute path to script
CMS_BOT_DIR=$(dirname ${SCRIPTPATH}) # To get CMS_BOT dir path
export SCRAM_PREFIX_PATH=${CMS_BOT_DIR}/das-utils
source ${CMS_BOT_DIR}/cmsrep.sh
CACHED=${WORKSPACE}/CACHED # Where cached PR metada etc are kept
PR_TESTING_DIR=${CMS_BOT_DIR}/pr_testing
COMMON=${CMS_BOT_DIR}/common
CONFIG_MAP=$CMS_BOT_DIR/config.map
[ "${USE_IB_TAG}" != "true" ] && export USE_IB_TAG=false
[ "${EXTRA_RELVALS_TESTS}" = "" ] && EXTRA_RELVALS_TESTS="GPU THREADING HIGH_STATS NANO"
EXTRA_RELVALS_TESTS=$(echo ${EXTRA_RELVALS_TESTS} | tr ' ' '\n' | grep -v THREADING | tr '\n' ' ')
# ---
# doc: Input variable
# PULL_REQUESTS # "cms-sw/cmsdist#4488,cms-sw/cmsdist#4480,cms-sw/cmsdist#4479,cms-sw/root#116"
# RELEASE_FORMAT # CMSSW_10_4_X_2018-11-26-2300
# PULL_REQUEST # CMSSW PR number, should avoid
# CMSDIST_PR # CMSDIST PR number, should avoid
# ARCHITECTURE # architecture (ex. slc6_amd64_gcc700)
# and some others
export CMSSW_GIT_REFERENCE=/cvmfs/cms.cern.ch/cmssw.git.daily
source ${PR_TESTING_DIR}/_helper_functions.sh # general helper functions
source ${CMS_BOT_DIR}/jenkins-artifacts
source ${COMMON}/github_reports.sh
if [ -z ${ARCHITECTURE} ] ; then
ARCHITECTURE=$(echo ${CONFIG_LINE} | sed 's/^.*SCRAM_ARCH=//' | sed 's/;.*//' )
fi
export SCRAM_ARCH=${ARCHITECTURE}
JENKINS_PREFIX=$(echo "${JENKINS_URL}" | sed 's|/*$||;s|.*/||')
if [ "X${JENKINS_PREFIX}" = "X" ] ; then JENKINS_PREFIX="jenkins"; fi
export JENKINS_PREFIX
RESULTS_FILE=${RESULTS_DIR}.txt
PR_NUMBER=$(echo ${PULL_REQUEST} | sed 's|.*#||')
PR_REPO=$(echo ${PULL_REQUEST} | sed 's|#.*||')
PR_NUM=$(echo ${PULL_REQUEST} | md5sum | sed 's| .*||' | cut -c27-33)
UPLOAD_UNIQ_ID=PR-${PR_NUM}/${BUILD_NUMBER}
PR_RESULT_URL="https://cmssdt.cern.ch/SDT/${JENKINS_PREFIX}-artifacts/pull-request-integration/${UPLOAD_UNIQ_ID}"
NCPU=$(${COMMON}/get_cpu_number.sh)
if [[ $NODE_NAME == *"cms-cmpwg-0"* ]]; then
let NCPU=${NCPU}/2
fi
let NCPU2=${NCPU}*2
rm -rf ${RESULTS_DIR} ${RESULTS_FILE}
mkdir ${RESULTS_DIR}
TEST_RELVALS_INPUT=false
DO_COMPARISON=false
DO_MB_COMPARISON=false
DO_DAS_QUERY=false
DO_CRAB_TESTS=false
DO_HLT_P2_TIMING=false
DO_HLT_P2_INTEGRATION=false
[ "${UPLOAD_TO_PACKAGE_STORE}" != "" ] || UPLOAD_TO_PACKAGE_STORE=true
[ $(echo ${ARCHITECTURE} | grep "_amd64_" | wc -l) -gt 0 ] && DO_COMPARISON=true
[ $(echo ${RELEASE_FORMAT} | grep 'SAN_X' | wc -l) -gt 0 ] && DO_COMPARISON=false
BUILD_VERBOSE=true
if [ "${BUILD_VERBOSE}" = "true" ] ; then
BUILD_VERBOSE="-v"
else
BUILD_VERBOSE=""
fi
PRODUCTION_RELEASE=false
CMSSW_BRANCH=$(echo "${CONFIG_LINE}" | sed 's|.*RELEASE_BRANCH=||;s|;.*||')
CMSSW_DEVEL_BRANCH=$(cd $CMS_BOT_DIR; ${CMSBOT_PYTHON_CMD} -c 'from releases import CMSSW_DEVEL_BRANCH; print(CMSSW_DEVEL_BRANCH)')
CMSSW_DEVEL_REL=false
CMSSW_DEVEL_PROD_ARCH=false
if [ "${CMSSW_BRANCH}" = "master" ] ; then
CMSSW_BRANCH=${CMSSW_DEVEL_BRANCH}
CMSSW_DEVEL_REL=true
fi
if [ $(echo "${CONFIG_LINE}" | grep "PROD_ARCH=1" | wc -l) -gt 0 ] ; then
if [ $(echo "${CONFIG_LINE}" | grep "ADDITIONAL_TESTS=" | wc -l) -gt 0 ] ; then
PRODUCTION_RELEASE=true
if ${CMSSW_DEVEL_REL} ; then
DO_DAS_QUERY=true
TEST_RELVALS_INPUT=true
CMSSW_DEVEL_PROD_ARCH=true
fi
fi
fi
# ----------
# -- MAIN --
# ----------
echo_section "Variable setup"
# Seperate script use different flag in order not to comment back to Github
NO_POST=
DRY_RUN=
if [ "X$AUTO_POST_MESSAGE" != Xtrue ]; then
NO_POST='--no-post'
DRY_RUN='--dry-run'
fi
export NO_POST ; export DRY_RUN
export PYTHONPATH=$CMS_BOT_DIR:$PYTHONPATH
# If RELEASE_FORMAT is not set, use the CMSSW_DEVEL_BRANCH.
# if someone starts jenkins job without scheduler directly from Jenkins
IS_DEV_BRANCH=false
DEV_BRANCH=$(grep '^ *CMSSW_DEVEL_BRANCH *= *' $CMS_BOT_DIR/releases.py | sed 's| ||g;s|.*=||;s|"||g')
if [ -z ${RELEASE_FORMAT} ]; then
RELEASE_FORMAT=$DEV_BRANCH
fi
DISABLE_CMS_DEPRECATED=false
DISABLE_GPU_TESTS=true
if [ $(uname -m) != "aarch64" ] ; then
DISABLE_GPU_TESTS=false
fi
CMSSW_QUEUE=$(echo ${RELEASE_FORMAT} | sed 's/_X.*/_X/') # RELEASE_FORMAT - CMSSW_10_4_X_2018-11-26-2300
PULL_REQUESTS=$(echo ${PULL_REQUESTS} | tr ',' ' ' | sed 's/ */ /g' | sed 's/^ *//;s/ *$//' ) # to make consistent separation in list
UNIQ_REPOS=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | sed 's|#.*||g' | sort | uniq | tr '\n' ' ' ) # Repos without pull number
UNIQ_REPO_NAMES=$(echo ${UNIQ_REPOS} | tr ' ' '\n' | sed 's|.*/||' )
UNIQ_REPO_NAMES_WITH_COUNT=$(echo ${UNIQ_REPO_NAMES} | sort | uniq -c )
RPM_UPLOAD_REPO=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep -v '/cmssw#' | grep -v '/cms-bot#' | sort | uniq | md5sum | sed 's| .*||')
if [ $(echo $CMSSW_QUEUE | cut -d_ -f2) -lt 13 ] ; then
export DISABLE_CMS_DEPRECATED=true
export DISABLE_GPU_TESTS=true
fi
let WEEK_NUM=$(tail -1 $CMS_BOT_DIR/ib-weeks | sed 's|.*-||;s|^0*||')%2 || true
CMS_WEEKLY_REPO=cms.week${WEEK_NUM}
# this is to automount directories in cvmfs, otherwise they wont show up
ls /cvmfs/cms.cern.ch
ls /cvmfs/cms-ib.cern.ch || true
which scram 2>/dev/null || source /cvmfs/cms.cern.ch/cmsset_default.sh
# Put hashcodes of last commits to a file. Mostly used for commenting back
COMMIT=$(${CMSBOT_PYTHON_CMD} ${CMS_BOT_DIR}/process-pull-request.py -c -r ${PR_REPO} ${PR_NUMBER})
echo "${PULL_REQUEST}=${COMMIT}" > ${WORKSPACE}/prs_commits
cp ${WORKSPACE}/prs_commits ${WORKSPACE}/prs_commits.txt
mark_commit_status_all_prs '' 'pending' -u "${BUILD_URL}" -d 'Setting up build environment' --reset
PR_COMMIT_STATUS="optional"
if $REQUIRED_TEST ; then PR_COMMIT_STATUS="required" ; fi
mark_commit_status_all_prs "${PR_COMMIT_STATUS}" 'success' -d 'OK' -u "${BUILD_URL}"
echo -n "**Summary**: ${PR_RESULT_URL}/summary.html" > ${RESULTS_DIR}/09-report.res
CMSSW_VERSION=${RELEASE_FORMAT} $CMS_BOT_DIR/report-pull-request-results GET_BASE_MESSAGE --report-url ${PR_RESULT_URL} \
--commit ${COMMIT} --report-file ${RESULTS_DIR}/09-report.res ${REPORT_OPTS}
echo "**User test area**: For local testing, you can use \`/cvmfs/cms-ci.cern.ch/week${WEEK_NUM}/${PR_REPO}/${PR_NUMBER}/${BUILD_NUMBER}/install.sh\` to create a dev area with all the needed externals and cmssw changes." >> ${RESULTS_DIR}/09-report.res
echo "" >> ${RESULTS_DIR}/09-report.res
echo_section "Pull request checks"
# Check if same organization/repo PRs
if [ $(echo ${UNIQ_REPO_NAMES_WITH_COUNT} | grep -v '1 ' | wc -w ) -gt 0 ]; then
echo "ERROR: multiple PRs from different organisations but same repos: ${UNIQ_REPO_NAMES_WITH_COUNT}" > ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${BUILD_URL}" -d "Multiple PRs from different repositories"
exit 0
fi
# Filter PR for specific repo and then check if its PRs point to same base branch
for U_REPO in ${UNIQ_REPOS}; do
FILTERED_PRS=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep ${U_REPO} | tr '\n' ' ' )
MASTER_LIST=""
for PR in ${FILTERED_PRS}; do
MASTER_LIST="${MASTER_LIST} $(get_base_branch ${PR})"
done
UNIQ_MASTERS=$(echo ${MASTER_LIST} | tr ' ' '\n' | sort | uniq )
if [ -z ${UNIQ_MASTERS} ]; then continue ; fi
NUMBER_U_M=$(echo ${UNIQ_MASTERS} | wc -l )
if [ ! ${NUMBER_U_M} -eq 1 ]; then
echo "ERROR: PRs for repo ${U_REPO} wants to merge to different branches: ${UNIQ_MASTERS}" > ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${BUILD_URL}" -d "Invalid branch merge."
exit 0
fi
done
CMSDIST_PR=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep '/cmsdist#' | head -n 1) # get 1st one
CMSSW_PR=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep '/cmssw#' | head -n 1)
CMSDIST_TAG= # To make sure no one set to different value by accident.
if [ ! -z ${CMSDIST_PR} ] ; then
CMSDIST_TAG=$(get_base_branch "$CMSDIST_PR" )
fi
if [[ ! -z ${CMSSW_PR} ]] ; then
CMSSW_BR=$(get_base_branch $CMSSW_PR)
fi
if [ "${CONFIG_LINE}" == "" ] ; then exit 1 ; fi
export CMSDIST_TAG=$(echo ${CONFIG_LINE} | sed 's/^.*CMSDIST_TAG=//' | sed 's/;.*//')
COMP_QUEUE=
case $CMSSW_QUEUE in
CMSSW_9_4_MAOD_X*|CMSSW_9_4_AN_X* ) COMP_QUEUE=$CMSSW_QUEUE ;;
* ) COMP_QUEUE=$(echo $CMSSW_QUEUE | cut -d_ -f1-3)_X;;
esac
if [ "X$DEV_BRANCH" = "X$COMP_QUEUE" ] ; then IS_DEV_BRANCH=true ; fi
CMSSW_IB="$RELEASE_FORMAT" # We are getting CMSSW_IB, so that we wont rebuild all the software
[ "$COMPARISON_ARCH" = "" ] && COMPARISON_ARCH=$(cat $CONFIG_MAP | grep "RELEASE_QUEUE=$COMP_QUEUE;" | grep -v "DISABLED=1" | grep 'PROD_ARCH=1' | sed 's|^.*SCRAM_ARCH=||;s|;.*$||')
if [[ $RELEASE_FORMAT != *-* ]]; then
grep '^CMSSW_' $CMS_BOT_DIR/ignore-releases-for-tests > $WORKSPACE/release-with-errors || true
echo "CMSSW_ERROR" >> $WORKSPACE/release-with-errors
if [ $(echo ${RELEASE_FORMAT} | grep _X | wc -l) -gt 0 ] ; then
CMSSW_IB=$(scram -a $SCRAM_ARCH l -c $RELEASE_FORMAT | grep '^CMSSW ' | grep -v -f "$WORKSPACE/release-with-errors" | awk '{print $2}' | sort -r | head -1)
fi
if [ "$CMSSW_IB" = "" ] ; then
CMSSW_IB=$(scram -a $SCRAM_ARCH l -c $CMSSW_QUEUE | grep '^CMSSW ' | grep -v -f "$WORKSPACE/release-with-errors" | awk '{print $2}' | sort -r | head -1)
if [ "$CMSSW_IB" = "" ] ; then
echo "I was not able to find a release to test this PR. See the Jenkins logs for more details" > ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${BUILD_URL}" -d "Unable to find CMSSW release for ${CMSSW_QUEUE}/${SCRAM_ARCH}"
exit 0
fi
fi
rm -f $WORKSPACE/release-with-errors
fi
if [ "$COMPARISON_REL" = "" ] ; then
if [ "$(echo $CMSSW_IB | sed 's|_X_.*|_X|')" = "$COMP_QUEUE" ] ; then
COMPARISON_REL=$CMSSW_IB
else
COMPARISON_REL=$(echo $CMSSW_IB | sed 's|_[A-Z][A-Z0-9]*_X_|_X_|')
fi
fi
[ "${RELEASE_FORMAT}" != "${CMSSW_IB}" ] && sed -i -e "s|${RELEASE_FORMAT}|${CMSSW_IB}|" ${RESULTS_DIR}/09-report.res
if [ "${USE_BASELINE}" = "self" ] ; then
COMPARISON_REL="$CMSSW_IB"
COMPARISON_ARCH="$SCRAM_ARCH"
fi
WORKFLOWS_PR_LABELS=""
scram -a $SCRAM_ARCH project $CMSSW_IB
if $DO_COMPARISON ; then
mkdir $WORKSPACE/ib-baseline-tests
pushd $WORKSPACE/ib-baseline-tests
COMP_OS=$(echo $COMPARISON_ARCH | sed 's|_.*||')
if [ "${COMP_OS}" = "slc7" ] ; then COMP_OS="cc7"; fi
echo "RELEASE_FORMAT=$COMPARISON_REL" > run-baseline-${BUILD_ID}-01.default
echo "ARCHITECTURE=$COMPARISON_ARCH" >> run-baseline-${BUILD_ID}-01.default
echo "DOCKER_IMG=cmssw/${COMP_OS}" >> run-baseline-${BUILD_ID}-01.default
echo "TEST_FLAVOR=" >> run-baseline-${BUILD_ID}-01.default
echo "REAL_ARCH=${RELVAL_REAL_ARCH}" >> run-baseline-${BUILD_ID}-01.default
echo "PRODUCTION_RELEASE=${PRODUCTION_RELEASE}" >> run-baseline-${BUILD_ID}-01.default
WF_LIST=$(get_pr_baseline_worklflow)
[ "${WF_LIST}" = "" ] || WF_LIST="-l ${WF_LIST}"
echo "WORKFLOWS=-s ${WF_LIST}" >> run-baseline-${BUILD_ID}-01.default
PR_LABELS=$(curl -s https://api.github.com/repos/${PR_REPO}/issues/${PR_NUMBER}/labels | grep '"name":' | sed 's|.*: *||;s|"||g;s|-pending||;s|-approved||;s|-rejected||' | tr ',\n' ' ' | tr '[a-z-]' '[A-Z_]')
EX_WFS=""
set +x
for l in ${PR_LABELS} ; do
EX_WFS="${EX_WFS},$(get_pr_baseline_worklflow _LAB_${l})"
done
EX_WFS=$(echo "${EX_WFS}" | sed 's|^,*||;s|,*$||;s|,,*|,|g')
if [ "${EX_WFS}" != "" ] ; then
(
set +x
cd $WORKSPACE/$CMSSW_IB
eval `scram run -sh`
runTheMatrix.py -n -e | grep '\[1\]:' | sed 's| .*||' > wfs.all
runTheMatrix.py -n -e -s ${WF_LIST} | grep '\[1\]:' | sed 's| .*||' > wfs.default
set -x
)
for wf in $(echo ${EX_WFS} | tr ',' '\n') ; do
if grep -q "^${wf}$" $WORKSPACE/$CMSSW_IB/wfs.all ; then
if ! grep -q "^${wf}$" $WORKSPACE/$CMSSW_IB/wfs.default ; then
WORKFLOWS_PR_LABELS="${WORKFLOWS_PR_LABELS},${wf}"
else
echo "WARNING: Workflow already part of default tests: $wf"
fi
else
echo "WARNING: No such workflow: $wf"
fi
done
WORKFLOWS_PR_LABELS=$(echo "${WORKFLOWS_PR_LABELS}" | sed 's|^,*||')
echo "WORKFLOWS_PR_LABELS=${WORKFLOWS_PR_LABELS}"
if [ "${WORKFLOWS_PR_LABELS}" != "" ] ; then
grep -v '^\(WORKFLOWS\|MATRIX_ARGS\)=' run-baseline-${BUILD_ID}-01.default > run-baseline-${BUILD_ID}-03.default
echo "WORKFLOWS=-l ${WORKFLOWS_PR_LABELS}" >> run-baseline-${BUILD_ID}-03.default
fi
fi
set -x
if [ "${MATRIX_EXTRAS}" != "" ] ; then
WF_LIST=$(order_workflow_list ${MATRIX_EXTRAS})
grep -v '^\(WORKFLOWS\|MATRIX_ARGS\)=' run-baseline-${BUILD_ID}-01.default > run-baseline-${BUILD_ID}-02.default
echo "WORKFLOWS=-l ${WF_LIST}" >> run-baseline-${BUILD_ID}-02.default
echo "MATRIX_ARGS=${EXTRA_MATRIX_ARGS}" >> run-baseline-${BUILD_ID}-02.default
fi
for ex_type in ${EXTRA_RELVALS_TESTS} ; do
[ $(echo ${ENABLE_BOT_TESTS} | tr ',' ' ' | tr ' ' '\n' | grep "^${ex_type}$" | wc -l) -gt 0 ] || continue
WF_LIST=$(get_pr_baseline_worklflow "_${ex_type}")
[ "$WF_LIST" != "" ] || continue
ex_type_lc=$(echo ${ex_type} | tr '[A-Z]' '[a-z]')
grep -v '^\(WORKFLOWS\|MATRIX_ARGS\|TEST_FLAVOR\)=' run-baseline-${BUILD_ID}-01.default > run-baseline-${BUILD_ID}-01.${ex_type_lc}
echo "WORKFLOWS=-l ${WF_LIST}" >> run-baseline-${BUILD_ID}-01.${ex_type_lc}
echo "TEST_FLAVOR=${ex_type_lc}" >> run-baseline-${BUILD_ID}-01.${ex_type_lc}
WF_LIST=$(order_workflow_list $(eval echo "\${MATRIX_EXTRAS_${ex_type}}"))
[ "${WF_LIST}" != "" ] || continue
WF_ARGS=$(eval echo "\${EXTRA_MATRIX_ARGS_${ex_type}}")
grep -v '^\(WORKFLOWS\|MATRIX_ARGS\)=' run-baseline-${BUILD_ID}-01.${ex_type_lc} > run-baseline-${BUILD_ID}-02.${ex_type_lc}
echo "WORKFLOWS=-l ${WF_LIST}" >> run-baseline-${BUILD_ID}-02.${ex_type_lc}
echo "MATRIX_ARGS=${WF_ARGS}" >> run-baseline-${BUILD_ID}-02.${ex_type_lc}
done
popd
send_jenkins_artifacts $WORKSPACE/ib-baseline-tests/ ib-baseline-tests/
rm -rf $WORKSPACE/ib-baseline-tests
fi
#Incase week is changed but tests were run for last week
let IB_WEEK=$(scram -a $SCRAM_ARCH list -c ${CMSSW_IB} | sed "s|/${SCRAM_ARCH}/.*||;s|^.*\([0-9]\)$|\1|")%2 || true
if [ "${IB_WEEK}" != "${WEEK_NUM}" ] ; then
sed -i -e "s|/week${WEEK_NUM}/|/week${IB_WEEK}/|" ${RESULTS_DIR}/09-report.res
WEEK_NUM=${IB_WEEK}
CMS_WEEKLY_REPO=cms.week${WEEK_NUM}
fi
PKG_TOOL_BRANCH=$(echo ${CONFIG_LINE} | sed 's/^.*PKGTOOLS_TAG=//' | sed 's/;.*//' )
PKG_TOOL_VERSION=$(echo ${PKG_TOOL_BRANCH} | cut -d- -f 2)
if [[ ${PKG_TOOL_VERSION} -lt 32 && ! -z $(echo ${UNIQ_REPO_NAMES} | tr ' ' '\n' | grep -v -w cmssw | grep -v -w cmsdist | grep -v -w cms-bot ) ]] ; then
echo "ERROR: RELEASE_FORMAT ${CMSSW_QUEUE} uses PKG_TOOL_BRANCH ${PKG_TOOL_BRANCH} which is lower then required to test externals." > ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${BUILD_URL}" -d "Invalid PKGTOOLS version to test external packages."
exit 0
fi
# Do git pull --rebase for each PR except for /cmssw
for U_REPO in $(echo ${UNIQ_REPOS} | tr ' ' '\n' | grep -v '/cmssw$' ); do
FILTERED_PRS=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep ${U_REPO} | tr '\n' ' ')
for PR in ${FILTERED_PRS}; do
ERR=false
git_clone_and_merge "$(get_cached_GH_JSON "${PR}")" || ERR=true
if [[ $(echo ${PR} | grep "cmsdist") ]]; then # Check for CRAB updates to trigger unit test
pushd cmsdist
UPDATES=$(git diff origin/${BASE_BRANCH} --name-only)
if [[ $(echo ${UPDATES} | grep -E 'crab-.*(spec|file)') ]]; then
echo "There is a CRAB update."
DO_CRAB_TESTS=true
fi
popd
fi
if ${ERR} ; then
echo "Failed to merge pull requests ${PR}." > ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${BUILD_URL}" -d "Failed to merge ${PR}"
exit 0
fi
done
done
# Preparations depending on from repo type
CMSSW_ORG='cms-sw'
BUILD_EXTERNAL=false
CMSDIST_ONLY=true # asume cmsdist only
CHECK_HEADER_TESTS=false
for U_REPO in ${UNIQ_REPOS}; do
PKG_REPO=$(echo ${U_REPO} | sed 's/#.*//')
PKG_NAME=$(echo ${U_REPO} | sed 's|.*/||')
case "$PKG_NAME" in # We do not care where the repo is kept (ex. cmssw organisation or other)
cmssw)
CMSSW_ORG=$(echo ${PKG_REPO} | sed 's|/.*||')
CMSDIST_ONLY=false
CHECK_HEADER_TESTS=true
;;
cms-bot)
# do nothing
;;
cmsdist|pkgtools)
BUILD_EXTERNAL=true
;;
*)
PKG_REPO=$(echo ${U_REPO} | sed 's/#.*//')
SPEC_NAMES=$( ${CMS_BOT_DIR}/pr_testing/get_external_name.sh ${PKG_REPO} )
BUILD_EXTERNAL=true
for SPEC_NAME in ${SPEC_NAMES} ; do
if ! ${PR_TESTING_DIR}/get_source_flag_for_cmsbuild.sh "$PKG_REPO" "$SPEC_NAME" "$CMSSW_QUEUE" "$ARCHITECTURE" "${CMS_WEEKLY_REPO}" "${BUILD_DIR}" ; then
echo "ERROR: There was an issue generating parameters for cmsBuild '--source' flag for spec file ${SPEC_NAME} from ${PKG_REPO} repo." > ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${BUILD_URL}" -d "Error getting source flag for ${PKG_REPO}, fix spec ${SPEC_NAME}"
exit 0
fi
done
;;
esac
done
if $CMSDIST_ONLY ; then
DO_DAS_QUERY=false
TEST_RELVALS_INPUT=false
fi
# Prepera html templates
cp $CMS_BOT_DIR/templates/PullRequestSummary.html $WORKSPACE/summary.html
sed -e "s|@JENKINS_PREFIX@|$JENKINS_PREFIX|g;" $CMS_BOT_DIR/templates/js/renderPRTests.js > $WORKSPACE/renderPRTests.js
mkdir -p ${RESULTS_DIR}
touch ${RESULTS_FILE} ${RESULTS_DIR}/comparison.txt
echo "PR_NUMBERS;$PULL_REQUESTS" >> ${RESULTS_FILE}
echo 'BASE_IB;'$CMSSW_IB >> ${RESULTS_FILE}
echo 'BUILD_NUMBER;'$BUILD_NUMBER >> ${RESULTS_FILE}
echo "PR_NUMBER;$PR_NUM" >> ${RESULTS_FILE}
echo "COMPARISON_IB;$COMPARISON_REL" >> ${RESULTS_FILE}
PR_EXTERNAL_REPO=""
TEST_DASGOCLIENT=false
SKIP_STATIC_CHECKS=false
[ $(echo ",${SKIP_TESTS}," | grep ',static,' | wc -l) -gt 0 ] && SKIP_STATIC_CHECKS=true
if ${BUILD_EXTERNAL} ; then
export USE_IB_TAG=false
mark_commit_status_all_prs '' 'pending' -u "${BUILD_URL}" -d "Building CMSSW externals" || true
if [ ! -d "pkgtools" ] ; then
git clone [email protected]:cms-sw/pkgtools -b $PKG_TOOL_BRANCH
fi
if [ ! -d "cmsdist" ] ; then
git clone [email protected]:cms-sw/cmsdist -b $CMSDIST_TAG
fi
echo_section "Building, testing and commenting status to github"
# add special flags for pkgtools/cmsbuild if version is high enough
REF_REPO=
SOURCE_FLAG=
if [ "X$USE_CMSPKG_REFERENCE" = "Xtrue" ] ; then
if [ ${PKG_TOOL_VERSION} -gt 31 ] ; then
REF_REPO="--reference "$(readlink /cvmfs/cms-ib.cern.ch/sw/$(uname -m)/$(echo $CMS_WEEKLY_REPO | sed 's|^cms.||'))
if [ -e ${WORKSPACE}/get_source_flag_result.txt ] ; then
SOURCE_FLAG=$(cat ${WORKSPACE}/get_source_flag_result.txt )
fi
fi
fi
# Not all packages are build with debug flag. If the current IB should be build with debug flag, we need to do some 'magic'
# otherwise everything will be rebuild.
if [ $( echo $CONFIG_LINE | grep ";ENABLE_DEBUG=" | wc -l) -eq 0 ] ; then
DEBUG_SUBPACKS=$(grep '^ *DEBUG_SUBPACKS=' $CMS_BOT_DIR/build-cmssw-ib-with-patch | sed 's|.*DEBUG_SUBPACKS="||;s|".*$||')
pushd ${WORKSPACE}/cmsdist
perl -p -i -e 's/^[\s]*%define[\s]+subpackageDebug[\s]+./#subpackage debug disabled/' $DEBUG_SUBPACKS
popd
fi
# Build the whole cmssw-tool-conf toolchain
CMSBUILD_ARGS="--tag ${PR_NUM} --define cmsswdata_version_link"
if [ ${PKG_TOOL_VERSION} -gt 31 ] ; then
CMSBUILD_ARGS="${CMSBUILD_ARGS} --monitor --log-deps --force-tag --tag hash --delete-build-directory --link-parent-repository"
fi
if [ "${ALLOW_VERSION_SUFFIX}" = "true" ] ; then
CMSBUILD_ARGS="${CMSBUILD_ARGS} --define allow_version_suffix"
fi
if [ $(echo "${CONFIG_LINE}" | grep "DEBUG_EXTERNALS=" | wc -l) -gt 0 ] ; then
dbg_pkgs=$(echo "${CONFIG_LINE}" | tr ';' '\n' | grep "^DEBUG_EXTERNALS=" | sed 's|.*=||')
CMSBUILD_ARGS="${CMSBUILD_ARGS} --define cms_debug_packages=${dbg_pkgs}"
fi
if [ $(grep 'upload-package-store-s3' pkgtools/cmsBuild | wc -l) -gt 0 ] ; then
if $UPLOAD_TO_PACKAGE_STORE ; then
CMSBUILD_ARGS="${CMSBUILD_ARGS} --upload-package-store-s3"
else
CMSBUILD_ARGS="${CMSBUILD_ARGS} --no-package-store"
fi
fi
#Process cmsdist Build options
BUILD_OPTS=$(echo $CONFIG_LINE | tr ';' '\n' | grep "^BUILD_OPTS=" | sed 's|^BUILD_OPTS=||')
MULTIARCH_OPTS=$(echo $CONFIG_LINE | tr ';' '\n' | grep "^MULTIARCH_OPTS=" | sed 's|^MULTIARCH_OPTS=||')
PKGS="cms-common cms-git-tools cmssw-tool-conf"
COMPILATION_CMD="PYTHONPATH= ./pkgtools/cmsBuild --server http://${CMSREP_IB_SERVER}/cgi-bin/cmspkg --upload-server ${CMSREP_IB_SERVER} \
${CMSBUILD_ARGS} --builders 3 -i $WORKSPACE/$BUILD_DIR $REF_REPO \
$SOURCE_FLAG --arch $ARCHITECTURE -j ${NCPU} $(cmsbuild_args "${BUILD_OPTS}" "${MULTIARCH_OPTS}" "${ARCHITECTURE}")"
PR_EXTERNAL_REPO="PR_$(echo ${RPM_UPLOAD_REPO}_${CMSSW_QUEUE}_${ARCHITECTURE} | md5sum | sed 's| .*||' | tail -c 9)"
if [ -e cmsdist/cmssw-tool-conf.spec ] ; then
echo "#PR ${PR_EXTERNAL_REPO}" >> cmsdist/cmssw-tool-conf.spec
else
echo "#PR ${PR_EXTERNAL_REPO}" >> cmsdist/cmssw-tool-conf.file
fi
UPLOAD_OPTS="--upload-tmp-repository ${PR_EXTERNAL_REPO}"
if [ $(curl -s --head http://${CMSREP_IB_SERVER}/cmssw/repos/${CMS_WEEKLY_REPO}.${PR_EXTERNAL_REPO}/${ARCHITECTURE}/latest/ 2>&1 | head -1 | grep " 200 OK" |wc -l) -gt 0 ] ; then
UPLOAD_OPTS="--sync-back"
COMPILATION_CMD="${COMPILATION_CMD} --repository ${CMS_WEEKLY_REPO}.${PR_EXTERNAL_REPO}"
else
COMPILATION_CMD="${COMPILATION_CMD} --repository ${CMS_WEEKLY_REPO}"
fi
if [ "${SOURCE_FLAG}" != "" ] ; then UPLOAD_OPTS="${UPLOAD_OPTS} --force-upload" ; fi
rm -rf $WORKSPACE/$BUILD_DIR
echo $COMPILATION_CMD build ${PKGS} > ${WORKSPACE}/cmsswtoolconf.log # log the command to be run
# run the command and both log it to file and display it
(eval $COMPILATION_CMD build ${PKGS} && echo 'ALL_OK') 2>&1 | tee -a $WORKSPACE/cmsswtoolconf.log
echo_section 'END OF BUILD LOG'
TEST_ERRORS=$(grep -E "Error [0-9]$" $WORKSPACE/cmsswtoolconf.log) || true
GENERAL_ERRORS=$(grep "ALL_OK" $WORKSPACE/cmsswtoolconf.log) || true
if [ -f "$WORKSPACE/$BUILD_DIR/tmp/bootstrap.log" ] ; then
mv $WORKSPACE/$BUILD_DIR/tmp/bootstrap.log $WORKSPACE/bootstrap.log
fi
#upload packages build
BLD_PKGS=$(ls $WORKSPACE/$BUILD_DIR/RPMS/${ARCHITECTURE}/ | grep '.rpm$' | cut -d+ -f2 | grep -v 'coral-debug' || true)
if [ "${BLD_PKGS}" != "" ] ; then eval $COMPILATION_CMD ${UPLOAD_OPTS} upload ${BLD_PKGS} ; fi
for d in bootstraptmp tmp RPMS SOURCES SPECS SRPMS WEB ; do
rm -rf $WORKSPACE/$BUILD_DIR/${d} || true
done
echo 'CMSSWTOOLCONF_LOGS;OK,External Build Logs,See Log,externals' >> ${RESULTS_DIR}/toolconf.txt
if [ $(grep 'RPM installation stderr' $WORKSPACE/cmsswtoolconf.log |wc -l) -gt 0 ] ; then
echo 'CMSSWTOOLCONF_INSTALL;ERROR,Externals Installation,See Log,cmsswtoolconf.log' >> ${RESULTS_DIR}/toolconf.txt
fi
if [ "X$TEST_ERRORS" != X ] || [ "X$GENERAL_ERRORS" == X ]; then
echo 'CMSSWTOOLCONF_RESULTS;ERROR,Externals compilation,See Log,cmsswtoolconf.log' >> ${RESULTS_DIR}/toolconf.txt
${CMS_BOT_DIR}/report-pull-request-results "PARSE_EXTERNAL_BUILD_FAIL" --unit-tests-file $WORKSPACE/cmsswtoolconf.log \
--report-url ${PR_RESULT_URL} ${NO_POST} --report-file ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${PR_RESULT_URL}" -d "Failed to build externals"
exit 0
else
echo 'CMSSWTOOLCONF_RESULTS;OK,Externals compilation,See Log,cmsswtoolconf.log' >> ${RESULTS_DIR}/toolconf.txt
fi
#Testing sourcing of cmsset_default: run only for CMSSW master IB/Proudcuton arch
if $CMSSW_DEVEL_PROD_ARCH ; then
CMSSET_DEFAULT_ERR=""
mkdir $WORKSPACE/cmsset_default
EL_OS=$(ls $WORKSPACE/$BUILD_DIR/common/cmssw-el* | sed 's|.*/common/cmssw-el|el|' | grep -v 'el5')
for sh in bash sh zsh ; do
for os in $EL_OS ; do
echo "Checking cmsset_default.sh for $sh under $os" >> $WORKSPACE/cmsset_default/run.log
if ! $WORKSPACE/$BUILD_DIR/common/cmssw-$os -- $sh -e $WORKSPACE/$BUILD_DIR/cmsset_default.sh >>$WORKSPACE/cmsset_default/run.log 2>&1 ; then
CMSSET_DEFAULT_ERR="${CMSSET_DEFAULT_ERR} $sh:$os"
echo "Failed" >> $WORKSPACE/cmsset_default/run.log
$WORKSPACE/$BUILD_DIR/common/cmssw-$os -- $sh -ex $WORKSPACE/$BUILD_DIR/cmsset_default.sh > $WORKSPACE/cmsset_default/${sh}-${os}.log 2>&1 || true
else
echo "OK" >> $WORKSPACE/cmsset_default/run.log
fi
done
done
if [ "${CMSSET_DEFAULT_ERR}" != "" ] ; then
echo "CMSSet_Default" >> ${RESULTS_DIR}/09-failed.res
echo 'CMSSET_DEFAULT_RESULTS;ERROR,Environment setup,See Log,cmsset_default' >> ${RESULTS_DIR}/toolconf.txt
echo "**Failed environment setup**: \`${CMSSET_DEFAULT_ERR}\`" >> ${RESULTS_DIR}/09-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${PR_RESULT_URL}" -d "Environment setup error"
exit 0
else
echo 'CMSSET_DEFAULT_RESULTS;OK,Environment setup,See Log,cmsset_default/run.log' >> ${RESULTS_DIR}/toolconf.txt
fi
fi
OLD_DASGOCLIENT=$(dasgoclient --version | tr ' ' '\n' | grep '^git=' | sed 's|^git=||')
# Create an appropriate CMSSW area
source $WORKSPACE/$BUILD_DIR/cmsset_default.sh
if [ -e $WORKSPACE/$BUILD_DIR/common/dasgoclient ] ; then
NEW_DASGOCLIENT=$($WORKSPACE/$BUILD_DIR/common/dasgoclient --version | tr ' ' '\n' | grep '^git=' | sed 's|^git=||')
XDAS=$(echo ${OLD_DASGOCLIENT} ${NEW_DASGOCLIENT} | tr ' ' '\n' | grep '^v' | sort | tail -1)
if [ "${OLD_DASGOCLIENT}" != "${XDAS}" ] ; then TEST_DASGOCLIENT=true ; fi
fi
echo /cvmfs/cms.cern.ch > $WORKSPACE/$BUILD_DIR/etc/scramrc/links.db
# To make sure we always pick scram from local area
rm -f $CMSSW_IB/config/scram_basedir
sver=$(grep '^lcg+SCRAMV1+' $WORKSPACE/cmsswtoolconf.log | head -1 | sed 's|^lcg+SCRAMV1+||;s| .*||')
echo $sver > $CMSSW_IB/config/scram_version
config_tag=$(grep '%define *configtag *V' $WORKSPACE/cmsdist/scram-project-build.file | sed 's|.*configtag *V|V|;s| *||g')
old_config_tag=$(cat $CMSSW_IB/config/config_tag)
if [ -d $WORKSPACE/config ] ; then
cp -r $WORKSPACE/config scram-buildrules
config_tag="${config_tag}-01"
else
if [ "${old_config_tag}" != "${config_tag}" ] ; then
git clone [email protected]:cms-sw/cmssw-config scram-buildrules
pushd scram-buildrules
git checkout ${config_tag}
popd
fi
fi
if [ -d scram-buildrules ] ; then
echo ${config_tag} > $WORKSPACE/$CMSSW_IB/config/config_tag
mv $CMSSW_IB/config/SCRAM $CMSSW_IB/config/SCRAM.orig
mv scram-buildrules/SCRAM $CMSSW_IB/config/SCRAM
if [ -d scram-buildrules/Projects/CMSSW ] ; then
cp -f scram-buildrules/Projects/CMSSW/BuildFile.xml $CMSSW_IB/config/BuildFile.xml
[ -e scram-buildrules/Projects/CMSSW/SCRAM_ExtraBuildRule.pm ] && cp -f scram-buildrules/Projects/CMSSW/SCRAM_ExtraBuildRule.pm $CMSSW_IB/config/SCRAM_ExtraBuildRule.pm
(
for x in SCRAM_COMPILER:DEFAULT_COMPILER PROJECT_GIT_HASH:CMSSW_GIT_HASH ENABLE_LTO:ENABLE_LTO ; do
e=$(echo $x | sed 's|:.*||');
k=$(echo $x | sed 's|.*:||');
export $e=$(grep "$k" $CMSSW_IB/config/Self.xml | tr ' ' '\n' | grep '=' | tail -1 | sed 's|[^"]*"||;s|".*||');\
done;
perl -p -i -e 's|\@([^@]*)\@|$ENV{$1}|g' scram-buildrules/Projects/CMSSW/Self.xml
)
if [ "$MULTIARCH_OPTS" != "" ] ; then
MULTIARCH_OPTSX=$(echo ${MULTIARCH_OPTS} | tr ',' ' ')
DEFAULT_TARGET=$(cmssw_default_target $CMSSW_IB)
sed -i -e "s| SCRAM_TARGETS=.*\"| SCRAM_TARGETS=\"${MULTIARCH_OPTSX}\"|" scram-buildrules/Projects/CMSSW/Self.xml
sed -i -e "s|</tool>| <runtime name=\"SCRAM_TARGET\" value=\"${DEFAULT_TARGET}\"/>\n <runtime name=\"USER_TARGETS_ALL\" value=\"1\"/>\n</tool>|" scram-buildrules/Projects/CMSSW/Self.xml
fi
cp scram-buildrules/Projects/CMSSW/Self.xml $CMSSW_IB/config/Self.xml
else
cp -f scram-buildrules/CMSSW_BuildFile.xml $CMSSW_IB/config/BuildFile.xml
[ -e scram-buildrules/CMSSW_SCRAM_ExtraBuildRule.pm ] && cp -f scram-buildrules/CMSSW_SCRAM_ExtraBuildRule.pm $CMSSW_IB/config/SCRAM_ExtraBuildRule.pm
fi
if [ -f $CMSSW_IB/config/SCRAM.orig/GMake/CXXModules.mk ] ; then
cp $WORKSPACE/cmsdist/CXXModules.mk.file $CMSSW_IB/config/SCRAM/GMake/CXXModules.mk
fi
fi
rm -rf scram-buildrules
cd $WORKSPACE/$CMSSW_IB/src
touch $WORKSPACE/cmsswtoolconf.log
CTOOLS=$WORKSPACE/$CMSSW_IB/config/toolbox/${ARCHITECTURE}/tools/selected
BTOOLS=${CTOOLS}.backup
mv ${CTOOLS} ${BTOOLS}
TOOL_CONF_VERSION=$(ls -d $WORKSPACE/$BUILD_DIR/$ARCHITECTURE/cms/cmssw-tool-conf/* | sed 's|.*/||')
echo "${CMS_WEEKLY_REPO}.${PR_EXTERNAL_REPO}/${TOOL_CONF_VERSION}" > $WORKSPACE/cmssw-tool-conf.txt
echo "CMSSWTOOLCONF_VERSION;OK,External tool conf,See log,cmssw-tool-conf.txt" >> ${RESULTS_DIR}/toolconf.txt
mv $WORKSPACE/$BUILD_DIR/$ARCHITECTURE/cms/cmssw-tool-conf/${TOOL_CONF_VERSION}/tools/selected ${CTOOLS}
if [ -e ${BTOOLS}/cmssw-config.xml ] ; then
cp ${BTOOLS}/cmssw-config.xml ${CTOOLS}/
if [ "${old_config_tag}" != "${config_tag}" ] ; then
sed -i -e "s|${old_config_tag}|${config_tag}|" ${CTOOLS}/cmssw-config.xml || true
fi
fi
#Copy extra available tools
if [ -d $WORKSPACE/$CMSSW_IB/config/toolbox/${ARCHITECTURE}/tools/available -a -d $WORKSPACE/$BUILD_DIR/$ARCHITECTURE/cms/cmssw-tool-conf/${TOOL_CONF_VERSION}/tools/available ] ; then
mv $WORKSPACE/$CMSSW_IB/config/toolbox/${ARCHITECTURE}/tools/{available,available.backup}
mv $WORKSPACE/$BUILD_DIR/$ARCHITECTURE/cms/cmssw-tool-conf/${TOOL_CONF_VERSION}/tools/available $WORKSPACE/$CMSSW_IB/config/toolbox/${ARCHITECTURE}/tools/available
fi
#Generate External Tools Status
echo '<html><head><link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet"></head>' > $WORKSPACE/upload/external-tools.html
echo '<body><h2>External tools build Statistics</h2><br/><table class="table table-striped"><tr><td>Tool Name</td><td>#Files(new)</td><td>#Files(old)</td><td>Size(new)</td><td>Size(old)</td></tr>' >> $WORKSPACE/upload/external-tools.html
for pkg in $(find ${WORKSPACE}/${BUILD_DIR}/BUILD/${ARCHITECTURE} -maxdepth 3 -mindepth 3 -type d | sed "s|$WORKSPACE/$BUILD_DIR/BUILD/||" | sort) ; do
ltpath="${WORKSPACE}/${BUILD_DIR}/${pkg}"
[ -d ${ltpath} ] || continue
l_tc=$(find ${ltpath} -follow | wc -l)
l_ts=$(du -shD ${ltpath} | awk '{print $1}')
tdir=$(dirname $pkg)
rtpath=$(grep -R ${tdir} ${BTOOLS} | grep '_BASE\|CMSSW_SEARCH_PATH' | tail -1 | sed 's|.* default="||;s|".*||')
if [ "${rtpath}" = "" ] || [ ! -d "${rtpath}" ] ; then
r_tc=0
r_ts=0
else
r_tc=$(find ${rtpath} -follow | wc -l)
r_ts=$(du -shD ${rtpath} | awk '{print $1}')
fi
tool=$(basename $tdir)
echo "<tr><td>${tool}</td><td>$l_tc</td><td>$r_tc</td><td>$l_ts</td><td>$r_ts</td></tr>" >> $WORKSPACE/upload/external-tools.html
done
echo "</table></body></html>" >> $WORKSPACE/upload/external-tools.html
echo 'CMSSWTOOLCONF_STATS;OK,External Build Stats,See Log,external-tools.html' >> ${RESULTS_DIR}/toolconf.txt
set +x
TOOL_SETUP=true
if [ "X$BUILD_FULL_CMSSW" != "Xtrue" ] ; then
# Setup all the toolfiles previously built
DEP_NAMES=
if [ -e "${BTOOLS}/cmssw.xml" ] ; then cp ${BTOOLS}/cmssw.xml ${CTOOLS}/cmssw.xml ; fi
RMV_CMSSW_EXTERNAL="$(ls -d $WORKSPACE/$CMSSW_IB/config/SCRAM/hooks/runtime/*-remove-release-external-lib 2>/dev/null || true)"
if [ "${RMV_CMSSW_EXTERNAL}" != "" ] ; then
chmod +x ${RMV_CMSSW_EXTERNAL}
fi
DEP_NAMES=""
#Fix for SCRAMV2 based releases were tools can have different capitalizations
ALL_NEW_TOOLS=$(ls ${CTOOLS}/ | tr '[A-Z]\n' '[a-z] ')
#In some releases libjpeg-turbo tool exists via libjpg
[ $(echo " ${ALL_NEW_TOOLS} " | grep " libjpg.xml " | wc -l) -gt 0 ] && ALL_NEW_TOOLS="${ALL_NEW_TOOLS} libjpeg-turbo.xml"
for xml in $(ls ${BTOOLS}/*.xml) ; do
name=$(basename $xml)
lcname=$(echo $name | tr '[A-Z]' '[a-z]')
if [ $(echo " ${ALL_NEW_TOOLS} " | grep " ${lcname} " |wc -l) -eq 0 ] ; then
tool=$(echo $name | sed 's|.xml$||')
echo "Removed tool $name"
DEP_NAMES="$DEP_NAMES echo_${tool}_USED_BY"
fi
done
if [ "${DEP_NAMES}" != "" ] ; then
CMSSW_DEP=$(scram build ${DEP_NAMES} | tr ' ' '\n' | grep '^cmssw/\|^self/' | cut -d"/" -f 2,3 | sort | uniq)
DEP_NAMES=""
echo "CMSSW_DEP=${CMSSW_DEP}"
fi
rm -rf $WORKSPACE/$CMSSW_IB/.SCRAM/$ARCHITECTURE/tools
for xml in $(ls ${CTOOLS}/*.xml) ; do
name=$(basename $xml)
tool=$(echo $name | sed 's|.xml$||')
echo "Checking tool $tool ($xml)"
if [ ! -e ${BTOOLS}/$name ] ; then
scram setup $xml >> $WORKSPACE/scram-tool-setup.log 2>&1 || TOOL_SETUP=false
continue
fi
nver=$(grep '<tool ' $xml | tr ' ' '\n' | grep 'version=' | sed 's|version="||;s|".*||g')
over=$(grep '<tool ' ${BTOOLS}/$name | tr ' ' '\n' | grep 'version=' | sed 's|version="||;s|".*||g')
nrev=$(grep '<tool ' $xml | tr ' ' '\n' | grep 'revision=' | sed 's|revision="||;s|".*||g')
orev=$(grep '<tool ' ${BTOOLS}/$name | tr ' ' '\n' | grep 'revision=' | sed 's|revision="||;s|".*||g')
echo "Checking version in release: ${over}(${orev}) vs $nver(${nrev})"
if [ "$nver" = "$over" -a "$nrev" = "$orev" ] ; then continue ; fi
echo "Setting up $name: Version: $over vs $nver / Revision: $orev vs $nrev"
DEP_NAMES="$DEP_NAMES echo_${tool}_USED_BY"
done
sed -i -e 's|.*/lib/python2.7/site-packages" .*||;s|.*/lib/python3.6/site-packages" .*||' ../config/Self.xml
touch $CTOOLS/*.xml
(scram setup && scram setup self && rm -rf $WORKSPACE/$CMSSW_IB/external && scram build -r echo_CXX) >> $WORKSPACE/scram-tool-setup.log 2>&1 || TOOL_SETUP=false
echo "DEP_NAMES=${DEP_NAMES}"
if $TOOL_SETUP ; then
if [ "${DEP_NAMES}" != "" ] ; then
CMSSW_DEPx=$(scram build ${DEP_NAMES} | tr ' ' '\n' | grep '^cmssw/\|^self/' | cut -d"/" -f 2,3 | sort | uniq)
CMSSW_DEP=$(echo ${CMSSW_DEP} ${CMSSW_DEPx} | tr ' ' '\n' | sort | uniq)
fi
echo "Final CMSSW_DEP=${CMSSW_DEP}"
if [ "$CMSSW_DEP" = "" ] ; then CMSSW_DEP="FWCore/Version" ; fi
fi
else
rm -f $WORKSPACE/$CMSSW_IB/.SCRAM/$ARCHITECTURE/Environment
rm -rf $WORKSPACE/$CMSSW_IB/.SCRAM/$ARCHITECTURE/tools
touch $CTOOLS/*.xml $WORKSPACE/$CMSSW_IB/config/Self.xml
scram tool remove cmssw || true
(scram setup && scram setup self && rm -rf $WORKSPACE/$CMSSW_IB/external && scram b clean && scram build -r echo_CXX) > $WORKSPACE/scram-tool-setup.log 2>&1 || TOOL_SETUP=false
CMSSW_DEP="*"
SKIP_STATIC_CHECKS=true
fi
if ! $TOOL_SETUP ; then
echo 'SCRAM_TOOLS_SETUP;ERROR,Scram tools setup,See Log,scram-tool-setup.log' >> ${RESULTS_DIR}/scramtools.txt
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${PR_RESULT_URL}" -d "Failed to setup externals"
exit 0
fi
eval $(scram runtime -sh)
set -x
echo $LD_LIBRARY_PATH
if [ -e $WORKSPACE/$CMSSW_IB/config/SCRAM/hooks/runtime/00-nvidia-drivers ] ; then
SCRAM=scram bash -ex $WORKSPACE/$CMSSW_IB/config/SCRAM/hooks/runtime/00-nvidia-drivers || true
fi
git cms-init --upstream-only
pushd $WORKSPACE/$CMSSW_IB/src
if [ "X$BUILD_FULL_CMSSW" = "Xtrue" ] ; then
git checkout $(echo "${CONFIG_LINE}" | sed 's|.*RELEASE_BRANCH=||;s|;.*||')
echo '/*/' >> .git/info/sparse-checkout
git read-tree -mu HEAD
else
git cms-checkout-topic --ssh $(git branch | grep '^ *CMSSW_') 2>&1 | tee -a $WORKSPACE/cmsswtoolconf.log
git cms-checkdeps -A -a 2>&1 | tee -a $WORKSPACE/cmsswtoolconf.log
git cms-addpkg --ssh "$CMSSW_DEP" 2>&1 | tee -a $WORKSPACE/cmsswtoolconf.log
fi
popd
rm -rf $WORKSPACE/$CMSSW_IB/external
scram b clean
scram b -r echo_CXX
fi # end of build external
echo_section "end of build external"
# This part responsible for testing CMSSW
echo_section "Testing CMSSW"
voms-proxy-init -voms cms -valid 24:00 || true # To get access to jenkins artifact machine
### to know at the end of the tests if everything went ok
ALL_OK=true
BUILD_OK=true
UNIT_TESTS_OK=true
RELVALS_OK=true
ADDON_OK=true
CLANG_BUILD_OK=true
PYTHON3_BUILD_OK=true
RUN_TESTS=true
cd $WORKSPACE/$CMSSW_IB
set +x
eval $(scram run -sh)
set -x
echo $LD_LIBRARY_PATH | tr ':' '\n'
BUILD_LOG_DIR="${CMSSW_BASE}/tmp/${SCRAM_ARCH}/cache/log"
USER_FLAGS=""
if $DISABLE_CMS_DEPRECATED ; then
#USER_FLAGS="USER_CXXFLAGS=-DUSE_CMS_DEPRECATED"
ANALOG_OPT="--ignoreWarning=Wdeprecated-declarations"
fi
ANALOG_CMD="scram build outputlog && (${CMS_PYTHON_TO_USE} $CMS_BOT_DIR/buildLogAnalyzer.py ${ANALOG_OPT} --logDir ${BUILD_LOG_DIR}/src || true)"
OK_ANALOG_CMD="true && (${CMS_PYTHON_TO_USE} $CMS_BOT_DIR/buildLogAnalyzer.py ${ANALOG_OPT} --logDir ${BUILD_LOG_DIR}/src || true)"
cd $WORKSPACE/$CMSSW_IB/src
git config --global --replace-all merge.renamelimit 2500 || true
GIT_MERGE_RESULT_FILE=$WORKSPACE/git-merge-result
RECENT_COMMITS_FILE=$WORKSPACE/git-recent-commits.json
RECENT_COMMITS_LOG_FILE=$WORKSPACE/git-log-recent-commits
echo '{}' > $RECENT_COMMITS_FILE
# use the branch name if necesary
touch $WORKSPACE/changed-files
if [ ! -d $WORKSPACE/cms-prs ] ; then git clone --depth 1 [email protected]:cms-sw/cms-prs $WORKSPACE/cms-prs ; fi
if ! $CMSDIST_ONLY ; then # If a CMSSW specific PR was specified #
if $USE_IB_TAG ; then git cms-init --upstream-only $CMSSW_IB ; fi
# this is to test several pull requests at the same time
for PR in $( echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmssw#"); do
echo 'I will add the following pull request to the test'
PR_NR=$(echo ${PR} | sed 's/.*#//' )
(git cms-merge-topic --debug --ssh -u ${CMSSW_ORG}:${PR_NR} && echo 'ALL_OK') 2>&1 | tee -a $GIT_MERGE_RESULT_FILE
done
if grep 'Automatic merge failed' $GIT_MERGE_RESULT_FILE; then
echo "This pull request cannot be automatically merged, could you please rebase it?" > ${RESULTS_DIR}/10-report.res
echo "You can see the log for git cms-merge-topic here: ${PR_RESULT_URL}/git-merge-result" >> ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${PR_RESULT_URL}" -d "Merge: Unable to merge CMSSW PRs"
exit 0
fi
if grep "Couldn't find remote ref" $GIT_MERGE_RESULT_FILE; then
echo "I had the issue <pre>could not find remote ref refs/pull/${PR_NUMBER}/head</pre>" > ${RESULTS_DIR}/10-report.res
echo 'Please restart the tests in jenkins providing the complete branch name' >> ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${PR_RESULT_URL}" -d "Merge: Unable to find remote reference."
exit 0
fi
git diff --name-only $CMSSW_VERSION > $WORKSPACE/changed-files
# look for any other error in general
if ! grep "ALL_OK" $GIT_MERGE_RESULT_FILE; then
echo "There was an issue with git-cms-merge-topic you can see the log here: ${PR_RESULT_URL}/git-merge-result" > ${RESULTS_DIR}/10-report.res
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${PR_RESULT_URL}" -d "Merge: Unknow error while merging."
exit 0
fi
if [[ "${PRODUCTION_RELEASE}" == "true" && "${PULL_REQUEST}" == *"/cmssw#"* ]]; then
pushd ${CMSSW_BASE}
mv src src.tmp && mkdir src
cd src
THRDS=""
git cms-init --upstream-only && git checkout -b codechecks $CMSSW_IB
git repack -h 2>&1 | grep '\-\-threads' && THRDS="--threads ${NCPU}" || true
git repack -a -d ${THRDS}
git repack -a -d ${THRDS}
OSIZE=$(du -sk .git/objects/pack | sed 's|\s.*||')
git cms-merge-topic --debug --ssh -u ${CMSSW_ORG}:${PR_NUMBER}
git repack -d ${THRDS}
NSIZE=$(du -sk .git/objects/pack | sed 's|\s.*||')
let DSIZE=${NSIZE}-${OSIZE} || DSIZE=0
if [ $DSIZE -gt 0 ]; then echo "**Size**: This PR adds an extra ${DSIZE}KB to repository" > ${RESULTS_DIR}/09-git-repo-size-report.res; fi
cd ..
rm -rf src && mv src.tmp src
popd
fi
#############################################
# Check if there are unwanted commits that came with the merge.
############################################
merged_prs=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmssw#" | sed 's|.*#||' | tr '\n' ',')
$SCRIPTPATH/get-merged-prs.py -r cms-sw/cmssw -i "${merged_prs}" -s $CMSSW_VERSION -e HEAD -g $CMSSW_BASE/src/.git -c $WORKSPACE/cms-prs -o $RECENT_COMMITS_FILE
echo "##### CMSSW Extra merges #####" >> $RECENT_COMMITS_LOG_FILE
git log ${CMSSW_IB}..HEAD --merges 2>&1 | tee -a $RECENT_COMMITS_LOG_FILE
if [ $DO_MB_COMPARISON -a $(grep 'Geometry' $WORKSPACE/changed-files | wc -l) -gt 0 ] ; then
has_jenkins_artifacts material-budget/$CMSSW_IB/$SCRAM_ARCH/Images || DO_MB_COMPARISON=false
else
DO_MB_COMPARISON=false
fi
elif [ "X$BUILD_FULL_CMSSW" = "Xtrue" ] ; then
$SCRIPTPATH/get-merged-prs.py -r cms-sw/cmssw -s $CMSSW_VERSION -e HEAD -g $CMSSW_BASE/src/.git -c $WORKSPACE/cms-prs -o $RECENT_COMMITS_FILE
echo "##### CMSSW Extra merges #####" >> $RECENT_COMMITS_LOG_FILE
git log ${CMSSW_IB}..HEAD --merges 2>&1 | tee -a $RECENT_COMMITS_LOG_FILE
fi
if ! scram build -r echo_CXX > $WORKSPACE/build.log 2>&1 ; then
echo "**ERROR**: SCRAM failed to generate build rules, there might be syntax errors in modified BuildFiles." > ${RESULTS_DIR}/10-report.res
echo "SCRAM_BUILD_CXX;ERROR,SCRAM Build Rules,See Log,build.log" > ${RESULTS_DIR}/scramb.txt
prepare_upload_results
mark_commit_status_all_prs '' 'error' -u "${PR_RESULT_URL}" -d "There might be syntax errors in BuildFiles."
exit 0
fi
if ${BUILD_EXTERNAL} ; then
pushd $WORKSPACE/cmsdist
CMSDIST_REL_TAG=$(git tag | grep '^'ALL/${CMSSW_VERSION}/${SCRAM_ARCH}'$' || true)
if [ "${CMSDIST_REL_TAG}" != "" ] ; then
merged_prs=$(echo ${PULL_REQUESTS} | tr ' ' '\n' | grep "/cmsdist#" | sed 's|.*#||' | tr '\n' ',')
$SCRIPTPATH/get-merged-prs.py -r cms-sw/cmsdist -i "${merged_prs}" -s ${CMSDIST_REL_TAG} -e HEAD -g $WORKSPACE/cmsdist/.git -c $WORKSPACE/cms-prs -o $RECENT_COMMITS_FILE
echo "##### CMSDIST Extra merges #####" >> $RECENT_COMMITS_LOG_FILE
git log ${CMSDIST_REL_TAG}..HEAD --merges 2>&1 | tee -a $RECENT_COMMITS_LOG_FILE
fi
popd
fi
$CMS_BOT_DIR/report-pull-request-results MERGE_COMMITS --recent-merges $RECENT_COMMITS_FILE --report-url ${PR_RESULT_URL} --report-file ${RESULTS_DIR}/09-report.res ${REPORT_OPTS}
# Don't do the following if we are only testing CMSDIST PR
if [ "X$CMSDIST_ONLY" == Xfalse ]; then
git log --oneline --merges ${CMSSW_VERSION}..
fi
# #############################################
# test compilation with Clang
# ############################################
echo 'test clang compilation'
NEED_CLANG_TEST=false
if cat $CONFIG_MAP | grep $CMSSW_QUEUE | grep PRS_TEST_CLANG= | grep SCRAM_ARCH=$ARCHITECTURE; then
NEED_CLANG_TEST=true
fi
if [ "X$TEST_CLANG_COMPILATION" = Xtrue -a $NEED_CLANG_TEST = true -a "X$CMSSW_PR" != X -a "$SKIP_STATIC_CHECKS" = "false" ]; then
#first, add the command to the log
CLANG_USER_CMD="USER_CUDA_FLAGS='--expt-relaxed-constexpr' USER_CXXFLAGS='-Wno-register -fsyntax-only' /usr/bin/time -v scram build -k -j ${NCPU2} COMPILER='llvm compile'"
CLANG_CMD="scram b vclean && ${CLANG_USER_CMD} BUILD_LOG=yes"
echo $CLANG_USER_CMD > $WORKSPACE/buildClang.log
(eval $CLANG_CMD && echo 'ALL_OK') >>$WORKSPACE/buildClang.log 2>&1 || true
#always run ${ANALOG_CMD} to print out the compile command which are normally printed when on runs full build
#in llvm case we are only doing compile (withoutlinking)
(eval ${ANALOG_CMD}) >>$WORKSPACE/buildClang.log 2>&1 || true
TEST_ERRORS=`grep -E "^gmake: .* Error [0-9]" $WORKSPACE/buildClang.log` || true
GENERAL_ERRORS=`grep "^ALL_OK$" $WORKSPACE/buildClang.log` || true
if [ -d ${BUILD_LOG_DIR}/html ] ; then
mv ${BUILD_LOG_DIR}/html $WORKSPACE/clang-logs
echo 'CLANG_LOG;OK,Clang warnings summary,See Log,clang-logs' >> ${RESULTS_DIR}/clang.txt
fi
if [ "X$TEST_ERRORS" != "X" -o "X$GENERAL_ERRORS" = "X" ]; then
echo "Errors when testing compilation with clang"
echo 'CLANG_COMPILATION_RESULTS;ERROR,Clang Compilation,See Log,buildClang.log' >> ${RESULTS_DIR}/clang.txt
RUN_TESTS=false
ALL_OK=false
CLANG_BUILD_OK=false
else
echo "the clang compilation had no errors/warnings!!"
echo 'CLANG_COMPILATION_RESULTS;OK,Clang Compilation,See Log,buildClang.log' >> ${RESULTS_DIR}/clang.txt
fi
fi
#Do QA checks
#Code Rules