diff --git a/categories_map.py b/categories_map.py index 3d7d0c8cc5c9..620636e2fc18 100644 --- a/categories_map.py +++ b/categories_map.py @@ -508,6 +508,7 @@ "DetectorDescription/Algorithm", "DetectorDescription/Base", "DetectorDescription/Core", + "DetectorDescription/DDCMS", "DetectorDescription/ExprAlgo", "DetectorDescription/OfflineDBLoader", "DetectorDescription/OnlineDBLoader", diff --git a/comparisons/matrix_RE.txt b/comparisons/matrix_RE.txt index 9093a85bd152..b4b42d89a667 100644 --- a/comparisons/matrix_RE.txt +++ b/comparisons/matrix_RE.txt @@ -53,6 +53,7 @@ HydjetQMinBiaswf140p0 140.0_HydjetQ_MinBias_2760GeV+HydjetQ_MinBias_2760GeV*/ste ZEEMMHIwf140p4 140.4_ZEEMM_13_HI+ZEEMM_13_HI*/step3.root RECO ZEEMMHIwf302p0 302.0_Pyquen_ZeemumuJets_pt10_2760GeV+Pyquen_ZeemumuJets_pt10_2760GeV+*/step3.root RECO EPOSPPb8160GeVwf281p0 281.0_EPOS_PPb_8160GeV_MinimumBias+*/step3.root RECO +HydjetQB12in2018wf150p0 150.0_HydjetQ_B12_5020GeV_*/step3.root RECO # # Run2 MC # @@ -73,6 +74,7 @@ SingleMu13Pt1000wf1322p0 1322.0_SingleMuPt1000_UP15+*/step3.root RECO TTbar13wf1325p0 1325.0_TTbar_13+TTbar_13*/step3.root RECO TTbar13wf11325p0 11325.0_TTbar_13_unsch+TTbar_13*/step3.root RECO TTbar13reMINIAODwf1325p5 1325.5_TTbar_13_reminiaod*/step2.root PAT +TTbar13nanoAODwf1325p7 1325.7_TTbar_13_94XNanoAODINPUT+*/step2.root DQM ZMM13TeVwf1330p0 1330.0_ZMM_13+*/step3.root RECO H125GG13TeVwf1332p0 1332.0_H125GGgluonfusion_13+*/step3.root RECO VBFH125BB13TeVwf1363p0 1363.0_VBFHToBB_M125_Pow_py8_Evt_13*/step3.root RECO @@ -116,6 +118,7 @@ RunJetHT2016EreMINIAODwf136p7611 136.7611_RunJetHT2016E_reminiaod+*/step2.root P RunMET2016Ewf136p762 136.762_RunMET2016E+*/step3.root reRECO RunSinglePh2016Ewf136p766 136.766_RunSinglePh2016E+*/step3.root reRECO RunJetHT2016Hwf136p772 136.772_RunJetHT2016H+*/step3.root reRECO +RunJetHT2016HreMINIAODwf136p7721 136.7721_RunJetHT2016H_reminiaod+*/step2.root PAT # # 2017 data # diff --git a/comparisons/validate.C b/comparisons/validate.C index 6ba9eb8222a6..46dbe966b335 100644 --- a/comparisons/validate.C +++ b/comparisons/validate.C @@ -347,10 +347,12 @@ void metVars(TString cName = "tcMet_", TString tName = "recoMETs_") { met("significance",cName,tName); } -void tau(TString var, TString cName = "hpsPFTauProducer_", TString tName = "recoPFTaus_", bool notafunction=false){ +void tau(TString var, TString cName = "hpsPFTauProducer_", TString tName = "recoPFTaus_", + bool log10Var = false, bool trycatch = false, bool notafunction = false){ TString v=notafunction ? tName+cName+"_"+recoS+".obj."+var: tName+cName+"_"+recoS+".obj."+var+"()"; - plotvar(v); + if (log10Var) v = "log10(" + v + ")"; + plotvar(v, "", trycatch); } void tauVars(TString cName = "hpsPFTauProducer_", TString tName = "recoPFTaus_"){ @@ -360,6 +362,41 @@ void tauVars(TString cName = "hpsPFTauProducer_", TString tName = "recoPFTaus_") tau("eta",cName,tName); tau("phi",cName,tName); if (tName!="patTaus_") tau("emFraction",cName,tName);//crashes now for patTaus + + if (tName == "patTaus_"){ + tau("dxy", cName, tName); + tau("dxy_error", cName, tName); + + tau("ip3d", cName, tName); + tau("ip3d_error", cName, tName); + tau("ecalEnergy", cName, tName); + tau("hcalEnergy", cName, tName); + tau("leadingTrackNormChi2", cName, tName); + tau("ecalEnergyLeadChargedHadrCand", cName, tName); + tau("hcalEnergyLeadChargedHadrCand", cName, tName); + tau("etaAtEcalEntrance", cName, tName); + tau("etaAtEcalEntranceLeadChargedCand", cName, tName); + tau("ptLeadChargedCand", cName, tName); + tau("emFraction_MVA", cName, tName); + + tau("userFloats_@.size", cName,tName); + for (int i = 0; i< 32; ++i){ + plotvar(tName+cName+"_"+recoS+Form(".obj[].userFloats_[%d]",i), "", true); + } + tau("userInts_@.size", cName,tName); + for (int i = 0; i< 32; ++i){ + plotvar(tName+cName+"_"+recoS+Form(".obj[].userInts_[%d]",i), "", true); + } + tau("userCands_@.size", cName,tName); + tau("isolations_@.size", cName,tName); + for (int i = 0; i< 12; ++i){ + plotvar(tName+cName+"_"+recoS+Form(".obj[].isolations_[%d]",i), "", true); + } + tau("tauIDs_@.size", cName,tName); + for (int i = 0; i< 82; ++i){ + plotvar(tName+cName+"_"+recoS+Form(".obj[].tauIDs_[%d].second",i), "", true); + } + } } void photon(TString var, TString cName = "photons_", TString tName = "recoPhotons_", bool notafunction=false){ @@ -2121,40 +2158,8 @@ void validateEvents(TString step, TString file, TString refFile, TString r="RECO tauVars("hpsPFTauProducer_"); // miniaod tauVars("slimmedTaus_","patTaus_"); - //pat::Tau specifics - plotvar("patTaus_slimmedTaus__"+recoS+".obj.dxy()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.dxy_error()"); - - plotvar("patTaus_slimmedTaus__"+recoS+".obj.ip3d()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.ip3d_error()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.ecalEnergy()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.hcalEnergy()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.leadingTrackNormChi2()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.ecalEnergyLeadChargedHadrCand()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.hcalEnergyLeadChargedHadrCand()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.etaAtEcalEntrance()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.etaAtEcalEntranceLeadChargedCand()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.ptLeadChargedCand()"); - plotvar("patTaus_slimmedTaus__"+recoS+".obj.emFraction_MVA()"); - // boosted tau reco - // miniaod tauVars("slimmedTausBoosted_","patTaus_"); - //pat::Tau specifics - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.dxy()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.dxy_error()"); - - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.ip3d()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.ip3d_error()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.ecalEnergy()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.hcalEnergy()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.leadingTrackNormChi2()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.ecalEnergyLeadChargedHadrCand()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.hcalEnergyLeadChargedHadrCand()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.etaAtEcalEntrance()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.etaAtEcalEntranceLeadChargedCand()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.ptLeadChargedCand()"); - plotvar("patTaus_slimmedTausBoosted__"+recoS+".obj.emFraction_MVA()"); //upstream discriminators plotvar("recoPFTauDiscriminator_hpsPFTauDiscriminationByIsolationMVArun2v1PWdR03oldDMwLTraw__"+recoS+".obj.data_"); diff --git a/config.map b/config.map index 25749af4150c..58e1b4239773 100644 --- a/config.map +++ b/config.map @@ -26,8 +26,8 @@ SCRAM_ARCH=slc6_amd64_gcc700;PKGTOOLS_TAG=V00-32-XX;CMSDIST_TAG=IB/CMSSW_10_0_X/ SCRAM_ARCH=slc6_amd64_gcc700;PKGTOOLS_TAG=V00-32-XX;CMSDIST_TAG=IB/CMSSW_10_0_X/gcc700;RELEASE_BRANCH=CMSSW_10_0_X;RELEASE_QUEUE=CMSSW_10_0_TSAN_X;BUILD_HOUR=00;DISABLED=1; SCRAM_ARCH=slc7_aarch64_gcc700;PKGTOOLS_TAG=V00-32-XX;CMSDIST_TAG=IB/CMSSW_10_0_X/gcc700;RELEASE_BRANCH=CMSSW_10_0_X;RELEASE_QUEUE=CMSSW_10_0_X;DISABLED=1; SCRAM_ARCH=slc6_amd64_gcc630;PKGTOOLS_TAG=V00-32-XX;CMSDIST_TAG=IB/CMSSW_10_0_X/gcc630;RELEASE_BRANCH=CMSSW_10_0_X;RELEASE_QUEUE=CMSSW_10_0_COVERAGE_X;DOCKER_IMG=cmssw/slc6-builder:latest;BUILD_HOUR=24;DISABLED=1; -SCRAM_ARCH=slc6_amd64_gcc630;PKGTOOLS_TAG=V00-31-XX;CMSDIST_TAG=IB/CMSSW_9_4_X/gcc630;RELEASE_BRANCH=CMSSW_9_4_MAOD_X;RELEASE_QUEUE=CMSSW_9_4_MAOD_X;ADDITIONAL_TESTS=HLT,baseline,static-checks,fwlite,valgrind,igprof,geometry,iwyu,lizard;PR_TESTS=1;PROD_ARCH=1;ENABLE_DEBUG=1;PRS_TEST_CLANG=1; -SCRAM_ARCH=slc6_amd64_gcc630;PKGTOOLS_TAG=V00-31-XX;CMSDIST_TAG=IB/CMSSW_9_4_X/gcc630;RELEASE_BRANCH=CMSSW_9_4_X;RELEASE_QUEUE=CMSSW_9_4_X;ADDITIONAL_TESTS=HLT,baseline,static-checks,fwlite,valgrind,igprof,geometry,iwyu,lizard;PR_TESTS=1;PROD_ARCH=1;ENABLE_DEBUG=1;PRS_TEST_CLANG=1; +SCRAM_ARCH=slc6_amd64_gcc630;PKGTOOLS_TAG=V00-31-XX;CMSDIST_TAG=IB/CMSSW_9_4_X/gcc630;RELEASE_BRANCH=CMSSW_9_4_MAOD_X;RELEASE_QUEUE=CMSSW_9_4_MAOD_X;ADDITIONAL_TESTS=HLT,baseline,static-checks,fwlite,valgrind,igprof,geometry,iwyu,lizard;PR_TESTS=1;PROD_ARCH=1;ENABLE_DEBUG=1;PRS_TEST_CLANG=1;BUILD_PATCH_RELEASE=1; +SCRAM_ARCH=slc6_amd64_gcc630;PKGTOOLS_TAG=V00-31-XX;CMSDIST_TAG=IB/CMSSW_9_4_X/gcc630;RELEASE_BRANCH=CMSSW_9_4_X;RELEASE_QUEUE=CMSSW_9_4_X;ADDITIONAL_TESTS=HLT,baseline,static-checks,fwlite,valgrind,igprof,geometry,iwyu,lizard;PR_TESTS=1;PROD_ARCH=1;ENABLE_DEBUG=1;PRS_TEST_CLANG=1;BUILD_PATCH_RELEASE=1; SCRAM_ARCH=slc7_amd64_gcc630;PKGTOOLS_TAG=V00-31-XX;CMSDIST_TAG=IB/CMSSW_9_4_X/gcc630;RELEASE_BRANCH=CMSSW_9_4_X;RELEASE_QUEUE=CMSSW_9_4_X;DOCKER_IMG=cmssw/slc7-builder:latest; SCRAM_ARCH=slc6_amd64_gcc630;PKGTOOLS_TAG=V00-31-XX;CMSDIST_TAG=IB/CMSSW_9_4_X/gcc630next;RELEASE_BRANCH=CMSSW_9_4_DEVEL_X;RELEASE_QUEUE=CMSSW_9_4_DEVEL_X;PR_TESTS=1;BUILD_HOUR=23,00;DOCKER_IMG=cmssw/slc6-builder:latest;DISABLED=1; SCRAM_ARCH=slc7_amd64_gcc630;PKGTOOLS_TAG=V00-31-XX;CMSDIST_TAG=IB/CMSSW_9_4_X/gcc630next;RELEASE_BRANCH=CMSSW_9_4_DEVEL_X;RELEASE_QUEUE=CMSSW_9_4_DEVEL_X;DOCKER_IMG=cmssw/slc7-builder:latest;BUILD_HOUR=23,00;DISABLED=1; diff --git a/docker_launcher.sh b/docker_launcher.sh index 555782445833..0a77fe19fe3c 100755 --- a/docker_launcher.sh +++ b/docker_launcher.sh @@ -14,6 +14,9 @@ if [ "X$DOCKER_IMG" != X -a "X$RUN_NATIVE" = "X" ]; then case $XUSER in cmsbld ) DOCKER_OPT=" -u $(id -u):$(id -g) -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group " ;; esac + if [ -e /etc/tnsnames.ora ] ; then + DOCKER_OPT="$DOCKER_OPT -v /etc/tnsnames.ora:/etc/tnsnames.ora " + fi DOCK_ARGS="voms-proxy-init -voms cms -valid 24:00|| true ; cd $WORKSPACE; $@" echo "Passing to docker the args: "$DOCK_ARGS docker run --rm -h `hostname -f` $DOCKER_OPT \ diff --git a/es_cmssdt_apache.py b/es_cmssdt_apache.py index e5789acf32c8..1eec3510a40f 100755 --- a/es_cmssdt_apache.py +++ b/es_cmssdt_apache.py @@ -3,7 +3,7 @@ from sys import exit from datetime import datetime from time import mktime -from es_utils import send_payload +from es_utils import send_payload_new from hashlib import sha1 from json import dumps from logwatch import logwatch, run_cmd, LOGWATCH_APACHE_IGNORE_AGENTS @@ -38,12 +38,12 @@ def process (line, count): payload["agent_type"]=agent.replace(" ","-").split("/",1)[0].upper() id = sha1(line).hexdigest() if (count%1000)==0: print "Processed entries",count - if not send_payload("apache-cmssdt-"+week,"access_log", id, dumps(payload), passwd_file="/data/es/es_secret"): + if not send_payload_new("apache-cmssdt-"+week,"access_log", id, dumps(payload), 'es-cmssdt.cern.ch:9203'): return False if payload["request"].startswith("/SDT/releases.map?release="): xpayload = dict(item.split("=") for item in payload["request"].split("?",1)[1].split("&")) for x in ["@timestamp","ip"]: xpayload[x] = payload[x] - return send_payload("scram-access-"+week,"cmssw-releases", id, dumps(xpayload), passwd_file="/data/es/es_secret") + return send_payload_new("scram-access-"+week,"cmssw-releases", id, dumps(xpayload), 'es-cmssdt.cern.ch:9203') return True count=run_cmd("pgrep -l -x -f '^python .*/es_cmssdt_apache.py$' | wc -l",False) diff --git a/es_utils.py b/es_utils.py index f401abf95548..ef11c74defe7 100755 --- a/es_utils.py +++ b/es_utils.py @@ -37,7 +37,7 @@ def send_payload_new(index,document,id,payload,es_server,passwd_file=None): except Exception as e: print "ERROR:",url,str(e) return False - print "OK",index,document + print "OK ",index return True def send_payload_old(index,document,id,payload,passwd_file=None): @@ -56,7 +56,7 @@ def send_payload_old(index,document,id,payload,passwd_file=None): except Exception as e: print "ERROR: ",url, str(e) return False - print "OK:",index,document + print "OK ",index return True def delete_hit(hit,passwd_file=None): diff --git a/get-pr-branch b/get-pr-branch index f2cf0f992ae9..16ae09a38bf5 100755 --- a/get-pr-branch +++ b/get-pr-branch @@ -1,7 +1,7 @@ #!/usr/bin/env python -from sys import exit, argv -from os.path import expanduser +from sys import exit, argv, path +from os.path import expanduser,dirname,abspath,join, exists from github import Github from socket import setdefaulttimeout setdefaulttimeout(120) @@ -26,4 +26,13 @@ if __name__ == "__main__": if not o: o = CMSSW_DEVEL_BRANCH print o else: - print pr.base.ref + pr_branch = pr.base.ref + try: + SCRIPT_DIR = dirname(abspath(argv[0])) + repo_dir = join(SCRIPT_DIR,'repos',repo.replace("-","_")) + if exists(join(repo_dir,"repo_config.py")): + path.insert(0,repo_dir) + import repo_config + pr_branch = repo_config.CMS_BRANCH_MAP[pr.base.ref] + except: pass + print pr_branch diff --git a/ib-weeks b/ib-weeks index 15ee6c22c167..cae51e6c6fb9 100644 --- a/ib-weeks +++ b/ib-weeks @@ -8,3 +8,4 @@ nweek-02508 nweek-02509 nweek-02510 nweek-02511 +nweek-02512 diff --git a/ignore-webhooks b/ignore-webhooks new file mode 100644 index 000000000000..fe4fc0ec3d18 --- /dev/null +++ b/ignore-webhooks @@ -0,0 +1,2 @@ +cms-sw/cmssdt-ib +cms-sw/logreader diff --git a/jobs/create-relval-jobs.py b/jobs/create-relval-jobs.py index f32d7036b552..4bbcc145fd03 100755 --- a/jobs/create-relval-jobs.py +++ b/jobs/create-relval-jobs.py @@ -49,15 +49,25 @@ def createJob(workflow, cmssw_ver, arch): #Get Workflow stats from ES print "Getting Workflow stats from ES....." -stats = es_query(index='relvals_stats_*', +stats = {} +release_cycle=cmssw_ver.split("_X_")[0]+"_X" +if release_cycle=="CMSSW_9_4_MAOD_X": release_cycle="CMSSW_9_4_X" +while True: + stats = es_query(index='relvals_stats_*', query=format('exit_code:0 AND release:%(release_cycle)s AND architecture:%(architecture)s AND (%(workflows)s)', - release_cycle=cmssw_ver.split("_X_")[0]+"_X_*", + release_cycle=release_cycle+"_*", architecture=arch, workflows=wf_query[4:] ), start_time=1000*int(time()-(86400*10)), end_time=1000*int(time())) - + if (not 'hits' in stats) or (not 'hits' in stats['hits']) or (not stats['hits']['hits']): + xrelease_cycle = "_".join(cmssw_ver.split("_",4)[0:3])+"_X" + if xrelease_cycle!=release_cycle: + release_cycle=xrelease_cycle + continue + break + wf_stats = es_workflow_stats(stats) #Create Jobs diff --git a/jobs/jobscheduler.py b/jobs/jobscheduler.py index 6c76d3585445..3b2b767fd509 100755 --- a/jobs/jobscheduler.py +++ b/jobs/jobscheduler.py @@ -8,8 +8,8 @@ from optparse import OptionParser from subprocess import Popen -simulation_time = 0 -simulation = False +global simulation_time +global simulation def gettime(addtime=0): if not simulation: return int(time()) global simulation_time @@ -148,7 +148,7 @@ def initJobs(jobs, resources, otype): parser.add_option("-M", "--max-jobs", dest="maxJobs", default=-1, type="int", help="Maximum jobs to run in parallel. Default is -1 which means no limit. Special value 0 means maximum jobs=CPU counts") parser.add_option("-s", "--simulate", dest="simulate", action="store_true", help="Do not run the jobs but simulate the timings.", default=False) opts, args = parser.parse_args() - global simulation + simulation_time = 0 simulation = opts.simulate if opts.memory>200: opts.memory=200 if opts.cpu>300: opts.cpu=300 diff --git a/process_pr.py b/process_pr.py index 7976cfe3661b..cbb4487c737a 100644 --- a/process_pr.py +++ b/process_pr.py @@ -50,10 +50,14 @@ def read_repo_file(repo_config, repo_file, default=None): # # creates a properties file to trigger the test of the pull request # -def create_properties_file_tests(repository, pr_number, cmsdist_pr, cmssw_prs, extra_wfs, dryRun, abort=False, req_type="tests"): +def create_properties_file_tests(repository, pr_number, cmsdist_pr, cmssw_prs, extra_wfs, dryRun, abort=False, req_type="tests", repo_config=None): if abort: req_type = "abort" repo_parts = repository.split("/") - if (req_type in "tests") and (not repo_parts[1] in [GH_CMSDIST_REPO,GH_CMSSW_REPO]): req_type = "user-"+req_type + if (req_type in "tests"): + try: + if (not repo_parts[1] in [GH_CMSDIST_REPO,GH_CMSSW_REPO]): req_type = "user-"+req_type + elif not repo_config.CMS_STANDARD_TESTS: req_type = "user-"+req_type + except: pass if (repo_parts[0] == GH_CMSSW_ORGANIZATION) and (repo_parts[1] in [GH_CMSDIST_REPO,GH_CMSSW_REPO]): repo_partsX=repo_parts[1] else: repo_partsX=repository.replace("/","-") out_file_name = 'trigger-%s-%s-%s.properties' % (req_type, repo_partsX, pr_number) @@ -70,6 +74,9 @@ def create_properties_file_tests(repository, pr_number, cmsdist_pr, cmssw_prs, e out_file.write( '%s=%s\n' % ( 'PULL_REQUEST', pr_number ) ) out_file.write( '%s=%s\n' % ( 'CMSDIST_PR', cmsdist_pr ) ) out_file.write( '%s=%s\n' % ( 'ADDITIONAL_PULL_REQUESTS', cmssw_prs ) ) + try: + if repo_config.JENKINS_SLAVE_LABEL: out_file.write( '%s=%s\n' % ('RUN_LABEL', repo_config.JENKINS_SLAVE_LABEL)) + except: pass out_file.close() # Update the milestone for a given issue. @@ -501,6 +508,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F ok, cmsdist_pr, cmssw_prs, extra_wfs = check_test_cmd(first_line) if ok: print 'Tests requested:', commenter, 'asked to test this PR with cmsdist_pr=%s, cmssw_prs=%s and workflows=%s' % (cmsdist_pr, cmssw_prs, extra_wfs) + print "Comment message:",first_line trigger_test_on_signature = False if tests_already_queued: print "Test results not obtained in ",comment.created_at-last_test_start_time @@ -730,7 +738,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F issue.create_comment( test_msg ) if cmsdist_issue: cmsdist_issue.create_comment(TRIGERING_TESTS_MSG+"\nUsing cmssw from "+CMSSW_REPO_NAME+"#"+str(prId)) if (not cmsdist_pr) or cmsdist_issue: - create_properties_file_tests( repository, prId, cmsdist_pr, cmssw_prs, extra_wfs, dryRun, abort=False) + create_properties_file_tests( repository, prId, cmsdist_pr, cmssw_prs, extra_wfs, dryRun, abort=False, repo_config=repo_config) elif abort_test: issue.create_comment( TRIGERING_TESTS_ABORT_MSG ) if cmsdist_issue: cmsdist_issue.create_comment( TRIGERING_TESTS_ABORT_MSG ) diff --git a/releases.map b/releases.map index 756718057418..eec704d9dfaf 100644 --- a/releases.map +++ b/releases.map @@ -2023,3 +2023,5 @@ architecture=slc7_amd64_gcc630;label=CMSSW_10_1_0_pre1;type=Development;state=An architecture=slc6_amd64_gcc700;label=CMSSW_10_1_0_pre1;type=Development;state=Announced;prodarch=0; architecture=slc7_amd64_gcc700;label=CMSSW_10_1_0_pre1;type=Development;state=Announced;prodarch=0; architecture=slc6_amd64_gcc630;label=CMSSW_10_1_0_pre1;type=Development;state=Announced;prodarch=1; +architecture=slc7_amd64_gcc630;label=CMSSW_9_3_6_patch2;type=Production;state=Announced;prodarch=0; +architecture=slc6_amd64_gcc630;label=CMSSW_9_3_6_patch2;type=Production;state=Announced;prodarch=1; diff --git a/repos/README.md b/repos/README.md new file mode 100644 index 000000000000..f2d992a71f7e --- /dev/null +++ b/repos/README.md @@ -0,0 +1,27 @@ +## Setting up Pull Requests testing for CMS user repositories +### Setup you repository +- Make a Pull Request to add your repository configuration in `cms-bot/repos/your_github_user/your_repository` + - If you have `-` in your github user or repository name then replace it with `_` +- It is better to copy existing configuration and change it accordingly e.g. copy `repos/smuzaffar/cmssw` in to `repos/your_repository` +- Allow `cmsbot` to update your repository + - If you have a github organization then please add github user `cmsbot` in to a team with write (or admin) rights + - If it is not a organization then please add `cmsbot` as Collaborators (under the Settings of your repository). +- Add github webhook so that bot can get notifications. + - If you have given admin rights to `cmsbot` and set `ADD_WEB_HOOK=True` in `repos/your_repo/repo_config.py` then bot can add web-hook + - If `cmsbot` does not have admin rights to your repository then please add yourself the github webhook (under Settings of your repository) and send us the "Secret" pass phrase so that cms-bot only recognize valid web hooks + - Please disable SSL Verificaton as github does not recognize cmssdt.cern.ch certificate + - Payload URL: https://cmssdt.cern.ch/SDT/cgi-bin/github_webhook + - Content type: application/json + - Secret: any password of your choice + - Disable SSL Verification + - Let me select individual events: Select + - Issues, Issue comment, Pull request + +### Pull request Testing: +- For `user/cmssw` and `user/cmsdist` repositories , bot can run standard PR tests. + - If you do not want to run standard cms PR tests then set `CMS_STANDARD_TESTS=False` in your `repo_config.py` file. +- For `user/non-cmssw` repository, you need to provide `repos/your_repository/run-pr-tests` script which bot can run. + - bot will clone your repository in `$WORKSPACE/userrepo` and will merge your pull request on top of your default branch + - A file `$WORKSPACE/changed-files.txt` will contain the list of changed file in the Pull Request + - If you want to upload job logs (max 1G) then copy them under `$WORKSPACE/upload` +- cmsbot commands are listed here http://cms-sw.github.io/cms-bot-cmssw-cmds.html diff --git a/repos/cms_patatrack/__init__.py b/repos/cms_patatrack/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/repos/cms_patatrack/cmssw/__init__.py b/repos/cms_patatrack/cmssw/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/repos/cms_patatrack/cmssw/categories.py b/repos/cms_patatrack/cmssw/categories.py new file mode 100644 index 000000000000..ff7cbf441e20 --- /dev/null +++ b/repos/cms_patatrack/cmssw/categories.py @@ -0,0 +1,29 @@ +from cms_static import GH_CMSSW_REPO as gh_cmssw +from cms_static import GH_CMSDIST_REPO as gh_cmsdist +from repo_config import GH_REPO_ORGANIZATION as gh_user +from repo_config import CMSBUILD_USER +from categories_map import CMSSW_CATEGORIES + +CMSSW_L1 = [] +APPROVE_BUILD_RELEASE = list(set([ ] + CMSSW_L1)) +REQUEST_BUILD_RELEASE = APPROVE_BUILD_RELEASE +TRIGGER_PR_TESTS = list(set(['smuzaffar'] + REQUEST_BUILD_RELEASE)) +PR_HOLD_MANAGERS = [ ] + +COMMON_CATEGORIES = [ "tests", "code-checks" ] +EXTERNAL_CATEGORIES = [ "externals" ] +EXTERNAL_REPOS = [] + +CMSSW_REPOS = [ gh_user+"/"+gh_cmssw ] +CMSDIST_REPOS = [ gh_user+"/"+gh_cmsdist ] +CMSSW_ISSUES_TRACKERS = list(set(CMSSW_L1)) +COMPARISON_MISSING_MAP = [] + +#github_user:[list of categories] +CMSSW_L2 = { + CMSBUILD_USER : ["tests", "code-checks" ], + 'fwyzard' : CMSSW_CATEGORIES.keys(), +} + +USERS_TO_TRIGGER_HOOKS = set(TRIGGER_PR_TESTS + CMSSW_ISSUES_TRACKERS + CMSSW_L2.keys()) +CMS_REPOS = set(CMSDIST_REPOS + CMSSW_REPOS + EXTERNAL_REPOS) diff --git a/repos/cms_patatrack/cmssw/category-watchers.yaml b/repos/cms_patatrack/cmssw/category-watchers.yaml new file mode 100644 index 000000000000..77dc27c61264 --- /dev/null +++ b/repos/cms_patatrack/cmssw/category-watchers.yaml @@ -0,0 +1,7 @@ +# Map between github username and the list of cmssw categories to watch e.g. alca, db, core etc. +# valid categories are available here http://cms-sw.github.io/categories.html +#Format +#gh_user: +#- category +#- category + diff --git a/repos/cms_patatrack/cmssw/groups.yaml b/repos/cms_patatrack/cmssw/groups.yaml new file mode 100644 index 000000000000..3ae066f99fed --- /dev/null +++ b/repos/cms_patatrack/cmssw/groups.yaml @@ -0,0 +1,7 @@ +#Map between groups and github users, format of this file is +#groupname: +# - gh_user +# - gh_user +#groupname: +# - gh_user +# - gh_user diff --git a/repos/cms_patatrack/cmssw/releases.py b/repos/cms_patatrack/cmssw/releases.py new file mode 100644 index 000000000000..a69f01107464 --- /dev/null +++ b/repos/cms_patatrack/cmssw/releases.py @@ -0,0 +1,13 @@ +#Default development branch +# Changes from master branch will be merge in to it +# Any PR open against this will be automatically closed by cms-bot (Pr should be made for master branch) +# For new release cycle just change this and make sure to add its milestone and production branches + +CMSSW_DEVEL_BRANCH = "CMSSW_10_1_X" +RELEASE_BRANCH_MILESTONE={} +RELEASE_BRANCH_CLOSED=[] +RELEASE_BRANCH_PRODUCTION=[] +SPECIAL_RELEASE_MANAGERS=[] +RELEASE_MANAGERS={} +USERS_TO_TRIGGER_HOOKS = set(SPECIAL_RELEASE_MANAGERS + [ m for rel in RELEASE_MANAGERS for m in rel ]) + diff --git a/repos/cms_patatrack/cmssw/repo_config.py b/repos/cms_patatrack/cmssw/repo_config.py new file mode 100644 index 000000000000..7ffe8ee2d72e --- /dev/null +++ b/repos/cms_patatrack/cmssw/repo_config.py @@ -0,0 +1,35 @@ +from cms_static import GH_CMSSW_ORGANIZATION,GH_CMSSW_REPO,CMSBUILD_GH_USER +from os.path import basename,dirname,abspath +#GH read/write token: Use default ~/.github-token-cmsbot +GH_TOKEN="~/.github-token-cmsbot" +#GH readonly token: Use default ~/.github-token-readonly +GH_TOKEN_READONLY="~/.github-token-readonly" +CONFIG_DIR=dirname(abspath(__file__)) +#GH bot user: Use default cmsbot +CMSBUILD_USER="cmsbot" +GH_REPO_ORGANIZATION="cms-patatrack" +GH_REPO_FULLNAME="cms-patatrack/cmssw" +CREATE_EXTERNAL_ISSUE=False +#Jenkins CI server: User default http://cmsjenkins05.cern.ch:8080/cms-jenkins +JENKINS_SERVER="http://cmsjenkins05.cern.ch:8080/cms-jenkins" +#GH Web hook pass phrase. This is encrypeted used bot keys. +GITHUB_WEBHOOK_TOKEN='''U2FsdGVkX19C9pvh4GUbgDDUy0G9tSJZu7pFoQ0QodGMQtb/h4AFOKPsBxKlORAz +KXg7+k1B6egPueUzlaJ9BA==''' +#Set to True if you want bot to add build/test labels to your repo +ADD_LABELS=True +#Set to True if you want bot to add GH webhooks. cmsbot needs admin rights +ADD_WEB_HOOK=False +#List of issues/pr which bot should ignore +IGNORE_ISSUES = [] +#Set the Jenkins slave label is your tests needs special machines to run. +JENKINS_SLAVE_LABEL="slc7_amd64 && GPU" +#For cmsdist/cmssw repos , set it to False if you do not want to run standard cms pr tests +CMS_STANDARD_TESTS=True +#Map your branches with cmssw branches for tests +#User Branch => CMSSW/CMSDIST Bracnh +CMS_BRANCH_MAP={ + 'CMSSW_10_1_X_Patatrack' : 'CMSSW_10_1_X' +} +#Valid Web hooks e.g. '.+' to match all event +VALID_WEB_HOOKS=['.+'] + diff --git a/repos/cms_patatrack/cmssw/super-users.yaml b/repos/cms_patatrack/cmssw/super-users.yaml new file mode 100644 index 000000000000..a41abbbcfb8b --- /dev/null +++ b/repos/cms_patatrack/cmssw/super-users.yaml @@ -0,0 +1,3 @@ +#List of super users, format is +#- ghuser +#- ghuser diff --git a/repos/cms_patatrack/cmssw/watchers.yaml b/repos/cms_patatrack/cmssw/watchers.yaml new file mode 100644 index 000000000000..2a1bf157228a --- /dev/null +++ b/repos/cms_patatrack/cmssw/watchers.yaml @@ -0,0 +1,9 @@ +# Map between github username and the list of packages to watch. +# Package is a regular expression which will be automatically delimited by ".*" +# Format of this file is +#ghuser: +# - subsystem1/ +# - subsystem/package1 +#ghuser: +# - subsystem2/package1 +# - subsystem2/package2 diff --git a/repos/smuzaffar/cmssw/repo_config.py b/repos/smuzaffar/cmssw/repo_config.py index bc46f08e528b..5d20e64c9d27 100644 --- a/repos/smuzaffar/cmssw/repo_config.py +++ b/repos/smuzaffar/cmssw/repo_config.py @@ -1,14 +1,33 @@ from cms_static import GH_CMSSW_ORGANIZATION,GH_CMSSW_REPO,CMSBUILD_GH_USER from os.path import basename,dirname,abspath +#GH read/write token: Use default ~/.github-token-cmsbot GH_TOKEN="~/.github-token-cmsbot" +#GH readonly token: Use default ~/.github-token-readonly GH_TOKEN_READONLY="~/.github-token-readonly" CONFIG_DIR=dirname(abspath(__file__)) +#GH bot user: Use default cmsbot CMSBUILD_USER="cmsbot" GH_REPO_ORGANIZATION=basename(dirname(CONFIG_DIR)) GH_REPO_FULLNAME="smuzaffar/cmssw" CREATE_EXTERNAL_ISSUE=False +#Jenkins CI server: User default http://cmsjenkins05.cern.ch:8080/cms-jenkins JENKINS_SERVER="http://cmsjenkins05.cern.ch:8080/cms-jenkins" +#GH Web hook pass phrase. This is encrypeted used bot keys. GITHUB_WEBHOOK_TOKEN='U2FsdGVkX1+GEHdp/Cmu73+ctvrzSGXc9OvL+8bZyjOe6ZPkqr/GIPgpJHiEp+hR' +#Set to True if you want bot to add build/test labels to your repo ADD_LABELS=True +#Set to True if you want bot to add GH webhooks. cmsbot needs admin rights ADD_WEB_HOOK=False +#List of issues/pr which bot should ignore IGNORE_ISSUES = [10] +#Set the Jenkins slave label is your tests needs special machines to run. +JENKINS_SLAVE_LABEL="" +#For cmsdist/cmssw repos , set it to False if you do not want to run standard cms pr tests +CMS_STANDARD_TESTS=True +#Map your branches with cmssw branches for tests +#User Branch => CMSSW/CMSDIST Bracnh +CMS_BRANCH_MAP={ +} +#Valid Web hooks e.g. '.+' to match all event +VALID_WEB_HOOKS=['.+'] + diff --git a/run-pr-comparisons b/run-pr-comparisons index 0af0a40cf031..a7ff941d5b70 100755 --- a/run-pr-comparisons +++ b/run-pr-comparisons @@ -17,6 +17,9 @@ WFS_WITH_ERRORS='' MAPPING_FILE=wf_mapping.txt get_jenkins_artifacts ${BASELINE_DIR}/$MAPPING_FILE $WORKSPACE/$MAPPING_FILE +get_jenkins_artifacts ${PR_BASELINE_JOBDIR}/testsResults.txt $WORKSPACE/results/testsResults.txt +BASE_IB=$(grep '^BASE_IB;' $WORKSPACE/results/testsResults.txt | sed 's|.*;||') +if [ "X${BASE_IB}" = "X" ] ; then BASE_IB=$RELEASE_FORMAT ; fi ERRORS_FILE=wf_errors.txt get_jenkins_artifacts ${BASELINE_DIR}/$ERRORS_FILE $WORKSPACE/$ERRORS_FILE @@ -59,9 +62,9 @@ cat /proc/cpuinfo | grep "model name" >> $DETAILS_FILE ls /cvmfs/cms-ib.cern.ch || true export SCRAM_ARCH=$ARCHITECTURE echo "Set up release area at `date`" -scram project $RELEASE_FORMAT +scram project $BASE_IB set +x -cd $RELEASE_FORMAT +cd $BASE_IB eval `scram run -sh` if [ -e $CMSSW_RELEASE_BASE/src/Utilities/RelMon ] ; then mkdir -p src/Utilities/RelMon @@ -119,8 +122,7 @@ echo $WFS_WITH_DAS_INCONSISTENCY >> $WORKSPACE/$DAS_NON_CONSISTENT_WFS_FILE echo "Finished downloading files at `date`:" -get_jenkins_artifacts ${PR_BASELINE_JOBDIR}/testsResults.txt $WORKSPACE/results/testsResults.txt -sed -i "s/COMPARISON;QUEUED/COMPARISON;RUNNING/g" $WORKSPACE/results/testsResults.txt +sed -i "s/COMPARISON;.*/COMPARISON;RUNNING/g" $WORKSPACE/results/testsResults.txt send_jenkins_artifacts $WORKSPACE/results/testsResults.txt ${PR_BASELINE_JOBDIR}/testsResults.txt # -------------------------------------------------------------------------- @@ -290,7 +292,7 @@ if [ -e results/files ] ; then done fi [ -e comparisonDetails.txt ] && mv comparisonDetails.txt upload/ -send_jenkins_artifacts $WORKSPACE/upload/ baseLineComparisons/${RELEASE_FORMAT}+${PULL_REQUEST_NUMBER}/${BUILD_NUMBER}/ +send_jenkins_artifacts $WORKSPACE/upload/ baseLineComparisons/${BASE_IB}+${PULL_REQUEST_NUMBER}/${BUILD_NUMBER}/ if [ -e $WORKSPACE/results/testsResults.txt ] ; then sed -i "s/COMPARISON;.*/COMPARISON;$BUILD_NUMBER/g" $WORKSPACE/results/testsResults.txt send_jenkins_artifacts $WORKSPACE/results/testsResults.txt ${PR_BASELINE_JOBDIR}/testsResults.txt diff --git a/run-pr-tests b/run-pr-tests index 73df07cef024..46b538a68ee1 100755 --- a/run-pr-tests +++ b/run-pr-tests @@ -50,7 +50,10 @@ REAL_ARCH=-`cat /proc/cpuinfo | grep vendor_id | head -n 1 | sed "s/.*: //"` export SCRAM_ARCH=$ARCHITECTURE COMPARISON_REL= -COMP_QUEUE=$(echo $RELEASE_FORMAT | sed 's|^\(CMSSW_[0-9]*_[0-9]*\)_.*|\1_X|') +case $RELEASE_FORMAT in + CMSSW_9_4_MAOD_X* ) COMP_QUEUE="CMSSW_9_4_MAOD_X" ;; + * ) COMP_QUEUE=$(echo $RELEASE_FORMAT | sed 's|^\(CMSSW_[0-9]*_[0-9]*\)_.*|\1_X|') ;; +esac #If a CMSSW area already exists use it as it has been prepared by the CMSDIST test script if [ ! -d CMSSW_* ]; then @@ -62,7 +65,11 @@ if [ ! -d CMSSW_* ]; then if [ "X$COMP_ARCH" = "X" ] ; then COMP_ARCH=$ARCHITECTURE ; fi fi for SCRAM_REL in $(scram -a $SCRAM_ARCH l -c $RELEASE_FORMAT | grep -v -f "$CMS_BOT_DIR/ignore-releases-for-tests" | awk '{print $2}' | sort -r) ; do - COMP_REL=$(echo $SCRAM_REL | sed 's|_[A-Z][A-Z0-9]*_X_|_X_|') + if [ "$(echo $SCRAM_REL | sed 's|_X_.*|_X|')" = "$COMP_QUEUE" ] ; then + COMP_REL=$SCRAM_REL + else + COMP_REL=$(echo $SCRAM_REL | sed 's|_[A-Z][A-Z0-9]*_X_|_X_|') + fi has_jenkins_artifacts ib-baseline-tests/$COMP_REL/$COMP_ARCH/$REAL_ARCH/matrix-results/wf_errors.txt || continue RELEASE_FORMAT=$SCRAM_REL COMPARISON_ARCH=$COMP_ARCH