From 6b7fa1b14064f8f999a7e0b9d4ffde1a016d0f63 Mon Sep 17 00:00:00 2001 From: mmusich Date: Tue, 2 Feb 2021 14:27:51 +0100 Subject: [PATCH] add diffGlobalTagsAtRun to conddb --- CondCore/Utilities/scripts/conddb | 240 ++++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) diff --git a/CondCore/Utilities/scripts/conddb b/CondCore/Utilities/scripts/conddb index 556c4ff25dabd..940ee63c66023 100755 --- a/CondCore/Utilities/scripts/conddb +++ b/CondCore/Utilities/scripts/conddb @@ -940,6 +940,237 @@ def listParentTags_(args): + +def diffGlobalTagsAtRun_(args): + connection = connect(args) + session = connection.session() + + IOV = session.get_dbtype(conddb.IOV) + TAG = session.get_dbtype(conddb.Tag) + GT = session.get_dbtype(conddb.GlobalTag) + GTMAP = session.get_dbtype(conddb.GlobalTagMap) + RUNINFO = session.get_dbtype(conddb.RunInfo) + + #################################### + # Get the time info for the test run + #################################### + + if(not args.lastIOV): + + if(args.testRunNumber<0): + raise Exception("Run %s (default) can't be matched with an existing run in the database. \n\t\t Please specify a run with the option --run." % args.testRunNumber) + + bestRun = session.query(RUNINFO.run_number, RUNINFO.start_time, RUNINFO.end_time).filter(RUNINFO.run_number == int(args.testRunNumber)).first() + if bestRun is None: + raise Exception("Run %s can't be matched with an existing run in the database." % args.testRunNumber) + + print("Run",args.testRunNumber," |Start time",bestRun[1]," |End time",bestRun[2],".") + + #################################### + # Get the Global Tag snapshots + #################################### + + refSnap = session.query(GT.snapshot_time).\ + filter(GT.name == args.refGT).all()[0][0] + + tarSnap = session.query(GT.snapshot_time).\ + filter(GT.name == args.tarGT).all()[0][0] + + print("reference GT (",args.refGT ,") snapshot: ",refSnap," | target GT (",args.tarGT,") snapshot",tarSnap) + + #################################### + # Get the Global Tag maps + #################################### + + GTMap_ref = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\ + filter(GTMAP.global_tag_name == args.refGT).\ + order_by(GTMAP.record, GTMAP.label).\ + all() + + GTMap_tar = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\ + filter(GTMAP.global_tag_name == args.tarGT).\ + order_by(GTMAP.record, GTMAP.label).\ + all() + + text_file = open(("diff_%s_vs_%s.twiki") % (args.refGT,args.tarGT), "w") + + differentTags = {} + + for element in GTMap_ref: + RefRecord = element[0] + RefLabel = element[1] + RefTag = element[2] + + for element2 in GTMap_tar: + if (RefRecord == element2[0] and RefLabel==element2[1]): + if RefTag != element2[2]: + differentTags[(RefRecord,RefLabel)]=(RefTag,element2[2]) + + #################################### + ## Search for Records,Label not-found in the other list + #################################### + + temp1 = [item for item in GTMap_ref if (item[0],item[1]) not in list(zip(list(zip(*GTMap_tar))[0],list(zip(*GTMap_tar))[1]))] + for elem in temp1: + differentTags[(elem[0],elem[1])]=(elem[2],"") + + temp2 = [item for item in GTMap_tar if (item[0],item[1]) not in list(zip(list(zip(*GTMap_ref))[0],list(zip(*GTMap_ref))[1]))] + for elem in temp2: + differentTags[(elem[0],elem[1])]=("",elem[2]) + + text_file.write("| *Record* | *"+args.refGT+"* | *"+args.tarGT+"* | Remarks | \n") + + t = PrettyTable() + + if(args.isVerbose): + t.field_names = ['/','',args.refGT,args.tarGT,refSnap,tarSnap] + else: + t.field_names = ['/','',args.refGT,args.tarGT] + + t.hrules=1 + + if(args.isVerbose): + t.add_row(['Record','label','Reference Tag','Target Tag','hash1:time1:since1','hash2:time2:since2']) + else: + t.add_row(['Record','label','Reference Tag','Target Tag']) + + isDifferent=False + + #################################### + # Loop on the difference + #################################### + + for Rcd in sorted(differentTags): + + # empty lists at the beginning + refTagIOVs=[] + tarTagIOVs=[] + + if( differentTags[Rcd][0]!=""): + refTagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == differentTags[Rcd][0]).all() + refTagInfo = session.query(TAG.synchronization,TAG.time_type).filter(TAG.name == differentTags[Rcd][0]).all()[0] + if( differentTags[Rcd][1]!=""): + tarTagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == differentTags[Rcd][1]).all() + tarTagInfo = session.query(TAG.synchronization,TAG.time_type).filter(TAG.name == differentTags[Rcd][1]).all()[0] + + if(differentTags[Rcd][0]!="" and differentTags[Rcd][1]!=""): + if(tarTagInfo[1] != refTagInfo[1]): + print(colors.bold_red+" *** Warning *** found mismatched time type for",Rcd,"entry. \n"+differentTags[Rcd][0],"has time type",refTagInfo[1],"while",differentTags[Rcd][1],"has time type",tarTagInfo[1]+". These need to be checked by hand. \n\n"+ colors.end) + + if(args.lastIOV): + + if(sorted(differentTags).index(Rcd)==0): + print("\n") + print(33 * "=") + print("|| COMPARING ONLY THE LAST IOV ||") + print(33 * "=") + print("\n") + + lastSinceRef=-1 + lastSinceTar=-1 + + hash_lastRefTagIOV = "" + time_lastRefTagIOV = "" + + hash_lastTarTagIOV = "" + time_lastTarTagIOV = "" + + for i in refTagIOVs: + if (i[0]>lastSinceRef): + lastSinceRef = i[0] + hash_lastRefTagIOV = i[1] + time_lastRefTagIOV = str(i[2]) + + for j in tarTagIOVs: + if (j[0]>lastSinceTar): + lastSinceTar = j[0] + hash_lastTarTagIOV = j[1] + time_lastTarTagIOV = str(j[2]) + + if(hash_lastRefTagIOV!=hash_lastTarTagIOV): + isDifferent=True + text_file.write("| ="+Rcd[0]+"= ("+Rcd[1]+") | =="+differentTags[Rcd][0]+"== | =="+differentTags[Rcd][1]+"== | | \n") + text_file.write("|^|"+hash_lastRefTagIOV+"
("+time_lastRefTagIOV+") "+ str(lastSinceRef) +" | "+hash_lastTarTagIOV+"
("+time_lastTarTagIOV+") " + str(lastSinceTar)+" | ^| \n") + + if(args.isVerbose): + t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0],differentTags[Rcd][1],str(hash_lastRefTagIOV)+"\n"+str(time_lastRefTagIOV)+"\n"+str(lastSinceRef),str(hash_lastTarTagIOV)+"\n"+str(time_lastTarTagIOV)+"\n"+str(lastSinceTar)]) + else: + t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0]+"\n"+str(hash_lastRefTagIOV),differentTags[Rcd][1]+"\n"+str(hash_lastTarTagIOV)]) + + else: + + ### reset all defaults + + theGoodRefIOV=-1 + theGoodTarIOV=-1 + sinceRefTagIOV=0 + sinceTarTagIOV=0 + + RefIOVtime = datetime.datetime(1970, 1, 1, 0, 0, 0) + TarIOVtime = datetime.datetime(1970, 1, 1, 0, 0, 0) + + theRefPayload="" + theTarPayload="" + theRefTime="" + theTarTime="" + + ### loop on the reference IOV list + for refIOV in refTagIOVs: + + ## logic for retrieving the the last payload active on a given IOV + ## - the new IOV since is less than the run under consideration + ## - the payload insertion time is lower than the GT snapshot + ## - finall either: + ## - the new IOV since is larger then the last saved + ## - the new IOV since is equal to the last saved but it has a more recent insertion time + + if ( (refIOV[0] <= int(args.testRunNumber)) and (refIOV[0]>sinceRefTagIOV) or ((refIOV[0]==sinceRefTagIOV) and (refIOV[2]>RefIOVtime)) and (refIOV[2]<=refSnap) ): + sinceRefTagIOV = refIOV[0] + RefIOVtime = refIOV[2] + theGoodRefIOV=sinceRefTagIOV + theRefPayload=refIOV[1] + theRefTime=str(refIOV[2]) + + ### loop on the target IOV list + for tarIOV in tarTagIOVs: + if ( (tarIOV[0] <= int(args.testRunNumber)) and (tarIOV[0]>sinceTarTagIOV) or ((tarIOV[0]==sinceTarTagIOV) and (tarIOV[2]>=TarIOVtime)) and (tarIOV[2]<=tarSnap) ): + sinceTarTagIOV = tarIOV[0] + tarIOVtime = tarIOV[2] + theGoodTarIOV=sinceTarTagIOV + theTarPayload=tarIOV[1] + theTarTime=str(tarIOV[2]) + + #print Rcd[0],theRefPayload,theTarPayload + + if(theRefPayload!=theTarPayload): + isDifferent=True + text_file.write("| ="+Rcd[0]+"= ("+Rcd[1]+") | =="+differentTags[Rcd][0]+"== | =="+differentTags[Rcd][1]+"== |\n") + text_file.write("|^|"+theRefPayload+" ("+theRefTime+") | "+theTarPayload+" ("+theTarTime+") |\n") + + ### determinte if it is to be shown + + isMatched=False + tokens=args.stringToMatch.split(",") + decisions = [bool(Rcd[0].find(x)!=-1) for x in tokens] + for decision in decisions: + isMatched = (isMatched or decision) + + if(args.isVerbose): + if (args.stringToMatch=="" or isMatched): + t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0],differentTags[Rcd][1],str(theRefPayload)+"\n"+str(theRefTime)+"\n"+str(theGoodRefIOV),str(theTarPayload)+"\n"+str(theTarTime)+"\n"+str(theGoodTarIOV)]) + else: + if (args.stringToMatch=="" or isMatched): + t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0]+"\n"+str(theRefPayload),differentTags[Rcd][1]+"\n"+str(theTarPayload)]) + + if(not isDifferent): + if(args.isVerbose): + t.add_row(["None","None","None","None","None","None"]) + else: + t.add_row(["None","None","None"]) + print(t) + + + def listGTsForTag_(args): connection = connect(args) session = connection.session() @@ -2214,6 +2445,15 @@ def main(): parser_listParentTags.add_argument('hash_name', help="Payload hash to match.") parser_listParentTags.set_defaults(func=listParentTags_) + parser_diffGlobalTagsAtRun = parser_subparsers.add_parser('diffGloablTagsAtRun', description='Diffs two global tags, but showing only the differences relevant for a given run number.') + parser_diffGlobalTagsAtRun.add_argument('--last', '-L', dest='lastIOV', action='store_true', default=False, help='Diff the Global tags at the last open IOV.') + parser_diffGlobalTagsAtRun.add_argument('--reference', '-R', dest='refGT', help="Reference Global Tag") + parser_diffGlobalTagsAtRun.add_argument('--target', '-T', dest='tarGT', help="Target Global Tag") + parser_diffGlobalTagsAtRun.add_argument('--run', '-r', dest='testRunNumber', help="target run to compare",default=-1) + parser_diffGlobalTagsAtRun.add_argument('--verbose','-v',help='returns more info', dest='isVerbose',action='store_true',default=False) + parser_diffGlobalTagsAtRun.add_argument('--match','-m',help='print only matching',dest='stringToMatch',action='store',default='') + parser_diffGlobalTagsAtRun.set_defaults(func=diffGlobalTagsAtRun_) + parser_listGTsForTag = parser_subparsers.add_parser('listGTsForTag', description='Lists the GTs which contain a given tag.') parser_listGTsForTag.add_argument('name', help="Name of the tag.") parser_listGTsForTag.set_defaults(func=listGTsForTag_)