add diffGlobalTagsAtRun to conddb

cms-sw · Feb 2, 2021 · 6b7fa1b · 6b7fa1b
1 parent c1d18d3
commit 6b7fa1b
Showing 1 changed file with 240 additions and 0 deletions.
diff --git a/CondCore/Utilities/scripts/conddb b/CondCore/Utilities/scripts/conddb
@@ -940,6 +940,237 @@ def listParentTags_(args):
 
 
 
+
+def diffGlobalTagsAtRun_(args):
+    connection = connect(args)
+    session = connection.session() 
+
+    IOV     = session.get_dbtype(conddb.IOV)
+    TAG     = session.get_dbtype(conddb.Tag)
+    GT      = session.get_dbtype(conddb.GlobalTag)
+    GTMAP   = session.get_dbtype(conddb.GlobalTagMap)
+    RUNINFO = session.get_dbtype(conddb.RunInfo)
+
+    ####################################
+    # Get the time info for the test run
+    ####################################
+
+    if(not args.lastIOV):
+
+        if(args.testRunNumber<0):
+            raise Exception("Run %s (default) can't be matched with an existing run in the database. \n\t\t Please specify a run with the option --run." % args.testRunNumber)
+
+        bestRun = session.query(RUNINFO.run_number, RUNINFO.start_time, RUNINFO.end_time).filter(RUNINFO.run_number == int(args.testRunNumber)).first()
+        if bestRun is None:
+            raise Exception("Run %s can't be matched with an existing run in the database." % args.testRunNumber)
+
+        print("Run",args.testRunNumber," |Start time",bestRun[1]," |End time",bestRun[2],".")
+
+    ####################################
+    # Get the Global Tag snapshots
+    ####################################
+
+    refSnap = session.query(GT.snapshot_time).\
+        filter(GT.name == args.refGT).all()[0][0]
+
+    tarSnap = session.query(GT.snapshot_time).\
+        filter(GT.name == args.tarGT).all()[0][0]
+
+    print("reference GT (",args.refGT ,") snapshot: ",refSnap," | target GT (",args.tarGT,") snapshot",tarSnap)
+
+    ####################################
+    # Get the Global Tag maps
+    ####################################
+
+    GTMap_ref = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\
+        filter(GTMAP.global_tag_name == args.refGT).\
+        order_by(GTMAP.record, GTMAP.label).\
+        all()
+
+    GTMap_tar = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\
+        filter(GTMAP.global_tag_name == args.tarGT).\
+        order_by(GTMAP.record, GTMAP.label).\
+        all()
+
+    text_file = open(("diff_%s_vs_%s.twiki") % (args.refGT,args.tarGT), "w")
+
+    differentTags = {}
+
+    for element in GTMap_ref:
+        RefRecord = element[0]
+        RefLabel  = element[1]
+        RefTag    = element[2]
+
+        for element2 in GTMap_tar:
+            if (RefRecord == element2[0] and RefLabel==element2[1]): 
+                if RefTag != element2[2]:
+                    differentTags[(RefRecord,RefLabel)]=(RefTag,element2[2])
+
+    ####################################
+    ## Search for Records,Label not-found in the other list
+    ####################################
+
+    temp1 = [item for item in GTMap_ref if (item[0],item[1]) not in list(zip(list(zip(*GTMap_tar))[0],list(zip(*GTMap_tar))[1]))]
+    for elem in temp1:
+        differentTags[(elem[0],elem[1])]=(elem[2],"")
+
+    temp2 = [item for item in GTMap_tar if (item[0],item[1]) not in list(zip(list(zip(*GTMap_ref))[0],list(zip(*GTMap_ref))[1]))]
+    for elem in temp2:
+        differentTags[(elem[0],elem[1])]=("",elem[2])    
+
+    text_file.write("| *Record* | *"+args.refGT+"* | *"+args.tarGT+"* | Remarks | \n")
+
+    t = PrettyTable()
+
+    if(args.isVerbose):
+        t.field_names = ['/','',args.refGT,args.tarGT,refSnap,tarSnap]
+    else:
+        t.field_names = ['/','',args.refGT,args.tarGT]
+
+    t.hrules=1
+
+    if(args.isVerbose):
+        t.add_row(['Record','label','Reference Tag','Target Tag','hash1:time1:since1','hash2:time2:since2'])
+    else:
+        t.add_row(['Record','label','Reference Tag','Target Tag'])
+
+    isDifferent=False
+
+    ####################################
+    # Loop on the difference
+    ####################################
+
+    for Rcd in sorted(differentTags):
+
+        # empty lists at the beginning
+        refTagIOVs=[]
+        tarTagIOVs=[]
+
+        if( differentTags[Rcd][0]!=""):
+            refTagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == differentTags[Rcd][0]).all()
+            refTagInfo = session.query(TAG.synchronization,TAG.time_type).filter(TAG.name == differentTags[Rcd][0]).all()[0]
+        if( differentTags[Rcd][1]!=""):
+            tarTagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == differentTags[Rcd][1]).all()
+            tarTagInfo = session.query(TAG.synchronization,TAG.time_type).filter(TAG.name == differentTags[Rcd][1]).all()[0]
+
+        if(differentTags[Rcd][0]!="" and differentTags[Rcd][1]!=""): 
+            if(tarTagInfo[1] != refTagInfo[1]):
+                print(colors.bold_red+" *** Warning *** found mismatched time type for",Rcd,"entry. \n"+differentTags[Rcd][0],"has time type",refTagInfo[1],"while",differentTags[Rcd][1],"has time type",tarTagInfo[1]+". These need to be checked by hand. \n\n"+ colors.end)
+
+        if(args.lastIOV):
+
+            if(sorted(differentTags).index(Rcd)==0):
+                print("\n")
+                print(33 * "=")
+                print("|| COMPARING ONLY THE LAST IOV ||")
+                print(33 * "=")
+                print("\n")
+
+            lastSinceRef=-1
+            lastSinceTar=-1
+
+            hash_lastRefTagIOV = ""
+            time_lastRefTagIOV = ""
+
+            hash_lastTarTagIOV = ""
+            time_lastTarTagIOV = ""
+
+            for i in refTagIOVs:            
+                if (i[0]>lastSinceRef):
+                    lastSinceRef = i[0]
+                    hash_lastRefTagIOV = i[1]
+                    time_lastRefTagIOV = str(i[2])
+
+            for j in tarTagIOVs:
+                if (j[0]>lastSinceTar):
+                    lastSinceTar = j[0]
+                    hash_lastTarTagIOV = j[1]
+                    time_lastTarTagIOV = str(j[2])
+
+            if(hash_lastRefTagIOV!=hash_lastTarTagIOV):
+                isDifferent=True
+                text_file.write("| ="+Rcd[0]+"= ("+Rcd[1]+") | =="+differentTags[Rcd][0]+"==  | =="+differentTags[Rcd][1]+"== | | \n")
+                text_file.write("|^|"+hash_lastRefTagIOV+" <br> ("+time_lastRefTagIOV+") "+ str(lastSinceRef) +" | "+hash_lastTarTagIOV+" <br> ("+time_lastTarTagIOV+") " + str(lastSinceTar)+" | ^| \n")
+
+                if(args.isVerbose):
+                    t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0],differentTags[Rcd][1],str(hash_lastRefTagIOV)+"\n"+str(time_lastRefTagIOV)+"\n"+str(lastSinceRef),str(hash_lastTarTagIOV)+"\n"+str(time_lastTarTagIOV)+"\n"+str(lastSinceTar)])
+                else:
+                    t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0]+"\n"+str(hash_lastRefTagIOV),differentTags[Rcd][1]+"\n"+str(hash_lastTarTagIOV)])
+
+        else:    
+
+            ### reset all defaults
+
+            theGoodRefIOV=-1
+            theGoodTarIOV=-1
+            sinceRefTagIOV=0
+            sinceTarTagIOV=0
+
+            RefIOVtime = datetime.datetime(1970, 1, 1, 0, 0, 0)
+            TarIOVtime = datetime.datetime(1970, 1, 1, 0, 0, 0)
+
+            theRefPayload=""
+            theTarPayload=""
+            theRefTime=""
+            theTarTime=""
+
+            ### loop on the reference IOV list
+            for refIOV in refTagIOVs:
+
+                ## logic for retrieving the the last payload active on a given IOV
+                ## - the new IOV since is less than the run under consideration
+                ## - the payload insertion time is lower than the GT snapshot
+                ## - finall either:
+                ##   - the new IOV since is larger then the last saved
+                ##   - the new IOV since is equal to the last saved but it has a more recent insertion time
+
+                if ( (refIOV[0] <= int(args.testRunNumber)) and (refIOV[0]>sinceRefTagIOV) or ((refIOV[0]==sinceRefTagIOV) and (refIOV[2]>RefIOVtime)) and (refIOV[2]<=refSnap) ):
+                    sinceRefTagIOV = refIOV[0]   
+                    RefIOVtime = refIOV[2]
+                    theGoodRefIOV=sinceRefTagIOV                
+                    theRefPayload=refIOV[1]
+                    theRefTime=str(refIOV[2])
+
+            ### loop on the target IOV list
+            for tarIOV in tarTagIOVs:
+                if ( (tarIOV[0] <= int(args.testRunNumber)) and (tarIOV[0]>sinceTarTagIOV) or ((tarIOV[0]==sinceTarTagIOV) and (tarIOV[2]>=TarIOVtime)) and (tarIOV[2]<=tarSnap) ):
+                    sinceTarTagIOV = tarIOV[0]
+                    tarIOVtime = tarIOV[2]
+                    theGoodTarIOV=sinceTarTagIOV
+                    theTarPayload=tarIOV[1]
+                    theTarTime=str(tarIOV[2])
+
+            #print Rcd[0],theRefPayload,theTarPayload
+
+            if(theRefPayload!=theTarPayload):
+                isDifferent=True
+                text_file.write("| ="+Rcd[0]+"= ("+Rcd[1]+") | =="+differentTags[Rcd][0]+"==  | =="+differentTags[Rcd][1]+"== |\n")
+                text_file.write("|^|"+theRefPayload+" ("+theRefTime+") | "+theTarPayload+" ("+theTarTime+") |\n")                       
+
+                ### determinte if it is to be shown
+
+                isMatched=False
+                tokens=args.stringToMatch.split(",")
+                decisions = [bool(Rcd[0].find(x)!=-1) for x in tokens]
+                for decision in decisions:
+                    isMatched = (isMatched or decision)
+
+                if(args.isVerbose):
+                    if (args.stringToMatch=="" or isMatched): 
+                        t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0],differentTags[Rcd][1],str(theRefPayload)+"\n"+str(theRefTime)+"\n"+str(theGoodRefIOV),str(theTarPayload)+"\n"+str(theTarTime)+"\n"+str(theGoodTarIOV)])
+                else:
+                    if (args.stringToMatch=="" or isMatched): 
+                        t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0]+"\n"+str(theRefPayload),differentTags[Rcd][1]+"\n"+str(theTarPayload)])
+
+    if(not isDifferent):
+        if(args.isVerbose):
+            t.add_row(["None","None","None","None","None","None"])
+        else:
+            t.add_row(["None","None","None"])
+    print(t)
+
+
+
 def listGTsForTag_(args):
     connection = connect(args)
     session = connection.session()
@@ -2214,6 +2445,15 @@ def main():
     parser_listParentTags.add_argument('hash_name', help="Payload hash to match.")
     parser_listParentTags.set_defaults(func=listParentTags_)
 
+    parser_diffGlobalTagsAtRun = parser_subparsers.add_parser('diffGloablTagsAtRun', description='Diffs two global tags, but showing only the differences relevant for a given run number.')
+    parser_diffGlobalTagsAtRun.add_argument('--last', '-L', dest='lastIOV', action='store_true', default=False, help='Diff the Global tags at the last open IOV.')
+    parser_diffGlobalTagsAtRun.add_argument('--reference', '-R', dest='refGT', help="Reference Global Tag")
+    parser_diffGlobalTagsAtRun.add_argument('--target', '-T', dest='tarGT', help="Target Global Tag")
+    parser_diffGlobalTagsAtRun.add_argument('--run', '-r', dest='testRunNumber', help="target run to compare",default=-1)
+    parser_diffGlobalTagsAtRun.add_argument('--verbose','-v',help='returns more info', dest='isVerbose',action='store_true',default=False)
+    parser_diffGlobalTagsAtRun.add_argument('--match','-m',help='print only matching',dest='stringToMatch',action='store',default='')
+    parser_diffGlobalTagsAtRun.set_defaults(func=diffGlobalTagsAtRun_)
+
     parser_listGTsForTag = parser_subparsers.add_parser('listGTsForTag', description='Lists the GTs which contain a given tag.')
     parser_listGTsForTag.add_argument('name', help="Name of the tag.")
     parser_listGTsForTag.set_defaults(func=listGTsForTag_)