Skip to content

Commit

Permalink
Merge pull request #32795 from mmusich/conddb_cli_update
Browse files Browse the repository at this point in the history 
Some additions to the conddb command line interface
  • Loading branch information
@cmsbuild
cmsbuild authored Feb 4, 2021
2 parents 0182ade + a72206f commit ea40acb
Showing 1 changed file with 283 additions and 0 deletions.
283 changes: 283 additions & 0 deletions CondCore/Utilities/scripts/conddb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import socket
import calendar
import sqlalchemy

from prettytable import PrettyTable

import CondCore.Utilities.conddblib as conddb
import CondCore.Utilities.cond2xml as cond2xml
import CondCore.Utilities.conddb_serialization_metadata as serialization_metadata
Expand Down Expand Up @@ -902,6 +904,273 @@ def listTags_(args):
return 0



def listParentTags_(args):
connection = connect(args)
session = connection.session()

IOV = session.get_dbtype(conddb.IOV)
Tag = session.get_dbtype(conddb.Tag)

query_result = session.query(IOV.tag_name).filter(IOV.payload_hash == args.hash_name).all()
tag_names = map(lambda entry : entry[0], query_result)

listOfOccur=[]

for tag in tag_names:
synchro = session.query(Tag.synchronization).filter(Tag.name == tag).all()
iovs = session.query(IOV.since).filter(IOV.tag_name == tag).filter(IOV.payload_hash == args.hash_name).all()
times = session.query(IOV.insertion_time).filter(IOV.tag_name == tag).filter(IOV.payload_hash == args.hash_name).all()

synchronization = [item[0] for item in synchro]
listOfIOVs = [item[0] for item in iovs]
listOfTimes = [str(item[0]) for item in times]

for iEntry in range(0,len(listOfIOVs)):
listOfOccur.append({"tag": tag,
"synchronization" : synchronization[0],
"since" : listOfIOVs[iEntry] ,
"insertion_time" : listOfTimes[iEntry] })

t = PrettyTable(['hash', 'since','tag','synch','insertion time'])
for element in listOfOccur:
t.add_row([args.hash_name,element['since'],element['tag'],element['synchronization'],element['insertion_time']])

print(t)




def diffGlobalTagsAtRun_(args):
connection = connect(args)
session = connection.session()

IOV = session.get_dbtype(conddb.IOV)
TAG = session.get_dbtype(conddb.Tag)
GT = session.get_dbtype(conddb.GlobalTag)
GTMAP = session.get_dbtype(conddb.GlobalTagMap)
RUNINFO = session.get_dbtype(conddb.RunInfo)

####################################
# Get the time info for the test run
####################################

if(not args.lastIOV):

if(args.testRunNumber<0):
raise Exception("Run %s (default) can't be matched with an existing run in the database. \n\t\t Please specify a run with the option --run." % args.testRunNumber)

bestRun = session.query(RUNINFO.run_number, RUNINFO.start_time, RUNINFO.end_time).filter(RUNINFO.run_number == int(args.testRunNumber)).first()
if bestRun is None:
raise Exception("Run %s can't be matched with an existing run in the database." % args.testRunNumber)

print("Run",args.testRunNumber," |Start time",bestRun[1]," |End time",bestRun[2],".")

####################################
# Get the Global Tag snapshots
####################################

refSnap = session.query(GT.snapshot_time).\
filter(GT.name == args.refGT).all()[0][0]

tarSnap = session.query(GT.snapshot_time).\
filter(GT.name == args.tarGT).all()[0][0]

print("reference GT (",args.refGT ,") snapshot: ",refSnap," | target GT (",args.tarGT,") snapshot",tarSnap)

####################################
# Get the Global Tag maps
####################################

GTMap_ref = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\
filter(GTMAP.global_tag_name == args.refGT).\
order_by(GTMAP.record, GTMAP.label).\
all()

GTMap_tar = session.query(GTMAP.record, GTMAP.label, GTMAP.tag_name).\
filter(GTMAP.global_tag_name == args.tarGT).\
order_by(GTMAP.record, GTMAP.label).\
all()

text_file = open(("diff_%s_vs_%s.twiki") % (args.refGT,args.tarGT), "w")

differentTags = {}

for element in GTMap_ref:
RefRecord = element[0]
RefLabel = element[1]
RefTag = element[2]

for element2 in GTMap_tar:
if (RefRecord == element2[0] and RefLabel==element2[1]):
if RefTag != element2[2]:
differentTags[(RefRecord,RefLabel)]=(RefTag,element2[2])

####################################
## Search for Records,Label not-found in the other list
####################################

temp1 = [item for item in GTMap_ref if (item[0],item[1]) not in list(zip(list(zip(*GTMap_tar))[0],list(zip(*GTMap_tar))[1]))]
for elem in temp1:
differentTags[(elem[0],elem[1])]=(elem[2],"")

temp2 = [item for item in GTMap_tar if (item[0],item[1]) not in list(zip(list(zip(*GTMap_ref))[0],list(zip(*GTMap_ref))[1]))]
for elem in temp2:
differentTags[(elem[0],elem[1])]=("",elem[2])

text_file.write("| *Record* | *"+args.refGT+"* | *"+args.tarGT+"* | Remarks | \n")

t = PrettyTable()

if(args.isVerbose):
t.field_names = ['/','',args.refGT,args.tarGT,refSnap,tarSnap]
else:
t.field_names = ['/','',args.refGT,args.tarGT]

t.hrules=1

if(args.isVerbose):
t.add_row(['Record','label','Reference Tag','Target Tag','hash1:time1:since1','hash2:time2:since2'])
else:
t.add_row(['Record','label','Reference Tag','Target Tag'])

isDifferent=False

####################################
# Loop on the difference
####################################

for Rcd in sorted(differentTags):

# empty lists at the beginning
refTagIOVs=[]
tarTagIOVs=[]

if( differentTags[Rcd][0]!=""):
refTagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == differentTags[Rcd][0]).all()
refTagInfo = session.query(TAG.synchronization,TAG.time_type).filter(TAG.name == differentTags[Rcd][0]).all()[0]
if( differentTags[Rcd][1]!=""):
tarTagIOVs = session.query(IOV.since,IOV.payload_hash,IOV.insertion_time).filter(IOV.tag_name == differentTags[Rcd][1]).all()
tarTagInfo = session.query(TAG.synchronization,TAG.time_type).filter(TAG.name == differentTags[Rcd][1]).all()[0]

if(differentTags[Rcd][0]!="" and differentTags[Rcd][1]!=""):
if(tarTagInfo[1] != refTagInfo[1]):
print(colors.bold_red+" *** Warning *** found mismatched time type for",Rcd,"entry. \n"+differentTags[Rcd][0],"has time type",refTagInfo[1],"while",differentTags[Rcd][1],"has time type",tarTagInfo[1]+". These need to be checked by hand. \n\n"+ colors.end)

if(args.lastIOV):

if(sorted(differentTags).index(Rcd)==0):
print("\n")
print(33 * "=")
print("|| COMPARING ONLY THE LAST IOV ||")
print(33 * "=")
print("\n")

lastSinceRef=-1
lastSinceTar=-1

hash_lastRefTagIOV = ""
time_lastRefTagIOV = ""

hash_lastTarTagIOV = ""
time_lastTarTagIOV = ""

for i in refTagIOVs:
if (i[0]>lastSinceRef):
lastSinceRef = i[0]
hash_lastRefTagIOV = i[1]
time_lastRefTagIOV = str(i[2])

for j in tarTagIOVs:
if (j[0]>lastSinceTar):
lastSinceTar = j[0]
hash_lastTarTagIOV = j[1]
time_lastTarTagIOV = str(j[2])

if(hash_lastRefTagIOV!=hash_lastTarTagIOV):
isDifferent=True
text_file.write("| ="+Rcd[0]+"= ("+Rcd[1]+") | =="+differentTags[Rcd][0]+"== | =="+differentTags[Rcd][1]+"== | | \n")
text_file.write("|^|"+hash_lastRefTagIOV+" <br> ("+time_lastRefTagIOV+") "+ str(lastSinceRef) +" | "+hash_lastTarTagIOV+" <br> ("+time_lastTarTagIOV+") " + str(lastSinceTar)+" | ^| \n")

if(args.isVerbose):
t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0],differentTags[Rcd][1],str(hash_lastRefTagIOV)+"\n"+str(time_lastRefTagIOV)+"\n"+str(lastSinceRef),str(hash_lastTarTagIOV)+"\n"+str(time_lastTarTagIOV)+"\n"+str(lastSinceTar)])
else:
t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0]+"\n"+str(hash_lastRefTagIOV),differentTags[Rcd][1]+"\n"+str(hash_lastTarTagIOV)])

else:

### reset all defaults

theGoodRefIOV=-1
theGoodTarIOV=-1
sinceRefTagIOV=0
sinceTarTagIOV=0

RefIOVtime = datetime.datetime(1970, 1, 1, 0, 0, 0)
TarIOVtime = datetime.datetime(1970, 1, 1, 0, 0, 0)

theRefPayload=""
theTarPayload=""
theRefTime=""
theTarTime=""

### loop on the reference IOV list
for refIOV in refTagIOVs:

## logic for retrieving the the last payload active on a given IOV
## - the new IOV since is less than the run under consideration
## - the payload insertion time is lower than the GT snapshot
## - finall either:
## - the new IOV since is larger then the last saved
## - the new IOV since is equal to the last saved but it has a more recent insertion time

if ( (refIOV[0] <= int(args.testRunNumber)) and (refIOV[0]>sinceRefTagIOV) or ((refIOV[0]==sinceRefTagIOV) and (refIOV[2]>RefIOVtime)) and (refIOV[2]<=refSnap) ):
sinceRefTagIOV = refIOV[0]
RefIOVtime = refIOV[2]
theGoodRefIOV=sinceRefTagIOV
theRefPayload=refIOV[1]
theRefTime=str(refIOV[2])

### loop on the target IOV list
for tarIOV in tarTagIOVs:
if ( (tarIOV[0] <= int(args.testRunNumber)) and (tarIOV[0]>sinceTarTagIOV) or ((tarIOV[0]==sinceTarTagIOV) and (tarIOV[2]>=TarIOVtime)) and (tarIOV[2]<=tarSnap) ):
sinceTarTagIOV = tarIOV[0]
tarIOVtime = tarIOV[2]
theGoodTarIOV=sinceTarTagIOV
theTarPayload=tarIOV[1]
theTarTime=str(tarIOV[2])

#print Rcd[0],theRefPayload,theTarPayload

if(theRefPayload!=theTarPayload):
isDifferent=True
text_file.write("| ="+Rcd[0]+"= ("+Rcd[1]+") | =="+differentTags[Rcd][0]+"== | =="+differentTags[Rcd][1]+"== |\n")
text_file.write("|^|"+theRefPayload+" ("+theRefTime+") | "+theTarPayload+" ("+theTarTime+") |\n")

### determinte if it is to be shown

isMatched=False
tokens=args.stringToMatch.split(",")
decisions = [bool(Rcd[0].find(x)!=-1) for x in tokens]
for decision in decisions:
isMatched = (isMatched or decision)

if(args.isVerbose):
if (args.stringToMatch=="" or isMatched):
t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0],differentTags[Rcd][1],str(theRefPayload)+"\n"+str(theRefTime)+"\n"+str(theGoodRefIOV),str(theTarPayload)+"\n"+str(theTarTime)+"\n"+str(theGoodTarIOV)])
else:
if (args.stringToMatch=="" or isMatched):
t.add_row([Rcd[0],Rcd[1],differentTags[Rcd][0]+"\n"+str(theRefPayload),differentTags[Rcd][1]+"\n"+str(theTarPayload)])

if(not isDifferent):
if(args.isVerbose):
t.add_row(["None","None","None","None","None","None"])
else:
t.add_row(["None","None","None"])
print(t)



def listGTsForTag_(args):
connection = connect(args)
session = connection.session()
Expand All @@ -926,6 +1195,7 @@ def listGTs_(args):
['GT_name', 'Description', 'Release', 'Snapshot_time', 'Insertion_time'],
)


def listRuns_(args):
connection = connect(args)
session = connection.session()
Expand Down Expand Up @@ -2154,6 +2424,19 @@ def main():
parser_listTags = parser_subparsers.add_parser('listTags', description='Lists all the Tags available in the DB.')
parser_listTags.set_defaults(func=listTags_)

parser_listParentTags = parser_subparsers.add_parser('listParentTags', description='Lists all the Tags available in the DB, containing the matched payload hash.')
parser_listParentTags.add_argument('hash_name', help="Payload hash to match.")
parser_listParentTags.set_defaults(func=listParentTags_)

parser_diffGlobalTagsAtRun = parser_subparsers.add_parser('diffGlobalTagsAtRun', description='Diffs two global tags, but showing only the differences relevant for a given run number.')
parser_diffGlobalTagsAtRun.add_argument('--last', '-L', dest='lastIOV', action='store_true', default=False, help='Diff the Global tags at the last open IOV.')
parser_diffGlobalTagsAtRun.add_argument('--reference', '-R', dest='refGT', help="Reference Global Tag")
parser_diffGlobalTagsAtRun.add_argument('--target', '-T', dest='tarGT', help="Target Global Tag")
parser_diffGlobalTagsAtRun.add_argument('--run', '-r', dest='testRunNumber', help="target run to compare",default=-1)
parser_diffGlobalTagsAtRun.add_argument('--verbose','-v',help='returns more info', dest='isVerbose',action='store_true',default=False)
parser_diffGlobalTagsAtRun.add_argument('--match','-m',help='print only matching',dest='stringToMatch',action='store',default='')
parser_diffGlobalTagsAtRun.set_defaults(func=diffGlobalTagsAtRun_)

parser_listGTsForTag = parser_subparsers.add_parser('listGTsForTag', description='Lists the GTs which contain a given tag.')
parser_listGTsForTag.add_argument('name', help="Name of the tag.")
parser_listGTsForTag.set_defaults(func=listGTsForTag_)
Expand Down

0 comments on commit ea40acb

Please sign in to comment.