Skip to content

Commit

Permalink
Merge pull request #43102 from kpedro88/PhysicsToolsArgParse
Browse files Browse the repository at this point in the history 
argparse migration for PhysicsTools and FWLite
  • Loading branch information
@cmsbuild
cmsbuild authored Oct 27, 2023
2 parents 666097a + 831b0fa commit fb0c11b
Show file tree
Hide file tree
Showing 17 changed files with 176 additions and 313 deletions.
2 changes: 0 additions & 2 deletions DataFormats/FWLite/python/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
#! /usr/bin/env python

from __future__ import print_function
import ROOT
import inspect
import sys
import optparse
from FWCore.ParameterSet.VarParsing import VarParsing
from builtins import int

Expand Down
25 changes: 10 additions & 15 deletions DataFormats/FWLite/scripts/edmLumisInFiles.py
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,19 @@
#! /usr/bin/env python3

from __future__ import print_function
from FWCore.PythonUtilities.LumiList import LumiList
import optparse

from FWCore.PythonUtilities.LumiList import LumiList
from argparse import ArgumentParser

if __name__ == '__main__':

parser = optparse.OptionParser ("Usage: %prog [--options] edm1.root [edm2.root...]",
description='Runs over input EDM files and prints out a list of contained lumi sections')
parser.add_option ('--intLumi', dest='intLumi', action='store_true',
help='print out total recorded and delivered integrated luminosity')
parser.add_option ('--output', dest='output', type='string',
help='save lumi sections output to file OUTPUT')
(options, args) = parser.parse_args()
parser = ArgumentParser(description='Runs over input EDM files and prints out a list of contained lumi sections')
parser.add_argument('--intLumi', dest='intLumi', action='store_true',
help='print out total recorded and delivered integrated luminosity')
parser.add_argument('--output', dest='output', type=str,
help='save lumi sections output to file OUTPUT')
parser.add_argument("edm", metavar="edm.root", type=str, nargs='+')
options = parser.parse_args()
# put this here after parsing the arguments since ROOT likes to
# grab command line arguments even when it shouldn't.
from DataFormats.FWLite import Lumis, Handle
if not args:
raise RuntimeError("Must provide at least one input file")

# do we want to get the luminosity summary?
if options.intLumi:
Expand All @@ -28,7 +23,7 @@
handle, lable = None, None

runsLumisDict = {}
lumis = Lumis (args)
lumis = Lumis (options.edm)
delivered = recorded = 0
for lum in lumis:
runList = runsLumisDict.setdefault (lum.aux().run(), [])
Expand Down
1 change: 0 additions & 1 deletion DataFormats/FWLite/test/chainEvent_python.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from __future__ import print_function
from builtins import range
from DataFormats.FWLite import Events, Handle
import optparse

print("starting python test")

Expand Down
30 changes: 13 additions & 17 deletions PhysicsTools/FWLite/scripts/newFWLiteAna.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#! /usr/bin/env python3

from __future__ import print_function
import optparse
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import os
import sys
import re
Expand Down Expand Up @@ -95,17 +95,16 @@ def addBuildPiece (targetBuild, buildPiece):
print("Error: You must have already setup a CMSSW release. Aborting.")
sys.exit()
# setup the options
parser = optparse.OptionParser('usage: %prog [options] '
'Package/SubPackage/name\n'
'Creates new analysis code')
parser.add_option ('--copy', dest='copy', type='string', default = 'blank',
help='Copies example. COPY should either be a file'
' _or_ an example in PhysicsTools/FWLite/examples')
parser.add_option ('--newPackage', dest='newPackage', action='store_true',
help='Will create Package/Subpackage folders if necessary')
parser.add_option ('--toTest', dest='toTest', action='store_true',
help='Will create files in test/ instead of bin/')
(options, args) = parser.parse_args()
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter, description='Creates new analysis code')
parser.add_argument('--copy', dest='copy', type=str, default = 'blank',
help='Copies example. COPY should either be a file'
' _or_ an example in PhysicsTools/FWLite/examples')
parser.add_argument('--newPackage', dest='newPackage', action='store_true',
help='Will create Package/Subpackage folders if necessary')
parser.add_argument('--toTest', dest='toTest', action='store_true',
help='Will create files in test/ instead of bin/')
parser.add_argument("name", metavar="Package/SubPackage/name", type=str)
options = parser.parse_args()
# get the name of the copy file and make sure we can find everything
copy = options.copy
if not re.search ('\.cc$', copy):
Expand All @@ -130,7 +129,7 @@ def addBuildPiece (targetBuild, buildPiece):
found = True
# Is there a Buildfile too?
if not os.path.exists (build):
print("Error: Found '%s', but no accompying " % name, \
print("Error: Found '%s', but no accompanying " % name, \
"Buildfile '%s'. Aborting" % build)
sys.exit()
fullName = name
Expand All @@ -139,10 +138,7 @@ def addBuildPiece (targetBuild, buildPiece):
if not found:
print("Error: Did not find '%s' to copy. Aborting." % copy)
sys.exit()
if len (args) != 1:
parser.print_usage()
sys.exit()
pieces = args[0].split('/')
pieces = options.name.split('/')
if len (pieces) != 3:
print("Error: Need to provide 'Package/SubPackage/name'")
sys.exit()
Expand Down
21 changes: 11 additions & 10 deletions PhysicsTools/NanoAOD/test/compare_branches.py
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
#!/usr/bin/env python3
from ROOT import TFile, TTree,TCut, gROOT,TH1F, TCanvas
import glob, re
from optparse import OptionParser
parser = OptionParser(usage="%prog [options] job1 job2 ...")
parser.add_option("--base", dest="base", default="step2.root", help="name of root file")
parser.add_option("--ref", dest="ref", default="ref/", help="path to the reference files")
parser.add_option("--png", dest="png", default="./", help="path to the plots")
parser.add_option("--onlydiff", dest="diff", default=False, action="store_true", help="print only the histograms with differences")
parser.add_option("--selection", dest="selection", default="", help="a selection of events to draw")
parser.add_option("--branch", dest="branch", default=".*", help="a regexp for selecting branches")
(options, args) = parser.parse_args()
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("--base", dest="base", default="step2.root", help="name of root file")
parser.add_argument("--ref", dest="ref", default="ref/", help="path to the reference files")
parser.add_argument("--png", dest="png", default="./", help="path to the plots")
parser.add_argument("--onlydiff", dest="diff", default=False, action="store_true", help="print only the histograms with differences")
parser.add_argument("--selection", dest="selection", default="", help="a selection of events to draw")
parser.add_argument("--branch", dest="branch", default=".*", help="a regexp for selecting branches")
parser.add_argument("job", type=str, nargs='+')
options = parser.parse_args()

def drawInOrder(t1,h1n, t2,h2n):
N1 = t1.Draw(f'{vname} >> {h1n}', options.selection, '')
Expand All @@ -33,7 +34,7 @@ def binContents( h ):
def emptyHist( h ):
return all([ b==0 for b in binContents(h) ])

for job in args:
for job in options.job:
fconn = glob.glob(f'{job}*/{options.base}')[0]
fcon = TFile.Open(fconn)
con = fcon.Get('Events')
Expand Down
17 changes: 9 additions & 8 deletions PhysicsTools/NanoAOD/test/compare_sizes_json.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
#!/usr/bin/env python3
import json

from optparse import OptionParser
parser = OptionParser(usage="%prog [options] job1 job2 ...")
parser.add_option("-f", "--format", dest="fmt", default="txt", help="output format")
parser.add_option("-H", "--header", dest="header", action="store_true", default=False, help="print headers")
parser.add_option("--base", dest="base", default="{}.json,{}_timing_report.json", help="coma separated base name for size and timing files")
parser.add_option("--ref", dest="ref", default="ref/", help="path to the reference files")
(options, args) = parser.parse_args()
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("-f", "--format", dest="fmt", default="txt", help="output format")
parser.add_argument("-H", "--header", dest="header", action="store_true", default=False, help="print headers")
parser.add_argument("--base", dest="base", default="{}.json,{}_timing_report.json", help="comma-separated base name for size and timing files")
parser.add_argument("--ref", dest="ref", default="ref/", help="path to the reference files")
parser.add_argument("job", type=str, nargs='+')
options = parser.parse_args()

headers = [ 'Sample' , 'kb/ev' , 'ref kb/ev' , 'diff kb/ev' , 'ev/s/thd' , 'ref ev/s/thd' , 'diff rate' , 'mem/thd' , 'ref mem/thd' ]
start, sep, end = "", "\t", ""
Expand All @@ -20,7 +21,7 @@ def prow(x):
first = True

size_pattern,timing_pattern = options.base.split(',')
for job in args:
for job in options.job:

label = job
size_json=size_pattern.format(job)
Expand Down
17 changes: 9 additions & 8 deletions PhysicsTools/NanoAOD/test/get_timing_from_jobreport.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,25 +4,26 @@
import json
import os

from optparse import OptionParser
parser = OptionParser(usage="%prog input.xml output.json")
(options, args) = parser.parse_args()
if len(args)!=2: raise RuntimeError
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument("input", metavar="input.xml", type=str)
parser.add_argument("output", metavar="output.json", type=str)
options = parser.parse_args()

timing={}

try:
tree = xml.etree.ElementTree.parse(args[0])
tree = xml.etree.ElementTree.parse(options.input)
timing['NumberEvents']=int(tree.find("./PerformanceReport/PerformanceSummary[@Metric='ProcessingSummary']/Metric[@Name='NumberEvents']").get('Value'))
for x in tree.findall("./PerformanceReport/PerformanceSummary[@Metric='Timing']/Metric"):
timing['Timing/'+x.get('Name')]=float(x.get('Value'))
for x in tree.findall("./PerformanceReport/PerformanceSummary[@Metric='ApplicationMemory']/Metric"):
timing['ApplicationMemory/'+x.get('Name')]=float(x.get('Value'))
except Exception as e:
print("Could not parse job report %s, content is:" % args[0])
os.system("cat %s" % args[0])
print("Could not parse job report %s, content is:" % options.input)
os.system("cat %s" % options.input)
raise e


with open(args[1],'w') as f:
with open(options.output,'w') as f:
json.dump(timing,f)
28 changes: 14 additions & 14 deletions PhysicsTools/NanoAOD/test/inspectNanoFile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -429,21 +429,21 @@ def _maybeOpen(filename):
return open(filename, 'w') if filename != "-" else sys.stdout

if __name__ == '__main__':
from optparse import OptionParser
parser = OptionParser(usage="%prog [options] inputFile")
parser.add_option("-j", "--json", dest="json", type="string", default=None, help="Write out json file")
parser.add_option("-d", "--doc", dest="doc", type="string", default=None, help="Write out html doc")
parser.add_option("-s", "--size", dest="size", type="string", default=None, help="Write out html size report")
parser.add_option("--docmd", dest="docmd", type="string", default=None, help="Write out markdown doc")
parser.add_option("--sizemd", dest="sizemd", type="string", default=None, help="Write out markdown size report")
(options, args) = parser.parse_args()
if len(args) != 1: raise RuntimeError("Please specify one input file")
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument("-j", "--json", dest="json", type=str, default=None, help="Write out json file")
parser.add_argument("-d", "--doc", dest="doc", type=str, default=None, help="Write out html doc")
parser.add_argument("-s", "--size", dest="size", type=str, default=None, help="Write out html size report")
parser.add_argument("--docmd", dest="docmd", type=str, default=None, help="Write out markdown doc")
parser.add_argument("--sizemd", dest="sizemd", type=str, default=None, help="Write out markdown size report")
parser.add_argument("inputFile", type=str)
options = parser.parse_args()

if args[0].endswith(".root"):
filedata = FileData(inspectRootFile(args[0]))
elif args[0].endswith(".json"):
filedata = FileData(json.load(open(args[0],'r')))
else: raise RuntimeError("Input file %s is not a root or json file" % args[0])
if options.inputFile.endswith(".root"):
filedata = FileData(inspectRootFile(options.inputFile))
elif options.inputFile.endswith(".json"):
filedata = FileData(json.load(open(options.inputFile,'r')))
else: raise RuntimeError("Input file %s is not a root or json file" % options.inputFile)

if options.json:
json.dump(filedata._json, _maybeOpen(options.json), indent=4)
Expand Down
81 changes: 38 additions & 43 deletions PhysicsTools/NanoAODTools/scripts/nano_postproc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,54 +7,49 @@
ROOT.PyConfig.IgnoreCommandLineOptions = True

if __name__ == "__main__":
from optparse import OptionParser
parser = OptionParser(usage="%prog [options] outputDir inputFiles")
parser.add_option("-s", "--postfix", dest="postfix", type="string", default=None,
help="Postfix which will be appended to the file name (default: _Friend for friends, _Skim for skims)")
parser.add_option("-J", "--json", dest="json", type="string",
default=None, help="Select events using this JSON file")
parser.add_option("-c", "--cut", dest="cut", type="string",
default=None, help="Cut string")
parser.add_option("-b", "--branch-selection", dest="branchsel",
type="string", default=None, help="Branch selection")
parser.add_option("--bi", "--branch-selection-input", dest="branchsel_in",
type="string", default=None, help="Branch selection input")
parser.add_option("--bo", "--branch-selection-output", dest="branchsel_out",
type="string", default=None, help="Branch selection output")
parser.add_option("--friend", dest="friend", action="store_true", default=False,
help="Produce friend trees in output (current default is to produce full trees)")
parser.add_option("--full", dest="friend", action="store_false", default=False,
help="Produce full trees in output (this is the current default)")
parser.add_option("--noout", dest="noOut", action="store_true",
default=False, help="Do not produce output, just run modules")
parser.add_option("-P", "--prefetch", dest="prefetch", action="store_true", default=False,
help="Prefetch input files locally instead of accessing them via xrootd")
parser.add_option("--long-term-cache", dest="longTermCache", action="store_true", default=False,
help="Keep prefetched files across runs instead of deleting them at the end")
parser.add_option("-N", "--max-entries", dest="maxEntries", type="long", default=None,
help="Maximum number of entries to process from any single given input tree")
parser.add_option("--first-entry", dest="firstEntry", type="long", default=0,
help="First entry to process in the three (to be used together with --max-entries)")
parser.add_option("--justcount", dest="justcount", default=False,
action="store_true", help="Just report the number of selected events")
parser.add_option("-I", "--import", dest="imports", type="string", default=[], action="append",
nargs=2, help="Import modules (python package, comma-separated list of ")
parser.add_option("-z", "--compression", dest="compression", type="string",
default=("LZMA:9"), help="Compression: none, or (algo):(level) ")

(options, args) = parser.parse_args()
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument("-s", "--postfix", dest="postfix", type=str, default=None,
help="Postfix which will be appended to the file name (default: _Friend for friends, _Skim for skims)")
parser.add_argument("-J", "--json", dest="json", type=str,
default=None, help="Select events using this JSON file")
parser.add_argument("-c", "--cut", dest="cut", type=str,
default=None, help="Cut string")
parser.add_argument("-b", "--branch-selection", dest="branchsel",
type=str, default=None, help="Branch selection")
parser.add_argument("--bi", "--branch-selection-input", dest="branchsel_in",
type=str, default=None, help="Branch selection input")
parser.add_argument("--bo", "--branch-selection-output", dest="branchsel_out",
type=str, default=None, help="Branch selection output")
parser.add_argument("--friend", dest="friend", action="store_true", default=False,
help="Produce friend trees in output (current default is to produce full trees)")
parser.add_argument("--full", dest="friend", action="store_false", default=False,
help="Produce full trees in output (this is the current default)")
parser.add_argument("--noout", dest="noOut", action="store_true",
default=False, help="Do not produce output, just run modules")
parser.add_argument("-P", "--prefetch", dest="prefetch", action="store_true", default=False,
help="Prefetch input files locally instead of accessing them via xrootd")
parser.add_argument("--long-term-cache", dest="longTermCache", action="store_true", default=False,
help="Keep prefetched files across runs instead of deleting them at the end")
parser.add_argument("-N", "--max-entries", dest="maxEntries", type=int, default=None,
help="Maximum number of entries to process from any single given input tree")
parser.add_argument("--first-entry", dest="firstEntry", type=int, default=0,
help="First entry to process in the three (to be used together with --max-entries)")
parser.add_argument("--justcount", dest="justcount", default=False,
action="store_true", help="Just report the number of selected events")
parser.add_argument("-I", "--import", dest="imports", type=str, default=[], action="append",
nargs=2, help="Import modules (python package, comma-separated list of ")
parser.add_argument("-z", "--compression", dest="compression", type=str,
default=("LZMA:9"), help="Compression: none, or (algo):(level) ")
parser.add_argument("outputDir", type=str)
parser.add_argument("inputFile", type=str, nargs='+')
options = parser.parse_args()

if options.friend:
if options.cut or options.json:
raise RuntimeError(
"Can't apply JSON or cut selection when producing friends")

if len(args) < 2:
parser.print_help()
sys.exit(1)
outdir = args[0]
args = args[1:]

modules = []
for mod, names in options.imports:
import_module(mod)
Expand All @@ -76,7 +71,7 @@
if options.branchsel != None:
options.branchsel_in = options.branchsel
options.branchsel_out = options.branchsel
p = PostProcessor(outdir, args,
p = PostProcessor(options.outputDir, options.inputFile,
cut=options.cut,
branchsel=options.branchsel_in,
modules=modules,
Expand Down
Loading

0 comments on commit fb0c11b

Please sign in to comment.