Skip to content

Commit

Permalink
update GPU attributes and runTheMatrix command
Browse files Browse the repository at this point in the history
  • Loading branch information
Norraphat committed Apr 16, 2021
1 parent 2e8888e commit 54954c4
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 40 deletions.
54 changes: 31 additions & 23 deletions Configuration/PyReleaseValidation/python/MatrixInjector.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,23 @@ def __init__(self,opt,mode='init',options=''):
self.batchTime = str(int(time.time()))
if(opt.batchName):
self.batchName = '__'+opt.batchName+'-'+self.batchTime
#GPU tuff
self.gpuClass = opt.gpuClass
self.gpuDriverVersion = opt.gpuDriverVersion
self.gpuRuntime = opt.gpuRuntime
self.gpuMemory = opt.gpuMemory
self.gpuRuntimeVersion = opt.gpuRuntimeVersion


####################################
# Checking and setting up GPU attributes
####################################
# require
self.RequiresGPU = opt.RequiresGPU
if self.RequiresGPU not in (0,1,2):
print('RequiresGPU should be 0,1,2. Now, reset to 0.')
self.RequiresGPU = 0 #reset if number is not 0,1,2
self.GPUMemory = opt.GPUMemory
self.CUDACapabilities = opt.CUDACapabilities.split(',')
self.CUDARuntime = opt.CUDARuntime
# optional
self.GPUName = opt.GPUName
self.CUDADriverVersion = opt.CUDADriverVersion
self.CUDARuntimeVersion = opt.CUDARuntimeVersion

# WMagent url
if not self.wmagent:
# Overwrite with env variable
Expand Down Expand Up @@ -138,11 +148,6 @@ def __init__(self,opt,mode='init',options=''):
"Memory" : 3000,
"SizePerEvent" : 1234,
"TimePerEvent" : 10,
"GPUClass": None,
"GPUDriverVersion": None,
"GPURuntime": None,
"GPUMemory": None,
"GPURuntimeVersion": None,
"PrepID": os.getenv('CMSSW_VERSION')
}

Expand Down Expand Up @@ -191,7 +196,17 @@ def __init__(self,opt,mode='init',options=''):
"nowmIO": {},
"Multicore" : opt.nThreads, # this is the per-taskchain Multicore; it's the default assigned to a task if it has no value specified
"EventStreams": self.numberOfStreams,
"KeepOutput" : False
"KeepOutput" : False,
"RequiresGPU" : 0,
"GPUParams": {None}
}
self.defaultGPUParams={
"GPUMemory": self.GPUMemory,
"CUDACapabilities": self.CUDACapabilities,
"CUDARuntime": self.CUDARuntime,
"GPUName": self.GPUName,
"CUDADriverVersion": self.CUDADriverVersion,
"CUDARuntimeVersion": self.CUDARuntimeVersion
}

self.chainDicts={}
Expand Down Expand Up @@ -332,16 +347,6 @@ def prepare(self, mReader, directories, mode='init'):
for (n,dir) in directories.items():
chainDict=copy.deepcopy(self.defaultChain)
print("inspecting",dir)
if self.gpuClass:
chainDict['GPUClass'] = self.gpuClass
if self.gpuDriverVersion:
chainDict['GPUDriverVersion'] =self.gpuDriverVersion
if self.gpuRuntime:
chainDict['GPURuntime'] =self.gpuRuntime
if self.gpuMemory:
chainDict['GPUMemory'] =self.gpuMemory
if self.gpuRuntimeVersion:
chainDict['GPURuntimeVersion'] =self.gpuRuntimeVersion
nextHasDSInput=None
for (x,s) in mReader.workFlowSteps.items():
#x has the format (num, prefix)
Expand Down Expand Up @@ -441,6 +446,9 @@ def prepare(self, mReader, directories, mode='init'):
chainDict['nowmTasklist'][-1]['LumisPerJob']=splitForThisWf
if step in wmsplit:
chainDict['nowmTasklist'][-1]['LumisPerJob']=wmsplit[step]
if 'GPU' in step and (self.RequiresGPU == 1 or self.RequiresGPU == 2):
chainDict['nowmTasklist'][-1]['RequiresGPU'] = self.RequiresGPU
chainDict['nowmTasklist'][-1]['GPUParams']=self.defaultGPUParams

# change LumisPerJob for Hadronizer steps.
if 'Hadronizer' in step:
Expand Down
83 changes: 66 additions & 17 deletions Configuration/PyReleaseValidation/scripts/runTheMatrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,26 +119,31 @@ def runSelected(opt):
dest='memoryOffset',
default=3000
)

parser.add_option('--addMemPerCore',
help='increase of memory per each n > 1 core: memory(n_core) = memoryOffset + (n_core-1) * memPerCore',
dest='memPerCore',
default=1500
)

parser.add_option('-j','--nproc',
help='number of processes. 0 Will use 4 processes, not execute anything but create the wfs',
dest='nProcs',
default=4
)

parser.add_option('-t','--nThreads',
help='number of threads per process to use in cmsRun.',
dest='nThreads',
default=1
)

parser.add_option('--nStreams',
help='number of streams to use in cmsRun.',
dest='nStreams',
default=0
)

parser.add_option('--numberEventsInLuminosityBlock',
help='number of events in a luminosity block',
dest='numberEventsInLuminosityBlock',
Expand All @@ -151,118 +156,141 @@ def runSelected(opt):
default=False,
action='store_true'
)

parser.add_option('-e','--extended',
help='Show details of workflows, used with --show',
dest='extended',
default=False,
action='store_true'
)

parser.add_option('-s','--selected',
help='Run a pre-defined selected matrix of wf. Deprecated, please use -l limited',
dest='restricted',
default=False,
action='store_true'
)

parser.add_option('-l','--list',
help='Coma separated list of workflow to be shown or ran. Possible keys are also '+str(predefinedSet.keys())+'. and wild card like muon, or mc',
dest='testList',
default=None
help='Coma separated list of workflow to be shown or ran. Possible keys are also '+str(predefinedSet.keys())+'. and wild card like muon, or mc',
dest='testList',
default=None
)

parser.add_option('-r','--raw',
help='Temporary dump the .txt needed for prodAgent interface. To be discontinued soon. Argument must be the name of the set (standard, pileup,...)',
dest='raw'
)

parser.add_option('-i','--useInput',
help='Use recyling where available. Either all, or a coma separated list of wf number.',
dest='useInput',
default=None
)

parser.add_option('-w','--what',
help='Specify the set to be used. Argument must be the name of a set (standard, pileup,...) or multiple sets separated by commas (--what standard,pileup )',
dest='what',
default='all'
)

parser.add_option('--step1',
help='Used with --raw. Limit the production to step1',
dest='step1Only',
default=False
)

parser.add_option('--maxSteps',
help='Only run maximum on maxSteps. Used when we are only interested in first n steps.',
dest='maxSteps',
default=9999,
type="int"
)

parser.add_option('--fromScratch',
help='Coma separated list of wf to be run without recycling. all is not supported as default.',
dest='fromScratch',
default=None
)

parser.add_option('--refRelease',
help='Allow to modify the recycling dataset version',
dest='refRel',
default=None
)

parser.add_option('--wmcontrol',
help='Create the workflows for injection to WMAgent. In the WORKING. -wmcontrol init will create the the workflows, -wmcontrol test will dryRun a test, -wmcontrol submit will submit to wmagent',
choices=['init','test','submit','force'],
dest='wmcontrol',
default=None,
)

parser.add_option('--revertDqmio',
help='When submitting workflows to wmcontrol, force DQM outout to use pool and not DQMIO',
choices=['yes','no'],
dest='revertDqmio',
default='no',
)

parser.add_option('--optionswm',
help='Specify a few things for wm injection',
default='',
dest='wmoptions')

parser.add_option('--keep',
help='allow to specify for which coma separated steps the output is needed',
default=None)

parser.add_option('--label',
help='allow to give a special label to the output dataset name',
default='')

parser.add_option('--command',
help='provide a way to add additional command to all of the cmsDriver commands in the matrix',
dest='command',
default=None
)

parser.add_option('--apply',
help='allow to use the --command only for 1 coma separeated',
dest='apply',
default=None)

parser.add_option('--workflow',
help='define a workflow to be created or altered from the matrix',
action='append',
dest='workflow',
default=None
)

parser.add_option('--dryRun',
help='do not run the wf at all',
action='store_true',
dest='dryRun',
default=False
)

parser.add_option('--testbed',
help='workflow injection to cmswebtest (you need dedicated rqmgr account)',
dest='testbed',
default=False,
action='store_true'
)

parser.add_option('--noCafVeto',
help='Run from any source, ignoring the CAF label',
dest='cafVeto',
default=True,
action='store_false'
)

parser.add_option('--overWrite',
help='Change the content of a step for another. List of pairs.',
dest='overWrite',
default=None
)

parser.add_option('--noRun',
help='Remove all run list selection from wfs',
dest='noRun',
Expand Down Expand Up @@ -292,6 +320,7 @@ def runSelected(opt):
dest='dasSites',
default='T2_CH_CERN',
action='store')

parser.add_option('--interactive',
help="Open the Matrix interactive shell",
action='store_true',
Expand All @@ -303,20 +332,39 @@ def runSelected(opt):
default=None,
action='store')

parser.add_option('--gpuClass',
parser.add_option('--RequiresGPU',
help='if GPU is reuired or not: 0 = Not (default), 1 = can use if available, 2 = must use. Default = 0.',
dest='RequiresGPU',
default=0)

parser.add_option('--GPUMemory',
help='to specify GPU memory. Default = 8000 MB (for RequiresGPU=2).',
dest='GPUMemory',
default='8000')

parser.add_option('--CUDACapabilities',
help='to specify CUDA capabilities. Default = 7.5 (for RequiresGPU=2).',
dest='CUDACapabilities',
default='7.5')

parser.add_option('--CUDARuntime',
help='to specify CUDA runtime. Default = 11.2 (for RequiresGPU=2).',
dest='CUDARuntime',
default='11.2')

parser.add_option('--GPUName',
help='to specify GPU class',
dest='GPUName',
default='')
parser.add_option('--gpuDriverVersion',
help='to specify GPU driver version',
default='')
parser.add_option('--gpuRuntime',
help='to specify GPU runtime',
default='')
parser.add_option('--gpuMemory',
help='to specify GPU memory',

parser.add_option('--CUDADriverVersion',
help='to specify CUDA driver version',
dest='CUDADriverVersion',
default='')
parser.add_option('--gpuRuntimeVersion',
help='to specify GPU runtime version',

parser.add_option('--CUDARuntimeVersion',
help='to specify CUDA runtime version',
dest='CUDARuntimeVersion',
default='')

opt,args = parser.parse_args()
Expand Down Expand Up @@ -385,9 +433,10 @@ def stepOrIndex(s):
if opt.nProcs: opt.nProcs=int(opt.nProcs)
if opt.nThreads: opt.nThreads=int(opt.nThreads)
if opt.nStreams: opt.nStreams=int(opt.nStreams)
if (opt.numberEventsInLuminosityBlock): opt.numberEventsInLuminosityBlock=int(opt.numberEventsInLuminosityBlock)
if (opt.memoryOffset): opt.memoryOffset=int(opt.memoryOffset)
if (opt.memPerCore): opt.memPerCore=int(opt.memPerCore)
if opt.numberEventsInLuminosityBlock: opt.numberEventsInLuminosityBlock=int(opt.numberEventsInLuminosityBlock)
if opt.memoryOffset: opt.memoryOffset=int(opt.memoryOffset)
if opt.memPerCore: opt.memPerCore=int(opt.memPerCore)
if opt.RequiresGPU: opt.RequiresGPU=int(opt.RequiresGPU)

if opt.wmcontrol:
performInjectionOptionTest(opt)
Expand Down

0 comments on commit 54954c4

Please sign in to comment.