diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index 7728919e058cd..9c1fed0fe384e 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -71,8 +71,8 @@ def notForGenOnly(key,specialType): stepList[specialType].append(stepMaker(key,frag[:-4],step.replace('RecoGlobal','HLT75e33'),specialWF.suffix)) # similar hacks for premixing if 'PMX' in specialType: - if 'GenSim' in step: - s = step.replace('GenSim','Premix')+'PU' # later processing requires to have PU here + if 'GenSim' in step or 'Gen' in step: + s = step.replace('GenSim','Premix').replace('Gen','Premix')+'PU' # later processing requires to have PU here if step in specialWF.PU: stepMade = stepMaker(key,'PREMIX',s,specialWF.suffix) # append for combined diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 88f70c48133ec..17d7e08815dea 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -164,7 +164,9 @@ def workflow(self, workflows, num, fragment, stepList, key, hasHarvest): def workflow_(self, workflows, num, fragment, stepList, key): fragmentTmp = [fragment, key] if len(self.suffix)>0: fragmentTmp.append(self.suffix) - workflows[num+self.offset] = [ fragmentTmp, stepList ] + # avoid spurious workflows (no steps modified) + if self.offset==0 or workflows[num][1]!=stepList: + workflows[num+self.offset] = [ fragmentTmp, stepList ] def condition(self, fragment, stepList, key, hasHarvest): return False def preventReuse(self, stepName, stepDict, k): @@ -560,7 +562,7 @@ def condition(self, fragment, stepList, key, hasHarvest): # WeightedMeanFitter vertexing workflows class UpgradeWorkflow_weightedVertex(UpgradeWorkflow): - def __init__(self, reco = {}, harvest = {}, **kwargs): + def __init__(self, **kwargs): # adapt the parameters for the UpgradeWorkflow init method super(UpgradeWorkflow_weightedVertex, self).__init__( steps = [ @@ -588,8 +590,6 @@ def __init__(self, reco = {}, harvest = {}, **kwargs): 'HARVESTNanoFakeHLT', ], **kwargs) - self.__reco = reco - self.__harvest = harvest def setup_(self, step, stepName, stepDict, k, properties): # temporarily remove trigger & downstream steps @@ -2235,7 +2235,7 @@ def setup_(self, step, stepName, stepDict, k, properties): stepDict[stepName][k] = None def condition(self, fragment, stepList, key, hasHarvest): - return ('2021' in key or '2023' in key or '2026' in key) + return fragment=="TTbar_14TeV" and ('2021' in key or '2023' in key or '2026' in key) upgradeWFs['ECALComponent'] = UpgradeWorkflow_ECalComponent( suffix = '_ecalComponent', @@ -2491,27 +2491,30 @@ def setup_(self, step, stepName, stepDict, k, properties): # just copy steps stepDict[stepName][k] = merge([stepDict[step][k]]) def setupPU_(self, step, stepName, stepDict, k, properties): - # setup for stage 1 - if "GenSim" in stepName: - stepNamePmx = stepName.replace('GenSim','Premix') - if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {} - stepDict[stepNamePmx][k] = merge([ - { - '-s': 'GEN,SIM,DIGI:pdigi_valid', - '--datatier': 'PREMIX', - '--eventcontent': 'PREMIX', - '--procModifiers': 'premix_stage1' - }, - stepDict[stepName][k] - ]) - if "ProdLike" in self.suffix: - stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]]) - # setup for stage 2 - elif "Digi" in step or "Reco" in step: - # go back to non-PU step version - d = merge([stepDict[self.getStepName(step)][k]]) - if d is None: return - if "Digi" in step: + # fastsim version + if 'FS' in k: + # setup for stage 1 fastsim + if "Gen" in stepName: + stepNamePmx = stepName.replace('Gen','Premix') + if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {} + stepDict[stepNamePmx][k] = merge([ + { + '-s': 'GEN,SIM,RECOBEFMIX,DIGI:pdigi_valid', + '--fast':'', + '--datatier': 'PREMIX', + '--eventcontent': 'PREMIX', + '--procModifiers': 'premix_stage1' + }, + stepDict[stepName][k] + ]) + if "ProdLike" in self.suffix: + # todo + pass + # setup for stage 2 fastsim + elif "FastSimRun3" in step: + # go back to non-PU step version + d = merge([stepDict[self.getStepName(step)][k]]) + if d is None: return tmpsteps = [] for s in d["-s"].split(","): if s == "DIGI" or "DIGI:" in s: @@ -2519,30 +2522,72 @@ def setupPU_(self, step, stepName, stepDict, k, properties): else: tmpsteps.append(s) d = merge([{"-s" : ",".join(tmpsteps), - "--datamix" : "PreMix", - "--procModifiers": "premix_stage2"}, + "--datamix" : "PreMix"}, d]) - # for combined stage1+stage2 - if "_PMXS1S2" in self.suffix: - d = merge([digiPremixLocalPileup, d]) - elif "Reco" in step: if "--procModifiers" in d: d["--procModifiers"] += ",premix_stage2" else: d["--procModifiers"] = "premix_stage2" - stepDict[stepName][k] = d - # Increase the input file step number by one for Nano in combined stage1+stage2 - elif "Nano"==step: - # go back to non-PU step version - d = merge([stepDict[self.getStepName(step)][k]]) - if "--filein" in d: - filein = d["--filein"] - m = re.search("step(?P\d+)_", filein) - if m: - d["--filein"] = filein.replace(m.group(), "step%d_"%(int(m.group("ind"))+1)) - stepDict[stepName][k] = d - # run2/3 WFs use Nano (not NanoPU) in PU WF - stepDict[self.getStepName(step)][k] = merge([d]) + # for combined stage1+stage2 + if "_PMXS1S2" in self.suffix: + d = merge([digiPremixLocalPileup, d]) + stepDict[stepName][k] = d + elif "HARVESTFastRun3" in step: + # increment input step number + stepDict[stepName][k] = merge([{'--filein':'file:step3_inDQM.root'},stepDict[stepName][k]]) + else: + # setup for stage 1 + if "GenSim" in stepName: + stepNamePmx = stepName.replace('GenSim','Premix') + if not stepNamePmx in stepDict: stepDict[stepNamePmx] = {} + stepDict[stepNamePmx][k] = merge([ + { + '-s': 'GEN,SIM,DIGI:pdigi_valid', + '--datatier': 'PREMIX', + '--eventcontent': 'PREMIX', + '--procModifiers': 'premix_stage1' + }, + stepDict[stepName][k] + ]) + if "ProdLike" in self.suffix: + stepDict[stepNamePmx][k] = merge([{'-s': 'GEN,SIM,DIGI'},stepDict[stepNamePmx][k]]) + # setup for stage 2 + elif "Digi" in step or "Reco" in step: + # go back to non-PU step version + d = merge([stepDict[self.getStepName(step)][k]]) + if d is None: return + if "Digi" in step: + tmpsteps = [] + for s in d["-s"].split(","): + if s == "DIGI" or "DIGI:" in s: + tmpsteps.extend([s, "DATAMIX"]) + else: + tmpsteps.append(s) + d = merge([{"-s" : ",".join(tmpsteps), + "--datamix" : "PreMix", + "--procModifiers": "premix_stage2"}, + d]) + # for combined stage1+stage2 + if "_PMXS1S2" in self.suffix: + d = merge([digiPremixLocalPileup, d]) + elif "Reco" in step: + if "--procModifiers" in d: + d["--procModifiers"] += ",premix_stage2" + else: + d["--procModifiers"] = "premix_stage2" + stepDict[stepName][k] = d + # separate nano step now only used in ProdLike workflows for Run3/Phase2 + elif "Nano"==step: + # go back to non-PU step version + d = merge([stepDict[self.getStepName(step)][k]]) + if "_PMXS1S2" in self.suffix and "--filein" in d: + filein = d["--filein"] + m = re.search("step(?P\\d+)", filein) + if m: + d["--filein"] = filein.replace(m.group(), "step%d"%(int(m.group("ind"))+1)) + stepDict[stepName][k] = d + # run2/3 WFs use Nano (not NanoPU) in PU WF + stepDict[self.getStepName(step)][k] = merge([d]) def condition(self, fragment, stepList, key, hasHarvest): if not 'PU' in key: return False @@ -2561,6 +2606,7 @@ def workflow_(self, workflows, num, fragment, stepList, key): steps = [ ], PU = [ + 'Gen', 'GenSim', 'GenSimHLBeamSpot', 'GenSimHLBeamSpot14', @@ -2581,6 +2627,8 @@ def workflow_(self, workflows, num, fragment, stepList, key): 'RecoNano', 'RecoNanoFakeHLT', 'Nano', + 'FastSimRun3', + 'HARVESTFastRun3', ], suffix = '_PMXS2', offset = 0.98, @@ -2589,6 +2637,7 @@ def workflow_(self, workflows, num, fragment, stepList, key): upgradeWFs['PMXS1S2'] = UpgradeWorkflowPremix( steps = [], PU = [ + 'Gen', 'GenSim', 'GenSimHLBeamSpot', 'GenSimHLBeamSpot14', @@ -2601,6 +2650,8 @@ def workflow_(self, workflows, num, fragment, stepList, key): 'RecoNano', 'RecoNanoFakeHLT', 'Nano', + 'FastSimRun3', + 'HARVESTFastRun3', ], suffix = '_PMXS1S2', offset = 0.99, @@ -2654,8 +2705,6 @@ def setup_(self, step, stepName, stepDict, k, properties): "--eventcontent": "PREMIXRAW"}, d]) stepDict[stepName][k] = d - if 'Nano'==step: - stepDict[stepName][k] = merge([{'--filein':'file:step5.root','-s':'NANO','--datatier':'NANOAODSIM','--eventcontent':'NANOEDMAODSIM'}, stepDict[step][k]]) def condition(self, fragment, stepList, key, hasHarvest): # use both conditions return UpgradeWorkflowPremix.condition(self, fragment, stepList, key, hasHarvest) and UpgradeWorkflow_ProdLike.condition(self, fragment, stepList, key, hasHarvest) @@ -2721,7 +2770,7 @@ def setup_(self, step, stepName, stepDict, k, properties): else: stepDict[stepName][k] = merge([stepDict[step][k]]) def condition(self, fragment, stepList, key, hasHarvest): - return ('2021FS' in key or '2023FS' in key) + return fragment=="TTbar_14TeV" and ('2021FS' in key or '2023FS' in key) upgradeWFs['Run3FStrackingOnly'] = UpgradeWorkflow_Run3FStrackingOnly( steps = [ 'Gen', @@ -2763,17 +2812,12 @@ def condition(self, fragment, stepList, key, hasHarvest): class UpgradeWorkflow_DD4hep(UpgradeWorkflow): def setup_(self, step, stepName, stepDict, k, properties): - if 'Run3' in stepDict[step][k]['--era'] and 'Fast' not in stepDict[step][k]['--era']: - if '2023' in stepDict[step][k]['--conditions']: - stepDict[stepName][k] = merge([{'--geometry': 'DD4hepExtended2023'}, stepDict[step][k]]) - else: - stepDict[stepName][k] = merge([{'--geometry': 'DD4hepExtended2021'}, stepDict[step][k]]) - elif 'Phase2' in stepDict[step][k]['--era']: + if 'Phase2' in stepDict[step][k]['--era']: dd4hepGeom="DD4hep" dd4hepGeom+=stepDict[step][k]['--geometry'] stepDict[stepName][k] = merge([{'--geometry' : dd4hepGeom, '--procModifiers': 'dd4hep'}, stepDict[step][k]]) def condition(self, fragment, stepList, key, hasHarvest): - return ('2021' in key or '2023' in key or '2026' in key) and ('FS' not in key) + return ('2026' in key) and ('FS' not in key) upgradeWFs['DD4hep'] = UpgradeWorkflow_DD4hep( steps = [ 'GenSim', @@ -2806,7 +2850,7 @@ def setup_(self, step, stepName, stepDict, k, properties): if 'Run3' in stepDict[step][k]['--era'] and 'Fast' not in stepDict[step][k]['--era']: stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2022_realistic', '--geometry': 'DB:Extended'}, stepDict[step][k]]) def condition(self, fragment, stepList, key, hasHarvest): - return '2021' in key and 'FS' not in key + return fragment=="TTbar_14TeV" and '2021' in key and 'FS' not in key upgradeWFs['DD4hepDB'] = UpgradeWorkflow_DD4hepDB( steps = [ 'GenSim', @@ -2842,7 +2886,7 @@ def setup_(self, step, stepName, stepDict, k, properties): tmp_eras = ','.join(tmp_eras) stepDict[stepName][k] = merge([{'--conditions': 'auto:phase1_2022_realistic_ddd', '--geometry': 'DB:Extended', '--era': tmp_eras}, stepDict[step][k]]) def condition(self, fragment, stepList, key, hasHarvest): - return '2021' in key and 'FS' not in key + return fragment=="TTbar_14TeV" and '2021' in key and 'FS' not in key upgradeWFs['DDDDB'] = UpgradeWorkflow_DDDDB( steps = [ 'GenSim',