From a51708ebf97deb6874503ca36cbaa4b81a5c31ba Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Mon, 12 Aug 2019 19:49:46 +0200 Subject: [PATCH 1/4] Add (combined) premixing workflows for Run 3 --- .../PyReleaseValidation/python/relval_2017.py | 12 ++++++------ .../PyReleaseValidation/python/relval_upgrade.py | 12 ++++++++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_2017.py b/Configuration/PyReleaseValidation/python/relval_2017.py index ac7f833eaa1c3..55f33022e2e40 100644 --- a/Configuration/PyReleaseValidation/python/relval_2017.py +++ b/Configuration/PyReleaseValidation/python/relval_2017.py @@ -22,9 +22,9 @@ # (TTbar trackingOnly, pixelTrackingOnly) # he collapse: TTbar, TTbar PU, TTbar design # ParkingBPH: TTbar -# 2021 (ZMM, TTbar, ZEE, MinBias, TTbar PU, ZEE PU, TTbar design) -# 2023 (TTbar, TTbar PU) -# 2024 (TTbar, TTbar PU) +# 2021 (ZMM, TTbar, ZEE, MinBias, TTbar PU, TTbar PU premix, ZEE PU, TTbar design) +# 2023 (TTbar, TTbar PU, TTbar PU premix) +# 2024 (TTbar, TTbar PU, TTbar PU premix) numWFIB = [10001.0,10002.0,10003.0,10004.0,10005.0,10006.0,10007.0,10008.0,10009.0,10059.0,10071.0, 10042.0,10024.0,10025.0,10026.0,10023.0,10224.0,10225.0,10424.0, 10024.1,10024.2,10024.3,10024.4,10024.5, @@ -33,9 +33,9 @@ 10824.1,10824.5, 10824.6,11024.6,11224.6, 10824.8, - 11650.0,11634.0,11646.0,11640.0,11834.0,11846.0,12024.0, - 12434.0,12634.0, - 12834.0,13034.0] + 11650.0,11634.0,11646.0,11640.0,11834.0,11834.99,11846.0,12024.0, + 12434.0,12634.0,12634.99, + 12834.0,13034.0,13034.99] for numWF in numWFIB: if not numWF in _upgrade_workflows: continue workflows[numWF] = _upgrade_workflows[numWF] diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index 0e17d1b965842..bf47e5f210d2e 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -105,12 +105,20 @@ def makeStepName(key,frag,step,suffix): if upgradeDatasetFromFragment[frag]=="TTbar_13" and '2018' in key: workflows[numWF+upgradeSteps['ParkingBPH']['offset']] = [ upgradeDatasetFromFragment[frag], stepList['ParkingBPH']] + inclPremix = 'PU' in key + if inclPremix: + inclPremix = False + for y in ['2021', '2023', '2024', '2026']: + if y in key: + inclPremix = True + continue + # premixing stage1, only for NuGun - if upgradeDatasetFromFragment[frag]=="NuGun" and 'PU' in key and '2026' in key: + if inclPremix and upgradeDatasetFromFragment[frag]=="NuGun": workflows[numWF+upgradeSteps['Premix']['offset']] = [upgradeDatasetFromFragment[frag], stepList['Premix']] # premixing stage2, only for ttbar for time being - if 'PU' in key and '2026' in key and upgradeDatasetFromFragment[frag]=="TTbar_14TeV": + if inclPremix and upgradeDatasetFromFragment[frag]=="TTbar_14TeV": slist = [] for step in stepList['baseline']: s = step From 9e47bb123227e05ea9ca5c03fab11399dd3167ae Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Mon, 12 Aug 2019 20:05:12 +0200 Subject: [PATCH 2/4] Drop NANO step from upgrade combined premixing workflow NANO is configured to use step3_inMINIAODSIM.root input file, but in combined premixing workflow the file is step4_inMINIAODSIM.root. Simplest option is to just drop the NANO step from these workflows. --- Configuration/PyReleaseValidation/python/relval_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index bf47e5f210d2e..4318ba2041cbf 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -132,6 +132,6 @@ def makeStepName(key,frag,step,suffix): [slist[0]] + # Start with signal generation stepList['Premix'] + # Premixing stage1 [slist[1].replace("PUPRMX", "PUPRMXCombined")] + # Premixing stage2, customized for the combined (defined in relval_steps.py) - slist[2:]] # Remaining standard steps + filter(lambda x: "nano" not in x.lower(), slist[2:])] # Remaining standard steps (excluding NANOAOD because of input file numbering mismatch) numWF+=1 From 71bc8fdf5456d1b90e1c97899bf8888a370f8259 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Wed, 14 Aug 2019 16:05:04 +0200 Subject: [PATCH 3/4] Increase the input file step number by 1 for Nano step in upgrade-style combined premixing workflows --- .../PyReleaseValidation/python/relval_steps.py | 12 ++++++++++-- .../PyReleaseValidation/python/relval_upgrade.py | 8 +++++++- .../python/upgradeWorkflowComponents.py | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index b7bad29633fdf..47eef932c3be3 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -3336,7 +3336,16 @@ def gen2018HiMix(fragment,howMuch): # For combined stage1+stage2 if "Digi" in step: upgradeStepDict[stepNamePUpmx+"Combined"][k] = merge([digiPremixLocalPileup, d]) - + # Increase the input file step number by one for Nano in combined stage1+stage2 + if "Nano" in step: + d = merge([upgradeStepDict[stepName][k]]) + if "--filein" in d: + filein = d["--filein"] + m = re.search("step(?P\d+)_", filein) + if m: + d["--filein"] = filein.replace(m.group(), "step%d_"%(int(m.group("ind"))+1)) + stepNamePUpmx = step + 'PUPRMX' + upgradeSteps[stepType]['suffix'] + upgradeStepDict[stepNamePUpmx+"Combined"][k] = d for step in upgradeStepDict.keys(): # we need to do this for each fragment if 'Sim' in step or 'Premix' in step: @@ -3360,4 +3369,3 @@ def gen2018HiMix(fragment,howMuch): steps[k]=None else: steps[k]=merge([upgradeStepDict[step][key]]) - diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index 4318ba2041cbf..42421b0fe6f91 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -128,10 +128,16 @@ def makeStepName(key,frag,step,suffix): workflows[numWF+premixS2_offset] = [upgradeDatasetFromFragment[frag], slist] # Combined stage1+stage2 + def nano(s): + if "Nano" in s: + if "_" in s: + return s.replace("_", "PUPRMXCombined_") + return s+"PUPRMXCombined" + return s workflows[numWF+premixS1S2_offset] = [upgradeDatasetFromFragment[frag], # Signal fragment [slist[0]] + # Start with signal generation stepList['Premix'] + # Premixing stage1 [slist[1].replace("PUPRMX", "PUPRMXCombined")] + # Premixing stage2, customized for the combined (defined in relval_steps.py) - filter(lambda x: "nano" not in x.lower(), slist[2:])] # Remaining standard steps (excluding NANOAOD because of input file numbering mismatch) + map(nano, slist[2:])] # Remaining standard steps numWF+=1 diff --git a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py index 8e15929faa61c..975f9fd06a767 100644 --- a/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py +++ b/Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py @@ -89,6 +89,7 @@ 'HARVESTFull', 'HARVESTFullGlobal', 'MiniAODFullGlobal', + 'NanoFull', ], 'suffix' : '', 'offset' : 0.0, From 60038c784ce8771fc1f45984e088bdedfea42d73 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Fri, 16 Aug 2019 20:55:57 +0200 Subject: [PATCH 4/4] Define stage2 premixing workflows for all datasets (for given years) --- .../PyReleaseValidation/python/relval_upgrade.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_upgrade.py b/Configuration/PyReleaseValidation/python/relval_upgrade.py index 42421b0fe6f91..c48247d4e08e8 100644 --- a/Configuration/PyReleaseValidation/python/relval_upgrade.py +++ b/Configuration/PyReleaseValidation/python/relval_upgrade.py @@ -112,13 +112,12 @@ def makeStepName(key,frag,step,suffix): if y in key: inclPremix = True continue + if inclPremix: + # premixing stage1, only for NuGun + if upgradeDatasetFromFragment[frag]=="NuGun": + workflows[numWF+upgradeSteps['Premix']['offset']] = [upgradeDatasetFromFragment[frag], stepList['Premix']] - # premixing stage1, only for NuGun - if inclPremix and upgradeDatasetFromFragment[frag]=="NuGun": - workflows[numWF+upgradeSteps['Premix']['offset']] = [upgradeDatasetFromFragment[frag], stepList['Premix']] - - # premixing stage2, only for ttbar for time being - if inclPremix and upgradeDatasetFromFragment[frag]=="TTbar_14TeV": + # premixing stage2 slist = [] for step in stepList['baseline']: s = step