Update HLT CPU vs GPU comparison workflow

DataFormats/ParticleFlowReco_Alpaka/src/alpaka/classes_cuda.h

@@ -1,6 +1,6 @@
 #include "DataFormats/Common/interface/DeviceProduct.h"
 #include "DataFormats/Common/interface/Wrapper.h"
-#include "DataFormats/Portable/interface/Product.h"
+//#include "DataFormats/Portable/interface/Product.h"
 #include "DataFormats/ParticleFlowReco_Alpaka/interface/PFRecHitSoA.h"
 #include "DataFormats/ParticleFlowReco_Alpaka/interface/alpaka/PFRecHitDeviceCollection.h"
 #include "DataFormats/ParticleFlowReco_Alpaka/interface/CaloRecHitSoA.h"

DataFormats/ParticleFlowReco_Alpaka/src/alpaka/classes_serial.h

@@ -1,5 +1,5 @@
 #include "DataFormats/Common/interface/Wrapper.h"
-#include "DataFormats/Portable/interface/Product.h"
+//#include "DataFormats/Portable/interface/Product.h"
 #include "DataFormats/ParticleFlowReco_Alpaka/interface/PFRecHitSoA.h"
 #include "DataFormats/ParticleFlowReco_Alpaka/interface/PFRecHitHostCollection.h"
 #include "DataFormats/ParticleFlowReco_Alpaka/interface/CaloRecHitSoA.h"

RecoParticleFlow/Configuration/test/run_HBHEandPF_onCPUandGPU.py

@@ -21,9 +21,6 @@
 process.load('Configuration.StandardSequences.EndOfProcess_cff')
 process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff')
 
-process.load("RecoParticleFlow.PFClusterProducer.pfhbheRecHitParamsGPUESProducer_cfi")
-process.load("RecoParticleFlow.PFClusterProducer.pfhbheTopologyGPUESProducer_cfi")
-
 process.maxEvents = cms.untracked.PSet(
     #input = cms.untracked.int32(5),
     input = cms.untracked.int32(100),
@@ -167,15 +164,41 @@
                 if z.detectorEnum == 1: # HB
                     z.detectorEnum = cms.uint32( 1 )
                     z.depth = cms.vuint32( 1, 2, 3, 4 )
+                    process.pfhbheRecHitParamsGPUESProducer.thresholdE_HB = z.threshold # propagate to process.pfhbheRecHitParamsGPUESProducer
                 if z.detectorEnum == 2: # HE
                     z.detectorEnum = cms.uint32( 2 )
                     z.depth = cms.vuint32( 1, 2, 3, 4, 5, 6, 7  )
+                    process.pfhbheRecHitParamsGPUESProducer.thresholdE_HE = z.threshold # propagate to process.pfhbheRecHitParamsGPUESProducer
 
 process.hltParticleFlowRecHitHBHEonGPU = cms.EDProducer("PFHBHERecHitProducerGPU", # instead of "PFRecHitProducer"
                                                    producers = _pset_hltParticleFlowRecHitHBHE_producers_mod,
                                                    navigator = process.hltParticleFlowRecHitHBHE.navigator
 )
 
+#
+# Propagate PFCluster parameters for CPU to GPU ES-based ones
+#
+_pset_GPU = process.pfClusteringParamsGPUESSource.initialClusteringStep.thresholdsByDetector
+_pset_CPU = process.hltParticleFlowClusterHBHE.initialClusteringStep.thresholdsByDetector
+for idx, x in enumerate(_pset_GPU):
+    for idy, y in enumerate(_pset_CPU):
+        if x.detector == y.detector:
+            x.gatheringThreshold = y.gatheringThreshold
+
+_pset_GPU = process.pfClusteringParamsGPUESSource.pfClusterBuilder.recHitEnergyNorms
+_pset_CPU = process.hltParticleFlowClusterHBHE.pfClusterBuilder.recHitEnergyNorms
+for idx, x in enumerate(_pset_GPU):
+    for idy, y in enumerate(_pset_CPU):
+        if x.detector == y.detector:
+            x.recHitEnergyNorm = y.recHitEnergyNorm
+
+_pset_GPU = process.pfClusteringParamsGPUESSource.seedFinder.thresholdsByDetector
+_pset_CPU = process.hltParticleFlowClusterHBHE.seedFinder.thresholdsByDetector
+for idx, x in enumerate(_pset_GPU):
+    for idy, y in enumerate(_pset_CPU):
+        if x.detector == y.detector:
+            x.seedingThreshold = y.seedingThreshold
+
 #
 # for CPU PFRecHitHBHE
 #

RecoParticleFlow/PFClusterProducer/plugins/PFHBHERecHitProducerCPU.cc

@@ -24,7 +24,6 @@
 #include "Geometry/Records/interface/CaloGeometryRecord.h"
 #include "HeterogeneousCore/CUDACore/interface/JobConfigurationGPURecord.h"
 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
-#include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
 #include "HeterogeneousCore/CUDAUtilities/interface/cudaCheck.h"
 #include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 #include "HeterogeneousCore/CUDAUtilities/interface/host_unique_ptr.h"

RecoParticleFlow/PFClusterProducer/plugins/PFHBHERecHitProducerGPU.cc

@@ -24,7 +24,6 @@
 #include "Geometry/Records/interface/CaloGeometryRecord.h"
 #include "HeterogeneousCore/CUDACore/interface/JobConfigurationGPURecord.h"
 #include "HeterogeneousCore/CUDACore/interface/ScopedContext.h"
-#include "HeterogeneousCore/CUDAServices/interface/CUDAService.h"
 #include "HeterogeneousCore/CUDAUtilities/interface/cudaCheck.h"
 #include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
 #include "HeterogeneousCore/CUDAUtilities/interface/host_unique_ptr.h"

RecoParticleFlow/PFClusterProducer/python/particleFlowClusterHBHE_cfi.py

@@ -35,7 +35,7 @@
         nNeighbours = cms.int32(4)
     ),
     initialClusteringStep = cms.PSet(
-        algoName = cms.string("Basic2DGenericTopoClusterizer"),    
+        algoName = cms.string("Basic2DGenericTopoClusterizer"),
         thresholdsByDetector = cms.VPSet(
         cms.PSet( detector = cms.string("HCAL_BARREL1"),
                   depths = cms.vint32(1, 2, 3, 4),
@@ -50,14 +50,14 @@
         ),
         useCornerCells = cms.bool(True)
     ),
-    
+
     pfClusterBuilder = cms.PSet(
            algoName = cms.string("Basic2DGenericPFlowClusterizer"),
            #pf clustering parameters
            minFractionToKeep = cms.double(1e-7),
            positionCalc = cms.PSet(
                  algoName = cms.string("Basic2DGenericPFlowPositionCalc"),
-                 minFractionInCalc = cms.double(1e-9),    
+                 minFractionInCalc = cms.double(1e-9),
                  posCalcNCrystals = cms.int32(5),
                  logWeightDenominatorByDetector = cms.VPSet(
                        cms.PSet( detector = cms.string("HCAL_BARREL1"),
@@ -73,7 +73,7 @@
            ),
            allCellsPositionCalc =cms.PSet(
                  algoName = cms.string("Basic2DGenericPFlowPositionCalc"),
-                 minFractionInCalc = cms.double(1e-9),    
+                 minFractionInCalc = cms.double(1e-9),
                  posCalcNCrystals = cms.int32(-1),
                  logWeightDenominatorByDetector = cms.VPSet(
                        cms.PSet( detector = cms.string("HCAL_BARREL1"),
@@ -87,7 +87,7 @@
                        ),
                  minAllowedNormalization = cms.double(1e-9)
            ),
-           
+
 
            timeSigmaEB = cms.double(10.),
            timeSigmaEE = cms.double(10.),
@@ -171,7 +171,7 @@
 
 # offline 2023
 from Configuration.Eras.Modifier_run3_egamma_2023_cff import run3_egamma_2023
-run3_egamma_2023.toModify(particleFlowClusterHBHE,
+run3_egamma_2023.toModify(_particleFlowClusterHBHE_cpu,
     seedFinder = dict(thresholdsByDetector = {0 : dict(seedingThreshold = _seedingThresholdsHBphase1_2023) } ),
     initialClusteringStep = dict(thresholdsByDetector = {0 : dict(gatheringThreshold = _thresholdsHBphase1_2023) } ),
     pfClusterBuilder = dict(
@@ -180,7 +180,24 @@
         allCellsPositionCalc = dict(logWeightDenominatorByDetector = {0 : dict(logWeightDenominator = _thresholdsHBphase1_2023) } ),
     ),
 )
-
+run3_egamma_2023.toModify(_particleFlowClusterHBHE_cuda,
+    seedFinder = dict(thresholdsByDetector = {0 : dict(seedingThreshold = _seedingThresholdsHBphase1_2023) } ),
+    initialClusteringStep = dict(thresholdsByDetector = {0 : dict(gatheringThreshold = _thresholdsHBphase1_2023) } ),
+    pfClusterBuilder = dict(
+        recHitEnergyNorms = {0 : dict(recHitEnergyNorm = _thresholdsHBphase1_2023) },
+        positionCalc = dict(logWeightDenominatorByDetector = {0 : dict(logWeightDenominator = _thresholdsHBphase1_2023) } ),
+        allCellsPositionCalc = dict(logWeightDenominatorByDetector = {0 : dict(logWeightDenominator = _thresholdsHBphase1_2023) } ),
+    ),
+)
+# GPU version actually uses ES values below
+from RecoParticleFlow.PFClusterProducer.pfClusteringParamsGPUESSource_cfi import pfClusteringParamsGPUESSource
+run3_egamma_2023.toModify(pfClusteringParamsGPUESSource,
+    seedFinder = dict(thresholdsByDetector = {0 : dict(seedingThreshold = _seedingThresholdsHBphase1_2023) } ),
+    initialClusteringStep = dict(thresholdsByDetector = {0 : dict(gatheringThreshold = _thresholdsHBphase1_2023) } ),
+    pfClusterBuilder = dict(
+        recHitEnergyNorms = {0 : dict(recHitEnergyNorm = _thresholdsHBphase1_2023) }
+    ),
+)
 
 # HCALonly WF
 # particleFlowClusterHBHEOnly = _particleFlowClusterHBHE_cpu.clone(
@@ -208,6 +225,6 @@
 )
 
 from Configuration.ProcessModifiers.gpu_cff import gpu
-gpu.toModify(particleFlowClusterHBHE, 
+gpu.toModify(particleFlowClusterHBHE,
     cuda = _particleFlowClusterHBHE_cuda.clone()
-)         
+)

RecoParticleFlow/PFClusterProducer/python/particleFlowRecHitHBHE_cfi.py

@@ -86,10 +86,19 @@
     producers = {0 : dict(qualityTests = {0 : dict(cuts = {0 : dict(threshold = _thresholdsHBphase1) } ) } ) },
 )
 
+# offline 2023
 from Configuration.Eras.Modifier_run3_egamma_2023_cff import run3_egamma_2023
-run3_egamma_2023.toModify(particleFlowRecHitHBHE,
+run3_egamma_2023.toModify(_particleFlowRecHitHBHE_cpu,
     producers = {0 : dict(qualityTests = {0 : dict(cuts = {0 : dict(threshold = _thresholdsHBphase1_2023) } ) } ) },
 )
+run3_egamma_2023.toModify(_particleFlowRecHitHBHE_cuda,
+    producers = {0 : dict(qualityTests = {0 : dict(cuts = {0 : dict(threshold = _thresholdsHBphase1_2023) } ) } ) },
+)
+# GPU version actually uses ES values below
+from RecoParticleFlow.PFClusterProducer.pfhbheRecHitParamsGPUESProducer_cfi import pfhbheRecHitParamsGPUESProducer
+run3_egamma_2023.toModify(pfhbheRecHitParamsGPUESProducer,
+                          thresholdE_HB = _thresholdsHBphase1_2023,
+)
 
 from HeterogeneousCore.CUDACore.SwitchProducerCUDA import SwitchProducerCUDA
 particleFlowRecHitHBHE = SwitchProducerCUDA(
@@ -110,7 +119,7 @@
     producers = { 0: dict(src = "hbhereco") }
 )
 
-# KenH: this overwrites the original above. I am not sure if that's a good thing.
+#
 particleFlowRecHitHBHEOnly = SwitchProducerCUDA(
     cpu = _particleFlowRecHitHBHE_cpu.clone()
 )

RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitHBHEParamsESProducer.cc

@@ -16,7 +16,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
   class PFRecHitHBHEParamsESProducer : public ESProducer {
   public:
-    PFRecHitHBHEParamsESProducer(edm::ParameterSet const& iConfig) :
+    PFRecHitHBHEParamsESProducer(edm::ParameterSet const& iConfig) : ESProducer(iConfig),
         energyThresholdsHB_(iConfig.getParameter<std::array<double, kMaxDepthHB>>("energyThresholdsHB")),
         energyThresholdsHE_(iConfig.getParameter<std::array<double, kMaxDepthHE>>("energyThresholdsHE")) {
       setWhatProduced(this);

RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitHBHETopologyESProducer.cc

@@ -26,8 +26,9 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
   class PFRecHitHBHETopologyESProducer : public ESProducer {
   public:
-    PFRecHitHBHETopologyESProducer(edm::ParameterSet const& iConfig)
-      : hcalEnums_(iConfig.getParameter<std::vector<int>>("hcalEnums")) {
+    PFRecHitHBHETopologyESProducer(edm::ParameterSet const& iConfig) :
+      ESProducer(iConfig) ,
+      hcalEnums_(iConfig.getParameter<std::vector<int>>("hcalEnums")) {
       auto cc = setWhatProduced(this);
       hcalToken_ = cc.consumes();
       geomToken_ = cc.consumes();