diff --git a/CUDADataFormats/SiPixelCluster/interface/gpuClusteringConstants.h b/CUDADataFormats/SiPixelCluster/interface/gpuClusteringConstants.h index 1430606ab6678..3075682870297 100644 --- a/CUDADataFormats/SiPixelCluster/interface/gpuClusteringConstants.h +++ b/CUDADataFormats/SiPixelCluster/interface/gpuClusteringConstants.h @@ -6,6 +6,8 @@ namespace pixelGPUConstants { #ifdef GPU_SMALL_EVENTS constexpr uint32_t maxNumberOfHits = 24 * 1024; +#elif defined(GPU_HION_EVENTS) + constexpr uint32_t maxNumberOfHits = 192 * 1024; #else constexpr uint32_t maxNumberOfHits = 48 * 1024; // data at pileup 50 has 18300 +/- 3500 hits; 40000 is around 6 sigma away diff --git a/CUDADataFormats/Track/interface/PixelTrackHeterogeneous.h b/CUDADataFormats/Track/interface/PixelTrackHeterogeneous.h index d462be2c5dd7b..e79a32c21daa0 100644 --- a/CUDADataFormats/Track/interface/PixelTrackHeterogeneous.h +++ b/CUDADataFormats/Track/interface/PixelTrackHeterogeneous.h @@ -16,7 +16,7 @@ class TrackSoAT { static constexpr int32_t stride() { return S; } using Quality = trackQuality::Quality; - using hindex_type = uint16_t; + using hindex_type = uint32_t; using HitContainer = cms::cuda::OneToManyAssoc; // Always check quality is at least loose! diff --git a/CUDADataFormats/TrackingRecHit/interface/TrackingRecHit2DSOAView.h b/CUDADataFormats/TrackingRecHit/interface/TrackingRecHit2DSOAView.h index 808feb2a4218f..8c9626c75df95 100644 --- a/CUDADataFormats/TrackingRecHit/interface/TrackingRecHit2DSOAView.h +++ b/CUDADataFormats/TrackingRecHit/interface/TrackingRecHit2DSOAView.h @@ -15,10 +15,10 @@ namespace pixelCPEforGPU { class TrackingRecHit2DSOAView { public: static constexpr uint32_t maxHits() { return gpuClustering::MaxNumClusters; } - using hindex_type = uint16_t; // if above is <=2^16 + using hindex_type = uint32_t; // if above is <=2^32 using Hist = - cms::cuda::HistoContainer; + cms::cuda::HistoContainer; using AverageGeometry = phase1PixelTopology::AverageGeometry; diff --git a/EventFilter/EcalRawToDigi/plugins/DeclsForKernels.h b/EventFilter/EcalRawToDigi/plugins/DeclsForKernels.h index a6429121adc82..05eff58b6d31d 100644 --- a/EventFilter/EcalRawToDigi/plugins/DeclsForKernels.h +++ b/EventFilter/EcalRawToDigi/plugins/DeclsForKernels.h @@ -17,7 +17,11 @@ namespace ecal { constexpr auto empty_event_size = EMPTYEVENTSIZE; constexpr uint32_t nfeds_max = 54; +#ifdef GPU_HION_EVENTS + constexpr uint32_t nbytes_per_fed_max = 32 * 1024; +#else constexpr uint32_t nbytes_per_fed_max = 10 * 1024; +#endif struct InputDataCPU { cms::cuda::host::unique_ptr data; diff --git a/HLTrigger/Configuration/python/customizeHLTforPatatrackHIon.py b/HLTrigger/Configuration/python/customizeHLTforPatatrackHIon.py new file mode 100644 index 0000000000000..09b6fda7649ce --- /dev/null +++ b/HLTrigger/Configuration/python/customizeHLTforPatatrackHIon.py @@ -0,0 +1,577 @@ +import copy +import FWCore.ParameterSet.Config as cms + +# customisation for running on CPUs, common parts +def customise_cpu_common(process): + + # Services + + process.CUDAService = cms.Service("CUDAService", + enabled = cms.untracked.bool(False) + ) + + + # done + return process + + +# customisation for offloading to GPUs, common parts +def customise_gpu_common(process): + + # Services + + process.CUDAService = cms.Service("CUDAService", + enabled = cms.untracked.bool(True), + allocator = cms.untracked.PSet( + devicePreallocate = cms.untracked.vuint32(), + ), + limits = cms.untracked.PSet( + cudaLimitDevRuntimePendingLaunchCount = cms.untracked.int32(-1), + cudaLimitDevRuntimeSyncDepth = cms.untracked.int32(-1), + cudaLimitMallocHeapSize = cms.untracked.int32(-1), + cudaLimitPrintfFifoSize = cms.untracked.int32(-1), + cudaLimitStackSize = cms.untracked.int32(-1) + ) + ) + + process.load("HeterogeneousCore.CUDAServices.NVProfilerService_cfi") + + # done + return process + + +# customisation for running the "Patatrack" pixel track reconstruction on CPUs +def customise_cpu_pixel(process): + + # FIXME replace the Sequences with empty ones to avoid exanding them during the (re)definition of Modules and EDAliases + + process.HLTRecoPixelTracksPPOnAASequence = cms.Sequence() + process.HLTRecopixelvertexingSequencePPOnAA = cms.Sequence() + + + # Event Setup + + process.load("RecoLocalTracker.SiPixelRecHits.PixelCPEFastESProducer_cfi") + process.PixelCPEFastESProducer.DoLorentz = True + + + # Modules and EDAliases + + # referenced in process.HLTRecoPixelTracksSequence + + from RecoLocalTracker.SiPixelRecHits.siPixelRecHitHostSoA_cfi import siPixelRecHitHostSoA as _siPixelRecHitHostSoA + process.hltSiPixelRecHitSoAPPOnAA = _siPixelRecHitHostSoA.clone( + beamSpot = "hltOnlineBeamSpot", + src = "hltSiPixelClustersPPOnAA", + convertToLegacy = True + ) + + from RecoPixelVertexing.PixelTriplets.caHitNtupletCUDA_cfi import caHitNtupletCUDA as _caHitNtupletCUDA + process.hltPixelTracksHitQuadrupletsPPOnAA = _caHitNtupletCUDA.clone( + idealConditions = False, + pixelRecHitSrc = "hltSiPixelRecHitSoAPPOnAA", + onGPU = False + ) + + process.hltPixelTracksSoAPPOnAA = cms.EDAlias( + hltPixelTracksHitQuadrupletsPPOnAA = cms.VPSet( + cms.PSet( + type = cms.string("32768TrackSoATHeterogeneousSoA") + ) + ) + ) + + from RecoPixelVertexing.PixelTrackFitting.pixelTrackProducerFromSoA_cfi import pixelTrackProducerFromSoA as _pixelTrackProducerFromSoA + process.hltPixelTracksPPOnAA = _pixelTrackProducerFromSoA.clone( + beamSpot = "hltOnlineBeamSpot", + pixelRecHitLegacySrc = "hltSiPixelRecHitsPPOnAA", + trackSrc = "hltPixelTracksSoAPPOnAA" + ) + + + # referenced in process.HLTRecopixelvertexingSequence + + from RecoPixelVertexing.PixelVertexFinding.pixelVertexCUDA_cfi import pixelVertexCUDA as _pixelVertexCUDA + process.hltPixelVerticesSoAPPOnAA = _pixelVertexCUDA.clone( + pixelTrackSrc = "hltPixelTracksSoAPPOnAA", + onGPU = False + ) + + from RecoPixelVertexing.PixelVertexFinding.pixelVertexFromSoA_cfi import pixelVertexFromSoA as _pixelVertexFromSoA + process.hltPixelVerticesPPOnAA = _pixelVertexFromSoA.clone( + TrackCollection = "hltPixelTracksPPOnAA", + beamSpot = "hltOnlineBeamSpot", + src = "hltPixelVerticesSoAPPOnAA" + ) + + + # Sequences + + process.HLTRecoPixelTracksSequencePPOnAA = cms.Sequence( + process.hltPixelTracksFitter # not used here, kept for compatibility with legacy sequences + + process.hltPixelTracksFilter # not used here, kept for compatibility with legacy sequences + + process.hltPixelTracksTrackingRegionsPPOnAA # from the original sequence + + process.hltSiPixelRecHitSoAPPOnAA # pixel rechits on cpu, converted to SoA + + process.hltPixelTracksHitQuadrupletsPPOnAA # pixel ntuplets on cpu, in SoA format + # process.hltPixelTracksSoA # alias for hltPixelTracksHitQuadruplets + + process.hltPixelTracksPPOnAA) # pixel tracks on cpu, with transfer and conversion to legacy + process.HLTRecoPixelTracksPPOnAASequence = cms.Sequence(process.HLTRecoPixelTracksSequencePPOnAA) + + process.HLTRecopixelvertexingSequencePPOnAA = cms.Sequence( + process.HLTRecoPixelTracksSequencePPOnAA + + process.hltPixelVerticesSoAPPOnAA # pixel vertices on cpu, in SoA format + + process.hltPixelVerticesPPOnAA # pixel vertices on cpu, in legacy format + + process.hltTrimmedPixelVerticesPPOnAA) # from the original sequence + process.HLTPixelVertexingPPOnAASequence = cms.Sequence(process.HLTRecopixelvertexingSequencePPOnAA) + process.HLTPixelVertexingSequencePPOnAA = cms.Sequence(process.HLTRecopixelvertexingSequencePPOnAA) + + # done + return process + + +# customisation for offloading the Pixel local reconstruction to GPUs +def customise_gpu_pixel(process): + + # FIXME replace the Sequences with empty ones to avoid exanding them during the (re)definition of Modules and EDAliases + + process.HLTDoLocalPixelSequencePPOnAA = cms.Sequence() + process.HLTRecoPixelTracksPPOnAASequence = cms.Sequence() + process.HLTRecopixelvertexingSequencePPOnAA = cms.Sequence() + process.HLTDoLocalPixelSequencePPOnAAForLowPt = cms.Sequence() + + + # Event Setup + + process.load("CalibTracker.SiPixelESProducers.siPixelGainCalibrationForHLTGPU_cfi") + process.load("RecoLocalTracker.SiPixelClusterizer.siPixelFedCablingMapGPUWrapper_cfi") + process.load("RecoLocalTracker.SiPixelRecHits.PixelCPEFastESProducer_cfi") + process.PixelCPEFastESProducer.DoLorentz = True + + + # Modules and EDAliases + + # referenced in process.HLTDoLocalPixelSequence + + process.hltOnlineBeamSpotCUDA = cms.EDProducer("BeamSpotToCUDA", + src = cms.InputTag("hltOnlineBeamSpot") + ) + + from RecoLocalTracker.SiPixelClusterizer.siPixelRawToClusterCUDA_cfi import siPixelRawToClusterCUDA as _siPixelRawToClusterCUDA + process.hltSiPixelClustersCUDA = _siPixelRawToClusterCUDA.clone() + + process.hltSiPixelRecHitsCUDA = cms.EDProducer("SiPixelRecHitCUDA", + CPE = cms.string("PixelCPEFast"), + beamSpot = cms.InputTag("hltOnlineBeamSpotCUDA"), + src = cms.InputTag("hltSiPixelClustersCUDA") + ) + + process.hltSiPixelDigisSoA = cms.EDProducer("SiPixelDigisSoAFromCUDA", + src = cms.InputTag("hltSiPixelClustersCUDA") + ) + + process.hltSiPixelDigisClusters = cms.EDProducer("SiPixelDigisClustersFromSoA", + src = cms.InputTag("hltSiPixelDigisSoA") + ) + + process.hltSiPixelDigiErrorsSoA = cms.EDProducer("SiPixelDigiErrorsSoAFromCUDA", + src = cms.InputTag("hltSiPixelClustersCUDA") + ) + + from EventFilter.SiPixelRawToDigi.siPixelDigiErrorsFromSoA_cfi import siPixelDigiErrorsFromSoA as _siPixelDigiErrorsFromSoA + process.hltSiPixelDigiErrors = _siPixelDigiErrorsFromSoA.clone( + UsePhase1 = True, + digiErrorSoASrc = "hltSiPixelDigiErrorsSoA" + ) + + process.hltSiPixelRecHitsPPOnAA = cms.EDProducer("SiPixelRecHitFromSOA", + pixelRecHitSrc = cms.InputTag("hltSiPixelRecHitsCUDA"), + src = cms.InputTag("hltSiPixelDigisClusters") + ) + process.hltSiPixelRecHitsPPOnAAForLowPt = process.hltSiPixelRecHitsPPOnAA.clone() + + process.hltSiPixelDigis = cms.EDAlias( + hltSiPixelDigisClusters = cms.VPSet( + cms.PSet( + type = cms.string("PixelDigiedmDetSetVector") + ) + ), + hltSiPixelDigiErrors = cms.VPSet( + cms.PSet( + type = cms.string("DetIdedmEDCollection") + ), + cms.PSet( + type = cms.string("SiPixelRawDataErroredmDetSetVector") + ), + cms.PSet( + type = cms.string("PixelFEDChanneledmNewDetSetVector") + ) + ) + ) + + process.hltSiPixelClustersPPOnAA = cms.EDAlias( + hltSiPixelDigisClusters = cms.VPSet( + cms.PSet( + type = cms.string("SiPixelClusteredmNewDetSetVector") + ) + ) + ) + process.hltSiPixelClustersPPOnAAForLowPt = process.hltSiPixelClustersPPOnAA.clone() + + # referenced in process.HLTRecoPixelTracksSequence + + from RecoPixelVertexing.PixelTriplets.caHitNtupletCUDA_cfi import caHitNtupletCUDA as _caHitNtupletCUDA + process.hltPixelTracksHitQuadrupletsPPOnAA = _caHitNtupletCUDA.clone( + idealConditions = False, + pixelRecHitSrc = "hltSiPixelRecHitsCUDA", + onGPU = True + ) + + process.hltPixelTracksSoAPPOnAA = cms.EDProducer("PixelTrackSoAFromCUDA", + src = cms.InputTag("hltPixelTracksHitQuadrupletsPPOnAA") + ) + + process.hltPixelTracksPPOnAA = cms.EDProducer("PixelTrackProducerFromSoA", + beamSpot = cms.InputTag("hltOnlineBeamSpot"), + minNumberOfHits = cms.int32(0), + pixelRecHitLegacySrc = cms.InputTag("hltSiPixelRecHitsPPOnAA"), + trackSrc = cms.InputTag("hltPixelTracksSoAPPOnAA") + ) + + # referenced in process.HLTRecopixelvertexingSequence + + from RecoPixelVertexing.PixelVertexFinding.pixelVertexCUDA_cfi import pixelVertexCUDA as _pixelVertexCUDA + process.hltPixelVerticesCUDAPPOnAA = _pixelVertexCUDA.clone( + pixelTrackSrc = "hltPixelTracksHitQuadrupletsPPOnAA", + onGPU = True + ) + + process.hltPixelVerticesSoAPPOnAA = cms.EDProducer("PixelVertexSoAFromCUDA", + src = cms.InputTag("hltPixelVerticesCUDAPPOnAA") + ) + + process.hltPixelVerticesPPOnAA = cms.EDProducer("PixelVertexProducerFromSoA", + src = cms.InputTag("hltPixelVerticesSoAPPOnAA"), + beamSpot = cms.InputTag("hltOnlineBeamSpot"), + TrackCollection = cms.InputTag("hltPixelTracksPPOnAA") + ) + + + # Sequences + + HLTLocalPixelGPUSequence = cms.Sequence( + process.hltOnlineBeamSpotCUDA # transfer the beamspot to the gpu + + process.hltSiPixelClustersCUDA # digis and clusters on gpu + + process.hltSiPixelRecHitsCUDA # rechits on gpu + + process.hltSiPixelDigisSoA # copy to host + + process.hltSiPixelDigisClusters # convert to legacy + + process.hltSiPixelDigiErrorsSoA # copy to host + + process.hltSiPixelDigiErrors) # convert to legacy + + process.HLTDoLocalPixelSequencePPOnAA = cms.Sequence( + HLTLocalPixelGPUSequence + # process.hltSiPixelDigis # replaced by an alias + # process.hltSiPixelClustersPPOnAA # replaced by an alias + + process.hltSiPixelClustersCachePPOnAA # not used here, kept for compatibility with legacy sequences + + process.hltSiPixelRecHitsPPOnAA) # convert to legacy + + process.HLTDoLocalPixelSequencePPOnAAForLowPt = cms.Sequence( + HLTLocalPixelGPUSequence + # process.hltSiPixelDigis # replaced by an alias + # process.hltSiPixelClustersPPOnAAForLowPt # replaced by an alias + + process.hltSiPixelClustersCachePPOnAAForLowPt # not used here, kept for compatibility with legacy sequences + + process.hltSiPixelRecHitsPPOnAAForLowPt) + + process.HLTRecoPixelTracksSequencePPOnAA = cms.Sequence( + process.hltPixelTracksFitter # not used here, kept for compatibility with legacy sequences + + process.hltPixelTracksFilter # not used here, kept for compatibility with legacy sequences + + process.hltPixelTracksTrackingRegionsPPOnAA # from the original sequence + + process.hltPixelTracksHitQuadrupletsPPOnAA # pixel ntuplets on gpu, in SoA format + + process.hltPixelTracksSoAPPOnAA # pixel ntuplets on cpu, in SoA format + + process.hltPixelTracksPPOnAA) # pixel tracks on gpu, with transfer and conversion to legacy + process.HLTRecoPixelTracksPPOnAASequence = cms.Sequence(process.HLTRecoPixelTracksSequencePPOnAA) + + process.HLTRecopixelvertexingSequencePPOnAA = cms.Sequence( + process.HLTRecoPixelTracksSequencePPOnAA + + process.hltPixelVerticesCUDAPPOnAA # pixel vertices on gpu, in SoA format + + process.hltPixelVerticesSoAPPOnAA # pixel vertices on cpu, in SoA format + + process.hltPixelVerticesPPOnAA # pixel vertices on cpu, in legacy format + + process.hltTrimmedPixelVerticesPPOnAA) # from the original sequence + process.HLTPixelVertexingPPOnAASequence = cms.Sequence(process.HLTRecopixelvertexingSequencePPOnAA) + process.HLTPixelVertexingSequencePPOnAA = cms.Sequence(process.HLTRecopixelvertexingSequencePPOnAA) + + + # done + + return process + + +# customisation for offloading the ECAL local reconstruction to GPUs +def customise_gpu_ecal(process): + + # FIXME replace the Sequences with empty ones to avoid exanding them during the (re)definition of Modules and EDAliases + + process.HLTDoFullUnpackingEgammaEcalMFSequence = cms.Sequence() + process.HLTDoFullUnpackingEgammaEcalWithoutPreshowerSequence = cms.Sequence() + process.HLTDoFullUnpackingEgammaEcalSequence = cms.Sequence() + + + # Event Setup + + process.load("EventFilter.EcalRawToDigi.ecalElectronicsMappingGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalGainRatiosGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalPedestalsGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalPulseCovariancesGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalPulseShapesGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalSamplesCorrelationGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalTimeBiasCorrectionsGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalTimeCalibConstantsGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalMultifitParametersGPUESProducer_cfi") + + process.load("RecoLocalCalo.EcalRecProducers.ecalRechitADCToGeVConstantGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalRechitChannelStatusGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalIntercalibConstantsGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalLaserAPDPNRatiosGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalLaserAPDPNRatiosRefGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalLaserAlphasGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalLinearCorrectionsGPUESProducer_cfi") + process.load("RecoLocalCalo.EcalRecProducers.ecalRecHitParametersGPUESProducer_cfi") + + + # Modules and EDAliases + + process.hltEcalDigisGPU = cms.EDProducer("EcalRawToDigiGPU", + InputLabel = cms.InputTag("rawDataCollector"), + FEDs = cms.vint32( + 601, 602, 603, 604, 605, + 606, 607, 608, 609, 610, + 611, 612, 613, 614, 615, + 616, 617, 618, 619, 620, + 621, 622, 623, 624, 625, + 626, 627, 628, 629, 630, + 631, 632, 633, 634, 635, + 636, 637, 638, 639, 640, + 641, 642, 643, 644, 645, + 646, 647, 648, 649, 650, + 651, 652, 653, 654 + ), + digisLabelEB = cms.string("ebDigis"), + digisLabelEE = cms.string("eeDigis"), + maxChannelsEB = cms.uint32(61200), + maxChannelsEE = cms.uint32(14648), + ) + + process.hltEcalDigis = cms.EDProducer("EcalCPUDigisProducer", + digisInLabelEB = cms.InputTag("hltEcalDigisGPU", "ebDigis"), + digisInLabelEE = cms.InputTag("hltEcalDigisGPU", "eeDigis"), + digisOutLabelEB = cms.string("ebDigis"), + digisOutLabelEE = cms.string("eeDigis"), + produceDummyIntegrityCollections = cms.bool(True) + ) + + from RecoLocalCalo.EcalRecProducers.ecalUncalibRecHitProducerGPU_cfi import ecalUncalibRecHitProducerGPU as _ecalUncalibRecHitProducerGPU + process.hltEcalUncalibRecHitGPU = _ecalUncalibRecHitProducerGPU.clone( + digisLabelEB = cms.InputTag("hltEcalDigisGPU", "ebDigis"), + digisLabelEE = cms.InputTag("hltEcalDigisGPU", "eeDigis") + ) + + process.hltEcalUncalibRecHitSoA = cms.EDProducer("EcalCPUUncalibRecHitProducer", + containsTimingInformation = cms.bool(False), + recHitsInLabelEB = cms.InputTag("hltEcalUncalibRecHitGPU", "EcalUncalibRecHitsEB"), + recHitsInLabelEE = cms.InputTag("hltEcalUncalibRecHitGPU", "EcalUncalibRecHitsEE"), + recHitsOutLabelEB = cms.string("EcalUncalibRecHitsEB"), + recHitsOutLabelEE = cms.string("EcalUncalibRecHitsEE") + ) + + process.hltEcalUncalibRecHit = cms.EDProducer("EcalUncalibRecHitConvertGPU2CPUFormat", + recHitsLabelGPUEB = cms.InputTag("hltEcalUncalibRecHitSoA", "EcalUncalibRecHitsEB"), + recHitsLabelGPUEE = cms.InputTag("hltEcalUncalibRecHitSoA", "EcalUncalibRecHitsEE"), + recHitsLabelCPUEB = cms.string("EcalUncalibRecHitsEB"), + recHitsLabelCPUEE = cms.string("EcalUncalibRecHitsEE") + ) + + # Reconstructing the ECAL calibrated rechits on GPU works, but is extremely slow. + # Disable it for the time being, until the performance has been addressed. + """ + process.hltEcalRecHitGPU = cms.EDProducer("EcalRecHitProducerGPU", + uncalibrecHitsInLabelEB = cms.InputTag("hltEcalUncalibRecHitGPU","EcalUncalibRecHitsEB"), + uncalibrecHitsInLabelEE = cms.InputTag("hltEcalUncalibRecHitGPU","EcalUncalibRecHitsEE"), + recHitsLabelEB = cms.string("EcalRecHitsEB"), + recHitsLabelEE = cms.string("EcalRecHitsEE"), + maxNumberHitsEB = cms.uint32(61200), + maxNumberHitsEE = cms.uint32(14648), + ChannelStatusToBeExcluded = cms.vstring( + "kDAC", + "kNoisy", + "kNNoisy", + "kFixedG6", + "kFixedG1", + "kFixedG0", + "kNonRespondingIsolated", + "kDeadVFE", + "kDeadFE", + "kNoDataNoTP"), + killDeadChannels = cms.bool(True), + EBLaserMIN = cms.double(0.01), + EELaserMIN = cms.double(0.01), + EBLaserMAX = cms.double(30.0), + EELaserMAX = cms.double(30.0), + flagsMapDBReco = cms.PSet( + kGood = cms.vstring("kOk","kDAC","kNoLaser","kNoisy"), + kNoisy = cms.vstring("kNNoisy","kFixedG6","kFixedG1"), + kNeighboursRecovered = cms.vstring("kFixedG0", "kNonRespondingIsolated", "kDeadVFE"), + kTowerRecovered = cms.vstring("kDeadFE"), + kDead = cms.vstring("kNoDataNoTP") + ), + recoverEBIsolatedChannels = cms.bool(False), + recoverEEIsolatedChannels = cms.bool(False), + recoverEBVFE = cms.bool(False), + recoverEEVFE = cms.bool(False), + recoverEBFE = cms.bool(True), + recoverEEFE = cms.bool(True), + ) + + process.hltEcalRecHitSoA = cms.EDProducer("EcalCPURecHitProducer", + recHitsInLabelEB = cms.InputTag("hltEcalRecHitGPU", "EcalRecHitsEB"), + recHitsInLabelEE = cms.InputTag("hltEcalRecHitGPU", "EcalRecHitsEE"), + recHitsOutLabelEB = cms.string("EcalRecHitsEB"), + recHitsOutLabelEE = cms.string("EcalRecHitsEE"), + containsTimingInformation = cms.bool(False), + ) + + process.hltEcalRecHit = cms.EDProducer("EcalRecHitConvertGPU2CPUFormat", + recHitsLabelGPUEB = cms.InputTag("hltEcalRecHitSoA", "EcalRecHitsEB"), + recHitsLabelGPUEE = cms.InputTag("hltEcalRecHitSoA", "EcalRecHitsEE"), + recHitsLabelCPUEB = cms.string("EcalRecHitsEB"), + recHitsLabelCPUEE = cms.string("EcalRecHitsEE"), + ) + """ + + + # Sequences + + process.HLTDoFullUnpackingEgammaEcalWithoutPreshowerSequence = cms.Sequence( + process.hltEcalDigisGPU # unpack ECAL digis on gpu + + process.hltEcalDigis # copy to host and convert to legacy format + + process.hltEcalUncalibRecHitGPU # run ECAL local reconstruction and multifit on gpu + + process.hltEcalUncalibRecHitSoA # needed by hltEcalPhiSymFilter - copy to host + + process.hltEcalUncalibRecHit # needed by hltEcalPhiSymFilter - convert to legacy format + # process.hltEcalRecHitGPU # make ECAL calibrated rechits on gpu + # process.hltEcalRecHitSoA # copy to host + + process.hltEcalDetIdToBeRecovered # legacy producer + + process.hltEcalRecHit) # legacy producer + + process.HLTDoFullUnpackingEgammaEcalSequence = cms.Sequence( + process.HLTDoFullUnpackingEgammaEcalWithoutPreshowerSequence + + process.hltEcalPreshowerDigis # unpack ECAL preshower digis on the host + + process.hltEcalPreshowerRecHit) # build ECAL preshower rechits on the host + + process.HLTDoFullUnpackingEgammaEcalMFSequence = copy.copy(process.HLTDoFullUnpackingEgammaEcalSequence) + + + # done + return process + +# customisation for offloading the HCAL local reconstruction to GPUs +def customise_gpu_hcal(process): + + # FIXME replace the Sequences with empty ones to avoid exanding them during the (re)definition of Modules and EDAliases + + process.HLTDoLocalHcalSequence = cms.Sequence() + process.HLTStoppedHSCPLocalHcalReco = cms.Sequence() + + + # Event Setup + + process.load("EventFilter.HcalRawToDigi.hcalElectronicsMappingGPUESProducer_cfi") + + process.load("RecoLocalCalo.HcalRecProducers.hcalGainsGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalGainWidthsGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalLUTCorrsGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalConvertedPedestalsGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalConvertedEffectivePedestalsGPUESProducer_cfi") + process.hcalConvertedEffectivePedestalsGPUESProducer.label0 = "withTopoEff" + process.load("RecoLocalCalo.HcalRecProducers.hcalConvertedPedestalWidthsGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalConvertedEffectivePedestalWidthsGPUESProducer_cfi") + process.hcalConvertedEffectivePedestalWidthsGPUESProducer.label0 = "withTopoEff" + process.hcalConvertedEffectivePedestalWidthsGPUESProducer.label1 = "withTopoEff" + process.load("RecoLocalCalo.HcalRecProducers.hcalQIECodersGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalRecoParamsWithPulseShapesGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalRespCorrsGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalTimeCorrsGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalQIETypesGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalSiPMParametersGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalSiPMCharacteristicsGPUESProducer_cfi") + process.load("RecoLocalCalo.HcalRecProducers.hcalMahiPulseOffsetsGPUESProducer_cfi") + + + # Modules and EDAliases + + # The HCAL unpacker running on the gpu supports only the HB and HE digis. + # So, run the legacy unacker on the cpu, then convert the HB and HE digis + # to SoA format and copy them to the gpu. + process.hltHcalDigisGPU = cms.EDProducer("HcalDigisProducerGPU", + hbheDigisLabel = cms.InputTag("hltHcalDigis"), + qie11DigiLabel = cms.InputTag("hltHcalDigis"), + digisLabelF01HE = cms.string(""), + digisLabelF5HB = cms.string(""), + digisLabelF3HB = cms.string(""), + maxChannelsF01HE = cms.uint32(10000), + maxChannelsF5HB = cms.uint32(10000), + maxChannelsF3HB = cms.uint32(10000), + #nsamplesF01HE = cms.uint32(8), + #nsamplesF5HB = cms.uint32(8), + #nsamplesF3HB = cms.uint32(8) + ) + + from RecoLocalCalo.HcalRecProducers.hbheRecHitProducerGPU_cfi import hbheRecHitProducerGPU as _hbheRecHitProducerGPU + process.hltHbherecoGPU = _hbheRecHitProducerGPU.clone( + digisLabelF01HE = "hltHcalDigisGPU", + digisLabelF5HB = "hltHcalDigisGPU", + digisLabelF3HB = "hltHcalDigisGPU", + recHitsLabelM0HBHE = "" + ) + + from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hcalCPURecHitsProducer + process.hltHbhereco = _hcalCPURecHitsProducer.clone( + recHitsM0LabelIn = "hltHbherecoGPU", + recHitsM0LabelOut = "", + recHitsLegacyLabelOut = "" + ) + + + # Sequences + + process.HLTDoLocalHcalSequence = cms.Sequence( + process.hltHcalDigis # legacy producer, unpack HCAL digis on cpu + + process.hltHcalDigisGPU # copy to gpu and convert to SoA format + + process.hltHbherecoGPU # run HCAL local reconstruction (Method 0 and MAHI) on gpu + + process.hltHbhereco # copy to host and convert to legacy format + + process.hltHfprereco # legacy producer + + process.hltHfreco # legacy producer + + process.hltHoreco) # legacy producer + + process.HLTStoppedHSCPLocalHcalReco = cms.Sequence( + process.hltHcalDigis # legacy producer, unpack HCAL digis on cpu + + process.hltHcalDigisGPU # copy to gpu and convert to SoA format + + process.hltHbherecoGPU # run HCAL local reconstruction (Method 0 and MAHI) on gpu + + process.hltHbhereco) # copy to host and convert to legacy format + + + # done + return process + + +# customisation for running on CPUs +def customise_for_Patatrack_on_cpu(process): + process = customise_cpu_common(process) + process = customise_cpu_pixel(process) + return process + + +# customisation for offloading to GPUs +def customise_for_Patatrack_on_gpu(process): + process = customise_gpu_common(process) + process = customise_gpu_pixel(process) + process = customise_gpu_ecal(process) + process = customise_gpu_hcal(process) + return process diff --git a/RecoLocalTracker/SiPixelClusterizer/plugins/SiPixelRawToClusterGPUKernel.h b/RecoLocalTracker/SiPixelClusterizer/plugins/SiPixelRawToClusterGPUKernel.h index d214e7784af48..2c4b61d66f047 100644 --- a/RecoLocalTracker/SiPixelClusterizer/plugins/SiPixelRawToClusterGPUKernel.h +++ b/RecoLocalTracker/SiPixelClusterizer/plugins/SiPixelRawToClusterGPUKernel.h @@ -51,7 +51,11 @@ namespace pixelgpudetails { const uint32_t numRowsInRoc = 80; const uint32_t numColsInRoc = 52; +#ifdef GPU_HION_EVENTS + const uint32_t MAX_WORD = 8000; +#else const uint32_t MAX_WORD = 2000; +#endif const uint32_t ADC_shift = 0; const uint32_t PXID_shift = ADC_shift + ADC_bits; diff --git a/RecoPixelVertexing/PixelTriplets/plugins/CAConstants.h b/RecoPixelVertexing/PixelTriplets/plugins/CAConstants.h index 0ebbdf3ed3705..54a49dd511c56 100644 --- a/RecoPixelVertexing/PixelTriplets/plugins/CAConstants.h +++ b/RecoPixelVertexing/PixelTriplets/plugins/CAConstants.h @@ -18,6 +18,8 @@ namespace CAConstants { #ifndef ONLY_PHICUT #ifdef GPU_SMALL_EVENTS constexpr uint32_t maxNumberOfTuples() { return 3 * 1024; } +#elif defined(GPU_HION_EVENTS) + constexpr uint32_t maxNumberOfTuples() { return 32 * 1024; } #else constexpr uint32_t maxNumberOfTuples() { return 24 * 1024; } #endif @@ -28,7 +30,11 @@ namespace CAConstants { #ifndef ONLY_PHICUT #ifndef GPU_SMALL_EVENTS constexpr uint32_t maxNumberOfDoublets() { return 512 * 1024; } +#ifndef GPU_HION_EVENTS constexpr uint32_t maxCellsPerHit() { return 128; } +#else + constexpr uint32_t maxCellsPerHit() { return 2 * 128; } +#endif #else constexpr uint32_t maxNumberOfDoublets() { return 128 * 1024; } constexpr uint32_t maxCellsPerHit() { return 128 / 2; } @@ -44,7 +50,7 @@ namespace CAConstants { constexpr uint32_t maxTuples() { return maxNumberOfTuples(); } // types - using hindex_type = uint16_t; // FIXME from siPixelRecHitsHeterogeneousProduct + using hindex_type = uint32_t; // FIXME from siPixelRecHitsHeterogeneousProduct using tindex_type = uint16_t; // for tuples #ifndef ONLY_PHICUT