From 699cf908ddc1a2677048514bea075c506dc22bcf Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 22 Feb 2023 10:13:37 +0100 Subject: [PATCH 01/13] Add simple RecHitSoA --- DataFormats/ParticleFlowReco/BuildFile.xml | 6 +++++ .../interface/RecHitHostCollection.h | 13 ++++++++++ .../ParticleFlowReco/interface/RecHitSoA.h | 25 +++++++++++++++++++ .../interface/alpaka/RecHitDeviceCollection.h | 22 ++++++++++++++++ .../src/alpaka/classes_cuda.h | 5 ++++ .../src/alpaka/classes_cuda_def.xml | 5 ++++ .../src/alpaka/classes_serial.h | 4 +++ .../src/alpaka/classes_serial_def.xml | 15 +++++++++++ DataFormats/ParticleFlowReco/src/classes_2.h | 1 + .../ParticleFlowReco/src/classes_def_2.xml | 2 ++ 10 files changed, 98 insertions(+) create mode 100644 DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h create mode 100644 DataFormats/ParticleFlowReco/interface/RecHitSoA.h create mode 100644 DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h create mode 100644 DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h create mode 100644 DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml create mode 100644 DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h create mode 100644 DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml diff --git a/DataFormats/ParticleFlowReco/BuildFile.xml b/DataFormats/ParticleFlowReco/BuildFile.xml index 70e9023625204..ed0c1dbc48d88 100644 --- a/DataFormats/ParticleFlowReco/BuildFile.xml +++ b/DataFormats/ParticleFlowReco/BuildFile.xml @@ -17,8 +17,14 @@ + + + + + + diff --git a/DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h b/DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h new file mode 100644 index 0000000000000..998777e1208a1 --- /dev/null +++ b/DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h @@ -0,0 +1,13 @@ +#ifndef ParticleFlowReco_RecHitHostCollection_h +#define ParticleFlowReco_RecHitHostCollection_h + +#include "DataFormats/Portable/interface/PortableHostCollection.h" +#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" + +namespace portableRecHitSoA { + + using RecHitHostCollection = PortableHostCollection; + +} // namespace portableRecHitSoA + +#endif \ No newline at end of file diff --git a/DataFormats/ParticleFlowReco/interface/RecHitSoA.h b/DataFormats/ParticleFlowReco/interface/RecHitSoA.h new file mode 100644 index 0000000000000..ecb562dbd2c5a --- /dev/null +++ b/DataFormats/ParticleFlowReco/interface/RecHitSoA.h @@ -0,0 +1,25 @@ +#ifndef ParticleFlowReco_RecHitSoA_h +#define ParticleFlowReco_RecHitSoA_h + +#include +#include "DataFormats/SoATemplate/interface/SoACommon.h" +#include "DataFormats/SoATemplate/interface/SoALayout.h" +#include "DataFormats/SoATemplate/interface/SoAView.h" + +namespace portableRecHitSoA { + + GENERATE_SOA_LAYOUT(RecHitSoALayout, + // columns: one value per element + SOA_COLUMN(double, x), + SOA_COLUMN(double, y), + SOA_COLUMN(double, z), + SOA_COLUMN(int32_t, id), + // scalars: one value for the whole structure + SOA_SCALAR(double, r) + ) + + using RecHitSoA = RecHitSoALayout<>; + +} // namespace portableRecHitSoA + +#endif diff --git a/DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h b/DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h new file mode 100644 index 0000000000000..fd272757fbb51 --- /dev/null +++ b/DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h @@ -0,0 +1,22 @@ +#ifndef ParticleFlowReco_RecHitDeviceCollection_h +#define ParticleFlowReco_RecHitDeviceCollection_h + +#include "DataFormats/Portable/interface/alpaka/PortableCollection.h" +#include "HeterogeneousCore/AlpakaInterface/interface/config.h" +#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" + +namespace ALPAKA_ACCELERATOR_NAMESPACE { + + namespace portableRecHitSoA { + + // make the names from the top-level portableRecHitSoA namespace visible for unqualified lookup + // inside the ALPAKA_ACCELERATOR_NAMESPACE::portableRecHitSoA namespace + using namespace ::portableRecHitSoA; + + using RecHitDeviceCollection = PortableCollection; + + } // namespace portableRecHitSoA + +} // namespace ALPAKA_ACCELERATOR_NAMESPACE + +#endif \ No newline at end of file diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h new file mode 100644 index 0000000000000..001916923f2cd --- /dev/null +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h @@ -0,0 +1,5 @@ +#include "DataFormats/Common/interface/DeviceProduct.h" +#include "DataFormats/Common/interface/Wrapper.h" +#include "DataFormats/Portable/interface/Product.h" +#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h" diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml new file mode 100644 index 0000000000000..fa0803a4188dd --- /dev/null +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h new file mode 100644 index 0000000000000..9cf9d13d1fa23 --- /dev/null +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h @@ -0,0 +1,4 @@ +#include "DataFormats/Common/interface/Wrapper.h" +#include "DataFormats/Portable/interface/Product.h" +#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" +#include "DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h" diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml new file mode 100644 index 0000000000000..4300ebb43737d --- /dev/null +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml @@ -0,0 +1,15 @@ + + + + + + + diff --git a/DataFormats/ParticleFlowReco/src/classes_2.h b/DataFormats/ParticleFlowReco/src/classes_2.h index 460acc941900c..bd8a8be7ab3d5 100644 --- a/DataFormats/ParticleFlowReco/src/classes_2.h +++ b/DataFormats/ParticleFlowReco/src/classes_2.h @@ -4,6 +4,7 @@ #include #include "DataFormats/Common/interface/ValueMap.h" +#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" #include "DataFormats/ParticleFlowReco/interface/PFCluster.h" #include "Math/Cartesian3D.h" #include "DataFormats/ParticleFlowReco/interface/PFRecHit.h" diff --git a/DataFormats/ParticleFlowReco/src/classes_def_2.xml b/DataFormats/ParticleFlowReco/src/classes_def_2.xml index 6c40541360768..6d390578ea7ab 100644 --- a/DataFormats/ParticleFlowReco/src/classes_def_2.xml +++ b/DataFormats/ParticleFlowReco/src/classes_def_2.xml @@ -1,5 +1,7 @@ + + From 26d16ffdf8413c7b6b8e40373f64a4baeea3a4f4 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 22 Feb 2023 11:50:28 +0100 Subject: [PATCH 02/13] Start adding RecHitSoAProducer --- .../ParticleFlowReco/interface/RecHitSoA.h | 10 +--- .../PFRecHitProducer/BuildFile.xml | 7 +++ .../PFRecHitProducer/plugins/BuildFile.xml | 31 ++++++++++ .../plugins/alpaka/RecHitSoAProducer.cc | 58 +++++++++++++++++++ 4 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 RecoParticleFlow/PFRecHitProducer/BuildFile.xml create mode 100644 RecoParticleFlow/PFRecHitProducer/plugins/BuildFile.xml create mode 100644 RecoParticleFlow/PFRecHitProducer/plugins/alpaka/RecHitSoAProducer.cc diff --git a/DataFormats/ParticleFlowReco/interface/RecHitSoA.h b/DataFormats/ParticleFlowReco/interface/RecHitSoA.h index ecb562dbd2c5a..f26cefe826eb3 100644 --- a/DataFormats/ParticleFlowReco/interface/RecHitSoA.h +++ b/DataFormats/ParticleFlowReco/interface/RecHitSoA.h @@ -9,13 +9,9 @@ namespace portableRecHitSoA { GENERATE_SOA_LAYOUT(RecHitSoALayout, - // columns: one value per element - SOA_COLUMN(double, x), - SOA_COLUMN(double, y), - SOA_COLUMN(double, z), - SOA_COLUMN(int32_t, id), - // scalars: one value for the whole structure - SOA_SCALAR(double, r) + SOA_COLUMN(uint32_t, detId), + SOA_COLUMN(float, energy), + SOA_COLUMN(float, time) ) using RecHitSoA = RecHitSoALayout<>; diff --git a/RecoParticleFlow/PFRecHitProducer/BuildFile.xml b/RecoParticleFlow/PFRecHitProducer/BuildFile.xml new file mode 100644 index 0000000000000..eee6052d26e2d --- /dev/null +++ b/RecoParticleFlow/PFRecHitProducer/BuildFile.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/BuildFile.xml b/RecoParticleFlow/PFRecHitProducer/plugins/BuildFile.xml new file mode 100644 index 0000000000000..a6ae3b3ee8f27 --- /dev/null +++ b/RecoParticleFlow/PFRecHitProducer/plugins/BuildFile.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/RecHitSoAProducer.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/RecHitSoAProducer.cc new file mode 100644 index 0000000000000..4decff5b8f0d0 --- /dev/null +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/RecHitSoAProducer.cc @@ -0,0 +1,58 @@ +#include "HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h" +#include "FWCore/Utilities/interface/StreamID.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" +#include "DataFormats/Common/interface/SortedCollection.h" +#include "DataFormats/HcalRecHit/interface/HBHERecHit.h" +#include "DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h" + +namespace ALPAKA_ACCELERATOR_NAMESPACE { + class RecHitSoAProducer : public global::EDProducer<> { + public: + RecHitSoAProducer(edm::ParameterSet const& config) : + recHitsToken(consumes(config.getParameter("src"))), + deviceToken(produces()) + {} + + void produce(edm::StreamID sid, device::Event& event, device::EventSetup const&) const override { + const edm::SortedCollection& recHits = event.get(recHitsToken); + const int32_t num_recHits = recHits.size(); + printf("Found %d recHits\n", num_recHits); + + portableRecHitSoA::RecHitHostCollection hostProduct{num_recHits, event.queue()}; + auto& view = hostProduct.view(); + + for(int i = 0; i < num_recHits; i++) + { + const HBHERecHit& rh = recHits[i]; + if(i < 10) + printf("recHit %4d %u %f %f\n", i, rh.id().rawId(), rh.energy(), rh.time()); + view[i].detId() = rh.id().rawId(); + view[i].energy() = rh.energy(); + view[i].time() = rh.time(); + } + + portableRecHitSoA::RecHitDeviceCollection deviceProduct{num_recHits, event.queue()}; + alpaka::memcpy(event.queue(), deviceProduct.buffer(), hostProduct.buffer()); + event.emplace(deviceToken, std::move(deviceProduct)); + } + + static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + desc.add("src"); + descriptions.addWithDefaultLabel(desc); + } + + private: + const edm::EDGetTokenT> recHitsToken; + const device::EDPutToken deviceToken; + }; + +} // namespace ALPAKA_ACCELERATOR_NAMESPACE + +#include "HeterogeneousCore/AlpakaCore/interface/MakerMacros.h" +DEFINE_FWK_ALPAKA_MODULE(RecHitSoAProducer); \ No newline at end of file From 0b274f4ef64cd748070fabd9cb50895b0a22496f Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 22 Feb 2023 11:55:50 +0100 Subject: [PATCH 03/13] Merge DataFormats/ParticleFlowReco classes 1 and 2 --- DataFormats/ParticleFlowReco/BuildFile.xml | 2 - .../src/{classes_2.h => classes.h} | 56 ++++++++++++++++ DataFormats/ParticleFlowReco/src/classes_1.h | 54 ---------------- .../{classes_def_2.xml => classes_def.xml} | 58 +++++++++++++++++ .../ParticleFlowReco/src/classes_def_1.xml | 64 ------------------- 5 files changed, 114 insertions(+), 120 deletions(-) rename DataFormats/ParticleFlowReco/src/{classes_2.h => classes.h} (51%) delete mode 100644 DataFormats/ParticleFlowReco/src/classes_1.h rename DataFormats/ParticleFlowReco/src/{classes_def_2.xml => classes_def.xml} (77%) delete mode 100644 DataFormats/ParticleFlowReco/src/classes_def_1.xml diff --git a/DataFormats/ParticleFlowReco/BuildFile.xml b/DataFormats/ParticleFlowReco/BuildFile.xml index ed0c1dbc48d88..faf3c4f3af35a 100644 --- a/DataFormats/ParticleFlowReco/BuildFile.xml +++ b/DataFormats/ParticleFlowReco/BuildFile.xml @@ -20,8 +20,6 @@ - - diff --git a/DataFormats/ParticleFlowReco/src/classes_2.h b/DataFormats/ParticleFlowReco/src/classes.h similarity index 51% rename from DataFormats/ParticleFlowReco/src/classes_2.h rename to DataFormats/ParticleFlowReco/src/classes.h index bd8a8be7ab3d5..7bcb938ac2d6d 100644 --- a/DataFormats/ParticleFlowReco/src/classes_2.h +++ b/DataFormats/ParticleFlowReco/src/classes.h @@ -4,6 +4,62 @@ #include #include "DataFormats/Common/interface/ValueMap.h" +#include "DataFormats/ParticleFlowReco/interface/PFCluster.h" +#include "Math/Cartesian3D.h" +#include "DataFormats/ParticleFlowReco/interface/PFRecHit.h" +#include "Math/Polar3D.h" +#include "Math/CylindricalEta3D.h" +#include "DataFormats/ParticleFlowReco/interface/PFRecTrack.h" +#include "DataFormats/ParticleFlowReco/interface/GsfPFRecTrack.h" +#include "Math/GenVector/PositionVector3D.h" +#include "DataFormats/Math/interface/Point3D.h" +#include "Rtypes.h" +#include "DataFormats/Math/interface/Vector3D.h" +#include "Math/PxPyPzE4D.h" +#include "DataFormats/DetId/interface/DetId.h" +#include "DataFormats/TrackReco/interface/Track.h" +#include "DataFormats/TrackReco/interface/TrackFwd.h" +#include "DataFormats/GsfTrackReco/interface/GsfTrack.h" +#include "DataFormats/GsfTrackReco/interface/GsfTrackExtra.h" +#include "DataFormats/ParticleFlowReco/interface/PFSimParticle.h" +#include "DataFormats/ParticleFlowReco/interface/PFTrajectoryPoint.h" +#include "DataFormats/ParticleFlowReco/interface/PFBlockElement.h" +#include "DataFormats/ParticleFlowReco/interface/PFBlockElementTrack.h" +#include "DataFormats/ParticleFlowReco/interface/PFBlockElementGsfTrack.h" //Daniele +#include "DataFormats/ParticleFlowReco/interface/PFBlockElementBrem.h" //Daniele +#include "DataFormats/ParticleFlowReco/interface/PFBlockElementSuperCluster.h" //Florian +#include "DataFormats/ParticleFlowReco/interface/PFBlockElementCluster.h" +#include "DataFormats/ParticleFlowReco/interface/PFBlock.h" +#include "DataFormats/ParticleFlowReco/interface/PFDisplacedTrackerVertex.h" +#include "DataFormats/ParticleFlowReco/interface/PFNuclearInteraction.h" +#include "DataFormats/ParticleFlowReco/interface/PFConversion.h" +#include "DataFormats/ParticleFlowReco/interface/PFConversionFwd.h" +#include "DataFormats/ParticleFlowReco/interface/PFV0.h" +#include "DataFormats/ParticleFlowReco/interface/ConvBremSeed.h" +#include "DataFormats/ParticleFlowReco/interface/ConvBremSeedFwd.h" +#include "DataFormats/ParticleFlowReco/interface/ParticleFiltrationDecision.h" + +#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexCandidate.h" +#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexSeed.h" +#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertex.h" +#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexCandidateFwd.h" +#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexSeedFwd.h" +#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexFwd.h" + +#include "DataFormats/ParticleFlowReco/interface/PreId.h" +#include "DataFormats/ParticleFlowReco/interface/PreIdFwd.h" +#include "DataFormats/ParticleFlowReco/interface/RecoPFClusterRefCandidate.h" +#include "DataFormats/ParticleFlowReco/interface/RecoPFClusterRefCandidateFwd.h" + +#include + + +#include "DataFormats/Common/interface/Wrapper.h" +#include "DataFormats/Common/interface/Ref.h" +#include +#include +#include "DataFormats/Common/interface/ValueMap.h" + #include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" #include "DataFormats/ParticleFlowReco/interface/PFCluster.h" #include "Math/Cartesian3D.h" diff --git a/DataFormats/ParticleFlowReco/src/classes_1.h b/DataFormats/ParticleFlowReco/src/classes_1.h deleted file mode 100644 index 874750a6bf83f..0000000000000 --- a/DataFormats/ParticleFlowReco/src/classes_1.h +++ /dev/null @@ -1,54 +0,0 @@ -#include "DataFormats/Common/interface/Wrapper.h" -#include "DataFormats/Common/interface/Ref.h" -#include -#include -#include "DataFormats/Common/interface/ValueMap.h" - -#include "DataFormats/ParticleFlowReco/interface/PFCluster.h" -#include "Math/Cartesian3D.h" -#include "DataFormats/ParticleFlowReco/interface/PFRecHit.h" -#include "Math/Polar3D.h" -#include "Math/CylindricalEta3D.h" -#include "DataFormats/ParticleFlowReco/interface/PFRecTrack.h" -#include "DataFormats/ParticleFlowReco/interface/GsfPFRecTrack.h" -#include "Math/GenVector/PositionVector3D.h" -#include "DataFormats/Math/interface/Point3D.h" -#include "Rtypes.h" -#include "DataFormats/Math/interface/Vector3D.h" -#include "Math/PxPyPzE4D.h" -#include "DataFormats/DetId/interface/DetId.h" -#include "DataFormats/TrackReco/interface/Track.h" -#include "DataFormats/TrackReco/interface/TrackFwd.h" -#include "DataFormats/GsfTrackReco/interface/GsfTrack.h" -#include "DataFormats/GsfTrackReco/interface/GsfTrackExtra.h" -#include "DataFormats/ParticleFlowReco/interface/PFSimParticle.h" -#include "DataFormats/ParticleFlowReco/interface/PFTrajectoryPoint.h" -#include "DataFormats/ParticleFlowReco/interface/PFBlockElement.h" -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementTrack.h" -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementGsfTrack.h" //Daniele -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementBrem.h" //Daniele -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementSuperCluster.h" //Florian -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementCluster.h" -#include "DataFormats/ParticleFlowReco/interface/PFBlock.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedTrackerVertex.h" -#include "DataFormats/ParticleFlowReco/interface/PFNuclearInteraction.h" -#include "DataFormats/ParticleFlowReco/interface/PFConversion.h" -#include "DataFormats/ParticleFlowReco/interface/PFConversionFwd.h" -#include "DataFormats/ParticleFlowReco/interface/PFV0.h" -#include "DataFormats/ParticleFlowReco/interface/ConvBremSeed.h" -#include "DataFormats/ParticleFlowReco/interface/ConvBremSeedFwd.h" -#include "DataFormats/ParticleFlowReco/interface/ParticleFiltrationDecision.h" - -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexCandidate.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexSeed.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertex.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexCandidateFwd.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexSeedFwd.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexFwd.h" - -#include "DataFormats/ParticleFlowReco/interface/PreId.h" -#include "DataFormats/ParticleFlowReco/interface/PreIdFwd.h" -#include "DataFormats/ParticleFlowReco/interface/RecoPFClusterRefCandidate.h" -#include "DataFormats/ParticleFlowReco/interface/RecoPFClusterRefCandidateFwd.h" - -#include diff --git a/DataFormats/ParticleFlowReco/src/classes_def_2.xml b/DataFormats/ParticleFlowReco/src/classes_def.xml similarity index 77% rename from DataFormats/ParticleFlowReco/src/classes_def_2.xml rename to DataFormats/ParticleFlowReco/src/classes_def.xml index 6d390578ea7ab..4f0f2233d60a2 100644 --- a/DataFormats/ParticleFlowReco/src/classes_def_2.xml +++ b/DataFormats/ParticleFlowReco/src/classes_def.xml @@ -1,5 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/DataFormats/ParticleFlowReco/src/classes_def_1.xml b/DataFormats/ParticleFlowReco/src/classes_def_1.xml deleted file mode 100644 index 58b21e59e3c35..0000000000000 --- a/DataFormats/ParticleFlowReco/src/classes_def_1.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From bef053e8b865e083e16458bf8746603587537fe4 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 22 Feb 2023 12:16:08 +0100 Subject: [PATCH 04/13] Rename RecHitSoA -> CaloRecHitSoA; remove namespace --- .../interface/CaloRecHitHostCollection.h | 9 ++++++++ .../interface/CaloRecHitSoA.h | 17 ++++++++++++++ .../interface/RecHitHostCollection.h | 13 ----------- .../ParticleFlowReco/interface/RecHitSoA.h | 21 ------------------ .../alpaka/CaloRecHitDeviceCollection.h | 12 ++++++++++ .../interface/alpaka/RecHitDeviceCollection.h | 22 ------------------- .../src/alpaka/classes_cuda.h | 4 ++-- .../src/alpaka/classes_cuda_def.xml | 6 ++--- .../src/alpaka/classes_serial.h | 4 ++-- .../src/alpaka/classes_serial_def.xml | 12 +++++----- DataFormats/ParticleFlowReco/src/classes.h | 2 +- .../ParticleFlowReco/src/classes_def.xml | 4 ++-- ...oAProducer.cc => CaloRecHitSoAProducer.cc} | 17 +++++++------- 13 files changed, 63 insertions(+), 80 deletions(-) create mode 100644 DataFormats/ParticleFlowReco/interface/CaloRecHitHostCollection.h create mode 100644 DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h delete mode 100644 DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h delete mode 100644 DataFormats/ParticleFlowReco/interface/RecHitSoA.h create mode 100644 DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h delete mode 100644 DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h rename RecoParticleFlow/PFRecHitProducer/plugins/alpaka/{RecHitSoAProducer.cc => CaloRecHitSoAProducer.cc} (75%) diff --git a/DataFormats/ParticleFlowReco/interface/CaloRecHitHostCollection.h b/DataFormats/ParticleFlowReco/interface/CaloRecHitHostCollection.h new file mode 100644 index 0000000000000..6fe805dc1eaac --- /dev/null +++ b/DataFormats/ParticleFlowReco/interface/CaloRecHitHostCollection.h @@ -0,0 +1,9 @@ +#ifndef ParticleFlowReco_CaloRecHitHostCollection_h +#define ParticleFlowReco_CaloRecHitHostCollection_h + +#include "DataFormats/Portable/interface/PortableHostCollection.h" +#include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" + +using CaloRecHitHostCollection = PortableHostCollection; + +#endif \ No newline at end of file diff --git a/DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h b/DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h new file mode 100644 index 0000000000000..6fb19a9f49599 --- /dev/null +++ b/DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h @@ -0,0 +1,17 @@ +#ifndef ParticleFlowReco_CaloRecHitSoA_h +#define ParticleFlowReco_CaloRecHitSoA_h + +#include +#include "DataFormats/SoATemplate/interface/SoACommon.h" +#include "DataFormats/SoATemplate/interface/SoALayout.h" +#include "DataFormats/SoATemplate/interface/SoAView.h" + +GENERATE_SOA_LAYOUT(CaloRecHitSoALayout, + SOA_COLUMN(uint32_t, detId), + SOA_COLUMN(float, energy), + SOA_COLUMN(float, time) +) + +using CaloRecHitSoA = CaloRecHitSoALayout<>; + +#endif diff --git a/DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h b/DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h deleted file mode 100644 index 998777e1208a1..0000000000000 --- a/DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef ParticleFlowReco_RecHitHostCollection_h -#define ParticleFlowReco_RecHitHostCollection_h - -#include "DataFormats/Portable/interface/PortableHostCollection.h" -#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" - -namespace portableRecHitSoA { - - using RecHitHostCollection = PortableHostCollection; - -} // namespace portableRecHitSoA - -#endif \ No newline at end of file diff --git a/DataFormats/ParticleFlowReco/interface/RecHitSoA.h b/DataFormats/ParticleFlowReco/interface/RecHitSoA.h deleted file mode 100644 index f26cefe826eb3..0000000000000 --- a/DataFormats/ParticleFlowReco/interface/RecHitSoA.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef ParticleFlowReco_RecHitSoA_h -#define ParticleFlowReco_RecHitSoA_h - -#include -#include "DataFormats/SoATemplate/interface/SoACommon.h" -#include "DataFormats/SoATemplate/interface/SoALayout.h" -#include "DataFormats/SoATemplate/interface/SoAView.h" - -namespace portableRecHitSoA { - - GENERATE_SOA_LAYOUT(RecHitSoALayout, - SOA_COLUMN(uint32_t, detId), - SOA_COLUMN(float, energy), - SOA_COLUMN(float, time) - ) - - using RecHitSoA = RecHitSoALayout<>; - -} // namespace portableRecHitSoA - -#endif diff --git a/DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h b/DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h new file mode 100644 index 0000000000000..1f5ffb848c5c6 --- /dev/null +++ b/DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h @@ -0,0 +1,12 @@ +#ifndef ParticleFlowReco_CaloRecHitDeviceCollection_h +#define ParticleFlowReco_CaloRecHitDeviceCollection_h + +#include "DataFormats/Portable/interface/alpaka/PortableCollection.h" +#include "HeterogeneousCore/AlpakaInterface/interface/config.h" +#include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" + +namespace ALPAKA_ACCELERATOR_NAMESPACE { + using CaloRecHitDeviceCollection = PortableCollection; +} // namespace ALPAKA_ACCELERATOR_NAMESPACE + +#endif \ No newline at end of file diff --git a/DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h b/DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h deleted file mode 100644 index fd272757fbb51..0000000000000 --- a/DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef ParticleFlowReco_RecHitDeviceCollection_h -#define ParticleFlowReco_RecHitDeviceCollection_h - -#include "DataFormats/Portable/interface/alpaka/PortableCollection.h" -#include "HeterogeneousCore/AlpakaInterface/interface/config.h" -#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" - -namespace ALPAKA_ACCELERATOR_NAMESPACE { - - namespace portableRecHitSoA { - - // make the names from the top-level portableRecHitSoA namespace visible for unqualified lookup - // inside the ALPAKA_ACCELERATOR_NAMESPACE::portableRecHitSoA namespace - using namespace ::portableRecHitSoA; - - using RecHitDeviceCollection = PortableCollection; - - } // namespace portableRecHitSoA - -} // namespace ALPAKA_ACCELERATOR_NAMESPACE - -#endif \ No newline at end of file diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h index 001916923f2cd..bfa90841e52c4 100644 --- a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h @@ -1,5 +1,5 @@ #include "DataFormats/Common/interface/DeviceProduct.h" #include "DataFormats/Common/interface/Wrapper.h" #include "DataFormats/Portable/interface/Product.h" -#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" -#include "DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h" +#include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h" diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml index fa0803a4188dd..2b62b1198812c 100644 --- a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml @@ -1,5 +1,5 @@ - - - + + + diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h index 9cf9d13d1fa23..bf7faed3d765c 100644 --- a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h @@ -1,4 +1,4 @@ #include "DataFormats/Common/interface/Wrapper.h" #include "DataFormats/Portable/interface/Product.h" -#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" -#include "DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h" +#include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" +#include "DataFormats/ParticleFlowReco/interface/CaloRecHitHostCollection.h" diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml index 4300ebb43737d..48de13d797280 100644 --- a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml @@ -1,15 +1,15 @@ - + - + diff --git a/DataFormats/ParticleFlowReco/src/classes.h b/DataFormats/ParticleFlowReco/src/classes.h index 7bcb938ac2d6d..61af79cdbaaf8 100644 --- a/DataFormats/ParticleFlowReco/src/classes.h +++ b/DataFormats/ParticleFlowReco/src/classes.h @@ -60,7 +60,7 @@ #include #include "DataFormats/Common/interface/ValueMap.h" -#include "DataFormats/ParticleFlowReco/interface/RecHitSoA.h" +#include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" #include "DataFormats/ParticleFlowReco/interface/PFCluster.h" #include "Math/Cartesian3D.h" #include "DataFormats/ParticleFlowReco/interface/PFRecHit.h" diff --git a/DataFormats/ParticleFlowReco/src/classes_def.xml b/DataFormats/ParticleFlowReco/src/classes_def.xml index 4f0f2233d60a2..02ca7fdcdbdc9 100644 --- a/DataFormats/ParticleFlowReco/src/classes_def.xml +++ b/DataFormats/ParticleFlowReco/src/classes_def.xml @@ -58,8 +58,8 @@ - - + + diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/RecHitSoAProducer.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc similarity index 75% rename from RecoParticleFlow/PFRecHitProducer/plugins/alpaka/RecHitSoAProducer.cc rename to RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc index 4decff5b8f0d0..ef4186b0dc769 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/RecHitSoAProducer.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc @@ -1,3 +1,4 @@ +#include "HeterogeneousCore/AlpakaServices/interface/alpaka/AlpakaService.h" #include "HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h" #include "FWCore/Utilities/interface/StreamID.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" @@ -7,13 +8,13 @@ #include "FWCore/Utilities/interface/InputTag.h" #include "DataFormats/Common/interface/SortedCollection.h" #include "DataFormats/HcalRecHit/interface/HBHERecHit.h" -#include "DataFormats/ParticleFlowReco/interface/RecHitHostCollection.h" -#include "DataFormats/ParticleFlowReco/interface/alpaka/RecHitDeviceCollection.h" +#include "DataFormats/ParticleFlowReco/interface/CaloRecHitHostCollection.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h" namespace ALPAKA_ACCELERATOR_NAMESPACE { - class RecHitSoAProducer : public global::EDProducer<> { + class CaloRecHitSoAProducer : public global::EDProducer<> { public: - RecHitSoAProducer(edm::ParameterSet const& config) : + CaloRecHitSoAProducer(edm::ParameterSet const& config) : recHitsToken(consumes(config.getParameter("src"))), deviceToken(produces()) {} @@ -23,7 +24,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { const int32_t num_recHits = recHits.size(); printf("Found %d recHits\n", num_recHits); - portableRecHitSoA::RecHitHostCollection hostProduct{num_recHits, event.queue()}; + CaloRecHitHostCollection hostProduct{num_recHits, event.queue()}; auto& view = hostProduct.view(); for(int i = 0; i < num_recHits; i++) @@ -36,7 +37,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { view[i].time() = rh.time(); } - portableRecHitSoA::RecHitDeviceCollection deviceProduct{num_recHits, event.queue()}; + CaloRecHitDeviceCollection deviceProduct{num_recHits, event.queue()}; alpaka::memcpy(event.queue(), deviceProduct.buffer(), hostProduct.buffer()); event.emplace(deviceToken, std::move(deviceProduct)); } @@ -49,10 +50,10 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { private: const edm::EDGetTokenT> recHitsToken; - const device::EDPutToken deviceToken; + const device::EDPutToken deviceToken; }; } // namespace ALPAKA_ACCELERATOR_NAMESPACE #include "HeterogeneousCore/AlpakaCore/interface/MakerMacros.h" -DEFINE_FWK_ALPAKA_MODULE(RecHitSoAProducer); \ No newline at end of file +DEFINE_FWK_ALPAKA_MODULE(CaloRecHitSoAProducer); \ No newline at end of file From 39ac9ac8c33ebc2b61a43afd68e7a57c39bc09a8 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 22 Feb 2023 12:49:13 +0100 Subject: [PATCH 05/13] Add configuration to test CaloRecHitSoAProducer --- .../test/pfRecHitProducer_CPU_vs_Alpaka.py | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py diff --git a/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py new file mode 100644 index 0000000000000..3b04abd4745dc --- /dev/null +++ b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py @@ -0,0 +1,223 @@ +# Auto generated configuration file +# using: +# Revision: 1.19 +# Source: /local/reps/CMSSW/CMSSW/Configuration/Applications/python/ConfigBuilder.py,v +# with command line options: reHLT --processName reHLT -s HLT:@relval2021 --conditions auto:phase1_2021_realistic --datatier GEN-SIM-DIGI-RAW -n 5 --eventcontent FEVTDEBUGHLT --geometry DB:Extended --era Run3 --customise=HLTrigger/Configuration/customizeHLTforPatatrack.customizeHLTforPatatrack --filein /store/relval/CMSSW_12_3_0_pre5/RelValTTbar_14TeV/GEN-SIM-DIGI-RAW/123X_mcRun3_2021_realistic_v6-v1/10000/2639d8f2-aaa6-4a78-b7c2-9100a6717e6c.root +import FWCore.ParameterSet.Config as cms + +from Configuration.Eras.Era_Run3_cff import Run3 + +process = cms.Process('rereHLT',Run3) + +# import of standard configurations +process.load('Configuration.StandardSequences.Services_cff') +process.load('SimGeneral.HepPDTESSource.pythiapdt_cfi') +process.load('FWCore.MessageService.MessageLogger_cfi') +process.load('Configuration.EventContent.EventContent_cff') +process.load('SimGeneral.MixingModule.mixNoPU_cfi') +process.load('Configuration.StandardSequences.GeometryRecoDB_cff') +process.load('Configuration.StandardSequences.MagneticField_cff') +process.load('HLTrigger.Configuration.HLT_GRun_cff') +process.load('Configuration.StandardSequences.EndOfProcess_cff') +process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff') + +process.load("RecoParticleFlow.PFClusterProducer.pfhbheRecHitParamsGPUESProducer_cfi") +process.load("RecoParticleFlow.PFClusterProducer.pfhbheTopologyGPUESProducer_cfi") + +process.load('Configuration.StandardSequences.Accelerators_cff') +process.load('HeterogeneousCore.AlpakaCore.ProcessAcceleratorAlpaka_cfi') + +process.maxEvents = cms.untracked.PSet( + #input = cms.untracked.int32(5), + input = cms.untracked.int32(100), + #input = cms.untracked.int32(1000), + output = cms.optional.untracked.allowed(cms.int32,cms.PSet) +) + +# Input source +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring('/store/relval/CMSSW_12_4_0/RelValTTbar_14TeV/GEN-SIM-DIGI-RAW/PU_124X_mcRun3_2022_realistic_v5-v1/10000/012eda92-aad5-4a95-8dbd-c79546b5f508.root'), + secondaryFileNames = cms.untracked.vstring() +) + +process.options = cms.untracked.PSet( + FailPath = cms.untracked.vstring(), + IgnoreCompletely = cms.untracked.vstring(), + Rethrow = cms.untracked.vstring(), + SkipEvent = cms.untracked.vstring(), + allowUnscheduled = cms.obsolete.untracked.bool, + canDeleteEarly = cms.untracked.vstring(), + deleteNonConsumedUnscheduledModules = cms.untracked.bool(True), + dumpOptions = cms.untracked.bool(False), + emptyRunLumiMode = cms.obsolete.untracked.string, + eventSetup = cms.untracked.PSet( + forceNumberOfConcurrentIOVs = cms.untracked.PSet( + allowAnyLabel_=cms.required.untracked.uint32 + ), + numberOfConcurrentIOVs = cms.untracked.uint32(0) + ), + fileMode = cms.untracked.string('FULLMERGE'), + forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False), + makeTriggerResults = cms.obsolete.untracked.bool, + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(0), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfStreams = cms.untracked.uint32(0), + numberOfThreads = cms.untracked.uint32(1), + printDependencies = cms.untracked.bool(False), + sizeOfStackForThreadsInKB = cms.optional.untracked.uint32, + throwIfIllegalParameter = cms.untracked.bool(True), + wantSummary = cms.untracked.bool(False) +) + +# Production Info +process.configurationMetadata = cms.untracked.PSet( + annotation = cms.untracked.string('reHLT nevts:5'), + name = cms.untracked.string('Applications'), + version = cms.untracked.string('$Revision: 1.19 $') +) + +# Output definition + +process.FEVTDEBUGHLToutput = cms.OutputModule("PoolOutputModule", + dataset = cms.untracked.PSet( + dataTier = cms.untracked.string('GEN-SIM-DIGI-RAW'), + filterName = cms.untracked.string('') + ), + fileName = cms.untracked.string('reHLT_HLT_Alpaka.root'), + outputCommands = process.FEVTDEBUGHLTEventContent.outputCommands, + splitLevel = cms.untracked.int32(0) +) + +# Additional output definition + +# Other statements +from HLTrigger.Configuration.CustomConfigs import ProcessName +process = ProcessName(process) + +from Configuration.AlCa.GlobalTag import GlobalTag +process.GlobalTag = GlobalTag(process.GlobalTag, 'auto:phase1_2022_realistic', '') + +# Path and EndPath definitions +process.endjob_step = cms.EndPath(process.endOfProcess) +process.FEVTDEBUGHLToutput_step = cms.EndPath(process.FEVTDEBUGHLToutput) + +# Schedule definition +# process.schedule imported from cff in HLTrigger.Configuration +process.schedule.extend([process.endjob_step,process.FEVTDEBUGHLToutput_step]) +from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask +associatePatAlgosToolsTask(process) + +# customisation of the process. + +# Automatic addition of the customisation function from HLTrigger.Configuration.customizeHLTforPatatrack +#from HLTrigger.Configuration.customizeHLTforPatatrack import customizeHLTforPatatrack, customiseCommon, customiseHcalLocalReconstruction + +# only enable Hcal GPU +#process = customiseCommon(process) +#process = customiseHcalLocalReconstruction(process) + +#call to customisation function customizeHLTforPatatrack imported from HLTrigger.Configuration.customizeHLTforPatatrack +#process = customizeHLTforPatatrack(process) + +# Automatic addition of the customisation function from HLTrigger.Configuration.customizeHLTforMC +from HLTrigger.Configuration.customizeHLTforMC import customizeHLTforMC + +#call to customisation function customizeHLTforMC imported from HLTrigger.Configuration.customizeHLTforMC +process = customizeHLTforMC(process) + +# End of customisation functions + + +# Customisation from command line + +# Add early deletion of temporary data products to reduce peak memory need +from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete +process = customiseEarlyDelete(process) +# End adding early deletion + +process.load( "HLTrigger.Timer.FastTimerService_cfi" ) +if 'MessageLogger' in process.__dict__: + process.MessageLogger.TriggerSummaryProducerAOD = cms.untracked.PSet() + process.MessageLogger.L1GtTrigReport = cms.untracked.PSet() + process.MessageLogger.L1TGlobalSummary = cms.untracked.PSet() + process.MessageLogger.HLTrigReport = cms.untracked.PSet() + process.MessageLogger.FastReport = cms.untracked.PSet() + process.MessageLogger.ThroughputService = cms.untracked.PSet() + process.MessageLogger.cerr.FastReport = cms.untracked.PSet( limit = cms.untracked.int32( 10000000 ) ) + +##################################### +## Configure CPU and GPU producers ## +##################################### + +# CPU producer +process.hltParticleFlowRecHitHBHE = cms.EDProducer("PFRecHitProducer", + navigator = cms.PSet( + hcalEnums = cms.vint32(1, 2), + name = cms.string('PFRecHitHCALDenseIdNavigator') + ), + producers = cms.VPSet(cms.PSet( + name = cms.string('PFHBHERecHitCreator'), + qualityTests = cms.VPSet( + cms.PSet( + cuts = cms.VPSet( + cms.PSet( + depth = cms.vint32(1, 2, 3, 4), + detectorEnum = cms.int32(1), + threshold = cms.vdouble(0.1, 0.2, 0.3, 0.3) + ), + cms.PSet( + depth = cms.vint32(1, 2, 3, 4, 5, 6, 7), + detectorEnum = cms.int32(2), + threshold = cms.vdouble(0.1, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2) + ) + ), + name = cms.string('PFRecHitQTestHCALThresholdVsDepth') + ), + cms.PSet( + cleaningThresholds = cms.vdouble(0.0), + flags = cms.vstring('Standard'), + maxSeverities = cms.vint32(11), + name = cms.string('PFRecHitQTestHCALChannel') + ) + ), + src = cms.InputTag("hltHbhereco") + )) +) + +# Alpaka producer +process.hltParticleFlowRecHitToSoA = cms.EDProducer("alpaka_serial_sync::CaloRecHitSoAProducer", +#process.hltParticleFlowRecHitToSoA = cms.EDProducer("CaloRecHitSoAProducer@alpaka", + src = cms.InputTag("hltHbhereco") +) +#process.hltParticleFlowRecHitHBHEAlpaka = cms.EDProducer("alpaka_serial_sync::PFHBHERecHitProducerAlpaka", +# navigator = process.hltParticleFlowRecHitHBHE.navigator, +# src = cms.InputTag("hltHbherecoFromGPU") +#) + +# +# Additional customization +process.FEVTDEBUGHLToutput.outputCommands = cms.untracked.vstring('drop *_*_*_*') +process.FEVTDEBUGHLToutput.outputCommands.append('keep *_*ParticleFlow*HBHE*_*_*') +process.FEVTDEBUGHLToutput.outputCommands.append('keep *_*HbherecoLegacy*_*_*') +process.FEVTDEBUGHLToutput.outputCommands.append('keep *_*HbherecoFromGPU*_*_*') +process.FEVTDEBUGHLToutput.outputCommands.append('keep *_*Hbhereco*_*_*') + +# +# Run only localreco, PFRecHit and PFCluster producers for HBHE only +#process.source.fileNames = cms.untracked.vstring('file:/cms/data/hatake/ana/PF/GPU/CMSSW_12_4_0_v2/src/test/v21/GPU/reHLT_HLT.root ') + +# Path/sequence definitions +process.HBHEPFCPUGPUTask = cms.Path( + process.hltHcalDigis + #+process.hltHcalDigisGPU + +process.hltHbherecoLegacy + #+process.hltHbherecoGPU + #+process.hltHbherecoFromGPU + +process.hltParticleFlowRecHitHBHE # Construct PFRecHits on CPU + #+process.hltParticleFlowRecHitHBHEAlpaka + +process.hltParticleFlowRecHitToSoA # Convert legacy CaloRecHits to SoA and copy to device +) +process.schedule = cms.Schedule(process.HBHEPFCPUGPUTask) +process.schedule.extend([process.endjob_step,process.FEVTDEBUGHLToutput_step]) + +process.options.numberOfThreads = cms.untracked.uint32(8) From a4c5e748312cf8378d930d0543cd037b5fd29a66 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 22 Feb 2023 15:52:24 +0100 Subject: [PATCH 06/13] Less printout --- .../plugins/alpaka/CaloRecHitSoAProducer.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc index ef4186b0dc769..215a526cd3a4b 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc @@ -11,6 +11,8 @@ #include "DataFormats/ParticleFlowReco/interface/CaloRecHitHostCollection.h" #include "DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h" +#define DEBUG false + namespace ALPAKA_ACCELERATOR_NAMESPACE { class CaloRecHitSoAProducer : public global::EDProducer<> { public: @@ -22,7 +24,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { void produce(edm::StreamID sid, device::Event& event, device::EventSetup const&) const override { const edm::SortedCollection& recHits = event.get(recHitsToken); const int32_t num_recHits = recHits.size(); - printf("Found %d recHits\n", num_recHits); + if(DEBUG) + printf("Found %d recHits\n", num_recHits); CaloRecHitHostCollection hostProduct{num_recHits, event.queue()}; auto& view = hostProduct.view(); @@ -30,7 +33,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { for(int i = 0; i < num_recHits; i++) { const HBHERecHit& rh = recHits[i]; - if(i < 10) + if (DEBUG && i < 10) printf("recHit %4d %u %f %f\n", i, rh.id().rawId(), rh.energy(), rh.time()); view[i].detId() = rh.id().rawId(); view[i].energy() = rh.energy(); @@ -56,4 +59,4 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { } // namespace ALPAKA_ACCELERATOR_NAMESPACE #include "HeterogeneousCore/AlpakaCore/interface/MakerMacros.h" -DEFINE_FWK_ALPAKA_MODULE(CaloRecHitSoAProducer); \ No newline at end of file +DEFINE_FWK_ALPAKA_MODULE(CaloRecHitSoAProducer); From 3cc4058fbd04e0839b5a826c9847b71c560512e8 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 22 Feb 2023 16:51:50 +0100 Subject: [PATCH 07/13] Add PFRecHitSoA --- DataFormats/ParticleFlowReco/BuildFile.xml | 1 + .../interface/CaloRecHitSoA.h | 6 +- .../interface/PFRecHitHostCollection.h | 9 +++ .../ParticleFlowReco/interface/PFRecHitSoA.h | 22 +++++++ .../alpaka/CaloRecHitDeviceCollection.h | 2 +- .../alpaka/PFRecHitDeviceCollection.h | 12 ++++ .../src/alpaka/classes_cuda.h | 2 + .../src/alpaka/classes_cuda_def.xml | 4 ++ .../src/alpaka/classes_serial.h | 2 + .../src/alpaka/classes_serial_def.xml | 14 +++++ DataFormats/ParticleFlowReco/src/classes.h | 57 +------------------ .../ParticleFlowReco/src/classes_def.xml | 2 + 12 files changed, 73 insertions(+), 60 deletions(-) create mode 100644 DataFormats/ParticleFlowReco/interface/PFRecHitHostCollection.h create mode 100644 DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h create mode 100644 DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h diff --git a/DataFormats/ParticleFlowReco/BuildFile.xml b/DataFormats/ParticleFlowReco/BuildFile.xml index faf3c4f3af35a..2b79abecfbea5 100644 --- a/DataFormats/ParticleFlowReco/BuildFile.xml +++ b/DataFormats/ParticleFlowReco/BuildFile.xml @@ -17,6 +17,7 @@ + diff --git a/DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h b/DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h index 6fb19a9f49599..eb10c7af55cab 100644 --- a/DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h +++ b/DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h @@ -7,9 +7,9 @@ #include "DataFormats/SoATemplate/interface/SoAView.h" GENERATE_SOA_LAYOUT(CaloRecHitSoALayout, - SOA_COLUMN(uint32_t, detId), - SOA_COLUMN(float, energy), - SOA_COLUMN(float, time) + SOA_COLUMN(uint32_t, detId), + SOA_COLUMN(float, energy), + SOA_COLUMN(float, time) ) using CaloRecHitSoA = CaloRecHitSoALayout<>; diff --git a/DataFormats/ParticleFlowReco/interface/PFRecHitHostCollection.h b/DataFormats/ParticleFlowReco/interface/PFRecHitHostCollection.h new file mode 100644 index 0000000000000..25fd87fb42c50 --- /dev/null +++ b/DataFormats/ParticleFlowReco/interface/PFRecHitHostCollection.h @@ -0,0 +1,9 @@ +#ifndef ParticleFlowReco_PFRecHitHostCollection_h +#define ParticleFlowReco_PFRecHitHostCollection_h + +#include "DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h" +#include "DataFormats/Portable/interface/PortableHostCollection.h" + +using PFRecHitHostCollection = PortableHostCollection; + +#endif \ No newline at end of file diff --git a/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h b/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h new file mode 100644 index 0000000000000..b0ca87145747f --- /dev/null +++ b/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h @@ -0,0 +1,22 @@ +#ifndef ParticleFlowReco_PFRecHitSoA_h +#define ParticleFlowReco_PFRecHitSoA_h + +#include +#include + +#include "DataFormats/SoATemplate/interface/SoACommon.h" +#include "DataFormats/SoATemplate/interface/SoALayout.h" +#include "DataFormats/SoATemplate/interface/SoAView.h" + +using Vec8 = Eigen::Matrix; +GENERATE_SOA_LAYOUT(PFRecHitSoALayout, + SOA_COLUMN(uint32_t, detId), + SOA_COLUMN(float, energy), + SOA_COLUMN(float, time), + SOA_COLUMN(int, depth), + SOA_EIGEN_COLUMN(Vec8, neighbours) +) + +using PFRecHitSoA = PFRecHitSoALayout<>; + +#endif diff --git a/DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h b/DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h index 1f5ffb848c5c6..ca20936c7fdd0 100644 --- a/DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h +++ b/DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h @@ -1,9 +1,9 @@ #ifndef ParticleFlowReco_CaloRecHitDeviceCollection_h #define ParticleFlowReco_CaloRecHitDeviceCollection_h +#include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" #include "DataFormats/Portable/interface/alpaka/PortableCollection.h" #include "HeterogeneousCore/AlpakaInterface/interface/config.h" -#include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" namespace ALPAKA_ACCELERATOR_NAMESPACE { using CaloRecHitDeviceCollection = PortableCollection; diff --git a/DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h b/DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h new file mode 100644 index 0000000000000..f4b53f7ba00f6 --- /dev/null +++ b/DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h @@ -0,0 +1,12 @@ +#ifndef ParticleFlowReco_PFRecHitDeviceCollection_h +#define ParticleFlowReco_PFRecHitDeviceCollection_h + +#include "DataFormats/Portable/interface/alpaka/PortableCollection.h" +#include "HeterogeneousCore/AlpakaInterface/interface/config.h" +#include "DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h" + +namespace ALPAKA_ACCELERATOR_NAMESPACE { + using PFRecHitDeviceCollection = PortableCollection; +} // namespace ALPAKA_ACCELERATOR_NAMESPACE + +#endif diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h index bfa90841e52c4..7c9adfa56d124 100644 --- a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda.h @@ -1,5 +1,7 @@ #include "DataFormats/Common/interface/DeviceProduct.h" #include "DataFormats/Common/interface/Wrapper.h" #include "DataFormats/Portable/interface/Product.h" +#include "DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" #include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" #include "DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h" diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml index 2b62b1198812c..dbc1b48817f3f 100644 --- a/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_cuda_def.xml @@ -2,4 +2,8 @@ + + + + diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h index bf7faed3d765c..122b9f803c5bd 100644 --- a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial.h @@ -1,4 +1,6 @@ #include "DataFormats/Common/interface/Wrapper.h" #include "DataFormats/Portable/interface/Product.h" +#include "DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h" +#include "DataFormats/ParticleFlowReco/interface/PFRecHitHostCollection.h" #include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" #include "DataFormats/ParticleFlowReco/interface/CaloRecHitHostCollection.h" diff --git a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml index 48de13d797280..fc121f1ea492c 100644 --- a/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml +++ b/DataFormats/ParticleFlowReco/src/alpaka/classes_serial_def.xml @@ -12,4 +12,18 @@ ]]> + + + + + + diff --git a/DataFormats/ParticleFlowReco/src/classes.h b/DataFormats/ParticleFlowReco/src/classes.h index 61af79cdbaaf8..177424ea74d6b 100644 --- a/DataFormats/ParticleFlowReco/src/classes.h +++ b/DataFormats/ParticleFlowReco/src/classes.h @@ -4,62 +4,7 @@ #include #include "DataFormats/Common/interface/ValueMap.h" -#include "DataFormats/ParticleFlowReco/interface/PFCluster.h" -#include "Math/Cartesian3D.h" -#include "DataFormats/ParticleFlowReco/interface/PFRecHit.h" -#include "Math/Polar3D.h" -#include "Math/CylindricalEta3D.h" -#include "DataFormats/ParticleFlowReco/interface/PFRecTrack.h" -#include "DataFormats/ParticleFlowReco/interface/GsfPFRecTrack.h" -#include "Math/GenVector/PositionVector3D.h" -#include "DataFormats/Math/interface/Point3D.h" -#include "Rtypes.h" -#include "DataFormats/Math/interface/Vector3D.h" -#include "Math/PxPyPzE4D.h" -#include "DataFormats/DetId/interface/DetId.h" -#include "DataFormats/TrackReco/interface/Track.h" -#include "DataFormats/TrackReco/interface/TrackFwd.h" -#include "DataFormats/GsfTrackReco/interface/GsfTrack.h" -#include "DataFormats/GsfTrackReco/interface/GsfTrackExtra.h" -#include "DataFormats/ParticleFlowReco/interface/PFSimParticle.h" -#include "DataFormats/ParticleFlowReco/interface/PFTrajectoryPoint.h" -#include "DataFormats/ParticleFlowReco/interface/PFBlockElement.h" -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementTrack.h" -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementGsfTrack.h" //Daniele -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementBrem.h" //Daniele -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementSuperCluster.h" //Florian -#include "DataFormats/ParticleFlowReco/interface/PFBlockElementCluster.h" -#include "DataFormats/ParticleFlowReco/interface/PFBlock.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedTrackerVertex.h" -#include "DataFormats/ParticleFlowReco/interface/PFNuclearInteraction.h" -#include "DataFormats/ParticleFlowReco/interface/PFConversion.h" -#include "DataFormats/ParticleFlowReco/interface/PFConversionFwd.h" -#include "DataFormats/ParticleFlowReco/interface/PFV0.h" -#include "DataFormats/ParticleFlowReco/interface/ConvBremSeed.h" -#include "DataFormats/ParticleFlowReco/interface/ConvBremSeedFwd.h" -#include "DataFormats/ParticleFlowReco/interface/ParticleFiltrationDecision.h" - -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexCandidate.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexSeed.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertex.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexCandidateFwd.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexSeedFwd.h" -#include "DataFormats/ParticleFlowReco/interface/PFDisplacedVertexFwd.h" - -#include "DataFormats/ParticleFlowReco/interface/PreId.h" -#include "DataFormats/ParticleFlowReco/interface/PreIdFwd.h" -#include "DataFormats/ParticleFlowReco/interface/RecoPFClusterRefCandidate.h" -#include "DataFormats/ParticleFlowReco/interface/RecoPFClusterRefCandidateFwd.h" - -#include - - -#include "DataFormats/Common/interface/Wrapper.h" -#include "DataFormats/Common/interface/Ref.h" -#include -#include -#include "DataFormats/Common/interface/ValueMap.h" - +#include "DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h" #include "DataFormats/ParticleFlowReco/interface/CaloRecHitSoA.h" #include "DataFormats/ParticleFlowReco/interface/PFCluster.h" #include "Math/Cartesian3D.h" diff --git a/DataFormats/ParticleFlowReco/src/classes_def.xml b/DataFormats/ParticleFlowReco/src/classes_def.xml index 02ca7fdcdbdc9..27a852aa17533 100644 --- a/DataFormats/ParticleFlowReco/src/classes_def.xml +++ b/DataFormats/ParticleFlowReco/src/classes_def.xml @@ -60,6 +60,8 @@ + + From 48fea4faed2d54c45857d214fe7958ef2feb179a Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 22 Feb 2023 16:55:17 +0100 Subject: [PATCH 08/13] Add dummy PFRecHitProducerAlpaka, that reads RecHits and writes PFRecHits --- .../plugins/alpaka/CaloRecHitSoAProducer.cc | 1 - .../plugins/alpaka/PFRecHitProducerAlpaka.cc | 43 +++++++++++++++++++ .../test/pfRecHitProducer_CPU_vs_Alpaka.py | 25 ++++++----- 3 files changed, 58 insertions(+), 11 deletions(-) create mode 100644 RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerAlpaka.cc diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc index 215a526cd3a4b..89723a7cb88a5 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc @@ -1,4 +1,3 @@ -#include "HeterogeneousCore/AlpakaServices/interface/alpaka/AlpakaService.h" #include "HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h" #include "FWCore/Utilities/interface/StreamID.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerAlpaka.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerAlpaka.cc new file mode 100644 index 0000000000000..3a1b422a0487f --- /dev/null +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerAlpaka.cc @@ -0,0 +1,43 @@ +#include "HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h" +#include "FWCore/Utilities/interface/StreamID.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h" + +#define DEBUG false + +namespace ALPAKA_ACCELERATOR_NAMESPACE { + class PFRecHitProducerAlpaka : public global::EDProducer<> { + public: + PFRecHitProducerAlpaka(edm::ParameterSet const& config) + : recHitsToken(consumes(config.getParameter("src"))), pfRecHitsToken(produces()) {} + + void produce(edm::StreamID sid, device::Event& event, device::EventSetup const&) const override { + const CaloRecHitDeviceCollection& recHits = event.get(recHitsToken); + PFRecHitDeviceCollection deviceProduct{42, event.queue()}; + + // TODO to something with recHits + (void)recHits; + + event.emplace(pfRecHitsToken, std::move(deviceProduct)); + } + + static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + desc.add("src"); + descriptions.addWithDefaultLabel(desc); + } + + private: + const device::EDGetToken recHitsToken; + const device::EDPutToken pfRecHitsToken; + }; + +} // namespace ALPAKA_ACCELERATOR_NAMESPACE + +#include "HeterogeneousCore/AlpakaCore/interface/MakerMacros.h" +DEFINE_FWK_ALPAKA_MODULE(PFRecHitProducerAlpaka); \ No newline at end of file diff --git a/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py index 3b04abd4745dc..4c761a57d6e32 100644 --- a/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py +++ b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py @@ -186,9 +186,11 @@ # Alpaka producer process.hltParticleFlowRecHitToSoA = cms.EDProducer("alpaka_serial_sync::CaloRecHitSoAProducer", -#process.hltParticleFlowRecHitToSoA = cms.EDProducer("CaloRecHitSoAProducer@alpaka", src = cms.InputTag("hltHbhereco") ) +process.hltParticleFlowPFRecHitAlpaka = cms.EDProducer("alpaka_serial_sync::PFRecHitProducerAlpaka", + src = cms.InputTag("hltParticleFlowRecHitToSoA") +) #process.hltParticleFlowRecHitHBHEAlpaka = cms.EDProducer("alpaka_serial_sync::PFHBHERecHitProducerAlpaka", # navigator = process.hltParticleFlowRecHitHBHE.navigator, # src = cms.InputTag("hltHbherecoFromGPU") @@ -199,8 +201,10 @@ process.FEVTDEBUGHLToutput.outputCommands = cms.untracked.vstring('drop *_*_*_*') process.FEVTDEBUGHLToutput.outputCommands.append('keep *_*ParticleFlow*HBHE*_*_*') process.FEVTDEBUGHLToutput.outputCommands.append('keep *_*HbherecoLegacy*_*_*') -process.FEVTDEBUGHLToutput.outputCommands.append('keep *_*HbherecoFromGPU*_*_*') +#process.FEVTDEBUGHLToutput.outputCommands.append('keep *_*HbherecoFromGPU*_*_*') process.FEVTDEBUGHLToutput.outputCommands.append('keep *_*Hbhereco*_*_*') +process.FEVTDEBUGHLToutput.outputCommands.append('keep *_hltParticleFlowRecHitToSoA_*_*') +process.FEVTDEBUGHLToutput.outputCommands.append('keep *_hltParticleFlowPFRecHitAlpaka_*_*') # # Run only localreco, PFRecHit and PFCluster producers for HBHE only @@ -208,14 +212,15 @@ # Path/sequence definitions process.HBHEPFCPUGPUTask = cms.Path( - process.hltHcalDigis - #+process.hltHcalDigisGPU - +process.hltHbherecoLegacy - #+process.hltHbherecoGPU - #+process.hltHbherecoFromGPU - +process.hltParticleFlowRecHitHBHE # Construct PFRecHits on CPU - #+process.hltParticleFlowRecHitHBHEAlpaka - +process.hltParticleFlowRecHitToSoA # Convert legacy CaloRecHits to SoA and copy to device + process.hltHcalDigis + #+process.hltHcalDigisGPU + +process.hltHbherecoLegacy + #+process.hltHbherecoGPU + #+process.hltHbherecoFromGPU + +process.hltParticleFlowRecHitHBHE # Construct PFRecHits on CPU + #+process.hltParticleFlowRecHitHBHEAlpaka + +process.hltParticleFlowRecHitToSoA # Convert legacy CaloRecHits to SoA and copy to device + +process.hltParticleFlowPFRecHitAlpaka # Constrcut PFRecHits on device ) process.schedule = cms.Schedule(process.HBHEPFCPUGPUTask) process.schedule.extend([process.endjob_step,process.FEVTDEBUGHLToutput_step]) From 4028cb40bbabd3aed7fa036c60b26643f8c0adb5 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 23 Feb 2023 09:59:47 +0100 Subject: [PATCH 09/13] Add PFRecHitProducer kernel, that applied dummy filtering of recHits --- .../ParticleFlowReco/interface/PFRecHitSoA.h | 3 +- .../interface/alpaka/PFRecHitProducerKernel.h | 17 +++++ .../plugins/alpaka/CaloRecHitSoAProducer.cc | 5 +- .../plugins/alpaka/PFRecHitProducerAlpaka.cc | 10 +-- .../alpaka/PFRecHitProducerKernel.dev.cc | 68 +++++++++++++++++++ .../test/pfRecHitProducer_CPU_vs_Alpaka.py | 2 +- 6 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitProducerKernel.h create mode 100644 RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc diff --git a/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h b/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h index b0ca87145747f..766eee67f7dad 100644 --- a/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h +++ b/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h @@ -14,7 +14,8 @@ GENERATE_SOA_LAYOUT(PFRecHitSoALayout, SOA_COLUMN(float, energy), SOA_COLUMN(float, time), SOA_COLUMN(int, depth), - SOA_EIGEN_COLUMN(Vec8, neighbours) + SOA_EIGEN_COLUMN(Vec8, neighbours), + SOA_SCALAR(uint32_t, size) ) using PFRecHitSoA = PFRecHitSoALayout<>; diff --git a/RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitProducerKernel.h b/RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitProducerKernel.h new file mode 100644 index 0000000000000..9ce8e84ba475b --- /dev/null +++ b/RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitProducerKernel.h @@ -0,0 +1,17 @@ +#ifndef RecoParticleFlow_PFRecHitProducer_PFRecHitProducerKernel_h +#define RecoParticleFlow_PFRecHitProducer_PFRecHitProducerKernel_h + +#include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h" +#include "HeterogeneousCore/AlpakaInterface/interface/config.h" + +namespace ALPAKA_ACCELERATOR_NAMESPACE { + + class PFRecHitProducerKernel { + public: + void execute(Queue& queue, const CaloRecHitDeviceCollection& recHits, PFRecHitDeviceCollection& collection) const; + }; + +} // namespace ALPAKA_ACCELERATOR_NAMESPACE + +#endif \ No newline at end of file diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc index 89723a7cb88a5..d09c28b3b78e2 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc @@ -32,11 +32,12 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { for(int i = 0; i < num_recHits; i++) { const HBHERecHit& rh = recHits[i]; - if (DEBUG && i < 10) - printf("recHit %4d %u %f %f\n", i, rh.id().rawId(), rh.energy(), rh.time()); view[i].detId() = rh.id().rawId(); view[i].energy() = rh.energy(); view[i].time() = rh.time(); + + if (DEBUG && i < 10) + printf("recHit %4d %u %f %f\n", i, rh.id().rawId(), rh.energy(), rh.time()); } CaloRecHitDeviceCollection deviceProduct{num_recHits, event.queue()}; diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerAlpaka.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerAlpaka.cc index 3a1b422a0487f..fe8c570669b88 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerAlpaka.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerAlpaka.cc @@ -7,6 +7,7 @@ #include "FWCore/Utilities/interface/InputTag.h" #include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" #include "DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h" +#include "RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitProducerKernel.h" #define DEBUG false @@ -18,12 +19,13 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { void produce(edm::StreamID sid, device::Event& event, device::EventSetup const&) const override { const CaloRecHitDeviceCollection& recHits = event.get(recHitsToken); - PFRecHitDeviceCollection deviceProduct{42, event.queue()}; + const int num_recHits = recHits->metadata().size(); + PFRecHitDeviceCollection pfRecHits{num_recHits, event.queue()}; - // TODO to something with recHits - (void)recHits; + PFRecHitProducerKernel kernel{}; + kernel.execute(event.queue(), recHits, pfRecHits); - event.emplace(pfRecHitsToken, std::move(deviceProduct)); + event.emplace(pfRecHitsToken, std::move(pfRecHits)); } static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) { diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc new file mode 100644 index 0000000000000..233d4db99cceb --- /dev/null +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc @@ -0,0 +1,68 @@ +#include + +#include "DataFormats/ParticleFlowReco/interface/alpaka/PFRecHitDeviceCollection.h" +#include "DataFormats/ParticleFlowReco/interface/alpaka/CaloRecHitDeviceCollection.h" +#include "HeterogeneousCore/AlpakaInterface/interface/config.h" +#include "HeterogeneousCore/AlpakaInterface/interface/workdivision.h" + +#include "RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitProducerKernel.h" + +namespace ALPAKA_ACCELERATOR_NAMESPACE { + + using namespace cms::alpakatools; + + class PFRecHitProducerKernelImpl { + public: + template >> + ALPAKA_FN_ACC void operator()(const TAcc& acc, + const CaloRecHitDeviceCollection::ConstView recHits, int32_t num_recHits, + PFRecHitDeviceCollection::View pfRecHits) const { + // global index of the thread within the grid + const int32_t thread = alpaka::getIdx(acc)[0u]; + + // set this only once in the whole kernel grid + int& num_pfRecHits = alpaka::declareSharedVar(acc); + if (thread == 0) { + num_pfRecHits = 0; + } + alpaka::syncBlockThreads(acc); + + // make a strided loop over the kernel grid, covering up to "size" elements + for (int32_t i : elements_with_stride(acc, num_recHits)) { + + // TODO real filtering + if(i % 2 == 1) + continue; + + const int32_t j = alpaka::atomicAdd(acc, &num_pfRecHits, 1, alpaka::hierarchy::Blocks{}); + pfRecHits[j].detId() = recHits[i].detId(); + pfRecHits[j].energy() = recHits[i].energy(); + pfRecHits[j].time() = recHits[i].time(); + pfRecHits[j].depth() = 0; + //pfRecHits[i].neighbours() = {0, 0, 0, 0, 0, 0, 0, 0}; + } + + alpaka::syncBlockThreads(acc); + + if (thread == 0) { + pfRecHits.size() = num_pfRecHits; + } + } + }; + + void PFRecHitProducerKernel::execute(Queue& queue, const CaloRecHitDeviceCollection& recHits, PFRecHitDeviceCollection& pfRecHits) const { + // use 64 items per group (this value is arbitrary, but it's a reasonable starting point) + const uint32_t items = 64; + + // use as many groups as needed to cover the whole problem + const uint32_t groups = 1;//divide_up_by(recHits->metadata().size(), items); + + // map items to + // - threads with a single element per thread on a GPU backend + // - elements within a single thread on a CPU backend + auto workDiv = make_workdiv(groups, items); + + alpaka::exec(queue, workDiv, PFRecHitProducerKernelImpl{}, recHits.view(), recHits->metadata().size(), pfRecHits.view()); + } + +} // namespace ALPAKA_ACCELERATOR_NAMESPACE \ No newline at end of file diff --git a/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py index 4c761a57d6e32..801004f5a7612 100644 --- a/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py +++ b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py @@ -220,7 +220,7 @@ +process.hltParticleFlowRecHitHBHE # Construct PFRecHits on CPU #+process.hltParticleFlowRecHitHBHEAlpaka +process.hltParticleFlowRecHitToSoA # Convert legacy CaloRecHits to SoA and copy to device - +process.hltParticleFlowPFRecHitAlpaka # Constrcut PFRecHits on device + +process.hltParticleFlowPFRecHitAlpaka # Construct PFRecHits on device ) process.schedule = cms.Schedule(process.HBHEPFCPUGPUTask) process.schedule.extend([process.endjob_step,process.FEVTDEBUGHLToutput_step]) From b6079a2ad7ee28312a9b0e490d3fa1395af53ca1 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 23 Feb 2023 10:56:27 +0100 Subject: [PATCH 10/13] Add validation tool for PFRecHits CPU vs Alpaka --- .../PFRecHitProducer/plugins/BuildFile.xml | 15 +-- .../plugins/PFRecHitProducerTest.cc | 107 ++++++++++++++++++ .../alpaka/PFRecHitProducerKernel.dev.cc | 35 +++++- .../test/pfRecHitProducer_CPU_vs_Alpaka.py | 7 ++ 4 files changed, 153 insertions(+), 11 deletions(-) create mode 100644 RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/BuildFile.xml b/RecoParticleFlow/PFRecHitProducer/plugins/BuildFile.xml index a6ae3b3ee8f27..7d108384c5d45 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/BuildFile.xml +++ b/RecoParticleFlow/PFRecHitProducer/plugins/BuildFile.xml @@ -1,18 +1,13 @@ - - - - - - - - - - + + + + + diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc b/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc new file mode 100644 index 0000000000000..6ee2a07210090 --- /dev/null +++ b/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc @@ -0,0 +1,107 @@ +#include "DQMServices/Core/interface/DQMEDAnalyzer.h" +#include "DQMServices/Core/interface/DQMStore.h" +#include "DataFormats/Common/interface/Handle.h" +#include "DataFormats/Common/interface/SortedCollection.h" +#include "DataFormats/DetId/interface/DetId.h" +#include "DataFormats/HcalDetId/interface/HcalDetId.h" +#include "DataFormats/HcalDetId/interface/HcalSubdetector.h" +#include "DataFormats/HcalRecHit/interface/HBHERecHit.h" +#include "DataFormats/ParticleFlowReco/interface/PFRecHit.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/EventSetup.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/PluginManager/interface/ModuleDef.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "DataFormats/ParticleFlowReco/interface/PFRecHitHostCollection.h" + +#include +#include +#include +#include + +class PFRecHitProducerTest : public DQMEDAnalyzer { +public: + PFRecHitProducerTest(edm::ParameterSet const& conf); + ~PFRecHitProducerTest() override = default; + void analyze(edm::Event const& e, edm::EventSetup const& c) override; + void bookHistograms(DQMStore::IBooker&, edm::Run const&, edm::EventSetup const&) override {}; + //static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); + +private: + edm::EDGetTokenT> recHitsToken; + edm::EDGetTokenT pfRecHitsTokenCPU; + edm::EDGetTokenT pfRecHitsTokenAlpaka; +}; + +PFRecHitProducerTest::PFRecHitProducerTest(const edm::ParameterSet& conf) + : recHitsToken( + consumes>(conf.getUntrackedParameter("recHitsSourceCPU"))), + pfRecHitsTokenCPU( + consumes(conf.getUntrackedParameter("pfRecHitsSourceCPU"))), + pfRecHitsTokenAlpaka( + consumes(conf.getUntrackedParameter("pfRecHitsSourceAlpaka"))) + {} + +void PFRecHitProducerTest::analyze(edm::Event const& event, edm::EventSetup const& c) { + static int cnt = 0; + if (cnt++ >= 1) + return; + + // Rec Hits + //edm::Handle> recHits; + //event.getByToken(recHitsToken, recHits); + //printf("Found %zd recHits\n", recHits->size()); + //fprintf(stderr, "Found %zd recHits\n", recHits->size()); + //for (size_t i = 0; i < recHits->size(); i++) + // printf("recHit %4lu %u\n", i, recHits->operator[](i).id().rawId()); + + // PF Rec Hits + // paste <(grep "^CPU" validation.log | sort -nk3) <(grep "^GPU" validation.log | sort -nk3) | awk '$3!=$13 || $4!=$14 || $5!=$15 || $6!=$16 || $9!=$19 {print}' | head + edm::Handle pfRecHitsCPU; + edm::Handle pfRecHitsAlpaka; + event.getByToken(pfRecHitsTokenCPU, pfRecHitsCPU); + event.getByToken(pfRecHitsTokenAlpaka, pfRecHitsAlpaka); + auto& view = pfRecHitsAlpaka->view(); + + fprintf(stdout, "Found %zd/%d pfRecHits with CPU/Alpaka\n", pfRecHitsCPU->size(), view.size()); + fprintf(stderr, "Found %zd/%d pfRecHits with CPU/Alpaka\n", pfRecHitsCPU->size(), view.size()); + for (size_t i = 0; i < pfRecHitsCPU->size(); i++) + printf("CPU %4lu %u %d %d %u : %f %f (%f,%f,%f)\n", + i, + pfRecHitsCPU->at(i).detId(), + pfRecHitsCPU->at(i).depth(), + pfRecHitsCPU->at(i).layer(), + pfRecHitsCPU->at(i).neighbours().size(), + pfRecHitsCPU->at(i).time(), + pfRecHitsCPU->at(i).energy(), + 0., //pfRecHitsCPU->at(i).position().x(), + 0., //pfRecHitsCPU->at(i).position().y(), + 0. //pfRecHitsCPU->at(i).position().z() + ); + for (size_t i = 0; i < view.size(); i++) + printf("Alpaka %4lu %u %d %d %u : %f %f (%f,%f,%f)\n", + i, + view[i].detId(), + view[i].depth(), + 0,//view[i].layer(), + 0,//view[i].neighbours().size(), + view[i].time(), + view[i].energy(), + 0., //pfRecHitsGPU->at(i).position().x(), + 0., //pfRecHitsGPU->at(i).position().y(), + 0. //pfRecHitsGPU->at(i).position().z() + ); +} + +// void PFRecHitProducerTest::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { +// edm::ParameterSetDescription desc; +// //desc.setUnknown(); +// desc.add("pfClusterToken_ref", edm::InputTag("particleFlowClusterHBHE")); +// desc.add("pfClusterToken_target", edm::InputTag("particleFlowClusterHBHEonGPU")); +// desc.addUntracked("pfCaloGPUCompDir", "pfClusterHBHEGPUv"); +// descriptions.addDefault(desc); +// } + +#include "FWCore/Framework/interface/MakerMacros.h" +DEFINE_FWK_MODULE(PFRecHitProducerTest); diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc index 233d4db99cceb..2db0700b1cc4b 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc @@ -7,6 +7,39 @@ #include "RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitProducerKernel.h" +#include "DataFormats/DetId/interface/DetId.h" +#include "DataFormats/HcalDetId/interface/HcalDetId.h" + +namespace { + // Get subdetector encoded in detId to narrow the range of reference table values to search + // https://cmssdt.cern.ch/lxr/source/DataFormats/DetId/interface/DetId.h#0048 + constexpr uint32_t getSubdet(uint32_t detId) { + return ((detId >> DetId::kSubdetOffset) & DetId::kSubdetMask); + } + + //https://cmssdt.cern.ch/lxr/source/DataFormats/HcalDetId/interface/HcalDetId.h#0163 + constexpr uint32_t getDepth(uint32_t detId) { + return ((detId >> HcalDetId::kHcalDepthOffset2) & HcalDetId::kHcalDepthMask2); + } + + //https://cmssdt.cern.ch/lxr/source/DataFormats/HcalDetId/interface/HcalDetId.h#0148 + constexpr uint32_t getIetaAbs(uint32_t detId) { + return ((detId >> HcalDetId::kHcalEtaOffset2) & HcalDetId::kHcalEtaMask2); + } + + //https://cmssdt.cern.ch/lxr/source/DataFormats/HcalDetId/interface/HcalDetId.h#0157 + constexpr uint32_t getIphi(uint32_t detId) { + return (detId & HcalDetId::kHcalPhiMask2); + } + + //https://cmssdt.cern.ch/lxr/source/DataFormats/HcalDetId/interface/HcalDetId.h#0141 + constexpr int getZside(uint32_t detId) { + return ((detId & HcalDetId::kHcalZsideMask2) ? (1) : (-1)); + } +} + + + namespace ALPAKA_ACCELERATOR_NAMESPACE { using namespace cms::alpakatools; @@ -38,7 +71,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { pfRecHits[j].detId() = recHits[i].detId(); pfRecHits[j].energy() = recHits[i].energy(); pfRecHits[j].time() = recHits[i].time(); - pfRecHits[j].depth() = 0; + pfRecHits[j].depth() = getDepth(recHits[i].detId()); //pfRecHits[i].neighbours() = {0, 0, 0, 0, 0, 0, 0, 0}; } diff --git a/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py index 801004f5a7612..44ae6585d737b 100644 --- a/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py +++ b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py @@ -191,6 +191,12 @@ process.hltParticleFlowPFRecHitAlpaka = cms.EDProducer("alpaka_serial_sync::PFRecHitProducerAlpaka", src = cms.InputTag("hltParticleFlowRecHitToSoA") ) +from DQMServices.Core.DQMEDAnalyzer import DQMEDAnalyzer +process.hltParticleFlowPFRecHitComparison = DQMEDAnalyzer("PFRecHitProducerTest", + recHitsSourceCPU = cms.untracked.InputTag("hltHbhereco"), + pfRecHitsSourceCPU = cms.untracked.InputTag("hltParticleFlowRecHitHBHE"), + pfRecHitsSourceAlpaka = cms.untracked.InputTag("hltParticleFlowPFRecHitAlpaka") +) #process.hltParticleFlowRecHitHBHEAlpaka = cms.EDProducer("alpaka_serial_sync::PFHBHERecHitProducerAlpaka", # navigator = process.hltParticleFlowRecHitHBHE.navigator, # src = cms.InputTag("hltHbherecoFromGPU") @@ -221,6 +227,7 @@ #+process.hltParticleFlowRecHitHBHEAlpaka +process.hltParticleFlowRecHitToSoA # Convert legacy CaloRecHits to SoA and copy to device +process.hltParticleFlowPFRecHitAlpaka # Construct PFRecHits on device + +process.hltParticleFlowPFRecHitComparison # Validate Alpaka vs CPU ) process.schedule = cms.Schedule(process.HBHEPFCPUGPUTask) process.schedule.extend([process.endjob_step,process.FEVTDEBUGHLToutput_step]) From 6c0d4e3f7d050fb459d6df3a1e6164275acbc3e6 Mon Sep 17 00:00:00 2001 From: Florian Date: Thu, 23 Feb 2023 11:34:47 +0100 Subject: [PATCH 11/13] Use hard-coded thresholds in PFRecHitProducer kernel --- .../ParticleFlowReco/interface/PFRecHitSoA.h | 3 ++ .../plugins/PFRecHitProducerTest.cc | 24 +++++----- .../alpaka/PFRecHitProducerKernel.dev.cc | 47 ++++++++++++++----- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h b/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h index 766eee67f7dad..99a767ba828a2 100644 --- a/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h +++ b/DataFormats/ParticleFlowReco/interface/PFRecHitSoA.h @@ -8,12 +8,15 @@ #include "DataFormats/SoATemplate/interface/SoALayout.h" #include "DataFormats/SoATemplate/interface/SoAView.h" +#include "DataFormats/ParticleFlowReco/interface/PFLayer.h" + using Vec8 = Eigen::Matrix; GENERATE_SOA_LAYOUT(PFRecHitSoALayout, SOA_COLUMN(uint32_t, detId), SOA_COLUMN(float, energy), SOA_COLUMN(float, time), SOA_COLUMN(int, depth), + SOA_COLUMN(PFLayer::Layer, layer), SOA_EIGEN_COLUMN(Vec8, neighbours), SOA_SCALAR(uint32_t, size) ) diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc b/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc index 6ee2a07210090..5ac4cc1e29a55 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc @@ -59,13 +59,13 @@ void PFRecHitProducerTest::analyze(edm::Event const& event, edm::EventSetup cons // PF Rec Hits // paste <(grep "^CPU" validation.log | sort -nk3) <(grep "^GPU" validation.log | sort -nk3) | awk '$3!=$13 || $4!=$14 || $5!=$15 || $6!=$16 || $9!=$19 {print}' | head edm::Handle pfRecHitsCPU; - edm::Handle pfRecHitsAlpaka; + edm::Handle pfRecHitsAlpakaSoA; event.getByToken(pfRecHitsTokenCPU, pfRecHitsCPU); - event.getByToken(pfRecHitsTokenAlpaka, pfRecHitsAlpaka); - auto& view = pfRecHitsAlpaka->view(); + event.getByToken(pfRecHitsTokenAlpaka, pfRecHitsAlpakaSoA); + auto& pfRecHitsAlpaka = pfRecHitsAlpakaSoA->view(); - fprintf(stdout, "Found %zd/%d pfRecHits with CPU/Alpaka\n", pfRecHitsCPU->size(), view.size()); - fprintf(stderr, "Found %zd/%d pfRecHits with CPU/Alpaka\n", pfRecHitsCPU->size(), view.size()); + fprintf(stdout, "Found %zd/%d pfRecHits with CPU/Alpaka\n", pfRecHitsCPU->size(), pfRecHitsAlpaka.size()); + fprintf(stderr, "Found %zd/%d pfRecHits with CPU/Alpaka\n", pfRecHitsCPU->size(), pfRecHitsAlpaka.size()); for (size_t i = 0; i < pfRecHitsCPU->size(); i++) printf("CPU %4lu %u %d %d %u : %f %f (%f,%f,%f)\n", i, @@ -79,15 +79,15 @@ void PFRecHitProducerTest::analyze(edm::Event const& event, edm::EventSetup cons 0., //pfRecHitsCPU->at(i).position().y(), 0. //pfRecHitsCPU->at(i).position().z() ); - for (size_t i = 0; i < view.size(); i++) + for (size_t i = 0; i < pfRecHitsAlpaka.size(); i++) printf("Alpaka %4lu %u %d %d %u : %f %f (%f,%f,%f)\n", i, - view[i].detId(), - view[i].depth(), - 0,//view[i].layer(), - 0,//view[i].neighbours().size(), - view[i].time(), - view[i].energy(), + pfRecHitsAlpaka[i].detId(), + pfRecHitsAlpaka[i].depth(), + pfRecHitsAlpaka[i].layer(), + -1,//pfRecHitsAlpaka[i].neighbours().size(), + pfRecHitsAlpaka[i].time(), + pfRecHitsAlpaka[i].energy(), 0., //pfRecHitsGPU->at(i).position().x(), 0., //pfRecHitsGPU->at(i).position().y(), 0. //pfRecHitsGPU->at(i).position().z() diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc index 2db0700b1cc4b..7cc1bc4ec2d33 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc @@ -9,6 +9,7 @@ #include "DataFormats/DetId/interface/DetId.h" #include "DataFormats/HcalDetId/interface/HcalDetId.h" +#include "DataFormats/HcalDetId/interface/HcalSubdetector.h" namespace { // Get subdetector encoded in detId to narrow the range of reference table values to search @@ -60,19 +61,43 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { } alpaka::syncBlockThreads(acc); + // TODO get these from config via ESProducer + const float thresholdE_HB[4] = {0.1, 0.2, 0.3, 0.3}; + const float thresholdE_HE[7] = {0.1, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2}; + // make a strided loop over the kernel grid, covering up to "size" elements for (int32_t i : elements_with_stride(acc, num_recHits)) { - - // TODO real filtering - if(i % 2 == 1) - continue; - - const int32_t j = alpaka::atomicAdd(acc, &num_pfRecHits, 1, alpaka::hierarchy::Blocks{}); - pfRecHits[j].detId() = recHits[i].detId(); - pfRecHits[j].energy() = recHits[i].energy(); - pfRecHits[j].time() = recHits[i].time(); - pfRecHits[j].depth() = getDepth(recHits[i].detId()); - //pfRecHits[i].neighbours() = {0, 0, 0, 0, 0, 0, 0, 0}; + const uint32_t detId = recHits[i].detId(); + const uint32_t subdet = getSubdet(detId); + const uint32_t depth = getDepth(detId); + const float energy = recHits[i].energy(); + + float threshold = 9999.; + if (subdet == HcalBarrel) { + threshold = thresholdE_HB[depth - 1]; + } else if (subdet == HcalEndcap) { + threshold = thresholdE_HE[depth - 1]; + } else { + printf("Rechit with detId %u has invalid subdetector %u!\n", detId, subdet); + } + + if (energy >= threshold) { + const int32_t j = alpaka::atomicAdd(acc, &num_pfRecHits, 1, alpaka::hierarchy::Blocks{}); + pfRecHits[j].detId() = detId; + pfRecHits[j].energy() = recHits[i].energy(); + pfRecHits[j].time() = recHits[i].time(); + + pfRecHits[j].depth() = depth; + + if (subdet == HcalBarrel) + pfRecHits[j].layer() = PFLayer::HCAL_BARREL1; + else if (subdet == HcalEndcap) + pfRecHits[j].layer() = PFLayer::HCAL_ENDCAP; + else + pfRecHits[j].layer() = PFLayer::NONE; + + //pfRecHits[i].neighbours() = {0, 0, 0, 0, 0, 0, 0, 0}; + } } alpaka::syncBlockThreads(acc); From 89c892f560e3a7ff725dabef8b0f2ae064b70018 Mon Sep 17 00:00:00 2001 From: Florian Lorkowski Date: Thu, 23 Feb 2023 12:00:45 +0100 Subject: [PATCH 12/13] Make PFRecHit CPU/Alpaka comparion more rigorous --- .../plugins/PFRecHitProducerTest.cc | 83 ++++++++++++++----- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc b/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc index 5ac4cc1e29a55..b732c24c61cb3 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc @@ -11,7 +11,6 @@ #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "FWCore/PluginManager/interface/ModuleDef.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" #include "DataFormats/ParticleFlowReco/interface/PFRecHitHostCollection.h" @@ -23,15 +22,18 @@ class PFRecHitProducerTest : public DQMEDAnalyzer { public: PFRecHitProducerTest(edm::ParameterSet const& conf); - ~PFRecHitProducerTest() override = default; + ~PFRecHitProducerTest() override; void analyze(edm::Event const& e, edm::EventSetup const& c) override; void bookHistograms(DQMStore::IBooker&, edm::Run const&, edm::EventSetup const&) override {}; //static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); private: + void DumpEvent(const reco::PFRecHitCollection& pfRecHitsCPU, const PFRecHitHostCollection::ConstView& pfRecHitsAlpaka); + edm::EDGetTokenT> recHitsToken; edm::EDGetTokenT pfRecHitsTokenCPU; edm::EDGetTokenT pfRecHitsTokenAlpaka; + int32_t num_events = 0, num_errors = 0; }; PFRecHitProducerTest::PFRecHitProducerTest(const edm::ParameterSet& conf) @@ -43,11 +45,11 @@ PFRecHitProducerTest::PFRecHitProducerTest(const edm::ParameterSet& conf) consumes(conf.getUntrackedParameter("pfRecHitsSourceAlpaka"))) {} -void PFRecHitProducerTest::analyze(edm::Event const& event, edm::EventSetup const& c) { - static int cnt = 0; - if (cnt++ >= 1) - return; +PFRecHitProducerTest::~PFRecHitProducerTest() { + printf("PFRecHitProducerTest has compared %u events and found %u problems\n", num_events, num_errors); +} +void PFRecHitProducerTest::analyze(edm::Event const& event, edm::EventSetup const& c) { // Rec Hits //edm::Handle> recHits; //event.getByToken(recHitsToken, recHits); @@ -57,27 +59,63 @@ void PFRecHitProducerTest::analyze(edm::Event const& event, edm::EventSetup cons // printf("recHit %4lu %u\n", i, recHits->operator[](i).id().rawId()); // PF Rec Hits - // paste <(grep "^CPU" validation.log | sort -nk3) <(grep "^GPU" validation.log | sort -nk3) | awk '$3!=$13 || $4!=$14 || $5!=$15 || $6!=$16 || $9!=$19 {print}' | head - edm::Handle pfRecHitsCPU; + edm::Handle pfRecHitsCPUlegacy; edm::Handle pfRecHitsAlpakaSoA; - event.getByToken(pfRecHitsTokenCPU, pfRecHitsCPU); + event.getByToken(pfRecHitsTokenCPU, pfRecHitsCPUlegacy); event.getByToken(pfRecHitsTokenAlpaka, pfRecHitsAlpakaSoA); - auto& pfRecHitsAlpaka = pfRecHitsAlpakaSoA->view(); + + const reco::PFRecHitCollection& pfRecHitsCPU = *pfRecHitsCPUlegacy; + const PFRecHitHostCollection::ConstView& pfRecHitsAlpaka = pfRecHitsAlpakaSoA->const_view(); - fprintf(stdout, "Found %zd/%d pfRecHits with CPU/Alpaka\n", pfRecHitsCPU->size(), pfRecHitsAlpaka.size()); - fprintf(stderr, "Found %zd/%d pfRecHits with CPU/Alpaka\n", pfRecHitsCPU->size(), pfRecHitsAlpaka.size()); - for (size_t i = 0; i < pfRecHitsCPU->size(); i++) + bool error = false; + if(pfRecHitsCPU.size() != pfRecHitsAlpaka.size()) + error = true; + else + { + for (size_t i = 0; i < pfRecHitsCPU.size(); i++) + { + const uint32_t detId = pfRecHitsCPU[i].detId(); + bool detId_found = false; + for (size_t j = 0; j < pfRecHitsAlpaka.size(); j++) + if(detId == pfRecHitsAlpaka[j].detId()) + { + if(detId_found) + error = true; + detId_found = true; + if(pfRecHitsCPU[i].depth() != pfRecHitsAlpaka[j].depth() + || pfRecHitsCPU[i].layer() != pfRecHitsAlpaka[j].layer() + || pfRecHitsCPU[i].time() != pfRecHitsAlpaka[j].time() + || pfRecHitsCPU[i].energy() != pfRecHitsAlpaka[j].energy()) + error = true; + } + if(!detId_found) + error = true; + } + } + + if(error) + { + //if(num_errors == 0) + // DumpEvent(pfRecHitsCPU, pfRecHitsAlpaka); + num_errors++; + } + num_events++; +} + +void PFRecHitProducerTest::DumpEvent(const reco::PFRecHitCollection& pfRecHitsCPU, const PFRecHitHostCollection::ConstView& pfRecHitsAlpaka) { + printf("Found %zd/%d pfRecHits with CPU/Alpaka\n", pfRecHitsCPU.size(), pfRecHitsAlpaka.size()); + for (size_t i = 0; i < pfRecHitsCPU.size(); i++) printf("CPU %4lu %u %d %d %u : %f %f (%f,%f,%f)\n", i, - pfRecHitsCPU->at(i).detId(), - pfRecHitsCPU->at(i).depth(), - pfRecHitsCPU->at(i).layer(), - pfRecHitsCPU->at(i).neighbours().size(), - pfRecHitsCPU->at(i).time(), - pfRecHitsCPU->at(i).energy(), - 0., //pfRecHitsCPU->at(i).position().x(), - 0., //pfRecHitsCPU->at(i).position().y(), - 0. //pfRecHitsCPU->at(i).position().z() + pfRecHitsCPU[i].detId(), + pfRecHitsCPU[i].depth(), + pfRecHitsCPU[i].layer(), + pfRecHitsCPU[i].neighbours().size(), + pfRecHitsCPU[i].time(), + pfRecHitsCPU[i].energy(), + 0., //pfRecHitsCPU[i].position().x(), + 0., //pfRecHitsCPU[i].position().y(), + 0. //pfRecHitsCPU[i].position().z() ); for (size_t i = 0; i < pfRecHitsAlpaka.size(); i++) printf("Alpaka %4lu %u %d %d %u : %f %f (%f,%f,%f)\n", @@ -94,6 +132,7 @@ void PFRecHitProducerTest::analyze(edm::Event const& event, edm::EventSetup cons ); } + // void PFRecHitProducerTest::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { // edm::ParameterSetDescription desc; // //desc.setUnknown(); From c24b26f0f45d61ac26774c7b67d1d76e70120074 Mon Sep 17 00:00:00 2001 From: Florian Lorkowski Date: Thu, 23 Feb 2023 12:45:25 +0100 Subject: [PATCH 13/13] Cleanup --- .../plugins/PFRecHitProducerTest.cc | 20 +++++++++---------- .../test/pfRecHitProducer_CPU_vs_Alpaka.py | 7 +++++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc b/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc index b732c24c61cb3..b9a5efc3d93b4 100644 --- a/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc +++ b/RecoParticleFlow/PFRecHitProducer/plugins/PFRecHitProducerTest.cc @@ -25,7 +25,7 @@ class PFRecHitProducerTest : public DQMEDAnalyzer { ~PFRecHitProducerTest() override; void analyze(edm::Event const& e, edm::EventSetup const& c) override; void bookHistograms(DQMStore::IBooker&, edm::Run const&, edm::EventSetup const&) override {}; - //static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); + static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); private: void DumpEvent(const reco::PFRecHitCollection& pfRecHitsCPU, const PFRecHitHostCollection::ConstView& pfRecHitsAlpaka); @@ -77,6 +77,7 @@ void PFRecHitProducerTest::analyze(edm::Event const& event, edm::EventSetup cons const uint32_t detId = pfRecHitsCPU[i].detId(); bool detId_found = false; for (size_t j = 0; j < pfRecHitsAlpaka.size(); j++) + { if(detId == pfRecHitsAlpaka[j].detId()) { if(detId_found) @@ -90,6 +91,7 @@ void PFRecHitProducerTest::analyze(edm::Event const& event, edm::EventSetup cons } if(!detId_found) error = true; + } } } @@ -132,15 +134,13 @@ void PFRecHitProducerTest::DumpEvent(const reco::PFRecHitCollection& pfRecHitsCP ); } - -// void PFRecHitProducerTest::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { -// edm::ParameterSetDescription desc; -// //desc.setUnknown(); -// desc.add("pfClusterToken_ref", edm::InputTag("particleFlowClusterHBHE")); -// desc.add("pfClusterToken_target", edm::InputTag("particleFlowClusterHBHEonGPU")); -// desc.addUntracked("pfCaloGPUCompDir", "pfClusterHBHEGPUv"); -// descriptions.addDefault(desc); -// } +void PFRecHitProducerTest::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + desc.addUntracked("recHitsSourceCPU"); + desc.addUntracked("pfRecHitsSourceCPU"); + desc.addUntracked("pfRecHitsSourceAlpaka"); + descriptions.addDefault(desc); +} #include "FWCore/Framework/interface/MakerMacros.h" DEFINE_FWK_MODULE(PFRecHitProducerTest); diff --git a/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py index 44ae6585d737b..fd0260b86e3cc 100644 --- a/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py +++ b/RecoParticleFlow/PFRecHitProducer/test/pfRecHitProducer_CPU_vs_Alpaka.py @@ -185,10 +185,13 @@ ) # Alpaka producer -process.hltParticleFlowRecHitToSoA = cms.EDProducer("alpaka_serial_sync::CaloRecHitSoAProducer", +alpaka_backend_str = "alpaka_serial_sync::%s" +#alpaka_backend_str = "alpaka_cuda_async::%s" +#alpaka_backend_str = "%s@alpaka" +process.hltParticleFlowRecHitToSoA = cms.EDProducer(alpaka_backend_str % "CaloRecHitSoAProducer", src = cms.InputTag("hltHbhereco") ) -process.hltParticleFlowPFRecHitAlpaka = cms.EDProducer("alpaka_serial_sync::PFRecHitProducerAlpaka", +process.hltParticleFlowPFRecHitAlpaka = cms.EDProducer(alpaka_backend_str % "PFRecHitProducerAlpaka", src = cms.InputTag("hltParticleFlowRecHitToSoA") ) from DQMServices.Core.DQMEDAnalyzer import DQMEDAnalyzer