Change CaloRecHitProducer to produce new HCAL RecHit SoA

DataFormats/HcalRecHit/BuildFile.xml

@@ -2,6 +2,10 @@
 <use name="DataFormats/Common"/>
 <use name="DataFormats/HcalDetId"/>
 <use name="DataFormats/HcalDigi"/>
+<use name="DataFormats/Portable"/>
+<use name="DataFormats/SoATemplate"/>
+<use name="HeterogeneousCore/AlpakaInterface"/>
+<flags ALPAKA_BACKENDS="!serial"/>
 <export>
   <lib name="1"/>
 </export>

DataFormats/HcalRecHit/interface/HcalRecHitHostCollection.h

@@ -0,0 +1,13 @@
+#ifndef DataFormats_HcalRecHit_HcalRecHitHostCollection_h
+#define DataFormats_HcalRecHit_HcalRecHitHostCollection_h
+
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "DataFormats/HcalRecHit/interface/HcalRecHitSoA.h"
+
+namespace hcal {
+
+  // HcalRecHitSoA in host memory
+  using RecHitHostCollection = PortableHostCollection<HcalRecHitSoA>;
+}  // namespace hcal
+
+#endif

DataFormats/HcalRecHit/interface/HcalRecHitSoA.h

@@ -0,0 +1,19 @@
+#ifndef DataFormats_HcalRecHit_HcalRecHitSoA_h
+#define DataFormats_HcalRecHit_HcalRecHitSoA_h
+
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+
+namespace hcal {
+
+  GENERATE_SOA_LAYOUT(HcalRecHitSoALayout,
+                      SOA_SCALAR(uint32_t, size),
+                      SOA_COLUMN(uint32_t, detId),
+                      SOA_COLUMN(float, energy),
+                      SOA_COLUMN(float, chi2),
+                      SOA_COLUMN(float, energyM0),
+                      SOA_COLUMN(float, timeM0))
+
+  using HcalRecHitSoA = HcalRecHitSoALayout<>;
+}  // namespace hcal
+
+#endif

DataFormats/HcalRecHit/interface/alpaka/HcalRecHitDeviceCollection.h

@@ -0,0 +1,22 @@
+#ifndef DataFormats_HcalRecHit_alpaka_HcalRecHitDeviceCollection_h
+#define DataFormats_HcalRecHit_alpaka_HcalRecHitDeviceCollection_h
+
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "DataFormats/HcalRecHit/interface/HcalRecHitSoA.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  namespace hcal {
+
+    // make the names from the top-level hcal namespace visible for unqualified lookup
+    // inside the ALPAKA_ACCELERATOR_NAMESPACE::hcal namespace
+    using namespace ::hcal;
+
+    // HcalRecHitSoA in device global memory
+    using RecHitDeviceCollection = PortableCollection<HcalRecHitSoA>;
+  }  // namespace hcal
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif

DataFormats/HcalRecHit/src/alpaka/classes_cuda.h

@@ -0,0 +1,4 @@
+#include "DataFormats/Common/interface/DeviceProduct.h"
+#include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/HcalRecHit/interface/HcalRecHitSoA.h"
+#include "DataFormats/HcalRecHit/interface/alpaka/HcalRecHitDeviceCollection.h"

DataFormats/HcalRecHit/src/alpaka/classes_cuda_def.xml

@@ -0,0 +1,5 @@
+<lcgdict>
+  <class name="alpaka_cuda_async::hcal::RecHitDeviceCollection" persistent="false"/>
+  <class name="edm::DeviceProduct<alpaka_cuda_async::hcal::RecHitDeviceCollection>" persistent="false"/>
+  <class name="edm::Wrapper<edm::DeviceProduct<alpaka_cuda_async::hcal::RecHitDeviceCollection>>" persistent="false"/>
+</lcgdict>

DataFormats/HcalRecHit/src/alpaka/classes_rocm.h

@@ -0,0 +1,4 @@
+#include "DataFormats/Common/interface/DeviceProduct.h"
+#include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/HcalRecHit/interface/HcalRecHitSoA.h"
+#include "DataFormats/HcalRecHit/interface/alpaka/HcalRecHitDeviceCollection.h"

DataFormats/HcalRecHit/src/alpaka/classes_rocm_def.xml

@@ -0,0 +1,5 @@
+<lcgdict>
+  <class name="alpaka_rocm_async::hcal::RecHitDeviceCollection" persistent="false"/>
+  <class name="edm::DeviceProduct<alpaka_rocm_async::hcal::RecHitDeviceCollection>" persistent="false"/>
+  <class name="edm::Wrapper<edm::DeviceProduct<alpaka_rocm_async::hcal::RecHitDeviceCollection>>" persistent="false"/>
+</lcgdict>

DataFormats/HcalRecHit/src/classes.cc

@@ -0,0 +1,4 @@
+#include "DataFormats/HcalRecHit/interface/HcalRecHitHostCollection.h"
+#include "DataFormats/Portable/interface/PortableHostCollectionReadRules.h"
+
+SET_PORTABLEHOSTCOLLECTION_READ_RULES(hcal::RecHitHostCollection);

DataFormats/HcalRecHit/src/classes.h

@@ -10,6 +10,8 @@
 #include "DataFormats/HcalRecHit/interface/HcalSourcePositionData.h"
 #include "DataFormats/HcalDetId/interface/HcalDetId.h"
 #include "DataFormats/HcalRecHit/interface/HcalRecHitCollections.h"
+#include "DataFormats/HcalRecHit/interface/HcalRecHitSoA.h"
+#include "DataFormats/HcalRecHit/interface/HcalRecHitHostCollection.h"
 #include "DataFormats/Common/interface/Wrapper.h"
 #include "DataFormats/Common/interface/Ref.h"
 #include "DataFormats/Common/interface/RefProd.h"

DataFormats/HcalRecHit/src/classes_def.xml

@@ -118,4 +118,9 @@
   <class name="edm::reftobase::Holder<CaloRecHit,edm::Ref<edm::SortedCollection<HFRecHit,edm::StrictWeakOrdering<HFRecHit> >,HFRecHit,edm::refhelper::FindUsingAdvance<edm::SortedCollection<HFRecHit,edm::StrictWeakOrdering<HFRecHit> >,HFRecHit> > >" />
   <class name="edm::reftobase::Holder<CaloRecHit,edm::Ref<edm::SortedCollection<HORecHit,edm::StrictWeakOrdering<HORecHit> >,HORecHit,edm::refhelper::FindUsingAdvance<edm::SortedCollection<HORecHit,edm::StrictWeakOrdering<HORecHit> >,HORecHit> > >" />
   <class name="edm::reftobase::Holder<CaloRecHit,edm::Ref<edm::SortedCollection<HBHERecHit,edm::StrictWeakOrdering<HBHERecHit> >,HBHERecHit,edm::refhelper::FindUsingAdvance<edm::SortedCollection<HBHERecHit,edm::StrictWeakOrdering<HBHERecHit> >,HBHERecHit> > >" />
-</lcgdict>
+
+  <class name="hcal::HcalRecHitSoA"/>
+  <class name="hcal::HcalRecHitSoA::View"/>
+  <class name="hcal::RecHitHostCollection" />
+  <class name="edm::Wrapper<hcal::RecHitHostCollection>" splitLevel="0"/>
+  </lcgdict>

RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CaloRecHitSoAProducer.cc

@@ -14,6 +14,9 @@
 #include "HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h"
 #include "CalorimeterDefinitions.h"
 
+#include "DataFormats/HcalRecHit/interface/HcalRecHitHostCollection.h"
+#include "DataFormats/HcalRecHit/interface/alpaka/HcalRecHitDeviceCollection.h"
+
 #define DEBUG false
 
 namespace ALPAKA_ACCELERATOR_NAMESPACE {
@@ -25,30 +28,25 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     CaloRecHitSoAProducer(edm::ParameterSet const& config)
         : recHitsToken_(consumes(config.getParameter<edm::InputTag>("src"))),
           deviceToken_(produces()),
-          synchronise_(config.getUntrackedParameter<bool>("synchronise")) {
-      // Workaround until the ProductID problem in issue https://github.com/cms-sw/cmssw/issues/44643 is fixed
-#ifdef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED
-      producesTemporarily("edm::DeviceProduct<alpaka_cuda_async::reco::CaloRecHitDeviceCollection>");
-#endif
-    }
+          synchronise_(config.getUntrackedParameter<bool>("synchronise")) {}
 
     void produce(edm::StreamID sid, device::Event& event, device::EventSetup const&) const override {
       const edm::SortedCollection<typename CAL::CaloRecHitType>& recHits = event.get(recHitsToken_);
       const int32_t num_recHits = recHits.size();
       if (DEBUG)
         printf("Found %d recHits\n", num_recHits);
 
-      reco::CaloRecHitHostCollection hostProduct{num_recHits, event.queue()};
+      hcal::RecHitHostCollection hostProduct{num_recHits, event.queue()};
       auto& view = hostProduct.view();
 
       for (int i = 0; i < num_recHits; i++) {
         convertRecHit(view[i], recHits[i]);
 
         if (DEBUG && i < 10)
-          printf("recHit %4d %u %f %f %08x\n", i, view.detId(i), view.energy(i), view.time(i), view.flags(i));
+          printf("recHit %4d %u %f %f\n", i, view.detId(i), view.energy(i), view.timeM0(i));
       }
 
-      reco::CaloRecHitDeviceCollection deviceProduct{num_recHits, event.queue()};
+      hcal::RecHitDeviceCollection deviceProduct{num_recHits, event.queue()};
       alpaka::memcpy(event.queue(), deviceProduct.buffer(), hostProduct.buffer());
       if (synchronise_)
         alpaka::wait(event.queue());
@@ -65,23 +63,29 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
   private:
     const edm::EDGetTokenT<edm::SortedCollection<typename CAL::CaloRecHitType>> recHitsToken_;
-    const device::EDPutToken<reco::CaloRecHitDeviceCollection> deviceToken_;
+    const device::EDPutToken<hcal::RecHitDeviceCollection> deviceToken_;
     const bool synchronise_;
 
-    static void convertRecHit(reco::CaloRecHitHostCollection::View::element to,
-                              const typename CAL::CaloRecHitType& from);
+    static void convertRecHit(hcal::RecHitHostCollection::View::element to, const typename CAL::CaloRecHitType& from);
   };
 
   template <>
-  void CaloRecHitSoAProducer<HCAL>::convertRecHit(reco::CaloRecHitHostCollection::View::element to,
+  void CaloRecHitSoAProducer<HCAL>::convertRecHit(hcal::RecHitHostCollection::View::element to,
                                                   const HCAL::CaloRecHitType& from) {
     // Fill SoA from HCAL rec hit
     to.detId() = from.id().rawId();
     to.energy() = from.energy();
-    to.time() = from.time();
-    to.flags() = from.flags();
+    to.timeM0() = from.time();
   }
 
+  /*
+  The ECALRecHitSoAProducer currently has no use, but is available via this
+  module. In the case where ECAL PF Clustering is moved to Alpaka, we can then
+  decide to use this converted solely for ECAL, or if the SoA is available
+  we can switch to using just the ECAL RecHit SoA.
+  */
+
+  /*
   template <>
   void CaloRecHitSoAProducer<ECAL>::convertRecHit(reco::CaloRecHitHostCollection::View::element to,
                                                   const ECAL::CaloRecHitType& from) {
@@ -91,11 +95,14 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     to.time() = from.time();
     to.flags() = from.flagsBits();
   }
+  */
 
   using HCALRecHitSoAProducer = CaloRecHitSoAProducer<HCAL>;
-  using ECALRecHitSoAProducer = CaloRecHitSoAProducer<ECAL>;
+
+  // Purposely commented out; see above.
+  //using ECALRecHitSoAProducer = CaloRecHitSoAProducer<ECAL>;
 }  // namespace ALPAKA_ACCELERATOR_NAMESPACE
 
 #include "HeterogeneousCore/AlpakaCore/interface/alpaka/MakerMacros.h"
 DEFINE_FWK_ALPAKA_MODULE(HCALRecHitSoAProducer);
-DEFINE_FWK_ALPAKA_MODULE(ECALRecHitSoAProducer);
+//DEFINE_FWK_ALPAKA_MODULE(ECALRecHitSoAProducer);

RecoParticleFlow/PFRecHitProducer/plugins/alpaka/CalorimeterDefinitions.h

@@ -1,6 +1,8 @@
 #ifndef RecoParticleFlow_PFRecHitProducer_interface_alpaka_CalorimeterDefinitions_h
 #define RecoParticleFlow_PFRecHitProducer_interface_alpaka_CalorimeterDefinitions_h
 
+#include <limits>
+
 #include "DataFormats/DetId/interface/DetId.h"
 #include "DataFormats/EcalDetId/interface/EcalSubdetector.h"
 #include "DataFormats/EcalRecHit/interface/EcalRecHit.h"
@@ -13,6 +15,9 @@
 #include "RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitParamsDeviceCollection.h"
 #include "RecoParticleFlow/PFRecHitProducer/interface/alpaka/PFRecHitTopologyDeviceCollection.h"
 
+#include "DataFormats/HcalRecHit/interface/HcalRecHitHostCollection.h"
+#include "DataFormats/HcalRecHit/interface/alpaka/HcalRecHitDeviceCollection.h"
+
 // Forward declaration of EventSetup records, to avoid propagating the dependency on framework headers to device code
 class PFRecHitHCALParamsRecord;
 class PFRecHitHCALTopologyRecord;
@@ -32,8 +37,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE::particleFlowRecHitProducer {
 
   struct HCAL {
     using CaloRecHitType = HBHERecHit;
-    using CaloRecHitSoATypeHost = reco::CaloRecHitHostCollection;
-    using CaloRecHitSoATypeDevice = reco::CaloRecHitDeviceCollection;
+    using CaloRecHitSoATypeHost = hcal::RecHitHostCollection;
+    using CaloRecHitSoATypeDevice = hcal::RecHitDeviceCollection;
     using ParameterType = reco::PFRecHitHCALParamsDeviceCollection;
     using ParameterRecordType = PFRecHitHCALParamsRecord;
     using TopologyTypeHost = reco::PFRecHitHCALTopologyHostCollection;
@@ -97,6 +102,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE::particleFlowRecHitProducer {
       return retval + kSizeBarrel;
     }
 
+    static constexpr uint32_t kInvalidDenseId = std::numeric_limits<uint32_t>::max();
+
     static constexpr uint32_t detId2denseId(uint32_t detId) {
       const uint32_t subdet = getSubdet(detId);
       if (subdet == HcalBarrel)
@@ -105,7 +112,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE::particleFlowRecHitProducer {
         return detId2denseIdHE(detId);
 
       printf("invalid detId: %u\n", detId);
-      return -1;
+      return kInvalidDenseId;
     }
   };
 
@@ -181,6 +188,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE::particleFlowRecHitProducer {
 
     static constexpr bool checkFlag(uint32_t flagBits, int flag) { return flagBits & (0x1 << flag); }
 
+    static constexpr uint32_t kInvalidDenseId = std::numeric_limits<uint32_t>::max();
+
     static constexpr uint32_t detId2denseId(uint32_t detId) {
       const uint32_t subdet = getSubdet(detId);
       if (subdet == EcalBarrel)
@@ -189,7 +198,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE::particleFlowRecHitProducer {
         return Barrel::kSize + Endcap::denseIndex(detId);
 
       printf("invalid detId: %u\n", detId);
-      return 0;
+      return kInvalidDenseId;
     }
 
     static constexpr bool detIdInRange(uint32_t detId) {

RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitProducerKernel.dev.cc

@@ -99,7 +99,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     pfrh.detId() = rh.detId();
     pfrh.denseId() = HCAL::detId2denseId(rh.detId());
     pfrh.energy() = rh.energy();
-    pfrh.time() = rh.time();
+    pfrh.time() = rh.timeM0();
     pfrh.depth() = HCAL::getDepth(pfrh.detId());
     const uint32_t subdet = getSubdet(pfrh.detId());
     if (subdet == HcalBarrel)
@@ -152,7 +152,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
         for (uint32_t n = 0; n < 8; n++) {
           pfRecHits.neighbours(i)(n) = -1;
           const uint32_t denseId_neighbour = topology.neighbours(denseId)(n);
-          if (denseId_neighbour != 0xffffffff) {
+          if (denseId_neighbour != CAL::kInvalidDenseId) {
             const uint32_t pfRecHit_neighbour = denseId2pfRecHit[denseId_neighbour];
             if (pfRecHit_neighbour != 0xffffffff)
               pfRecHits.neighbours(i)(n) = (int32_t)pfRecHit_neighbour;

RecoParticleFlow/PFRecHitProducer/plugins/alpaka/PFRecHitTopologyESProducer.cc

@@ -96,7 +96,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
             if (CAL::detIdInRange(neighDetId))
               view.neighbours(denseId)(n) = CAL::detId2denseId(neighDetId);
             else
-              view.neighbours(denseId)(n) = 0xffffffff;
+              view.neighbours(denseId)(n) = CAL::kInvalidDenseId;
           }
         }
       }
@@ -107,13 +107,13 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
           for (auto const detId : geom.getValidDetIds(CAL::kDetectorId, subdet)) {
             const uint32_t denseId = CAL::detId2denseId(detId);
             for (uint32_t n = 0; n < 8; n++) {
-              if (view.neighbours(denseId)[n] == 0xffffffff)
+              if (view.neighbours(denseId)[n] == CAL::kInvalidDenseId)
                 continue;
               const ::reco::PFRecHitsTopologyNeighbours& neighboursOfNeighbour =
                   view.neighbours(view.neighbours(denseId)[n]);
               if (std::find(neighboursOfNeighbour.begin(), neighboursOfNeighbour.end(), denseId) ==
                   neighboursOfNeighbour.end())
-                view.neighbours(denseId)[n] = 0xffffffff;
+                view.neighbours(denseId)[n] = CAL::kInvalidDenseId;
             }
           }
       }