HCAL local reconstruction in Alpaka

  - first implementation of MAHI in alpaka, running on CPU and GPU
  - implement and use atomicMaxPair function
  - implement HCAL portable conditions
  - implement HCAL digis portable SoA
  - update the HCAL offline reconstruction using the alpaka modifier
  - HLT customisation for alpaka-based HCAL reconstruction,
    skipping the legacy conversion for HCAL PF clusters
  - various fixes from the code review

CondFormats/DataRecord/interface/HcalMahiConditionsRcd.h

@@ -0,0 +1,35 @@
+#ifndef CondFormats_DataRecord_HcalMahiConditionsRcd_h
+#define CondFormats_DataRecord_HcalMahiConditionsRcd_h
+
+#include "FWCore/Framework/interface/DependentRecordImplementation.h"
+
+#include "CondFormats/DataRecord/interface/HcalRecoParamsRcd.h"
+#include "CondFormats/DataRecord/interface/HcalPedestalsRcd.h"
+#include "CondFormats/DataRecord/interface/HcalGainsRcd.h"
+#include "CondFormats/DataRecord/interface/HcalLUTCorrsRcd.h"
+#include "CondFormats/DataRecord/interface/HcalRespCorrsRcd.h"
+#include "CondFormats/DataRecord/interface/HcalTimeCorrsRcd.h"
+#include "CondFormats/DataRecord/interface/HcalPedestalWidthsRcd.h"
+#include "CondFormats/DataRecord/interface/HcalGainWidthsRcd.h"
+#include "CondFormats/DataRecord/interface/HcalChannelQualityRcd.h"
+#include "CondFormats/DataRecord/interface/HcalQIETypesRcd.h"
+#include "CondFormats/DataRecord/interface/HcalQIEDataRcd.h"
+#include "CondFormats/DataRecord/interface/HcalSiPMParametersRcd.h"
+#include "Geometry/Records/interface/HcalRecNumberingRecord.h"
+
+class HcalMahiConditionsRcd
+    : public edm::eventsetup::DependentRecordImplementation<HcalMahiConditionsRcd,
+                                                            edm::mpl::Vector<HcalRecoParamsRcd,
+                                                                             HcalPedestalsRcd,
+                                                                             HcalGainsRcd,
+                                                                             HcalLUTCorrsRcd,
+                                                                             HcalRespCorrsRcd,
+                                                                             HcalTimeCorrsRcd,
+                                                                             HcalPedestalWidthsRcd,
+                                                                             HcalGainWidthsRcd,
+                                                                             HcalChannelQualityRcd,
+                                                                             HcalQIETypesRcd,
+                                                                             HcalQIEDataRcd,
+                                                                             HcalSiPMParametersRcd,
+                                                                             HcalRecNumberingRecord>> {};
+#endif

CondFormats/DataRecord/src/HcalMahiConditionsRcd.cc

@@ -0,0 +1,4 @@
+#include "CondFormats/DataRecord/interface/HcalMahiConditionsRcd.h"
+#include "FWCore/Framework/interface/eventsetuprecord_registration_macro.h"
+
+EVENTSETUP_RECORD_REG(HcalMahiConditionsRcd);

CondFormats/HcalObjects/BuildFile.xml

@@ -3,13 +3,18 @@
 </ifrelease>
 <use name="boost_serialization"/>
 <use name="CondFormats/Serialization"/>
+<use name="DataFormats/Portable"/>
 <use name="DataFormats/DetId"/>
 <use name="DataFormats/HcalDetId"/>
+<use name="DataFormats/SoATemplate"/>
 <use name="FWCore/MessageLogger"/>
 <use name="FWCore/Utilities"/>
 <use name="Geometry/CaloTopology"/>
 <use name="HeterogeneousCore/CUDACore"/>
 <use name="HeterogeneousCore/CUDAUtilities"/>
+<use name="HeterogeneousCore/AlpakaCore"/>
+<use name="HeterogeneousCore/AlpakaInterface"/>
+<flags ALPAKA_BACKENDS="1"/>
 <export>
   <lib name="1"/>
 </export>

CondFormats/HcalObjects/interface/HcalMahiConditionsHost.h

@@ -0,0 +1,11 @@
+#ifndef CondFormats_HcalObjects_interface_HcalMahiConditionsHost_h
+#define CondFormats_HcalObjects_interface_HcalMahiConditionsHost_h
+
+#include "CondFormats/HcalObjects/interface/HcalMahiConditionsSoA.h"
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+
+namespace hcal {
+  using HcalMahiConditionsPortableHost = PortableHostCollection<HcalMahiConditionsSoA>;
+}
+
+#endif

CondFormats/HcalObjects/interface/HcalMahiConditionsSoA.h

@@ -0,0 +1,71 @@
+#ifndef CondFormats_HcalObjects_HcalMahiConditionsSoA_h
+#define CondFormats_HcalObjects_HcalMahiConditionsSoA_h
+
+#include "RecoLocalCalo/HcalRecAlgos/interface/HcalConstants.h"
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+#include <array>
+
+namespace hcal {
+
+  static constexpr uint32_t numValuesPerChannel = 16;
+  using HcalPedestalArray = std::array<float, 4>;                     // 4 capIds
+  using HcalQIECodersArray = std::array<float, numValuesPerChannel>;  // QIEData
+
+  GENERATE_SOA_LAYOUT(HcalMahiConditionsSoALayout,
+                      SOA_COLUMN(HcalPedestalArray, pedestals_value),
+                      SOA_COLUMN(HcalPedestalArray, pedestals_width),
+                      SOA_COLUMN(HcalPedestalArray, gains_value),
+                      SOA_COLUMN(HcalPedestalArray, effectivePedestals),
+                      SOA_COLUMN(HcalPedestalArray, effectivePedestalWidths),
+                      SOA_COLUMN(float, lutCorrs_values),
+                      SOA_COLUMN(float, respCorrs_values),
+                      SOA_COLUMN(float, timeCorrs_values),
+                      // Use EIGEN_COLUMN for matrix?
+                      SOA_COLUMN(float, pedestalWidths_sigma00),
+                      SOA_COLUMN(float, pedestalWidths_sigma01),
+                      SOA_COLUMN(float, pedestalWidths_sigma02),
+                      SOA_COLUMN(float, pedestalWidths_sigma03),
+                      SOA_COLUMN(float, pedestalWidths_sigma10),
+                      SOA_COLUMN(float, pedestalWidths_sigma11),
+                      SOA_COLUMN(float, pedestalWidths_sigma12),
+                      SOA_COLUMN(float, pedestalWidths_sigma13),
+                      SOA_COLUMN(float, pedestalWidths_sigma20),
+                      SOA_COLUMN(float, pedestalWidths_sigma21),
+                      SOA_COLUMN(float, pedestalWidths_sigma22),
+                      SOA_COLUMN(float, pedestalWidths_sigma23),
+                      SOA_COLUMN(float, pedestalWidths_sigma30),
+                      SOA_COLUMN(float, pedestalWidths_sigma31),
+                      SOA_COLUMN(float, pedestalWidths_sigma32),
+                      SOA_COLUMN(float, pedestalWidths_sigma33),
+                      SOA_COLUMN(float, gainWidths_value0),
+                      SOA_COLUMN(float, gainWidths_value1),
+                      SOA_COLUMN(float, gainWidths_value2),
+                      SOA_COLUMN(float, gainWidths_value3),
+                      SOA_COLUMN(uint32_t, channelQuality_status),
+                      SOA_COLUMN(HcalQIECodersArray, qieCoders_offsets),
+                      SOA_COLUMN(HcalQIECodersArray, qieCoders_slopes),
+                      SOA_COLUMN(int, qieTypes_values),
+                      SOA_COLUMN(int, sipmPar_type),
+                      SOA_COLUMN(int, sipmPar_auxi1),
+                      SOA_COLUMN(float, sipmPar_fcByPE),
+                      SOA_COLUMN(float, sipmPar_darkCurrent),
+                      SOA_COLUMN(float, sipmPar_auxi2),
+                      SOA_SCALAR(int, maxDepthHB),
+                      SOA_SCALAR(int, maxDepthHE),
+                      SOA_SCALAR(int, maxPhiHE),
+                      SOA_SCALAR(int, firstHBRing),
+                      SOA_SCALAR(int, lastHBRing),
+                      SOA_SCALAR(int, firstHERing),
+                      SOA_SCALAR(int, lastHERing),
+                      SOA_SCALAR(int, nEtaHB),
+                      SOA_SCALAR(int, nEtaHE),
+                      SOA_SCALAR(uint32_t, offsetForHashes))
+  using HcalMahiConditionsSoA = HcalMahiConditionsSoALayout<>;
+
+}  // namespace hcal
+
+#endif

CondFormats/HcalObjects/interface/HcalMahiPulseOffsetsHost.h

@@ -0,0 +1,10 @@
+#ifndef CondFormats_HcalObjects_interface_HcalMahiPulseOffsetsPortable_h
+#define CondFormats_HcalObjects_interface_HcalMahiPulseOffsetsPortable_h
+
+#include "CondFormats/HcalObjects/interface/HcalMahiPulseOffsetsSoA.h"
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+
+namespace hcal {
+  using HcalMahiPulseOffsetsPortableHost = PortableHostCollection<HcalMahiPulseOffsetsSoA>;
+}
+#endif

CondFormats/HcalObjects/interface/HcalMahiPulseOffsetsSoA.h

@@ -0,0 +1,13 @@
+#ifndef CondFormats_HcalObjects_HcalMahiPulseOffsetsSoA_h
+#define CondFormats_HcalObjects_HcalMahiPulseOffsetsSoA_h
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+namespace hcal {
+  GENERATE_SOA_LAYOUT(HcalMahiPulseOffsetsSoALayout, SOA_COLUMN(int, offsets))
+  using HcalMahiPulseOffsetsSoA = HcalMahiPulseOffsetsSoALayout<>;
+
+}  // namespace hcal
+#endif

CondFormats/HcalObjects/interface/HcalRecoParamWithPulseShapeHost.h

@@ -0,0 +1,25 @@
+#ifndef CondFormats_HcalObjects_interface_HcalRecoParamWithPulseShapeHost_h
+#define CondFormats_HcalObjects_interface_HcalRecoParamWithPulseShapeHost_h
+
+#include "CondFormats/HcalObjects/interface/HcalRecoParamWithPulseShapeT.h"
+#include "DataFormats/Portable/interface/PortableCollection.h"
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+
+namespace hcal {
+  using HcalRecoParamWithPulseShapeHost = HcalRecoParamWithPulseShapeT<alpaka::DevCpu>;
+}
+namespace cms::alpakatools {
+  template <>
+  struct CopyToDevice<::hcal::HcalRecoParamWithPulseShapeHost> {
+    template <typename TQueue>
+    static auto copyAsync(TQueue& queue, ::hcal::HcalRecoParamWithPulseShapeHost const& hostProduct) {
+      using RecoParamCopy = CopyToDevice<::hcal::HcalRecoParamWithPulseShapeHost::RecoParamCollection>;
+      using PulseShapeCopy = CopyToDevice<::hcal::HcalRecoParamWithPulseShapeHost::PulseShapeCollection>;
+      using Device = alpaka::Dev<TQueue>;
+      return ::hcal::HcalRecoParamWithPulseShapeT<Device>(RecoParamCopy::copyAsync(queue, hostProduct.recoParam()),
+                                                          PulseShapeCopy::copyAsync(queue, hostProduct.pulseShape()));
+    }
+  };
+}  // namespace cms::alpakatools
+
+#endif

CondFormats/HcalObjects/interface/HcalRecoParamWithPulseShapeSoA.h

@@ -0,0 +1,31 @@
+#ifndef CondFormats_HcalObjects_HcalRecoParamWithPulseShapeSoA_h
+#define CondFormats_HcalObjects_HcalRecoParamWithPulseShapeSoA_h
+
+#include "RecoLocalCalo/HcalRecAlgos/interface/HcalConstants.h"
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+#include <array>
+
+namespace hcal {
+  using HcalPSfunctorArray = std::array<float, hcal::constants::maxPSshapeBin>;  // 256
+  using HcalPSfunctorBXarray = std::array<float, hcal::constants::nsPerBX>;      // 25
+
+  GENERATE_SOA_LAYOUT(HcalRecoParamSoALayout,
+                      SOA_COLUMN(uint32_t, param1),
+                      SOA_COLUMN(uint32_t, param2),
+                      SOA_COLUMN(uint32_t, ids))
+  GENERATE_SOA_LAYOUT(HcalPulseShapeSoALayout,
+                      SOA_COLUMN(HcalPSfunctorArray, acc25nsVec),
+                      SOA_COLUMN(HcalPSfunctorArray, diff25nsItvlVec),
+                      SOA_COLUMN(HcalPSfunctorBXarray, accVarLenIdxMinusOneVec),
+                      SOA_COLUMN(HcalPSfunctorBXarray, diffVarItvlIdxMinusOneVec),
+                      SOA_COLUMN(HcalPSfunctorBXarray, accVarLenIdxZEROVec),
+                      SOA_COLUMN(HcalPSfunctorBXarray, diffVarItvlIdxZEROVec))
+
+  using HcalRecoParamSoA = HcalRecoParamSoALayout<>;
+  using HcalPulseShapeSoA = HcalPulseShapeSoALayout<>;
+}  // namespace hcal
+#endif

CondFormats/HcalObjects/interface/HcalRecoParamWithPulseShapeT.h

@@ -0,0 +1,57 @@
+#ifndef CondFormats_HcalObjects_interface_HcalRecoParamWithPulseShapeHostT_h
+#define CondFormats_HcalObjects_interface_HcalRecoParamWithPulseShapeHostT_h
+
+#include "CondFormats/HcalObjects/interface/HcalRecoParamWithPulseShapeSoA.h"
+#include "DataFormats/Portable/interface/PortableCollection.h"
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+
+namespace hcal {
+  template <typename TDev>
+  class HcalRecoParamWithPulseShapeT {
+  public:
+    using RecoParamCollection = PortableCollection<HcalRecoParamSoA, TDev>;
+    using PulseShapeCollection = PortableCollection<HcalPulseShapeSoA, TDev>;
+
+    using PulseShapeConstElement = typename PulseShapeCollection::ConstView::const_element;
+
+    class ConstView {
+    public:
+      using RecoParamConstView = typename RecoParamCollection::ConstView;
+      using PulseShapeConstView = typename PulseShapeCollection::ConstView;
+      constexpr ConstView(RecoParamConstView recoView, PulseShapeConstView psView)
+          : recoParamView_{recoView}, pulseShapeView_{psView} {};
+
+      ALPAKA_FN_ACC PulseShapeConstElement getPulseShape(uint32_t const hashedId) const {
+        auto const recoPulseShapeId = recoParamView_.ids()[hashedId];
+        return pulseShapeView_[recoPulseShapeId];
+      }
+
+      constexpr typename RecoParamCollection::ConstView recoParamView() { return recoParamView_; }
+
+    private:
+      typename RecoParamCollection::ConstView recoParamView_;
+      typename PulseShapeCollection::ConstView pulseShapeView_;
+    };
+
+    HcalRecoParamWithPulseShapeT(size_t recoSize, size_t pulseSize, TDev const& dev)
+        : recoParam_(recoSize, dev), pulseShape_(pulseSize, dev) {}
+    template <typename TQueue, typename = std::enable_if_t<alpaka::isQueue<TQueue>>>
+    HcalRecoParamWithPulseShapeT(size_t recoSize, size_t pulseSize, TQueue const& queue)
+        : recoParam_(recoSize, queue), pulseShape_(pulseSize, queue) {}
+    HcalRecoParamWithPulseShapeT(RecoParamCollection reco, PulseShapeCollection pulse)
+        : recoParam_(std::move(reco)), pulseShape_(std::move(pulse)) {}
+
+    const RecoParamCollection& recoParam() const { return recoParam_; }
+    const PulseShapeCollection& pulseShape() const { return pulseShape_; }
+
+    typename RecoParamCollection::View recoParamView() { return recoParam_.view(); }
+    typename PulseShapeCollection::View pulseShapeView() { return pulseShape_.view(); }
+
+    ConstView const_view() const { return ConstView(recoParam_.view(), pulseShape_.view()); }
+
+  private:
+    RecoParamCollection recoParam_;
+    PulseShapeCollection pulseShape_;
+  };
+}  // namespace hcal
+#endif

CondFormats/HcalObjects/interface/HcalSiPMCharacteristicsHost.h

@@ -0,0 +1,12 @@
+#ifndef CondFormats_HcalObjects_interface_HcalSiPMCharacteristicsPortable_h
+#define CondFormats_HcalObjects_interface_HcalSiPMCharacteristicsPortable_h
+
+#include "CondFormats/HcalObjects/interface/HcalSiPMCharacteristicsSoA.h"
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/memory.h"
+
+namespace hcal {
+  using HcalSiPMCharacteristicsPortableHost = PortableHostCollection<HcalSiPMCharacteristicsSoA>;
+}
+#endif

CondFormats/HcalObjects/interface/HcalSiPMCharacteristicsSoA.h

@@ -0,0 +1,15 @@
+#ifndef CondFormats_HcalObjects_HcalSiPMCharacteristicsSoA_h
+#define CondFormats_HcalObjects_HcalSiPMCharacteristicsSoA_h
+
+#include "CondFormats/HcalObjects/interface/HcalSiPMCharacteristics.h"
+
+#include "DataFormats/SoATemplate/interface/SoACommon.h"
+#include "DataFormats/SoATemplate/interface/SoALayout.h"
+#include "DataFormats/SoATemplate/interface/SoAView.h"
+
+namespace hcal {
+  GENERATE_SOA_LAYOUT(HcalSiPMCharacteristicsSoALayout,
+                      SOA_COLUMN(HcalSiPMCharacteristics::PrecisionItem, precisionItem))
+  using HcalSiPMCharacteristicsSoA = HcalSiPMCharacteristicsSoALayout<>;
+}  // namespace hcal
+#endif

CondFormats/HcalObjects/interface/alpaka/HcalMahiConditionsDevice.h

@@ -0,0 +1,20 @@
+#ifndef CondFormats_HcalObjects_interface_alpaka_HcalMahiConditionsDevice_h
+#define CondFormats_HcalObjects_interface_alpaka_HcalMahiConditionsDevice_h
+
+#include "CondFormats/HcalObjects/interface/HcalMahiConditionsHost.h"
+#include "CondFormats/HcalObjects/interface/HcalMahiConditionsSoA.h"
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/memory.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  namespace hcal {
+
+    using ::hcal::HcalMahiConditionsPortableHost;
+    using HcalMahiConditionsPortableDevice = PortableCollection<::hcal::HcalMahiConditionsSoA>;
+  }  // namespace hcal
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif

CondFormats/HcalObjects/interface/alpaka/HcalMahiPulseOffsetsDevice.h

@@ -0,0 +1,20 @@
+#ifndef CondFormats_HcalObjects_interface_alpaka_HcalMahiPulseOffsetsPortable_h
+#define CondFormats_HcalObjects_interface_alpaka_HcalMahiPulseOffsetsPortable_h
+
+#include "CondFormats/HcalObjects/interface/HcalMahiPulseOffsetsHost.h"
+#include "CondFormats/HcalObjects/interface/HcalMahiPulseOffsetsSoA.h"
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/memory.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  namespace hcal {
+    using ::hcal::HcalMahiPulseOffsetsPortableHost;
+    using HcalMahiPulseOffsetsPortableDevice = PortableCollection<::hcal::HcalMahiPulseOffsetsSoA>;
+
+  }  // namespace hcal
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif

CondFormats/HcalObjects/interface/alpaka/HcalRecoParamWithPulseShapeDevice.h

@@ -0,0 +1,13 @@
+#ifndef CondFormats_HcalObjects_interface_alpaka_HcalRecoParamWithPulseShapeDevice_h
+#define CondFormats_HcalObjects_interface_alpaka_HcalRecoParamWithPulseShapeDevice_h
+
+#include "CondFormats/HcalObjects/interface/HcalRecoParamWithPulseShapeT.h"
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+  namespace hcal {
+    using HcalRecoParamWithPulseShapeDevice = ::hcal::HcalRecoParamWithPulseShapeT<Device>;
+  }
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif

CondFormats/HcalObjects/interface/alpaka/HcalSiPMCharacteristicsDevice.h

@@ -0,0 +1,19 @@
+#ifndef CondFormats_HcalObjects_interface_alpaka_HcalSiPMCharacteristicsPortable_h
+#define CondFormats_HcalObjects_interface_alpaka_HcalSiPMCharacteristicsPortable_h
+
+#include "CondFormats/HcalObjects/interface/HcalSiPMCharacteristicsHost.h"
+#include "CondFormats/HcalObjects/interface/HcalSiPMCharacteristicsSoA.h"
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/memory.h"
+
+namespace ALPAKA_ACCELERATOR_NAMESPACE {
+
+  namespace hcal {
+    using ::hcal::HcalSiPMCharacteristicsPortableHost;
+    using HcalSiPMCharacteristicsPortableDevice = PortableCollection<::hcal::HcalSiPMCharacteristicsSoA>;
+  }  // namespace hcal
+
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
+#endif

CondFormats/HcalObjects/src/ES_HcalMahiConditionsHost.cc

@@ -0,0 +1,4 @@
+#include "CondFormats/HcalObjects/interface/HcalMahiConditionsHost.h"
+#include "FWCore/Utilities/interface/typelookup.h"
+
+TYPELOOKUP_DATA_REG(hcal::HcalMahiConditionsPortableHost);

CondFormats/HcalObjects/src/ES_HcalMahiPulseOffsetsHost.cc

@@ -0,0 +1,4 @@
+#include "CondFormats/HcalObjects/interface/HcalMahiPulseOffsetsHost.h"
+#include "FWCore/Utilities/interface/typelookup.h"
+
+TYPELOOKUP_DATA_REG(hcal::HcalMahiPulseOffsetsPortableHost);