Merge pull request cms-sw#21 from fwyzard/hgcal-calibration_13_1_0_pr…

…e2_based

Rewrite the HGCalRecHitProducer and HGCalRecHitCalibrationAlgorithms to fix various problems

DataFormats/HGCalDigi/BuildFile.xml

@@ -1,9 +1,10 @@
-<use name="alpaka"/>
 <use name="eigen"/>
+<use name="rootcore"/>
 <use name="DataFormats/Common"/>
 <use name="DataFormats/ForwardDetId"/>
 <use name="DataFormats/DetId"/>
-<use name="HeterogeneousCore/AlpakaInterface" source_only="1"/>
+<use name="HeterogeneousCore/AlpakaInterface"/>
+<flags ALPAKA_BACKENDS="cuda rocm"/>
 <export>
   <lib name="1"/>
 </export>

DataFormats/HGCalDigi/src/alpaka/classes_cuda.h

@@ -1,5 +1,4 @@
 #include "DataFormats/Common/interface/DeviceProduct.h"
 #include "DataFormats/Common/interface/Wrapper.h"
-#include "DataFormats/Portable/interface/Product.h"
 #include "DataFormats/HGCalDigi/interface/HGCalDigiSoA.h"
 #include "DataFormats/HGCalDigi/interface/alpaka/HGCalDeviceDigiCollection.h"

DataFormats/HGCalDigi/src/alpaka/classes_rocm.h

@@ -1,5 +1,4 @@
 #include "DataFormats/Common/interface/DeviceProduct.h"
 #include "DataFormats/Common/interface/Wrapper.h"
-#include "DataFormats/Portable/interface/Product.h"
 #include "DataFormats/HGCalDigi/interface/HGCalDigiSoA.h"
 #include "DataFormats/HGCalDigi/interface/alpaka/HGCalDeviceDigiCollection.h"

DataFormats/HGCalRecHit/BuildFile.xml

@@ -1,9 +1,11 @@
-<use name="alpaka"/>
 <use name="eigen"/>
+<use name="rootcore"/>
 <use name="DataFormats/Common"/>
-<use name="DataFormats/ForwardDetId"/>
+<use name="DataFormats/Portable"/>
 <use name="DataFormats/DetId"/>
-<use name="HeterogeneousCore/AlpakaInterface" source_only="1"/>
+<use name="DataFormats/ForwardDetId"/>
+<use name="HeterogeneousCore/AlpakaInterface"/>
+<flags ALPAKA_BACKENDS="cuda rocm"/>
 <export>
   <lib name="1"/>
 </export>

DataFormats/HGCalRecHit/src/alpaka/classes_cuda.h

@@ -1,5 +1,4 @@
 #include "DataFormats/Common/interface/DeviceProduct.h"
 #include "DataFormats/Common/interface/Wrapper.h"
-#include "DataFormats/Portable/interface/Product.h"
 #include "DataFormats/HGCalRecHit/interface/HGCalRecHitSoA.h"
 #include "DataFormats/HGCalRecHit/interface/alpaka/HGCalDeviceRecHitCollection.h"

DataFormats/HGCalRecHit/src/alpaka/classes_rocm.h

@@ -1,5 +1,4 @@
 #include "DataFormats/Common/interface/DeviceProduct.h"
 #include "DataFormats/Common/interface/Wrapper.h"
-#include "DataFormats/Portable/interface/Product.h"
 #include "DataFormats/HGCalRecHit/interface/HGCalRecHitSoA.h"
 #include "DataFormats/HGCalRecHit/interface/alpaka/HGCalDeviceRecHitCollection.h"

DataFormats/HGCalRecHit/src/classes.h

@@ -1,4 +1,3 @@
-#include <vector>
 #include "DataFormats/Common/interface/Wrapper.h"
 #include "DataFormats/HGCalRecHit/interface/HGCalHostRecHitCollection.h"
 #include "DataFormats/HGCalRecHit/interface/HGCalRecHitSoA.h"

HeterogeneousCore/AlpakaRecHitTest/plugins/alpaka/AlpakaRecHitTest.cc

@@ -1,4 +1,3 @@
-#include "DataFormats/Portable/interface/Product.h"
 //#include "DataFormats/PortableTestObjects/interface/alpaka/TestDeviceCollection.h"
 #include "DataFormats/PortableTestObjects/interface/alpaka/TestDeviceRecHitCollection.h"
 #include "FWCore/Framework/interface/Frameworkfwd.h"

RecoLocalCalo/HGCalRecAlgos/plugins/BuildFile.xml

@@ -1,28 +1,8 @@
-<!-- host-only plugins -->
-<!-- <library file="*.cc" name="RecoLocalCaloHGCalRecAlgosPlugins">
-  <use name="alpaka"/>
-  <use name="fmt"/>
-  <use name="DataFormats/PortableTestObjects"/>
-  <use name="FWCore/Framework"/>
-  <use name="FWCore/MessageLogger"/>
-  <use name="FWCore/ParameterSet"/>
-  <use name="FWCore/Utilities"/>
-  <use name="RecoLocalCalo/HGCalRecAlgos"/>
-  <flags EDM_PLUGIN="1"/>
-</library> -->
-
-
-
 <!-- alpaka-based portable plugins -->
 <library file="alpaka/*.cc" name="RecoLocalCaloHGCalRecAlgosPluginsPortable">
   <use name="alpaka"/>
-  <!--
-  The dependency on "DataFormats/PortableTestObjects" automatically expands to include
-  the host-only library (if it exists) and the corresponding Alpaka libraries (if they exist)
-  -->
-  <use name="DataFormats/PortableTestObjects"/>
-  <use name="DataFormats/HGCalRecHit"/>
   <use name="DataFormats/HGCalDigi"/>
+  <use name="DataFormats/HGCalRecHit"/>
   <use name="DataFormats/TestObjects"/>
   <use name="FWCore/Framework"/>
   <use name="FWCore/ParameterSet"/>
@@ -32,4 +12,4 @@
   <use name="RecoLocalCalo/HGCalRecAlgos"/>
   <flags ALPAKA_BACKENDS="1"/>
   <flags EDM_PLUGIN="1"/>
-</library>
+</library>

RecoLocalCalo/HGCalRecAlgos/plugins/alpaka/HGCalRecHitCalibrationAlgorithms.dev.cc

@@ -9,17 +9,13 @@
 namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
   using namespace cms::alpakatools;
-  using namespace std;
 
   class HGCalRecHitCalibrationKernel_digisToRecHits {
   public:
-    template <typename TAcc, typename T>
-    ALPAKA_FN_ACC void operator()(TAcc const& acc, const T &digis,
-                                  HGCalDeviceRecHitCollection::View recHits) const {
-
-
-      auto ToA_to_time = [&](uint16_t ToA){return float(ToA);};
-      auto ADC_to_energy = [&](uint16_t ADC){return float(ADC);};
+    template <typename TAcc>
+    ALPAKA_FN_ACC void operator()(TAcc const& acc, HGCalDeviceDigiCollection::ConstView digis, HGCalDeviceRecHitCollection::View recHits) const {
+      auto ToA_to_time = [&](uint16_t ToA) { return float(ToA); };
+      auto ADC_to_energy = [&](uint16_t ADC) { return float(ADC); };
       // auto ToT_to_energy = [&](uint16_t ToT){return float(ToT);};
 
       // dummy digis -> rechits conversion (to be replaced by the actual formula)
@@ -33,99 +29,107 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
   };
 
   struct HGCalRecHitCalibrationKernel_pedestalCorrection {
-    template <typename TAcc, typename T>
-    ALPAKA_FN_ACC void operator()(TAcc const& acc, T recHits, float pedestalValue) const {
+    template <typename TAcc>
+    ALPAKA_FN_ACC void operator()(TAcc const& acc, HGCalDeviceRecHitCollection::View recHits, float pedestalValue) const {
       for (auto index : elements_with_stride(acc, recHits.metadata().size())) {
         recHits[index].energy() -= pedestalValue;
       }
     }
   };
 
   struct HGCalRecHitCalibrationKernel_commonModeCorrection {
-    template <typename TAcc, typename T>
-    ALPAKA_FN_ACC void operator()(TAcc const& acc, T recHits, float commonModeValue) const {
+    template <typename TAcc>
+    ALPAKA_FN_ACC void operator()(TAcc const& acc, HGCalDeviceRecHitCollection::View recHits, float commonModeValue) const {
       for (auto index : elements_with_stride(acc, recHits.metadata().size())) {
         recHits[index].energy() -= commonModeValue;
       }
     }
   };
 
   struct HGCalRecHitCalibrationKernel_ADCmCorrection {
-    template <typename TAcc, typename T>
-    ALPAKA_FN_ACC void operator()(TAcc const& acc, T recHits, float ADCmValue) const {
+    template <typename TAcc>
+    ALPAKA_FN_ACC void operator()(TAcc const& acc, HGCalDeviceRecHitCollection::View recHits, float ADCmValue) const {
       for (auto index : elements_with_stride(acc, recHits.metadata().size())) {
         recHits[index].energy() -= ADCmValue;
       }
     }
   };
 
-unique_ptr<HGCalDeviceRecHitCollection> HGCalRecHitCalibrationAlgorithms::calibrate(const HGCalHostDigiCollection &digis)
-{
-  cout<<"\n\nINFO -- Start of calibrate\n\n"<<endl;
-
-  auto queue = Queue(devices<Platform>()[0]); // should pick the right device somehow...
-  auto grid = make_workdiv<Acc1D>(4, 4);
-
-  cout << "Input digis: " << endl;
-  print(digis, 10);
-
-  cout<<"\n\nINFO -- converting digis to rechits"<<endl;
-  auto recHits = make_unique<HGCalDeviceRecHitCollection>(digis.view().metadata().size(), queue);
-
-  alpaka::exec<Acc1D>(queue, 
-                      grid, 
-                      HGCalRecHitCalibrationKernel_digisToRecHits{}, 
-                      digis.view(),
-                      recHits->view()
-  );
-
-  cout << "RecHits before calibration: " << endl;
-  print(recHits, 10);
-
-  float pedestalValue = 10;
-  alpaka::exec<Acc1D>(queue, grid, HGCalRecHitCalibrationKernel_pedestalCorrection{}, recHits->view(), pedestalValue);
-  cout << "RecHits after pedestal calibration: " << endl;
-  print(recHits, 10);
-
-  float commonModeValue = 10;
-  alpaka::exec<Acc1D>(queue, grid, HGCalRecHitCalibrationKernel_commonModeCorrection{}, recHits->view(), commonModeValue);
-  cout << "RecHits after CM calibration: " << endl;
-  print(recHits, 10);
-
-  float ADCmValue = 10;
-  alpaka::exec<Acc1D>(queue, grid, HGCalRecHitCalibrationKernel_ADCmCorrection{}, recHits->view(), ADCmValue);
-  cout << "RecHits after ADCm calibration: " << endl;
-  print(recHits, 10);
-
-  return recHits;
-}
-
-void HGCalRecHitCalibrationAlgorithms::print(const HGCalHostDigiCollection &digis, int max){
-  int max_ = max > 0 ? max : digis.view().metadata().size();
-  for(int i=0; i<max_; i++){
-    cout<<i;
-    cout<<"\t"<<digis.view()[i].electronicsId();
-    cout<<"\t"<<digis.view()[i].raw();
-    cout<<"\t"<<digis.view()[i].cm();
-    cout<<"\t"<<digis.view()[i].flags();
-    cout<<endl;
+  struct HGCalRecHitCalibrationKernel_printRecHits {
+    template <typename TAcc>
+    ALPAKA_FN_ACC void operator()(TAcc const& acc, HGCalDeviceRecHitCollection::ConstView view, int size) const {
+      for (int i = 0; i < size; ++i) {
+        auto const& rechit = view[i];
+        printf("%d\t%d\t%f\t%f\t%d\n", i, rechit.detid(), rechit.energy(), rechit.time(), rechit.flags());
+      }
+      printf("\n");
+    }
+  };
+
+  std::unique_ptr<HGCalDeviceRecHitCollection> HGCalRecHitCalibrationAlgorithms::calibrate(Queue& queue, HGCalHostDigiCollection const& digis) {
+    std::cout << "\n\nINFO -- Start of calibrate\n\n" << std::endl;
+
+    auto grid = make_workdiv<Acc1D>(4, 4);
+
+    std::cout << "Input digis: " << std::endl;
+    print(digis, 10);
+
+    std::cout << "\n\nINFO -- allocating rechits buffer" << std::endl;
+    auto recHits = std::make_unique<HGCalDeviceRecHitCollection>(digis.view().metadata().size(), queue);
+    print(queue, *recHits, 10);
+
+    std::cout << "\n\nINFO -- copying the digis to the device\n\n" << std::endl;
+    HGCalDeviceDigiCollection device_digis(digis.view().metadata().size(), queue);
+    alpaka::memcpy(queue, device_digis.buffer(), digis.const_buffer());
+
+    std::cout << "\n\nINFO -- converting digis to rechits" << std::endl;
+    alpaka::exec<Acc1D>(queue, grid, HGCalRecHitCalibrationKernel_digisToRecHits{}, device_digis.const_view(), recHits->view());
+
+    std::cout << "RecHits before calibration: " << std::endl;
+    print(queue, *recHits, 10);
+
+    float pedestalValue = 10;
+    alpaka::exec<Acc1D>(queue, grid, HGCalRecHitCalibrationKernel_pedestalCorrection{}, recHits->view(), pedestalValue);
+
+    std::cout << "RecHits after pedestal calibration: " << std::endl;
+    print(queue, *recHits, 10);
+
+    float commonModeValue = 10;
+    alpaka::exec<Acc1D>(queue, grid, HGCalRecHitCalibrationKernel_commonModeCorrection{}, recHits->view(), commonModeValue);
+
+    std::cout << "RecHits after CM calibration: " << std::endl;
+    print(queue, *recHits, 10);
+
+    float ADCmValue = 10;
+    alpaka::exec<Acc1D>(queue, grid, HGCalRecHitCalibrationKernel_ADCmCorrection{}, recHits->view(), ADCmValue);
+
+    std::cout << "RecHits after ADCm calibration: " << std::endl;
+    print(queue, *recHits, 10);
+
+    return recHits;
   }
-}
-
-void HGCalRecHitCalibrationAlgorithms::print(const unique_ptr<HGCalDeviceRecHitCollection> &recHits, int max){
-  int max_ = max > 0 ? max : recHits->view().metadata().size();
-  for(int i=0; i<max_; i++){
-    cout<<i;
-    cout<<"\t"<<recHits->view()[i].detid();
-    cout<<"\t"<<recHits->view()[i].energy();
-    cout<<"\t"<<recHits->view()[i].time();
-    cout<<"\t"<<recHits->view()[i].flags();
-    cout<<endl;
+
+  void HGCalRecHitCalibrationAlgorithms::print(HGCalHostDigiCollection const& digis, int max) const {
+    int max_ = max > 0 ? max : digis.view().metadata().size();
+    for (int i = 0; i < max_; i++) {
+      std::cout << i;
+      std::cout << "\t" << digis.view()[i].electronicsId();
+      std::cout << "\t" << digis.view()[i].raw();
+      std::cout << "\t" << digis.view()[i].cm();
+      std::cout << "\t" << digis.view()[i].flags();
+      std::cout << std::endl;
+    }
   }
-}
 
-} // namespace ALPAKA_ACCELERATOR_NAMESPACE
+  void HGCalRecHitCalibrationAlgorithms::print(Queue& queue, HGCalDeviceRecHitCollection const& recHits, int max) const {
+    auto grid = make_workdiv<Acc1D>(1, 1);
+    auto size = max > 0 ? max : recHits.view().metadata().size();
+    alpaka::exec<Acc1D>(queue, grid, HGCalRecHitCalibrationKernel_printRecHits{}, recHits.view(), size);
+    // ensure that the print operations are complete before returning
+    alpaka::wait(queue);
+  }
 
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
 
 //
 // Some potentially useful code snippets:
@@ -144,4 +148,4 @@ void HGCalRecHitCalibrationAlgorithms::print(const unique_ptr<HGCalDeviceRecHitC
 //     auto workDiv = make_workdiv<Acc1D>(groups, items);
 
 //     alpaka::exec<Acc1D>(queue, workDiv, HGCalRecHitCalibrationKernel{}, collection.view(), collection->metadata().size(), xvalue);
-//   }
+//   }

RecoLocalCalo/HGCalRecAlgos/plugins/alpaka/HGCalRecHitCalibrationAlgorithms.h

@@ -21,17 +21,17 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
   class HGCalRecHitCalibrationAlgorithms {
   public:
-    std::unique_ptr<HGCalDeviceRecHitCollection> calibrate(const HGCalHostDigiCollection &digis);
+    std::unique_ptr<HGCalDeviceRecHitCollection> calibrate(Queue& queue, HGCalHostDigiCollection const& digis);
 
     // if converting host digis to device rechits turns out too slow, we should copy host digis to device digis and then
     // convert to device rechits on device
-    // std::unique_ptr<HGCalDeviceRecHitCollection> calibrate(const std::unique_ptr<HGCalDeviceDigiCollection> &digis);
+    // std::unique_ptr<HGCalDeviceRecHitCollection> calibrate(Queue& queue, const std::unique_ptr<HGCalDeviceDigiCollection> &digis);
 
   private:
-    void print(const std::unique_ptr<HGCalDeviceRecHitCollection> &recHits, int max=-1);
-    void print(const HGCalHostDigiCollection &digis, int max=-1);
+    void print(Queue& queue, HGCalDeviceRecHitCollection const& recHits, int max = -1) const;
+    void print(HGCalHostDigiCollection const& digis, int max = -1) const;
   };
 
-} // namespace ALPAKA_ACCELERATOR_NAMESPACE
+}  // namespace ALPAKA_ACCELERATOR_NAMESPACE
 
-#endif // __HGCalRecHitCalibrationAlgorithms_H__
+#endif  // __HGCalRecHitCalibrationAlgorithms_H__