Merge pull request #49 from SegmentLinking/ESProducer_fixes

ESProducer fixes

RecoTracker/LST/interface/LSTOutput.h

@@ -1,5 +1,5 @@
-#ifndef RecoTracker_LST_LSTOutput_h
-#define RecoTracker_LST_LSTOutput_h
+#ifndef RecoTracker_LST_interface_LSTOutput_h
+#define RecoTracker_LST_interface_LSTOutput_h
 
 #include <memory>
 #include <vector>

RecoTracker/LST/interface/LSTPhase2OTHitsInput.h

@@ -1,5 +1,5 @@
-#ifndef RecoTracker_LST_LSTPhase2OTHitsInput_h
-#define RecoTracker_LST_LSTPhase2OTHitsInput_h
+#ifndef RecoTracker_LST_interface_LSTPhase2OTHitsInput_h
+#define RecoTracker_LST_interface_LSTPhase2OTHitsInput_h
 
 #include <memory>
 #include <vector>

RecoTracker/LST/interface/LSTPixelSeedInput.h

@@ -1,5 +1,5 @@
-#ifndef RecoTracker_LST_LSTPixelSeedInput_h
-#define RecoTracker_LST_LSTPixelSeedInput_h
+#ifndef RecoTracker_LST_interface_LSTPixelSeedInput_h
+#define RecoTracker_LST_interface_LSTPixelSeedInput_h
 
 #include <memory>
 #include <vector>

RecoTracker/LST/plugins/alpaka/LSTModulesDevESProducer.cc

@@ -15,41 +15,18 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
   class LSTModulesDevESProducer : public ESProducer {
   public:
-    LSTModulesDevESProducer(const edm::ParameterSet& iConfig);
+    LSTModulesDevESProducer(edm::ParameterSet const& iConfig) : ESProducer(iConfig) { setWhatProduced(this); }
 
-    static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
+    static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
+      edm::ParameterSetDescription desc;
+      descriptions.addWithDefaultLabel(desc);
+    }
 
-    std::unique_ptr<SDL::LSTESHostData<SDL::Dev>> produceHost(TrackerRecoGeometryRecord const& iRecord);
-    std::unique_ptr<SDL::LSTESDeviceData<SDL::Dev>> produceDevice(
-        device::Record<TrackerRecoGeometryRecord> const& iRecord);
-
-  private:
-    edm::ESGetToken<SDL::LSTESHostData<SDL::Dev>, TrackerRecoGeometryRecord> lstESHostToken_;
+    std::unique_ptr<SDL::LSTESData<SDL::DevHost>> produce(TrackerRecoGeometryRecord const& iRecord) {
+      return SDL::loadAndFillESHost();
+    }
   };
 
-  LSTModulesDevESProducer::LSTModulesDevESProducer(const edm::ParameterSet& iConfig) : ESProducer(iConfig) {
-    setWhatProduced(this, &LSTModulesDevESProducer::produceHost);
-    auto cc = setWhatProduced(this, &LSTModulesDevESProducer::produceDevice);
-    lstESHostToken_ = cc.consumes();
-  }
-
-  void LSTModulesDevESProducer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
-    edm::ParameterSetDescription desc;
-    descriptions.addWithDefaultLabel(desc);
-  }
-
-  std::unique_ptr<SDL::LSTESHostData<SDL::Dev>> LSTModulesDevESProducer::produceHost(
-      TrackerRecoGeometryRecord const& iRecord) {
-    return SDL::loadAndFillESHost();
-  }
-
-  std::unique_ptr<SDL::LSTESDeviceData<SDL::Dev>> LSTModulesDevESProducer::produceDevice(
-      device::Record<TrackerRecoGeometryRecord> const& iRecord) {
-    auto const& lstESHostData = iRecord.get(lstESHostToken_);
-    SDL::QueueAcc& queue = iRecord.queue();
-    return SDL::loadAndFillESDevice(queue, &lstESHostData);
-  }
-
 }  // namespace ALPAKA_ACCELERATOR_NAMESPACE
 
 DEFINE_FWK_EVENTSETUP_ALPAKA_MODULE(LSTModulesDevESProducer);

RecoTracker/LST/plugins/alpaka/LSTProducer.cc

@@ -87,7 +87,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
   private:
     edm::EDGetTokenT<LSTPixelSeedInput> lstPixelSeedInputToken_;
     edm::EDGetTokenT<LSTPhase2OTHitsInput> lstPhase2OTHitsInputToken_;
-    device::ESGetToken<SDL::LSTESDeviceData<SDL::Dev>, TrackerRecoGeometryRecord> lstESToken_;
+    device::ESGetToken<SDL::LSTESData<Device>, TrackerRecoGeometryRecord> lstESToken_;
     const bool verbose_, nopLSDupClean_, tcpLSTriplets_;
     edm::EDPutTokenT<LSTOutput> lstOutputToken_;
 

RecoTracker/LST/src/alpaka/ES_ModulesDev.cc

@@ -3,8 +3,5 @@
 
 // Temporary hack: The DevHost instantiation is needed in both CPU and GPU plugins,
 // whereas the (non-host-)Device instantiation only in the GPU plugin
-TYPELOOKUP_DATA_REG(SDL::LSTESHostData<SDL::Dev>);
-TYPELOOKUP_DATA_REG(SDL::LSTESDeviceData<SDL::DevHost>);
-TYPELOOKUP_DATA_REG(ALPAKA_ACCELERATOR_NAMESPACE::ESDeviceProduct<std::unique_ptr<SDL::LSTESHostData<SDL::Dev>>>);
-TYPELOOKUP_ALPAKA_TEMPLATED_DATA_REG(SDL::LSTESHostData);
-TYPELOOKUP_ALPAKA_TEMPLATED_DATA_REG(SDL::LSTESDeviceData);
+TYPELOOKUP_DATA_REG(SDL::LSTESData<SDL::DevHost>);
+TYPELOOKUP_ALPAKA_TEMPLATED_DATA_REG(SDL::LSTESData);

RecoTracker/LSTCore/BuildFile.xml

@@ -2,7 +2,7 @@
 <use name="boost_header"/>
 <use name="root"/>
 <use name="HeterogeneousCore/AlpakaInterface"/>
-<flags CXXFLAGS="-DCACHE_ALLOC -DT4FromT3 -DUSE_RZCHI2 -DUSE_T5_DNN -DPT_CUT=0.8 -DDUP_pLS -DDUP_T5 -DDUP_pT5 -DDUP_pT3 -DCrossclean_T5 -DCrossclean_pT3 -Wshadow"/>
+<flags CXXFLAGS="-DCACHE_ALLOC -DT4FromT3 -DUSE_RZCHI2 -DUSE_T5_DNN -DPT_CUT=0.8 -DDUP_pLS -DDUP_T5 -DDUP_pT5 -DDUP_pT3 -DCrossclean_T5 -DCrossclean_pT3"/>
 <flags ALPAKA_BACKENDS="1"/>
 <export>
   <lib name="1"/>

RecoTracker/LSTCore/interface/alpaka/Constants.h

@@ -1,5 +1,5 @@
-#ifndef Constants_cuh
-#define Constants_cuh
+#ifndef RecoTracker_LSTCore_interface_alpaka_Constants_h
+#define RecoTracker_LSTCore_interface_alpaka_Constants_h
 
 #include <alpaka/alpaka.hpp>
 

RecoTracker/LSTCore/interface/alpaka/EndcapGeometryBuffer.h

@@ -0,0 +1,57 @@
+#ifndef RecoTracker_LSTCore_interface_alpaka_EndcapGeometryBuffers_h
+#define RecoTracker_LSTCore_interface_alpaka_EndcapGeometryBuffers_h
+
+#include <map>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <stdexcept>
+
+#include "HeterogeneousCore/AlpakaInterface/interface/host.h"
+
+#include "RecoTracker/LSTCore/interface/alpaka/Constants.h"
+
+namespace SDL {
+
+  struct EndcapGeometryDev {
+    const unsigned int* geoMapDetId;
+    const float* geoMapPhi;
+
+    template <typename TBuff>
+    void setData(const TBuff& endcapgeombuf) {
+      geoMapDetId = alpaka::getPtrNative(endcapgeombuf.geoMapDetId_buf);
+      geoMapPhi = alpaka::getPtrNative(endcapgeombuf.geoMapPhi_buf);
+    }
+  };
+
+  template <typename TDev>
+  struct EndcapGeometryBuffer : EndcapGeometryDev {
+    Buf<TDev, unsigned int> geoMapDetId_buf;
+    Buf<TDev, float> geoMapPhi_buf;
+
+    EndcapGeometryBuffer(TDev const& dev, unsigned int nEndCapMap)
+        : geoMapDetId_buf(allocBufWrapper<unsigned int>(dev, nEndCapMap)),
+          geoMapPhi_buf(allocBufWrapper<float>(dev, nEndCapMap)) {
+      setData(*this);
+    }
+
+    template <typename TQueue, typename TDevSrc>
+    inline void copyFromSrc(TQueue queue, const EndcapGeometryBuffer<TDevSrc>& src) {
+      alpaka::memcpy(queue, geoMapDetId_buf, src.geoMapDetId_buf);
+      alpaka::memcpy(queue, geoMapPhi_buf, src.geoMapPhi_buf);
+    }
+
+    template <typename TQueue, typename TDevSrc>
+    EndcapGeometryBuffer(TQueue queue, const EndcapGeometryBuffer<TDevSrc>& src, unsigned int nEndCapMap)
+        : EndcapGeometryBuffer(alpaka::getDev(queue), nEndCapMap) {
+      copyFromSrc(queue, src);
+    }
+
+    inline SDL::EndcapGeometryDev const* data() const { return this; }
+  };
+
+}  // namespace SDL
+
+#endif

RecoTracker/LSTCore/interface/alpaka/LST.h

@@ -1,5 +1,5 @@
-#ifndef LST_H
-#define LST_H
+#ifndef RecoTracker_LSTCore_interface_alpaka_LST_h
+#define RecoTracker_LSTCore_interface_alpaka_LST_h
 
 #include "RecoTracker/LSTCore/interface/alpaka/Constants.h"
 #include "RecoTracker/LSTCore/interface/alpaka/LSTESData.h"
@@ -23,7 +23,7 @@ namespace SDL {
 
     void run(QueueAcc& queue,
              bool verbose,
-             const LSTESDeviceData<Dev>* deviceESData,
+             const LSTESData<Dev>* deviceESData,
              const std::vector<float> see_px,
              const std::vector<float> see_py,
              const std::vector<float> see_pz,

RecoTracker/LSTCore/interface/alpaka/LSTESData.h

@@ -1,7 +1,9 @@
-#ifndef LSTESData_H
-#define LSTESData_H
+#ifndef RecoTracker_LSTCore_interface_alpaka_LSTESData_h
+#define RecoTracker_LSTCore_interface_alpaka_LSTESData_h
 
 #include "RecoTracker/LSTCore/interface/alpaka/Constants.h"
+#include "RecoTracker/LSTCore/interface/alpaka/EndcapGeometryBuffer.h"
+#include "RecoTracker/LSTCore/interface/alpaka/Module.h"
 
 #include "HeterogeneousCore/AlpakaInterface/interface/CopyToDevice.h"
 
@@ -12,77 +14,56 @@ namespace SDL {
 
   struct pixelMap;
 
-  template <typename>
-  class TiltedGeometry;
-
-  template <typename>
-  class ModuleConnectionMap;
-  using MapPLStoLayer = std::array<std::array<ModuleConnectionMap<Dev>, 4>, 3>;
-
-  template <typename>
-  struct modulesBuffer;
-
-  template <typename>
-  class EndcapGeometryHost;
-
-  template <typename>
-  class EndcapGeometry;
-
-  template <typename TDev>
-  struct LSTESHostData;
-
-  // FIXME: This shouldn't be a templated struct
-  template <>
-  struct LSTESHostData<Dev> {
-    std::shared_ptr<const MapPLStoLayer> mapPLStoLayer;
-    std::shared_ptr<const EndcapGeometryHost<Dev>> endcapGeometry;
-    std::shared_ptr<const TiltedGeometry<Dev>> tiltedGeometry;
-    std::shared_ptr<const ModuleConnectionMap<Dev>> moduleConnectionMap;
-
-    LSTESHostData(std::shared_ptr<MapPLStoLayer> mapPLStoLayerIn,
-                  std::shared_ptr<EndcapGeometryHost<Dev>> endcapGeometryIn,
-                  std::shared_ptr<TiltedGeometry<Dev>> tiltedGeometryIn,
-                  std::shared_ptr<ModuleConnectionMap<Dev>> moduleConnectionMapIn)
-        : mapPLStoLayer(mapPLStoLayerIn),
-          endcapGeometry(endcapGeometryIn),
-          tiltedGeometry(tiltedGeometryIn),
-          moduleConnectionMap(moduleConnectionMapIn) {}
-  };
-
   template <typename TDev>
-  struct LSTESDeviceData {
+  struct LSTESData {
     uint16_t nModules;
     uint16_t nLowerModules;
     unsigned int nPixels;
+    unsigned int nEndCapMap;
     std::shared_ptr<const modulesBuffer<TDev>> modulesBuffers;
-    std::shared_ptr<const EndcapGeometry<TDev>> endcapGeometry;
+    std::shared_ptr<const EndcapGeometryBuffer<TDev>> endcapGeometryBuffers;
     std::shared_ptr<const pixelMap> pixelMapping;
 
-    LSTESDeviceData(uint16_t nModulesIn,
-                    uint16_t nLowerModulesIn,
-                    unsigned int nPixelsIn,
-                    std::shared_ptr<modulesBuffer<TDev>> modulesBuffersIn,
-                    std::shared_ptr<EndcapGeometry<TDev>> endcapGeometryIn,
-                    std::shared_ptr<pixelMap> pixelMappingIn)
+    LSTESData(uint16_t const& nModulesIn,
+              uint16_t const& nLowerModulesIn,
+              unsigned int const& nPixelsIn,
+              unsigned int const& nEndCapMapIn,
+              std::shared_ptr<const modulesBuffer<TDev>> const& modulesBuffersIn,
+              std::shared_ptr<const EndcapGeometryBuffer<TDev>> const& endcapGeometryBuffersIn,
+              std::shared_ptr<const pixelMap> const& pixelMappingIn)
         : nModules(nModulesIn),
           nLowerModules(nLowerModulesIn),
           nPixels(nPixelsIn),
+          nEndCapMap(nEndCapMapIn),
           modulesBuffers(modulesBuffersIn),
-          endcapGeometry(endcapGeometryIn),
+          endcapGeometryBuffers(endcapGeometryBuffersIn),
           pixelMapping(pixelMappingIn) {}
   };
 
-  std::unique_ptr<LSTESHostData<Dev>> loadAndFillESHost();
-  std::unique_ptr<LSTESDeviceData<Dev>> loadAndFillESDevice(SDL::QueueAcc& queue, const LSTESHostData<Dev>* hostData);
+  std::unique_ptr<LSTESData<alpaka_common::DevHost>> loadAndFillESHost();
 
 }  // namespace SDL
 
 namespace cms::alpakatools {
   template <>
-  struct CopyToDevice<SDL::LSTESHostData<SDL::Dev>> {
+  struct CopyToDevice<SDL::LSTESData<alpaka_common::DevHost>> {
     template <typename TQueue>
-    static auto copyAsync(TQueue& queue, SDL::LSTESHostData<SDL::Dev> const& hostData) {
-      return std::make_unique<SDL::LSTESHostData<SDL::Dev>>(hostData);
+    static SDL::LSTESData<alpaka::Dev<TQueue>> copyAsync(TQueue& queue,
+                                                         SDL::LSTESData<alpaka_common::DevHost> const& srcData) {
+      auto deviceModulesBuffers = std::make_shared<SDL::modulesBuffer<alpaka::Dev<TQueue>>>(
+          alpaka::getDev(queue), srcData.nModules, srcData.nPixels);
+      deviceModulesBuffers->copyFromSrc(queue, *srcData.modulesBuffers);
+      auto deviceEndcapGeometryBuffers =
+          std::make_shared<SDL::EndcapGeometryBuffer<alpaka::Dev<TQueue>>>(alpaka::getDev(queue), srcData.nEndCapMap);
+      deviceEndcapGeometryBuffers->copyFromSrc(queue, *srcData.endcapGeometryBuffers);
+
+      return SDL::LSTESData<alpaka::Dev<TQueue>>(srcData.nModules,
+                                                 srcData.nLowerModules,
+                                                 srcData.nPixels,
+                                                 srcData.nEndCapMap,
+                                                 deviceModulesBuffers,
+                                                 deviceEndcapGeometryBuffers,
+                                                 srcData.pixelMapping);
     }
   };
 }  // namespace cms::alpakatools

RecoTracker/LSTCore/interface/alpaka/Module.h

@@ -1,5 +1,5 @@
-#ifndef Module_cuh
-#define Module_cuh
+#ifndef RecoTracker_LSTCore_interface_alpaka_Module_h
+#define RecoTracker_LSTCore_interface_alpaka_Module_h
 
 #include <alpaka/alpaka.hpp>
 
@@ -355,8 +355,8 @@ namespace SDL {
       alpaka::wait(queue);
     }
 
-    template <typename TQueue>
-    modulesBuffer(TQueue queue, const modulesBuffer<alpaka::DevCpu>& src, unsigned int nMod, unsigned int nPixs)
+    template <typename TQueue, typename TDevSrc>
+    modulesBuffer(TQueue queue, const modulesBuffer<TDevSrc>& src, unsigned int nMod, unsigned int nPixs)
         : modulesBuffer(alpaka::getDev(queue), nMod, nPixs) {
       copyFromSrc(queue, src);
     }

RecoTracker/LSTCore/src/alpaka/EndcapGeometry.dev.cc

@@ -1,16 +1,8 @@
 #include "EndcapGeometry.h"
 
-SDL::EndcapGeometry<SDL::Dev>::EndcapGeometry(SDL::Dev const& devAccIn,
-                                              SDL::QueueAcc& queue,
-                                              SDL::EndcapGeometryHost<SDL::Dev> const& endcapGeometryIn)
-    : geoMapDetId_buf(allocBufWrapper<unsigned int>(devAccIn, endcapGeometryIn.centroid_phis_.size())),
-      geoMapPhi_buf(allocBufWrapper<float>(devAccIn, endcapGeometryIn.centroid_phis_.size())) {
-  dxdy_slope_ = endcapGeometryIn.dxdy_slope_;
-  centroid_phis_ = endcapGeometryIn.centroid_phis_;
-  fillGeoMapArraysExplicit(queue);
-}
+SDL::EndcapGeometry<SDL::Dev>::EndcapGeometry(std::string filename) { load(filename); }
 
-void SDL::EndcapGeometryHost<SDL::Dev>::load(std::string filename) {
+void SDL::EndcapGeometry<SDL::Dev>::load(std::string filename) {
   dxdy_slope_.clear();
   centroid_phis_.clear();
 
@@ -38,35 +30,22 @@ void SDL::EndcapGeometryHost<SDL::Dev>::load(std::string filename) {
       }
     }
   }
-}
 
-void SDL::EndcapGeometry<SDL::Dev>::fillGeoMapArraysExplicit(SDL::QueueAcc& queue) {
-  unsigned int phi_size = centroid_phis_.size();
+  fillGeoMapArraysExplicit();
+}
 
-  // Allocate buffers on host
-  SDL::DevHost const& devHost = cms::alpakatools::host();
-  auto mapPhi_host_buf = allocBufWrapper<float>(devHost, phi_size);
-  auto mapDetId_host_buf = allocBufWrapper<unsigned int>(devHost, phi_size);
+void SDL::EndcapGeometry<SDL::Dev>::fillGeoMapArraysExplicit() {
+  nEndCapMap = centroid_phis_.size();
 
-  // Access the raw pointers of the buffers
-  float* mapPhi = alpaka::getPtrNative(mapPhi_host_buf);
-  unsigned int* mapDetId = alpaka::getPtrNative(mapDetId_host_buf);
+  geoMapDetId_buf.reserve(nEndCapMap);
+  geoMapPhi_buf.reserve(nEndCapMap);
 
-  unsigned int counter = 0;
   for (auto it = centroid_phis_.begin(); it != centroid_phis_.end(); ++it) {
     unsigned int detId = it->first;
     float Phi = it->second;
-    mapPhi[counter] = Phi;
-    mapDetId[counter] = detId;
-    counter++;
+    geoMapPhi_buf.push_back(Phi);
+    geoMapDetId_buf.push_back(detId);
   }
-
-  nEndCapMap = counter;
-
-  // Copy data from host to device buffers
-  alpaka::memcpy(queue, geoMapPhi_buf, mapPhi_host_buf);
-  alpaka::memcpy(queue, geoMapDetId_buf, mapDetId_host_buf);
-  alpaka::wait(queue);
 }
 
 float SDL::EndcapGeometry<SDL::Dev>::getdxdy_slope(unsigned int detid) const {
@@ -76,10 +55,3 @@ float SDL::EndcapGeometry<SDL::Dev>::getdxdy_slope(unsigned int detid) const {
     return 0;
   }
 }
-float SDL::EndcapGeometryHost<SDL::Dev>::getdxdy_slope(unsigned int detid) const {
-  if (dxdy_slope_.find(detid) != dxdy_slope_.end()) {
-    return dxdy_slope_.at(detid);
-  } else {
-    return 0;
-  }
-}