diff --git a/RecoTracker/LST/plugins/alpaka/LSTModulesDevESProducer.cc b/RecoTracker/LST/plugins/alpaka/LSTModulesDevESProducer.cc index e6b3c42b7ed10..6e7b14d4f0c88 100644 --- a/RecoTracker/LST/plugins/alpaka/LSTModulesDevESProducer.cc +++ b/RecoTracker/LST/plugins/alpaka/LSTModulesDevESProducer.cc @@ -15,41 +15,18 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class LSTModulesDevESProducer : public ESProducer { public: - LSTModulesDevESProducer(const edm::ParameterSet& iConfig); + LSTModulesDevESProducer(edm::ParameterSet const& iConfig) : ESProducer(iConfig) { setWhatProduced(this); } - static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); + static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + descriptions.addWithDefaultLabel(desc); + } - std::unique_ptr> produceHost(TrackerRecoGeometryRecord const& iRecord); - std::unique_ptr> produceDevice( - device::Record const& iRecord); - - private: - edm::ESGetToken, TrackerRecoGeometryRecord> lstESHostToken_; + std::unique_ptr> produce(TrackerRecoGeometryRecord const& iRecord) { + return SDL::loadAndFillESHost(); + } }; - LSTModulesDevESProducer::LSTModulesDevESProducer(const edm::ParameterSet& iConfig) : ESProducer(iConfig) { - setWhatProduced(this, &LSTModulesDevESProducer::produceHost); - auto cc = setWhatProduced(this, &LSTModulesDevESProducer::produceDevice); - lstESHostToken_ = cc.consumes(); - } - - void LSTModulesDevESProducer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { - edm::ParameterSetDescription desc; - descriptions.addWithDefaultLabel(desc); - } - - std::unique_ptr> LSTModulesDevESProducer::produceHost( - TrackerRecoGeometryRecord const& iRecord) { - return SDL::loadAndFillESHost(); - } - - std::unique_ptr> LSTModulesDevESProducer::produceDevice( - device::Record const& iRecord) { - auto const& lstESHostData = iRecord.get(lstESHostToken_); - SDL::QueueAcc& queue = iRecord.queue(); - return SDL::loadAndFillESDevice(queue, &lstESHostData); - } - } // namespace ALPAKA_ACCELERATOR_NAMESPACE DEFINE_FWK_EVENTSETUP_ALPAKA_MODULE(LSTModulesDevESProducer); diff --git a/RecoTracker/LST/plugins/alpaka/LSTProducer.cc b/RecoTracker/LST/plugins/alpaka/LSTProducer.cc index 0875eebe9f620..79896ad11e3f6 100644 --- a/RecoTracker/LST/plugins/alpaka/LSTProducer.cc +++ b/RecoTracker/LST/plugins/alpaka/LSTProducer.cc @@ -81,7 +81,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { private: edm::EDGetTokenT lstPixelSeedInputToken_; edm::EDGetTokenT lstPhase2OTHitsInputToken_; - device::ESGetToken, TrackerRecoGeometryRecord> lstESToken_; + device::ESGetToken, TrackerRecoGeometryRecord> lstESToken_; const int verbose_; edm::EDPutTokenT lstOutputToken_; diff --git a/RecoTracker/LST/src/alpaka/ES_ModulesDev.cc b/RecoTracker/LST/src/alpaka/ES_ModulesDev.cc index 9c66dda420db0..3964d050f924b 100644 --- a/RecoTracker/LST/src/alpaka/ES_ModulesDev.cc +++ b/RecoTracker/LST/src/alpaka/ES_ModulesDev.cc @@ -3,8 +3,5 @@ // Temporary hack: The DevHost instantiation is needed in both CPU and GPU plugins, // whereas the (non-host-)Device instantiation only in the GPU plugin -TYPELOOKUP_DATA_REG(SDL::LSTESHostData); -TYPELOOKUP_DATA_REG(SDL::LSTESDeviceData); -TYPELOOKUP_DATA_REG(ALPAKA_ACCELERATOR_NAMESPACE::ESDeviceProduct>>); -TYPELOOKUP_ALPAKA_TEMPLATED_DATA_REG(SDL::LSTESHostData); -TYPELOOKUP_ALPAKA_TEMPLATED_DATA_REG(SDL::LSTESDeviceData); +TYPELOOKUP_DATA_REG(SDL::LSTESData); +TYPELOOKUP_ALPAKA_TEMPLATED_DATA_REG(SDL::LSTESData); diff --git a/RecoTracker/LSTCore/interface/alpaka/LST.h b/RecoTracker/LSTCore/interface/alpaka/LST.h index 7d9d11745ab3c..9846e400d93f6 100644 --- a/RecoTracker/LSTCore/interface/alpaka/LST.h +++ b/RecoTracker/LSTCore/interface/alpaka/LST.h @@ -28,7 +28,7 @@ namespace SDL { void run(QueueAcc& queue, bool verbose, - const LSTESDeviceData* deviceESData, + const LSTESData* deviceESData, const std::vector see_px, const std::vector see_py, const std::vector see_pz, diff --git a/RecoTracker/LSTCore/interface/alpaka/LSTESData.h b/RecoTracker/LSTCore/interface/alpaka/LSTESData.h index 29ae19a5484e8..d48d2e5c6e78c 100644 --- a/RecoTracker/LSTCore/interface/alpaka/LSTESData.h +++ b/RecoTracker/LSTCore/interface/alpaka/LSTESData.h @@ -16,45 +16,14 @@ namespace SDL { struct pixelMap; - template - class TiltedGeometry; - - template - class ModuleConnectionMap; - using MapPLStoLayer = std::array, 4>, 3>; - template struct modulesBuffer; - template - class EndcapGeometryHost; - template class EndcapGeometry; template - struct LSTESHostData; - - // FIXME: This shouldn't be a templated struct - template <> - struct LSTESHostData { - std::shared_ptr mapPLStoLayer; - std::shared_ptr> endcapGeometry; - std::shared_ptr> tiltedGeometry; - std::shared_ptr> moduleConnectionMap; - - LSTESHostData(std::shared_ptr mapPLStoLayerIn, - std::shared_ptr> endcapGeometryIn, - std::shared_ptr> tiltedGeometryIn, - std::shared_ptr> moduleConnectionMapIn) - : mapPLStoLayer(mapPLStoLayerIn), - endcapGeometry(endcapGeometryIn), - tiltedGeometry(tiltedGeometryIn), - moduleConnectionMap(moduleConnectionMapIn) {} - }; - - template - struct LSTESDeviceData { + struct LSTESData { uint16_t nModules; uint16_t nLowerModules; unsigned int nPixels; @@ -62,12 +31,12 @@ namespace SDL { std::shared_ptr> endcapGeometry; std::shared_ptr pixelMapping; - LSTESDeviceData(uint16_t nModulesIn, - uint16_t nLowerModulesIn, - unsigned int nPixelsIn, - std::shared_ptr> modulesBuffersIn, - std::shared_ptr> endcapGeometryIn, - std::shared_ptr pixelMappingIn) + LSTESData(uint16_t const& nModulesIn, + uint16_t const& nLowerModulesIn, + unsigned int const& nPixelsIn, + std::shared_ptr> const& modulesBuffersIn, + std::shared_ptr> const& endcapGeometryIn, + std::shared_ptr const& pixelMappingIn) : nModules(nModulesIn), nLowerModules(nLowerModulesIn), nPixels(nPixelsIn), @@ -76,18 +45,15 @@ namespace SDL { pixelMapping(pixelMappingIn) {} }; - std::unique_ptr> loadAndFillESHost(); - std::unique_ptr> loadAndFillESDevice(SDL::QueueAcc& queue, const LSTESHostData* hostData); + std::unique_ptr> loadAndFillESHost(); } // namespace SDL namespace cms::alpakatools { template <> - struct CopyToDevice> { + struct CopyToDevice> { template - static auto copyAsync(TQueue& queue, SDL::LSTESHostData const& hostData) { - return std::make_unique>(hostData); - } + static SDL::LSTESData> copyAsync(TQueue& queue, SDL::LSTESData const& srcData); }; } // namespace cms::alpakatools diff --git a/RecoTracker/LSTCore/src/alpaka/EndcapGeometry.dev.cc b/RecoTracker/LSTCore/src/alpaka/EndcapGeometry.dev.cc index 2b5be62ec94bc..b2dbacaa0244d 100644 --- a/RecoTracker/LSTCore/src/alpaka/EndcapGeometry.dev.cc +++ b/RecoTracker/LSTCore/src/alpaka/EndcapGeometry.dev.cc @@ -1,16 +1,23 @@ #include "EndcapGeometry.h" -SDL::EndcapGeometry::EndcapGeometry(SDL::Dev const& devAccIn, - SDL::QueueAcc& queue, - SDL::EndcapGeometryHost const& endcapGeometryIn) - : geoMapDetId_buf(allocBufWrapper(devAccIn, endcapGeometryIn.centroid_phis_.size())), - geoMapPhi_buf(allocBufWrapper(devAccIn, endcapGeometryIn.centroid_phis_.size())) { - dxdy_slope_ = endcapGeometryIn.dxdy_slope_; - centroid_phis_ = endcapGeometryIn.centroid_phis_; - fillGeoMapArraysExplicit(queue); +template +SDL::EndcapGeometry::EndcapGeometry(TDev const& devAccIn) + : geoMapDetId_buf(allocBufWrapper(devAccIn, 0)), geoMapPhi_buf(allocBufWrapper(devAccIn, 0)) {} + +template +template +SDL::EndcapGeometry::EndcapGeometry(TQueue& queue, SDL::EndcapGeometry const& endcapGeometrySrc) + : dxdy_slope_(endcapGeometrySrc.dxdy_slope_), + centroid_phis_(endcapGeometrySrc.centroid_phis_), + geoMapDetId_buf(allocBufWrapper(alpaka::getDev(queue), endcapGeometrySrc.nEndCapMap)), + geoMapPhi_buf(allocBufWrapper(alpaka::getDev(queue), endcapGeometrySrc.nEndCapMap)), + nEndCapMap(endcapGeometrySrc.nEndCapMap) { + alpaka::memcpy(queue, geoMapPhi_buf, endcapGeometrySrc.geoMapPhi_buf); + alpaka::memcpy(queue, geoMapDetId_buf, endcapGeometrySrc.geoMapDetId_buf); } -void SDL::EndcapGeometryHost::load(std::string filename) { +template +void SDL::EndcapGeometry::load(std::string filename) { dxdy_slope_.clear(); centroid_phis_.clear(); @@ -38,19 +45,22 @@ void SDL::EndcapGeometryHost::load(std::string filename) { } } } + + fillGeoMapArraysExplicitHost(); } -void SDL::EndcapGeometry::fillGeoMapArraysExplicit(SDL::QueueAcc& queue) { +template +void SDL::EndcapGeometry::fillGeoMapArraysExplicitHost() { unsigned int phi_size = centroid_phis_.size(); // Allocate buffers on host SDL::DevHost const& devHost = cms::alpakatools::host(); - auto mapPhi_host_buf = allocBufWrapper(devHost, phi_size); - auto mapDetId_host_buf = allocBufWrapper(devHost, phi_size); + geoMapPhi_buf = allocBufWrapper(devHost, phi_size); + geoMapDetId_buf = allocBufWrapper(devHost, phi_size); // Access the raw pointers of the buffers - float* mapPhi = alpaka::getPtrNative(mapPhi_host_buf); - unsigned int* mapDetId = alpaka::getPtrNative(mapDetId_host_buf); + float* mapPhi = alpaka::getPtrNative(geoMapPhi_buf); + unsigned int* mapDetId = alpaka::getPtrNative(geoMapDetId_buf); unsigned int counter = 0; for (auto it = centroid_phis_.begin(); it != centroid_phis_.end(); ++it) { @@ -62,24 +72,18 @@ void SDL::EndcapGeometry::fillGeoMapArraysExplicit(SDL::QueueAcc& queu } nEndCapMap = counter; - - // Copy data from host to device buffers - alpaka::memcpy(queue, geoMapPhi_buf, mapPhi_host_buf); - alpaka::memcpy(queue, geoMapDetId_buf, mapDetId_host_buf); - alpaka::wait(queue); } -float SDL::EndcapGeometry::getdxdy_slope(unsigned int detid) const { - if (dxdy_slope_.find(detid) != dxdy_slope_.end()) { - return dxdy_slope_.at(detid); - } else { - return 0; - } -} -float SDL::EndcapGeometryHost::getdxdy_slope(unsigned int detid) const { +template +float SDL::EndcapGeometry::getdxdy_slope(unsigned int detid) const { if (dxdy_slope_.find(detid) != dxdy_slope_.end()) { return dxdy_slope_.at(detid); } else { return 0; } } + +// We need to instantiate these to ensure that the compiler generates them +template class SDL::EndcapGeometry; +template SDL::EndcapGeometry::EndcapGeometry(SDL::QueueAcc& queue, + SDL::EndcapGeometry const& endcapGeometrySrc); \ No newline at end of file diff --git a/RecoTracker/LSTCore/src/alpaka/EndcapGeometry.h b/RecoTracker/LSTCore/src/alpaka/EndcapGeometry.h index 93da945c00b33..3caaa4c152627 100644 --- a/RecoTracker/LSTCore/src/alpaka/EndcapGeometry.h +++ b/RecoTracker/LSTCore/src/alpaka/EndcapGeometry.h @@ -19,43 +19,29 @@ namespace SDL { - // FIXME: Need to separate this better into host and device classes - // This is only needed for host, but we template it to avoid symbol conflicts template - class EndcapGeometryHost; - - template <> - class EndcapGeometryHost { - public: - std::map dxdy_slope_; // dx/dy slope - std::map centroid_phis_; // centroid phi - - EndcapGeometryHost() = default; - ~EndcapGeometryHost() = default; - - void load(std::string); - float getdxdy_slope(unsigned int detid) const; - }; - - template - class EndcapGeometry; - - template <> - class EndcapGeometry { + class EndcapGeometry { private: std::map dxdy_slope_; // dx/dy slope std::map centroid_phis_; // centroid phi + // Friend all other instantiations of this template + template + friend class EndcapGeometry; + public: - Buf geoMapDetId_buf; - Buf geoMapPhi_buf; + Buf geoMapDetId_buf; + Buf geoMapPhi_buf; unsigned int nEndCapMap; - EndcapGeometry(Dev const& devAccIn, QueueAcc& queue, SDL::EndcapGeometryHost const& endcapGeometryIn); + EndcapGeometry(TDev const& devAccIn); + template + EndcapGeometry(TQueue& queue, EndcapGeometry const& endcapGeometrySrc); ~EndcapGeometry() = default; - void fillGeoMapArraysExplicit(QueueAcc& queue); + void load(std::string); + void fillGeoMapArraysExplicitHost(); float getdxdy_slope(unsigned int detid) const; }; } // namespace SDL diff --git a/RecoTracker/LSTCore/src/alpaka/Event.h b/RecoTracker/LSTCore/src/alpaka/Event.h index 3d301c2c65069..7cb957f0e893d 100644 --- a/RecoTracker/LSTCore/src/alpaka/Event.h +++ b/RecoTracker/LSTCore/src/alpaka/Event.h @@ -97,7 +97,7 @@ namespace SDL { public: // Constructor used for CMSSW integration. Uses an external queue. template - Event(bool verbose, TQueue const& q, const LSTESDeviceData* deviceESData) + Event(bool verbose, TQueue const& q, const LSTESData* deviceESData) : queue(q), devAcc(alpaka::getDev(q)), devHost(cms::alpakatools::host()), diff --git a/RecoTracker/LSTCore/src/alpaka/LST.dev.cc b/RecoTracker/LSTCore/src/alpaka/LST.dev.cc index 9eb11503123df..ec00bf1c7b3b9 100644 --- a/RecoTracker/LSTCore/src/alpaka/LST.dev.cc +++ b/RecoTracker/LSTCore/src/alpaka/LST.dev.cc @@ -11,7 +11,7 @@ using XYZVector = ROOT::Math::XYZVector; void SDL::LST::run(SDL::QueueAcc& queue, bool verbose, - const LSTESDeviceData* deviceESData, + const LSTESData* deviceESData, const std::vector see_px, const std::vector see_py, const std::vector see_pz, diff --git a/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc b/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc index 616c6f4bf3894..505807d301f89 100644 --- a/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc +++ b/RecoTracker/LSTCore/src/alpaka/LSTESData.dev.cc @@ -44,7 +44,7 @@ namespace { } void loadMapsHost(SDL::MapPLStoLayer& pLStoLayer, - std::shared_ptr> endcapGeometry, + std::shared_ptr> endcapGeometry, std::shared_ptr> tiltedGeometry, std::shared_ptr> moduleConnectionMap) { // Module orientation information (DrDz or phi angles) @@ -81,30 +81,21 @@ namespace { } } // namespace -std::unique_ptr> SDL::loadAndFillESHost() { - auto pLStoLayer = std::make_shared(); - auto endcapGeometry = std::make_shared>(); - auto tiltedGeometry = std::make_shared>(); - auto moduleConnectionMap = std::make_shared>(); - ::loadMapsHost(*pLStoLayer, endcapGeometry, tiltedGeometry, moduleConnectionMap); - return std::make_unique>(pLStoLayer, endcapGeometry, tiltedGeometry, moduleConnectionMap); -} - -std::unique_ptr> SDL::loadAndFillESDevice(SDL::QueueAcc& queue, - const LSTESHostData* hostData) { - SDL::Dev const& devAccIn = alpaka::getDev(queue); +std::unique_ptr> SDL::loadAndFillESHost() { uint16_t nModules; uint16_t nLowerModules; unsigned int nPixels; - std::shared_ptr> modulesBuffers = nullptr; - auto endcapGeometry = std::make_shared>(devAccIn, queue, *hostData->endcapGeometry); + std::shared_ptr> modulesBuffers = nullptr; + auto pLStoLayer = std::make_shared(); + auto endcapGeometry = std::make_shared>(cms::alpakatools::host()); + auto tiltedGeometry = std::make_shared>(); auto pixelMapping = std::make_shared(); - auto moduleConnectionMap = hostData->moduleConnectionMap; + auto moduleConnectionMap = std::make_shared>(); + ::loadMapsHost(*pLStoLayer, endcapGeometry, tiltedGeometry, moduleConnectionMap); auto path = get_absolute_path_after_check_file_exists(trackLooperDir() + "/data/OT800_IT615_pt0.8/sensor_centroids.bin"); - SDL::loadModulesFromFile(queue, - hostData->mapPLStoLayer.get(), + SDL::loadModulesFromFile(pLStoLayer.get(), path.c_str(), nModules, nLowerModules, @@ -112,8 +103,30 @@ std::unique_ptr> SDL::loadAndFillESDevice(SDL::Qu modulesBuffers, pixelMapping.get(), endcapGeometry.get(), - hostData->tiltedGeometry.get(), + tiltedGeometry.get(), moduleConnectionMap.get()); - return std::make_unique>( + return std::make_unique>( nModules, nLowerModules, nPixels, modulesBuffers, endcapGeometry, pixelMapping); } + +template +SDL::LSTESData> cms::alpakatools::CopyToDevice>::copyAsync( + TQueue& queue, SDL::LSTESData const& srcData) { + auto deviceModulesBuffers = std::make_shared>>( + alpaka::getDev(queue), srcData.nModules, srcData.nPixels); + deviceModulesBuffers->copyFromSrc(queue, *srcData.modulesBuffers); + auto deviceEndcapGeometry = + std::make_shared>>(queue, *srcData.endcapGeometry); + + return SDL::LSTESData>(srcData.nModules, + srcData.nLowerModules, + srcData.nPixels, + deviceModulesBuffers, + deviceEndcapGeometry, + srcData.pixelMapping); +} + +// Make sure it is compiled +template struct cms::alpakatools::CopyToDevice>; +template SDL::LSTESData> cms::alpakatools::CopyToDevice< + SDL::LSTESData>::copyAsync(SDL::QueueAcc&, SDL::LSTESData const&); diff --git a/RecoTracker/LSTCore/src/alpaka/ModuleMethods.h b/RecoTracker/LSTCore/src/alpaka/ModuleMethods.h index dc0eb0fc1f3b8..75b3ff08fe039 100644 --- a/RecoTracker/LSTCore/src/alpaka/ModuleMethods.h +++ b/RecoTracker/LSTCore/src/alpaka/ModuleMethods.h @@ -30,7 +30,7 @@ namespace SDL { }; template - inline void fillPixelMap(std::shared_ptr>& modulesBuf, + inline void fillPixelMap(std::shared_ptr>& modulesBuf, uint16_t nModules, unsigned int& nPixels, pixelMap& pixelMapping, @@ -88,12 +88,11 @@ namespace SDL { nPixels = connectedPix_size; // Now we can initialize modulesBuf + DevHost const& devHost = cms::alpakatools::host(); if (modulesBuf == nullptr) { - SDL::Dev const& devAcc = alpaka::getDev(queue); - modulesBuf = std::make_shared>(devAcc, nModules, nPixels); + modulesBuf = std::make_shared>(devHost, nModules, nPixels); } - DevHost const& devHost = cms::alpakatools::host(); auto connectedPixels_buf = allocBufWrapper(devHost, connectedPix_size); unsigned int* connectedPixels = alpaka::getPtrNative(connectedPixels_buf); @@ -225,18 +224,16 @@ namespace SDL { nModules = counter; }; - template - void loadModulesFromFile(TQueue& queue, - const MapPLStoLayer* pLStoLayer, - const char* moduleMetaDataFilePath, - uint16_t& nModules, - uint16_t& nLowerModules, - unsigned int& nPixels, - std::shared_ptr>& modulesBuf, - pixelMap* pixelMapping, - const EndcapGeometry* endcapGeometry, - const TiltedGeometry* tiltedGeometry, - const ModuleConnectionMap* moduleConnectionMap) { + inline void loadModulesFromFile(const MapPLStoLayer* pLStoLayer, + const char* moduleMetaDataFilePath, + uint16_t& nModules, + uint16_t& nLowerModules, + unsigned int& nPixels, + std::shared_ptr>& modulesBuf, + pixelMap* pixelMapping, + const EndcapGeometry* endcapGeometry, + const TiltedGeometry* tiltedGeometry, + const ModuleConnectionMap* moduleConnectionMap) { ModuleMetaData mmd; loadCentroidsFromFile(moduleMetaDataFilePath, mmd, nModules); @@ -381,6 +378,9 @@ namespace SDL { } } + // TODO: We don't need a queue, but this code needs to be refactored + alpaka::QueueCpuBlocking queue(cms::alpakatools::host()); + // modulesBuf is initialized in fillPixelMap since both nModules and nPix will be known fillPixelMap(modulesBuf, nModules, nPixels, *pixelMapping, queue, *pLStoLayer, mmd); diff --git a/RecoTracker/LSTCore/standalone/bin/sdl.cc b/RecoTracker/LSTCore/standalone/bin/sdl.cc index 3aed3e3c4d3b7..6f3f7e29d9a49 100644 --- a/RecoTracker/LSTCore/standalone/bin/sdl.cc +++ b/RecoTracker/LSTCore/standalone/bin/sdl.cc @@ -295,7 +295,7 @@ void run_sdl() { TStopwatch full_timer; full_timer.Start(); auto hostESData = SDL::loadAndFillESHost(); - auto deviceESData = SDL::loadAndFillESDevice(queues[0], hostESData.get()); + auto deviceESData = cms::alpakatools::CopyToDevice>::copyAsync(queues[0], *hostESData.get()); float timeForMapLoading = full_timer.RealTime() * 1000; if (ana.do_write_ntuple) { @@ -373,7 +373,7 @@ void run_sdl() { full_timer.Start(); std::vector *> events; for (int s = 0; s < ana.streams; s++) { - SDL::Event *event = new SDL::Event(ana.verbose >= 2, queues[s], deviceESData.get()); + SDL::Event *event = new SDL::Event(ana.verbose >= 2, queues[s], &deviceESData); events.push_back(event); } float timeForEventCreation = full_timer.RealTime() * 1000;