Skip to content

Commit

Permalink
Merge pull request #77 from slava77/CMSSW_14_1_0_pre5/LSTb5-alpaka-ca…
Browse files Browse the repository at this point in the history 
…lls-review

batch5: alpaka::wait and use views to local host data instead of buffers
  • Loading branch information
@ariostas
ariostas authored Aug 16, 2024
2 parents c1767fa + 7889093 commit 4ff4aaf
Show file tree
Hide file tree
Showing 21 changed files with 957 additions and 1,021 deletions.
16 changes: 7 additions & 9 deletions RecoTracker/LSTCore/interface/Constants.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,21 @@ namespace lst {
using Buf = alpaka::Buf<TDev, TData, alpaka_common::Dim1D, alpaka_common::Idx>;

// Allocation wrapper function to make integration of the caching allocator easier and reduce code boilerplate.
template <typename T, typename TAcc, typename TSize, typename TQueue>
ALPAKA_FN_HOST ALPAKA_FN_INLINE Buf<alpaka::Dev<TAcc>, T> allocBufWrapper(TAcc const& devAccIn,
TSize nElements,
TQueue queue) {
template <typename T, typename TDev, typename TSize, typename TQueue>
ALPAKA_FN_HOST ALPAKA_FN_INLINE Buf<TDev, T> allocBufWrapper(TDev const& dev, TSize nElements, TQueue queue) {
#ifdef CACHE_ALLOC
return cms::alpakatools::allocCachedBuf<T, alpaka_common::Idx>(
devAccIn, queue, alpaka_common::Vec1D(static_cast<alpaka_common::Idx>(nElements)));
dev, queue, alpaka_common::Vec1D(static_cast<alpaka_common::Idx>(nElements)));
#else
return alpaka::allocBuf<T, alpaka_common::Idx>(devAccIn,
return alpaka::allocBuf<T, alpaka_common::Idx>(dev,
alpaka_common::Vec1D(static_cast<alpaka_common::Idx>(nElements)));
#endif
}

// Second allocation wrapper function when queue is not given. Reduces code boilerplate.
template <typename T, typename TAcc, typename TSize>
ALPAKA_FN_HOST ALPAKA_FN_INLINE Buf<alpaka::Dev<TAcc>, T> allocBufWrapper(TAcc const& devAccIn, TSize nElements) {
return alpaka::allocBuf<T, alpaka_common::Idx>(devAccIn,
template <typename T, typename TDev, typename TSize>
ALPAKA_FN_HOST ALPAKA_FN_INLINE Buf<TDev, T> allocBufWrapper(TDev const& dev, TSize nElements) {
return alpaka::allocBuf<T, alpaka_common::Idx>(dev,
alpaka_common::Vec1D(static_cast<alpaka_common::Idx>(nElements)));
}

Expand Down
4 changes: 2 additions & 2 deletions RecoTracker/LSTCore/interface/EndcapGeometryBuffer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ namespace lst {

template <typename TBuff>
void setData(TBuff const& buf) {
geoMapDetId = alpaka::getPtrNative(buf.geoMapDetId_buf);
geoMapPhi = alpaka::getPtrNative(buf.geoMapPhi_buf);
geoMapDetId = buf.geoMapDetId_buf.data();
geoMapPhi = buf.geoMapPhi_buf.data();
}
};

Expand Down
59 changes: 29 additions & 30 deletions RecoTracker/LSTCore/interface/Module.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,44 +72,44 @@ namespace lst {
} else {
return false;
}
};
}

static bool parseIsLower(bool isInvertedx, unsigned int detId) {
return (isInvertedx) ? !(detId & 1) : (detId & 1);
};
}

static unsigned int parsePartnerModuleId(unsigned int detId, bool isLowerx, bool isInvertedx) {
return isLowerx ? (isInvertedx ? detId - 1 : detId + 1) : (isInvertedx ? detId + 1 : detId - 1);
};
}

template <typename TBuff>
void setData(TBuff const& buf) {
detIds = alpaka::getPtrNative(buf.detIds_buf);
moduleMap = alpaka::getPtrNative(buf.moduleMap_buf);
mapdetId = alpaka::getPtrNative(buf.mapdetId_buf);
mapIdx = alpaka::getPtrNative(buf.mapIdx_buf);
nConnectedModules = alpaka::getPtrNative(buf.nConnectedModules_buf);
drdzs = alpaka::getPtrNative(buf.drdzs_buf);
dxdys = alpaka::getPtrNative(buf.dxdys_buf);
nModules = alpaka::getPtrNative(buf.nModules_buf);
nLowerModules = alpaka::getPtrNative(buf.nLowerModules_buf);
partnerModuleIndices = alpaka::getPtrNative(buf.partnerModuleIndices_buf);

layers = alpaka::getPtrNative(buf.layers_buf);
rings = alpaka::getPtrNative(buf.rings_buf);
modules = alpaka::getPtrNative(buf.modules_buf);
rods = alpaka::getPtrNative(buf.rods_buf);
subdets = alpaka::getPtrNative(buf.subdets_buf);
sides = alpaka::getPtrNative(buf.sides_buf);
eta = alpaka::getPtrNative(buf.eta_buf);
r = alpaka::getPtrNative(buf.r_buf);
isInverted = alpaka::getPtrNative(buf.isInverted_buf);
isLower = alpaka::getPtrNative(buf.isLower_buf);
isAnchor = alpaka::getPtrNative(buf.isAnchor_buf);
moduleType = alpaka::getPtrNative(buf.moduleType_buf);
moduleLayerType = alpaka::getPtrNative(buf.moduleLayerType_buf);
lstLayers = alpaka::getPtrNative(buf.lstLayers_buf);
connectedPixels = alpaka::getPtrNative(buf.connectedPixels_buf);
detIds = buf.detIds_buf.data();
moduleMap = buf.moduleMap_buf.data();
mapdetId = buf.mapdetId_buf.data();
mapIdx = buf.mapIdx_buf.data();
nConnectedModules = buf.nConnectedModules_buf.data();
drdzs = buf.drdzs_buf.data();
dxdys = buf.dxdys_buf.data();
nModules = buf.nModules_buf.data();
nLowerModules = buf.nLowerModules_buf.data();
partnerModuleIndices = buf.partnerModuleIndices_buf.data();

layers = buf.layers_buf.data();
rings = buf.rings_buf.data();
modules = buf.modules_buf.data();
rods = buf.rods_buf.data();
subdets = buf.subdets_buf.data();
sides = buf.sides_buf.data();
eta = buf.eta_buf.data();
r = buf.r_buf.data();
isInverted = buf.isInverted_buf.data();
isLower = buf.isLower_buf.data();
isAnchor = buf.isAnchor_buf.data();
moduleType = buf.moduleType_buf.data();
moduleLayerType = buf.moduleLayerType_buf.data();
lstLayers = buf.lstLayers_buf.data();
connectedPixels = buf.connectedPixels_buf.data();
}
};

Expand Down Expand Up @@ -212,7 +212,6 @@ namespace lst {
alpaka::memcpy(queue, lstLayers_buf, src.lstLayers_buf);
alpaka::memcpy(queue, connectedPixels_buf, src.connectedPixels_buf);
}
alpaka::wait(queue);
}

template <typename TQueue, typename TDevSrc>
Expand Down
27 changes: 13 additions & 14 deletions RecoTracker/LSTCore/interface/alpaka/Constants.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,25 +38,24 @@ namespace lst {
#endif

// Adjust grid and block sizes based on backend configuration
template <typename Vec>
ALPAKA_FN_HOST ALPAKA_FN_INLINE WorkDiv3D createWorkDiv(const Vec& blocksPerGrid,
const Vec& threadsPerBlock,
const Vec& elementsPerThreadArg) {
template <typename Vec, typename TAcc = ALPAKA_ACCELERATOR_NAMESPACE::Acc<typename Vec::Dim>>
ALPAKA_FN_HOST ALPAKA_FN_INLINE WorkDiv<typename Vec::Dim> createWorkDiv(const Vec& blocksPerGrid,
const Vec& threadsPerBlock,
const Vec& elementsPerThreadArg) {
Vec adjustedBlocks = blocksPerGrid;
Vec adjustedThreads = threadsPerBlock;

// Serial execution, so all launch parameters set to 1.
#if defined(ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED)
adjustedBlocks = Vec::all(static_cast<Idx>(1));
adjustedThreads = Vec::all(static_cast<Idx>(1));
#endif
// special overrides for CPU/host cases
if constexpr (std::is_same_v<Platform, alpaka::PlatformCpu>) {
adjustedBlocks = Vec::all(static_cast<Idx>(1));

// Threads enabled, set number of blocks to 1.
#if defined(ALPAKA_ACC_CPU_B_SEQ_T_THREADS_ENABLED)
adjustedBlocks = Vec::all(static_cast<Idx>(1));
#endif
if constexpr (alpaka::accMatchesTags<TAcc, alpaka::TagCpuSerial>) {
// Serial execution, set threads to 1 as well
adjustedThreads = Vec::all(static_cast<Idx>(1)); // probably redundant
}
}

return WorkDiv3D(adjustedBlocks, adjustedThreads, elementsPerThreadArg);
return WorkDiv<typename Vec::Dim>(adjustedBlocks, adjustedThreads, elementsPerThreadArg);
}

// The constants below are usually used in functions like alpaka::math::min(),
Expand Down
4 changes: 2 additions & 2 deletions RecoTracker/LSTCore/src/LSTESData.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,10 @@ std::unique_ptr<lst::LSTESData<alpaka_common::DevHost>> lst::loadAndFillESHost()

auto endcapGeometryBuffers =
EndcapGeometryBuffer<alpaka_common::DevHost>(cms::alpakatools::host(), endcapGeometry.nEndCapMap);
std::memcpy(alpaka::getPtrNative(endcapGeometryBuffers.geoMapDetId_buf),
std::memcpy(endcapGeometryBuffers.geoMapDetId_buf.data(),
endcapGeometry.geoMapDetId_buf.data(),
endcapGeometry.nEndCapMap * sizeof(unsigned int));
std::memcpy(alpaka::getPtrNative(endcapGeometryBuffers.geoMapPhi_buf),
std::memcpy(endcapGeometryBuffers.geoMapPhi_buf.data(),
endcapGeometry.geoMapPhi_buf.data(),
endcapGeometry.nEndCapMap * sizeof(float));

Expand Down
65 changes: 33 additions & 32 deletions RecoTracker/LSTCore/src/ModuleMethods.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include "RecoTracker/LSTCore/interface/PixelMap.h"

#include "HeterogeneousCore/AlpakaInterface/interface/host.h"
#include "HeterogeneousCore/AlpakaInterface/interface/memory.h"

namespace lst {
struct ModuleMetaData {
Expand Down Expand Up @@ -80,10 +81,10 @@ namespace lst {
nPixels = connectedPix_size;

// Now we re-initialize connectedPixels_buf since nPixels is now known
modulesBuf.connectedPixels_buf = allocBufWrapper<unsigned int>(cms::alpakatools::host(), nPixels);
modulesBuf.connectedPixels_buf = cms::alpakatools::make_host_buffer<unsigned int[]>(nPixels);
modulesBuf.data_.setData(modulesBuf);

unsigned int* connectedPixels = alpaka::getPtrNative(modulesBuf.connectedPixels_buf);
unsigned int* connectedPixels = modulesBuf.connectedPixels_buf.data();

for (unsigned int icondet = 0; icondet < totalSizes; icondet++) {
connectedPixels[icondet] = mmd.detIdToIndex.at(connectedModuleDetIds[icondet]);
Expand All @@ -94,13 +95,13 @@ namespace lst {
for (unsigned int icondet = 0; icondet < totalSizes_neg; icondet++) {
connectedPixels[icondet + totalSizes + totalSizes_pos] = mmd.detIdToIndex.at(connectedModuleDetIds_neg[icondet]);
}
};
}

inline void fillConnectedModuleArrayExplicit(ModulesBuffer<alpaka_common::DevHost>& modulesBuf,
ModuleMetaData const& mmd,
ModuleConnectionMap const& moduleConnectionMap) {
uint16_t* moduleMap = alpaka::getPtrNative(modulesBuf.moduleMap_buf);
uint16_t* nConnectedModules = alpaka::getPtrNative(modulesBuf.nConnectedModules_buf);
uint16_t* moduleMap = modulesBuf.moduleMap_buf.data();
uint16_t* nConnectedModules = modulesBuf.nConnectedModules_buf.data();

for (auto it = mmd.detIdToIndex.begin(); it != mmd.detIdToIndex.end(); ++it) {
unsigned int detId = it->first;
Expand All @@ -111,11 +112,11 @@ namespace lst {
moduleMap[index * max_connected_modules + i] = mmd.detIdToIndex.at(connectedModules[i]);
}
}
};
}

inline void fillMapArraysExplicit(ModulesBuffer<alpaka_common::DevHost>& modulesBuf, ModuleMetaData const& mmd) {
uint16_t* mapIdx = alpaka::getPtrNative(modulesBuf.mapIdx_buf);
unsigned int* mapdetId = alpaka::getPtrNative(modulesBuf.mapdetId_buf);
uint16_t* mapIdx = modulesBuf.mapIdx_buf.data();
unsigned int* mapdetId = modulesBuf.mapdetId_buf.data();

unsigned int counter = 0;
for (auto it = mmd.detIdToIndex.begin(); it != mmd.detIdToIndex.end(); ++it) {
Expand All @@ -125,7 +126,7 @@ namespace lst {
mapdetId[counter] = detId;
counter++;
}
};
}

inline void setDerivedQuantities(unsigned int detId,
unsigned short& layer,
Expand All @@ -148,7 +149,7 @@ namespace lst {

r = std::sqrt(m_x * m_x + m_y * m_y + m_z * m_z);
eta = ((m_z > 0) - (m_z < 0)) * std::acosh(r / std::sqrt(m_x * m_x + m_y * m_y));
};
}

inline void loadCentroidsFromFile(const char* filePath, ModuleMetaData& mmd, uint16_t& nModules) {
std::ifstream ifile(filePath, std::ios::binary);
Expand Down Expand Up @@ -185,7 +186,7 @@ namespace lst {
mmd.detIdToIndex[1] = counter; //pixel module is the last module in the module list
counter++;
nModules = counter;
};
}

inline ModulesBuffer<alpaka_common::DevHost> loadModulesFromFile(MapPLStoLayer const& pLStoLayer,
const char* moduleMetaDataFilePath,
Expand All @@ -205,26 +206,26 @@ namespace lst {
ModulesBuffer<alpaka_common::DevHost> modulesBuf(cms::alpakatools::host(), nModules, 0);

// Getting the underlying data pointers
unsigned int* host_detIds = alpaka::getPtrNative(modulesBuf.detIds_buf);
short* host_layers = alpaka::getPtrNative(modulesBuf.layers_buf);
short* host_rings = alpaka::getPtrNative(modulesBuf.rings_buf);
short* host_rods = alpaka::getPtrNative(modulesBuf.rods_buf);
short* host_modules = alpaka::getPtrNative(modulesBuf.modules_buf);
short* host_subdets = alpaka::getPtrNative(modulesBuf.subdets_buf);
short* host_sides = alpaka::getPtrNative(modulesBuf.sides_buf);
float* host_eta = alpaka::getPtrNative(modulesBuf.eta_buf);
float* host_r = alpaka::getPtrNative(modulesBuf.r_buf);
bool* host_isInverted = alpaka::getPtrNative(modulesBuf.isInverted_buf);
bool* host_isLower = alpaka::getPtrNative(modulesBuf.isLower_buf);
bool* host_isAnchor = alpaka::getPtrNative(modulesBuf.isAnchor_buf);
ModuleType* host_moduleType = alpaka::getPtrNative(modulesBuf.moduleType_buf);
ModuleLayerType* host_moduleLayerType = alpaka::getPtrNative(modulesBuf.moduleLayerType_buf);
float* host_dxdys = alpaka::getPtrNative(modulesBuf.dxdys_buf);
float* host_drdzs = alpaka::getPtrNative(modulesBuf.drdzs_buf);
uint16_t* host_nModules = alpaka::getPtrNative(modulesBuf.nModules_buf);
uint16_t* host_nLowerModules = alpaka::getPtrNative(modulesBuf.nLowerModules_buf);
uint16_t* host_partnerModuleIndices = alpaka::getPtrNative(modulesBuf.partnerModuleIndices_buf);
int* host_lstLayers = alpaka::getPtrNative(modulesBuf.lstLayers_buf);
unsigned int* host_detIds = modulesBuf.detIds_buf.data();
short* host_layers = modulesBuf.layers_buf.data();
short* host_rings = modulesBuf.rings_buf.data();
short* host_rods = modulesBuf.rods_buf.data();
short* host_modules = modulesBuf.modules_buf.data();
short* host_subdets = modulesBuf.subdets_buf.data();
short* host_sides = modulesBuf.sides_buf.data();
float* host_eta = modulesBuf.eta_buf.data();
float* host_r = modulesBuf.r_buf.data();
bool* host_isInverted = modulesBuf.isInverted_buf.data();
bool* host_isLower = modulesBuf.isLower_buf.data();
bool* host_isAnchor = modulesBuf.isAnchor_buf.data();
ModuleType* host_moduleType = modulesBuf.moduleType_buf.data();
ModuleLayerType* host_moduleLayerType = modulesBuf.moduleLayerType_buf.data();
float* host_dxdys = modulesBuf.dxdys_buf.data();
float* host_drdzs = modulesBuf.drdzs_buf.data();
uint16_t* host_nModules = modulesBuf.nModules_buf.data();
uint16_t* host_nLowerModules = modulesBuf.nLowerModules_buf.data();
uint16_t* host_partnerModuleIndices = modulesBuf.partnerModuleIndices_buf.data();
int* host_lstLayers = modulesBuf.lstLayers_buf.data();

//reassign detIdToIndex indices here
nLowerModules = (nModules - 1) / 2;
Expand Down Expand Up @@ -335,6 +336,6 @@ namespace lst {
fillMapArraysExplicit(modulesBuf, mmd);

return modulesBuf;
};
}
} // namespace lst
#endif
Loading

0 comments on commit 4ff4aaf

Please sign in to comment.