Skip to content

Commit

Permalink
switch to TrackCandidates = ::lst::TrackCandidatesSoA::View
Browse files Browse the repository at this point in the history
  • Loading branch information
@slava77devel
slava77devel committed Sep 25, 2024
1 parent af1f68f commit 155fc82
Show file tree
Hide file tree
Showing 4 changed files with 119 additions and 143 deletions.
118 changes: 66 additions & 52 deletions RecoTracker/LSTCore/src/alpaka/Event.dev.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ void Event::resetEventSync() {
tripletsBuffers_.reset();
quintupletsInGPU_.reset();
quintupletsBuffers_.reset();
trackCandidatesInGPU_.reset();
trackCandidatesD_ = nullptr;
trackCandidatesDC_.reset();
pixelTripletsInGPU_.reset();
pixelTripletsBuffers_.reset();
Expand Down Expand Up @@ -476,12 +476,11 @@ void Event::createTriplets() {
}

void Event::createTrackCandidates(bool no_pls_dupclean, bool tc_pls_triplets) {
if (!trackCandidatesInGPU_) {
trackCandidatesInGPU_.emplace();
if (!trackCandidatesDC_) {
trackCandidatesDC_.emplace(n_max_nonpixel_track_candidates + n_max_pixel_track_candidates, queue_);
auto buf = trackCandidatesDC_->buffer();
alpaka::memset(queue, buf, 0u);
trackCandidatesInGPU_->setData(trackCandidatesDC_->view());
alpaka::memset(queue_, buf, 0u);
trackCandidatesD_ = &trackCandidatesDC_->view();
}

Vec3D const threadsPerBlock_crossCleanpT3{1, 16, 64};
Expand All @@ -505,7 +504,7 @@ void Event::createTrackCandidates(bool no_pls_dupclean, bool tc_pls_triplets) {
AddpT3asTrackCandidatesInGPU{},
nLowerModules_,
*pixelTripletsInGPU_,
*trackCandidatesInGPU_,
*trackCandidatesD_,
*segmentsInGPU_,
*rangesInGPU_);

Expand Down Expand Up @@ -550,7 +549,7 @@ void Event::createTrackCandidates(bool no_pls_dupclean, bool tc_pls_triplets) {
AddT5asTrackCandidateInGPU{},
nLowerModules_,
*quintupletsInGPU_,
*trackCandidatesInGPU_,
*trackCandidatesD_,
*rangesInGPU_);

if (!no_pls_dupclean) {
Expand All @@ -573,7 +572,7 @@ void Event::createTrackCandidates(bool no_pls_dupclean, bool tc_pls_triplets) {
*modulesBuffers_.data(),
*rangesInGPU_,
*pixelTripletsInGPU_,
*trackCandidatesInGPU_,
*trackCandidatesD_,
*segmentsInGPU_,
*mdsInGPU_,
*hitsInGPU_,
Expand All @@ -588,7 +587,7 @@ void Event::createTrackCandidates(bool no_pls_dupclean, bool tc_pls_triplets) {
addpLSasTrackCandidateInGPU_workDiv,
AddpLSasTrackCandidateInGPU{},
nLowerModules_,
*trackCandidatesInGPU_,
*trackCandidatesD_,
*segmentsInGPU_,
tc_pls_triplets);

Expand All @@ -598,12 +597,13 @@ void Event::createTrackCandidates(bool no_pls_dupclean, bool tc_pls_triplets) {
auto nTrackCanpLSHost_buf = allocBufWrapper<unsigned int>(cms::alpakatools::host(), 1, queue_);
auto nTrackCanT5Host_buf = allocBufWrapper<unsigned int>(cms::alpakatools::host(), 1, queue_);
alpaka::memcpy(
queue_, nTrackCanpT5Host_buf, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidatespT5, 1u));
queue_, nTrackCanpT5Host_buf, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidatespT5(), 1u));
alpaka::memcpy(
queue_, nTrackCanpT3Host_buf, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidatespT3, 1u));
queue_, nTrackCanpT3Host_buf, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidatespT3(), 1u));
alpaka::memcpy(
queue_, nTrackCanpLSHost_buf, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidatespLS, 1u));
alpaka::memcpy(queue_, nTrackCanT5Host_buf, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidatesT5, 1u));
queue_, nTrackCanpLSHost_buf, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidatespLS(), 1u));
alpaka::memcpy(
queue_, nTrackCanT5Host_buf, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidatesT5(), 1u));
alpaka::wait(queue_); // wait to get the values before using them

auto nTrackCandidatespT5 = *nTrackCanpT5Host_buf.data();
Expand Down Expand Up @@ -824,12 +824,11 @@ void Event::createPixelQuintuplets() {
pixelQuintupletsBuffers_.emplace(n_max_pixel_quintuplets, devAcc_, queue_);
pixelQuintupletsInGPU_->setData(*pixelQuintupletsBuffers_);
}
if (!trackCandidatesInGPU_) {
trackCandidatesInGPU_.emplace();
if (!trackCandidatesDC_) {
trackCandidatesDC_.emplace(n_max_nonpixel_track_candidates + n_max_pixel_track_candidates, queue_);
auto buf = trackCandidatesDC_->buffer();
alpaka::memset(queue, buf, 0u);
trackCandidatesInGPU_->setData(trackCandidatesDC_->view());
alpaka::memset(queue_, buf, 0u);
trackCandidatesD_ = &trackCandidatesDC_->view();
}

auto superbins_buf = allocBufWrapper<int>(cms::alpakatools::host(), n_max_pixel_segments_per_module, queue_);
Expand Down Expand Up @@ -938,7 +937,7 @@ void Event::createPixelQuintuplets() {
AddpT5asTrackCandidateInGPU{},
nLowerModules_,
*pixelQuintupletsInGPU_,
*trackCandidatesInGPU_,
*trackCandidatesD_,
*segmentsInGPU_,
*rangesInGPU_);

Expand Down Expand Up @@ -1216,7 +1215,8 @@ unsigned int Event::getNumberOfQuintupletsByLayerEndcap(unsigned int layer) {
int Event::getNumberOfTrackCandidates() {
auto nTrackCandidates_buf_h = cms::alpakatools::make_host_buffer<unsigned int[]>(queue_, 1u);

alpaka::memcpy(queue_, nTrackCandidates_buf_h, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidates, 1u));
alpaka::memcpy(
queue_, nTrackCandidates_buf_h, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidates(), 1u));
alpaka::wait(queue_);

return *nTrackCandidates_buf_h.data();
Expand All @@ -1226,7 +1226,7 @@ int Event::getNumberOfPT5TrackCandidates() {
auto nTrackCandidatesPT5_buf_h = cms::alpakatools::make_host_buffer<unsigned int[]>(queue_, 1u);

alpaka::memcpy(
queue_, nTrackCandidatesPT5_buf_h, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidatespT5, 1u));
queue_, nTrackCandidatesPT5_buf_h, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidatespT5(), 1u));
alpaka::wait(queue_);

return *nTrackCandidatesPT5_buf_h.data();
Expand All @@ -1236,7 +1236,7 @@ int Event::getNumberOfPT3TrackCandidates() {
auto nTrackCandidatesPT3_buf_h = cms::alpakatools::make_host_buffer<unsigned int[]>(queue_, 1u);

alpaka::memcpy(
queue_, nTrackCandidatesPT3_buf_h, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidatespT3, 1u));
queue_, nTrackCandidatesPT3_buf_h, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidatespT3(), 1u));
alpaka::wait(queue_);

return *nTrackCandidatesPT3_buf_h.data();
Expand All @@ -1246,7 +1246,7 @@ int Event::getNumberOfPLSTrackCandidates() {
auto nTrackCandidatesPLS_buf_h = cms::alpakatools::make_host_buffer<unsigned int[]>(queue_, 1u);

alpaka::memcpy(
queue_, nTrackCandidatesPLS_buf_h, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidatespLS, 1u));
queue_, nTrackCandidatesPLS_buf_h, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidatespLS(), 1u));
alpaka::wait(queue_);

return *nTrackCandidatesPLS_buf_h.data();
Expand All @@ -1256,9 +1256,10 @@ int Event::getNumberOfPixelTrackCandidates() {
auto nTrackCandidates_buf_h = cms::alpakatools::make_host_buffer<unsigned int[]>(queue_, 1u);
auto nTrackCandidatesT5_buf_h = cms::alpakatools::make_host_buffer<unsigned int[]>(queue_, 1u);

alpaka::memcpy(queue_, nTrackCandidates_buf_h, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidates, 1u));
alpaka::memcpy(
queue_, nTrackCandidatesT5_buf_h, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidatesT5, 1u));
queue_, nTrackCandidates_buf_h, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidates(), 1u));
alpaka::memcpy(
queue_, nTrackCandidatesT5_buf_h, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidatesT5(), 1u));
alpaka::wait(queue_);

return (*nTrackCandidates_buf_h.data()) - (*nTrackCandidatesT5_buf_h.data());
Expand All @@ -1268,7 +1269,7 @@ int Event::getNumberOfT5TrackCandidates() {
auto nTrackCandidatesT5_buf_h = cms::alpakatools::make_host_buffer<unsigned int[]>(queue_, 1u);

alpaka::memcpy(
queue_, nTrackCandidatesT5_buf_h, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidatesT5, 1u));
queue_, nTrackCandidatesT5_buf_h, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidatesT5(), 1u));
alpaka::wait(queue_);

return *nTrackCandidatesT5_buf_h.data();
Expand Down Expand Up @@ -1556,7 +1557,8 @@ const TrackCandidatesHostCollection& Event::getTrackCandidates(bool sync) {
if (!trackCandidatesHC_) {
// Get nTrackCanHost parameter to initialize host based instance
auto nTrackCanHost_buf_h = cms::alpakatools::make_host_buffer<unsigned int[]>(queue_, 1u);
alpaka::memcpy(queue_, nTrackCanHost_buf_h, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidates, 1u));
alpaka::memcpy(
queue_, nTrackCanHost_buf_h, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidates(), 1u));
trackCandidatesHC_.emplace(n_max_nonpixel_track_candidates + n_max_pixel_track_candidates, queue_);
alpaka::wait(queue_); // wait here before we get nTrackCanHost and trackCandidatesInCPU becomes usable

Expand All @@ -1565,24 +1567,32 @@ const TrackCandidatesHostCollection& Event::getTrackCandidates(bool sync) {
trackCandidatesHC_->view().nTrackCandidates() = nTrackCanHost;
alpaka::memcpy(
queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().hitIndices()->data(), Params_pT5::kHits * nTrackCanHost),
alpaka::createView(devAcc, trackCandidatesInGPU_->hitIndices, Params_pT5::kHits * nTrackCanHost));
alpaka::memcpy(queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().pixelSeedIndex(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesInGPU_->pixelSeedIndex, nTrackCanHost));
alpaka::createView(cms::alpakatools::host(),
trackCandidatesHC_->view().hitIndices()->data(),
Params_pT5::kHits * nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesD_->hitIndices()->data(), Params_pT5::kHits * nTrackCanHost));
alpaka::memcpy(
queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().logicalLayers()->data(), Params_pT5::kLayers * nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesInGPU_->logicalLayers, Params_pT5::kLayers * nTrackCanHost));
alpaka::memcpy(queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().directObjectIndices(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesInGPU_->directObjectIndices, nTrackCanHost));
alpaka::memcpy(queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().objectIndices()->data(), 2 * nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesInGPU_->objectIndices, 2 * nTrackCanHost));
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().pixelSeedIndex(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesD_->pixelSeedIndex(), nTrackCanHost));
alpaka::memcpy(
queue_,
alpaka::createView(cms::alpakatools::host(),
trackCandidatesHC_->view().logicalLayers()->data(),
Params_pT5::kLayers * nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesD_->logicalLayers()->data(), Params_pT5::kLayers * nTrackCanHost));
alpaka::memcpy(
queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().directObjectIndices(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesD_->directObjectIndices(), nTrackCanHost));
alpaka::memcpy(queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().trackCandidateType(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesInGPU_->trackCandidateType, nTrackCanHost));
alpaka::createView(
cms::alpakatools::host(), trackCandidatesHC_->view().objectIndices()->data(), 2 * nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesD_->objectIndices()->data(), 2 * nTrackCanHost));
alpaka::memcpy(
queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().trackCandidateType(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesD_->trackCandidateType(), nTrackCanHost));
if (sync)
alpaka::wait(queue_); // host consumers expect filled data
}
Expand All @@ -1593,24 +1603,28 @@ const TrackCandidatesHostCollection& Event::getTrackCandidatesInCMSSW(bool sync)
if (!trackCandidatesHC_) {
// Get nTrackCanHost parameter to initialize host based instance
auto nTrackCanHost_buf_h = cms::alpakatools::make_host_buffer<unsigned int[]>(queue_, 1u);
alpaka::memcpy(queue_, nTrackCanHost_buf_h, alpaka::createView(devAcc_, trackCandidatesInGPU_->nTrackCandidates, 1u));
trackCandidatesHC_ =
new ::lst::TrackCandidatesHostCollection(n_max_nonpixel_track_candidates + n_max_pixel_track_candidates, queue_);
alpaka::memcpy(
queue_, nTrackCanHost_buf_h, alpaka::createView(devAcc_, &trackCandidatesD_->nTrackCandidates(), 1u));
trackCandidatesHC_.emplace(n_max_nonpixel_track_candidates + n_max_pixel_track_candidates, queue_);
alpaka::wait(queue_); // wait for the value before using and trackCandidatesInCPU becomes usable

auto const nTrackCanHost = *nTrackCanHost_buf_h.data();

trackCandidatesHC_->view().nTrackCandidates() = nTrackCanHost;
alpaka::memcpy(
queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().hitIndices()->data(), Params_pT5::kHits * nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesInGPU_->hitIndices, Params_pT5::kHits * nTrackCanHost));
alpaka::memcpy(queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().pixelSeedIndex(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesInGPU_->pixelSeedIndex, nTrackCanHost));
alpaka::memcpy(queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().trackCandidateType(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesInGPU_->trackCandidateType, nTrackCanHost));
alpaka::createView(cms::alpakatools::host(),
trackCandidatesHC_->view().hitIndices()->data(),
Params_pT5::kHits * nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesD_->hitIndices()->data(), Params_pT5::kHits * nTrackCanHost));
alpaka::memcpy(
queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().pixelSeedIndex(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesD_->pixelSeedIndex(), nTrackCanHost));
alpaka::memcpy(
queue_,
alpaka::createView(cms::alpakatools::host(), trackCandidatesHC_->view().trackCandidateType(), nTrackCanHost),
alpaka::createView(devAcc_, trackCandidatesD_->trackCandidateType(), nTrackCanHost));
if (sync)
alpaka::wait(queue_); // host consumers expect filled data
}
Expand Down
2 changes: 1 addition & 1 deletion RecoTracker/LSTCore/src/alpaka/Event.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE::lst {
std::optional<TripletsBuffer<Device>> tripletsBuffers_;
std::optional<Quintuplets> quintupletsInGPU_;
std::optional<QuintupletsBuffer<Device>> quintupletsBuffers_;
std::optional<TrackCandidates> trackCandidatesInGPU_;
TrackCandidates* trackCandidatesD_; // not owned, view from trackCandidatesDC_
std::optional<TrackCandidatesDeviceCollection> trackCandidatesDC_;
std::optional<PixelTriplets> pixelTripletsInGPU_;
std::optional<PixelTripletsBuffer<Device>> pixelTripletsBuffers_;
Expand Down
4 changes: 2 additions & 2 deletions RecoTracker/LSTCore/src/alpaka/LST.dev.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,8 +254,8 @@ void LST::getOutput(Event& event) {
std::vector<int> tc_seedIdx;
std::vector<short> tc_trackCandidateType;

HitsBuffer<alpaka::DevCpu>& hitsBuffer = (*event.getHitsInCMSSW(false)); // sync on next line
auto const& trackCandidates = event.getTrackCandidatesInCMSSW()->const_view();
HitsBuffer<alpaka::DevCpu>& hitsBuffer = event.getHitsInCMSSW(false); // sync on next line
auto const& trackCandidates = event.getTrackCandidatesInCMSSW().const_view();

unsigned int nTrackCandidates = trackCandidates.nTrackCandidates();

Expand Down
Loading

0 comments on commit 155fc82

Please sign in to comment.