Skip to content

Commit

Permalink
Works!
Browse files Browse the repository at this point in the history
  • Loading branch information
@AdrianoDee
AdrianoDee committed Apr 26, 2024
1 parent 427fc3b commit 7b2edff
Show file tree
Hide file tree
Showing 11 changed files with 169 additions and 51 deletions.
10 changes: 7 additions & 3 deletions DataFormats/SiPixelDigiSoA/interface/SiPixelDigisDevice.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,23 +15,27 @@ class SiPixelDigisDevice : public PortableDeviceCollection<SiPixelDigisSoA, TDev
SiPixelDigisDevice() = default;
template <typename TQueue>
explicit SiPixelDigisDevice(size_t maxFedWords, TQueue queue)
: PortableDeviceCollection<SiPixelDigisSoA, TDev>(maxFedWords + 1, queue) {}
: PortableDeviceCollection<SiPixelDigisSoA, TDev>(maxFedWords + 1, queue), nDigis_h(maxFedWords) {}

// Constructor which specifies the SoA size
explicit SiPixelDigisDevice(size_t maxFedWords, TDev const &device)
: PortableDeviceCollection<SiPixelDigisSoA, TDev>(maxFedWords + 1, device) {}
: PortableDeviceCollection<SiPixelDigisSoA, TDev>(maxFedWords + 1, device), nDigis_h(maxFedWords) {}

void setNModulesDigis(uint32_t nModules, uint32_t nDigis) {
nModules_h = nModules;
nDigis_h = nDigis;
}

void setNModules(uint32_t nModules) {
nModules_h = nModules;
}

uint32_t nModules() const { return nModules_h; }
uint32_t nDigis() const { return nDigis_h; }

private:
uint32_t nModules_h = 0;
uint32_t nDigis_h = 0;
uint32_t nModules_h = 0;
};

#endif // DataFormats_SiPixelDigiSoA_interface_SiPixelDigisDevice_h
8 changes: 6 additions & 2 deletions DataFormats/SiPixelDigiSoA/interface/SiPixelDigisHost.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,23 @@ class SiPixelDigisHost : public PortableHostCollection<SiPixelDigisSoA> {
SiPixelDigisHost() = default;
template <typename TQueue>
explicit SiPixelDigisHost(size_t maxFedWords, TQueue queue)
: PortableHostCollection<SiPixelDigisSoA>(maxFedWords + 1, queue) {}
: PortableHostCollection<SiPixelDigisSoA>(maxFedWords + 1, queue), nDigis_h(maxFedWords) {}

void setNModulesDigis(uint32_t nModules, uint32_t nDigis) {
nModules_h = nModules;
nDigis_h = nDigis;
}

void setNModules(uint32_t nModules) {
nModules_h = nModules;
}

uint32_t nModules() const { return nModules_h; }
uint32_t nDigis() const { return nDigis_h; }

private:
uint32_t nModules_h = 0;
uint32_t nDigis_h = 0;
uint32_t nModules_h = 0;
};

#endif // DataFormats_SiPixelDigiSoA_interface_SiPixelDigisHost_h
12 changes: 11 additions & 1 deletion DataFormats/SiPixelDigiSoA/interface/alpaka/SiPixelDigisSoACollection.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,19 @@ namespace cms::alpakatools {
struct CopyToHost<SiPixelDigisDevice<TDevice>> {
template <typename TQueue>
static auto copyAsync(TQueue &queue, SiPixelDigisDevice<TDevice> const &srcData) {
SiPixelDigisHost dstData(srcData.view().metadata().size(), queue);

printf("METADATA: %d NDIGIS: %d DIFF: %d\n",srcData.view().metadata().size(),srcData.nDigis(),srcData.view().metadata().size()-srcData.nDigis());

SiPixelDigisHost dstData(srcData.nDigis(), queue);

alpaka::memcpy(queue, dstData.buffer(), srcData.buffer());
dstData.setNModulesDigis(srcData.nModules(), srcData.nDigis());

auto digi_view = dstData.view();
for(int i = 0; i<srcData.view().metadata().size();i++)
printf("Host: %d %d %d %d %d \n",i,digi_view[i].rawIdArr(),digi_view[i].clus(),digi_view[i].pdigi(),digi_view[i].adc());

alpaka::wait(queue);
return dstData;
}
};
Expand Down
3 changes: 2 additions & 1 deletion DataFormats/SiPixelDigiSoA/src/classes_def.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
<class name="SiPixelDigisSoA"/>
<class name="SiPixelDigisSoA::View"/>
<class name="PortableHostCollection<SiPixelDigisSoA>"/>
<class name="SiPixelDigisHost" ClassVersion="3">
<class name="SiPixelDigisHost" ClassVersion="4">
<version ClassVersion="4" checksum="649262062"/>
<version ClassVersion="3" checksum="3022474662"/>
</class>
<class name="edm::Wrapper<SiPixelDigisHost>" splitLevel="0"/>
Expand Down
18 changes: 18 additions & 0 deletions DataFormats/SiPixelDigiSoA/src/classes_def.xml.generated
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
<lcgdict>
<class name="SiPixelDigisSoA"/>
<class name="SiPixelDigisSoA::View"/>
<class name="PortableHostCollection<SiPixelDigisSoA>"/>
<class name="SiPixelDigisHost" ClassVersion="4">
<version ClassVersion="4" checksum="649262062"/>
<version ClassVersion="3" checksum="3022474662"/>
</class>
<class name="edm::Wrapper<SiPixelDigisHost>" splitLevel="0"/>

<class name="SiPixelDigiErrorsSoA"/>
<class name="SiPixelDigiErrorsSoA::View"/>
<class name="PortableHostCollection<SiPixelDigiErrorsSoA>"/>
<class name="SiPixelDigiErrorsHost" ClassVersion="3">
<version ClassVersion="3" checksum="958593711"/>
</class>
<class name="edm::Wrapper<SiPixelDigiErrorsHost>" splitLevel="0"/>
</lcgdict>
2 changes: 1 addition & 1 deletion Geometry/CommonTopologies/interface/SimplePixelTopology.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ namespace pixelTopology {
static constexpr uint16_t numberOfModules = 3892;

// 1024 bins, 10 bits
static constexpr uint16_t clusterBinning = 1024;
static constexpr uint16_t clusterBinning = 896;
static constexpr uint16_t clusterBits = 10;

static constexpr uint16_t numberOfModulesInBarrel = 756;
Expand Down
4 changes: 2 additions & 2 deletions RecoLocalTracker/SiPixelClusterizer/plugins/SiPixelDigisClustersFromSoAAlpaka.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
// local include(s)
#include "PixelClusterizerBase.h"

//#define EDM_ML_DEBUG
//#define GPU_DEBUG
#define EDM_ML_DEBUG
#define GPU_DEBUG

template <typename TrackerTraits>
class SiPixelDigisClustersFromSoAAlpaka : public edm::global::EDProducer<> {
Expand Down
50 changes: 42 additions & 8 deletions RecoLocalTracker/SiPixelClusterizer/plugins/alpaka/ClusterChargeCut.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@ namespace pixelClustering {
SiPixelClusterThresholds clusterThresholds,
const uint32_t numElements) const {
constexpr int32_t maxNumClustersPerModules = TrackerTraits::maxNumClustersPerModules;


if (cms::alpakatools::once_per_grid(acc))
{
for(uint32_t i = 0; i<numElements;i++)
printf("PRE: %d %d %d %d %d \n",i,digi_view[i].rawIdArr(),digi_view[i].clus(),digi_view[i].pdigi(),digi_view[i].adc());
}
auto& charge = alpaka::declareSharedVar<int32_t[maxNumClustersPerModules], __COUNTER__>(acc);
auto& ok = alpaka::declareSharedVar<uint8_t[maxNumClustersPerModules], __COUNTER__>(acc);
auto& newclusId = alpaka::declareSharedVar<uint16_t[maxNumClustersPerModules], __COUNTER__>(acc);
Expand All @@ -35,8 +40,11 @@ namespace pixelClustering {

ALPAKA_ASSERT_ACC(TrackerTraits::numberOfModules < maxNumModules);
ALPAKA_ASSERT_ACC(startBPIX2 < TrackerTraits::numberOfModules);

auto endModule = clus_view[0].moduleStart();

// if (cms::alpakatools::once_per_block(acc))
// printf("%d %d\n",__LINE__,endModule);
for (auto module : cms::alpakatools::independent_groups(acc, endModule)) {
auto firstPixel = clus_view[1 + module].moduleStart();
auto thisModuleId = digi_view[firstPixel].moduleId();
Expand All @@ -58,6 +66,8 @@ namespace pixelClustering {
uint32_t nclus = clus_view[thisModuleId].clusInModule();
if (nclus == 0)
return;
// if (cms::alpakatools::once_per_block(acc))
// printf("%d %d \n",module,__LINE__);

if (cms::alpakatools::once_per_block(acc) && nclus > maxNumClustersPerModules)
printf("Warning: too many clusters in module %u in block %u: %u > %d\n",
Expand All @@ -80,6 +90,8 @@ namespace pixelClustering {
}
nclus = maxNumClustersPerModules;
}
// if (cms::alpakatools::once_per_block(acc))
// printf("%d %d \n",module,__LINE__);

#ifdef GPU_DEBUG
if (thisModuleId % 100 == 1)
Expand All @@ -92,7 +104,8 @@ namespace pixelClustering {
charge[i] = 0;
}
alpaka::syncBlockThreads(acc);

// if (cms::alpakatools::once_per_block(acc))
// printf("%d \n",__LINE__);
for (auto i : cms::alpakatools::independent_group_elements(acc, firstPixel, numElements)) {
if (digi_view[i].moduleId() == invalidModuleId)
continue; // not valid
Expand All @@ -104,19 +117,25 @@ namespace pixelClustering {
alpaka::hierarchy::Threads{});
}
alpaka::syncBlockThreads(acc);
// if (cms::alpakatools::once_per_block(acc))
// printf("%d %d \n",module,__LINE__);

auto chargeCut = clusterThresholds.getThresholdForLayerOnCondition(thisModuleId < startBPIX2);

bool good = true;
for (auto i : cms::alpakatools::independent_group_elements(acc, nclus)) {
newclusId[i] = ok[i] = (charge[i] >= chargeCut) ? 1 : 0;
if (0 == ok[i])
good = false;
good &= false;
printf("cut %d %d %d %d %d %d %d \n",i, thisModuleId,newclusId[i],ok[i],charge[i],chargeCut,good);
}

// if (cms::alpakatools::once_per_block(acc))
// printf("%d %d \n",module,__LINE__);
// if all clusters are above threshold, do nothing
if (alpaka::syncBlockThreadsPredicate<alpaka::BlockAnd>(acc, good))
continue;
// if (alpaka::syncBlockThreadsPredicate<alpaka::BlockAnd>(acc, good))
// continue;
// if (cms::alpakatools::once_per_block(acc))
// printf("%d %d \n",module,__LINE__);

// renumber
auto& ws = alpaka::declareSharedVar<uint16_t[32], __COUNTER__>(acc);
Expand All @@ -136,6 +155,9 @@ namespace pixelClustering {
alpaka::syncBlockThreads(acc);
}
}
// if (cms::alpakatools::once_per_block(acc))
// printf("%d %d \n",module,__LINE__);

ALPAKA_ASSERT_ACC(nclus >= newclusId[nclus - 1]);

clus_view[thisModuleId].clusInModule() = newclusId[nclus - 1];
Expand All @@ -147,13 +169,25 @@ namespace pixelClustering {
if (digi_view[i].moduleId() != thisModuleId)
break; // end of module
if (0 == ok[digi_view[i].clus()])
digi_view[i].moduleId() = digi_view[i].clus() = invalidModuleId;
{
digi_view[i].moduleId() = digi_view[i].clus() = invalidModuleId;
digi_view[i].rawIdArr() = 0;
}
else
digi_view[i].clus() = newclusId[digi_view[i].clus()] - 1;
}
// if (cms::alpakatools::once_per_block(acc))
// printf("%d %d \n",module,__LINE__);

// done
alpaka::syncBlockThreads(acc);

if (cms::alpakatools::once_per_grid(acc))
{
for(uint32_t i = 0; i<numElements;i++)
printf("AFTER: %d %d %d %d %d \n",i,digi_view[i].rawIdArr(),digi_view[i].clus(),digi_view[i].pdigi(),digi_view[i].adc());
}

}
}
};
Expand Down
2 changes: 1 addition & 1 deletion RecoLocalTracker/SiPixelClusterizer/plugins/alpaka/PixelClustering.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
#include "HeterogeneousCore/AlpakaInterface/interface/SimpleVector.h"
#include "HeterogeneousCore/AlpakaInterface/interface/config.h"

//#define GPU_DEBUG
#define GPU_DEBUG

namespace ALPAKA_ACCELERATOR_NAMESPACE::pixelClustering {

Expand Down
73 changes: 47 additions & 26 deletions RecoLocalTracker/SiPixelClusterizer/plugins/alpaka/SiPixelPhase2DigiToCluster.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,33 +45,33 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
const edm::EDGetTokenT<edm::DetSetVector<PixelDigi>> pixelDigiToken_;

device::EDPutToken<SiPixelDigisSoACollection> digiPutToken_;
device::EDPutToken<SiPixelDigiErrorsSoACollection> digiErrorPutToken_;
// device::EDPutToken<SiPixelDigiErrorsSoACollection> digiErrorPutToken_;
device::EDPutToken<SiPixelClustersSoACollection> clusterPutToken_;

Algo Algo_;

const bool includeErrors_;
// const bool includeErrors_;
const SiPixelClusterThresholds clusterThresholds_;
uint32_t nDigis_ = 0;

SiPixelDigisSoACollection digis_d;
// std::optional<SiPixelDigisSoACollection> digis_d;
};

SiPixelPhase2DigiToCluster::SiPixelPhase2DigiToCluster(const edm::ParameterSet& iConfig)
: geomToken_(esConsumes()),
pixelDigiToken_(consumes<edm::DetSetVector<PixelDigi>>(iConfig.getParameter<edm::InputTag>("InputDigis"))),
digiPutToken_(produces()),
clusterPutToken_(produces()),
includeErrors_(iConfig.getParameter<bool>("IncludeErrors")),
// includeErrors_(iConfig.getParameter<bool>("IncludeErrors")),
clusterThresholds_{iConfig.getParameter<int32_t>("clusterThreshold_layer1"),
iConfig.getParameter<int32_t>("clusterThreshold_otherLayers"),
static_cast<float>(iConfig.getParameter<double>("ElectronPerADCGain")),
static_cast<int8_t>(iConfig.getParameter<int>("Phase2ReadoutMode")),
static_cast<uint16_t>(iConfig.getParameter<uint32_t>("Phase2DigiBaseline")),
static_cast<uint8_t>(iConfig.getParameter<uint32_t>("Phase2KinkADC"))} {
if (includeErrors_) {
digiErrorPutToken_ = produces();
}
// if (includeErrors_) {
// digiErrorPutToken_ = produces();
// }
}

void SiPixelPhase2DigiToCluster::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
Expand All @@ -90,21 +90,26 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
}

void SiPixelPhase2DigiToCluster::acquire(device::Event const& iEvent, device::EventSetup const& iSetup) {

std::cout << "SiPixelPhase2DigiToCluster " << __LINE__ << std::endl;
auto const& input = iEvent.get(pixelDigiToken_);

const TrackerGeometry* geom_ = &iSetup.getData(geomToken_);

uint32_t nDigis = 0;

for (const auto& det : input) {
nDigis += det.size();
// nDigis += det.size();
for (auto const& px : det) {
nDigis++;
}
}

if (nDigis_ == 0)
if (nDigis == 0)
return;

SiPixelDigisHost digis_h(nDigis, iEvent.queue());
nDigis_ = nDigis;
SiPixelDigisHost digis_h(nDigis_, iEvent.queue());

nDigis = 0;
for (const auto& det : input) {
Expand All @@ -119,6 +124,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
digis_h.view()[nDigis].yy() = uint16_t(px.column());
digis_h.view()[nDigis].adc() = uint16_t(px.adc());

digis_h.view()[nDigis].clus() = 0;

digis_h.view()[nDigis].pdigi() = uint32_t(px.packedData());

digis_h.view()[nDigis].rawIdArr() = uint32_t(detid);
Expand All @@ -127,30 +134,44 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
}
}

digis_d = SiPixelDigisSoACollection(nDigis, iEvent.queue());
alpaka::memcpy(iEvent.queue(), digis_d.buffer(), digis_h.buffer());
// digis_d = SiPixelDigisSoACollection(nDigis_, iEvent.queue());
// alpaka::memcpy(iEvent.queue(), digis_d->buffer(), digis_h.buffer());

Algo_.makePhase2ClustersAsync(iEvent.queue(), clusterThresholds_, digis_d.view(), nDigis);
// alpaka::wait(iEvent.queue());
Algo_.makePhase2ClustersAsync(iEvent.queue(), clusterThresholds_, digis_h.view(), nDigis_);

// digis_d->setNModules(Algo_.nModules());
std::cout << "SiPixelPhase2DigiToCluster " << __LINE__ << std::endl;
}

void SiPixelPhase2DigiToCluster::produce(device::Event& iEvent, device::EventSetup const& iSetup) {
std::cout << "SiPixelPhase2DigiToCluster " << __LINE__ << std::endl;
if (nDigis_ == 0) {
SiPixelClustersSoACollection clusters_d{pixelTopology::Phase1::numberOfModules, iEvent.queue()};
iEvent.emplace(digiPutToken_, std::move(digis_d));
iEvent.emplace(clusterPutToken_, std::move(clusters_d));
if (includeErrors_) {
iEvent.emplace(digiErrorPutToken_, SiPixelDigiErrorsSoACollection());
}
// SiPixelClustersSoACollection clusters_d{pixelTopology::Phase2::numberOfModules, iEvent.queue()};
// SiPixelDigisSoACollection digis_d{nDigis_, iEvent.queue()};
// iEvent.emplace(digiPutToken_, std::move(*digis_d));
// iEvent.emplace(clusterPutToken_, std::move(clusters_d));
// if (includeErrors_) {
// iEvent.emplace(digiErrorPutToken_, SiPixelDigiErrorsSoACollection());
// }
return;
}

digis_d.setNModulesDigis(Algo_.nModules(), nDigis_);

iEvent.emplace(digiPutToken_, std::move(digis_d));
// std::cout << "SiPixelPhase2DigiToCluster " << __LINE__ << std::endl;
// digis_d.setNModulesDigis(Algo_.nModules(), nDigis_);
// std::cout << "SiPixelPhase2DigiToCluster " << __LINE__ << std::endl;
// // iEvent.emplace(digiPutToken_, std::move(digis_d));
// iEvent.emplace(clusterPutToken_, Algo_.getClusters());
// if (includeErrors_) {
// iEvent.emplace(digiErrorPutToken_, Algo_.getErrors());
// }

iEvent.emplace(digiPutToken_, Algo_.getDigis());
iEvent.emplace(clusterPutToken_, Algo_.getClusters());
if (includeErrors_) {
iEvent.emplace(digiErrorPutToken_, Algo_.getErrors());
}
// if (includeErrors_) {
// iEvent.emplace(digiErrorPutToken_, Algo_.getErrors());
// }

std::cout << "SiPixelPhase2DigiToCluster " << __LINE__ << std::endl;
}

} // namespace ALPAKA_ACCELERATOR_NAMESPACE
Expand Down
Loading

0 comments on commit 7b2edff

Please sign in to comment.