Fixes for Phase2 pixel Alpaka reco

- nDigis = 0
- limit to
- actual buffer size for SiPixelDigisSoA (also for Phase1)

DataFormats/SiPixelDigiSoA/interface/SiPixelDigisDevice.h

@@ -15,11 +15,11 @@ class SiPixelDigisDevice : public PortableDeviceCollection<SiPixelDigisSoA, TDev
   SiPixelDigisDevice() = default;
   template <typename TQueue>
   explicit SiPixelDigisDevice(size_t maxFedWords, TQueue queue)
-      : PortableDeviceCollection<SiPixelDigisSoA, TDev>(maxFedWords + 1, queue) {}
+      : PortableDeviceCollection<SiPixelDigisSoA, TDev>(maxFedWords + 1, queue), nDigis_h(maxFedWords) {}
 
   // Constructor which specifies the SoA size
   explicit SiPixelDigisDevice(size_t maxFedWords, TDev const &device)
-      : PortableDeviceCollection<SiPixelDigisSoA, TDev>(maxFedWords + 1, device) {}
+      : PortableDeviceCollection<SiPixelDigisSoA, TDev>(maxFedWords + 1, device), nDigis_h(maxFedWords) {}
 
   void setNModulesDigis(uint32_t nModules, uint32_t nDigis) {
     nModules_h = nModules;
@@ -30,8 +30,8 @@ class SiPixelDigisDevice : public PortableDeviceCollection<SiPixelDigisSoA, TDev
   uint32_t nDigis() const { return nDigis_h; }
 
 private:
-  uint32_t nModules_h = 0;
   uint32_t nDigis_h = 0;
+  uint32_t nModules_h = 0;
 };
 
 #endif  // DataFormats_SiPixelDigiSoA_interface_SiPixelDigisDevice_h

DataFormats/SiPixelDigiSoA/interface/SiPixelDigisHost.h

@@ -12,7 +12,7 @@ class SiPixelDigisHost : public PortableHostCollection<SiPixelDigisSoA> {
   SiPixelDigisHost() = default;
   template <typename TQueue>
   explicit SiPixelDigisHost(size_t maxFedWords, TQueue queue)
-      : PortableHostCollection<SiPixelDigisSoA>(maxFedWords + 1, queue) {}
+      : PortableHostCollection<SiPixelDigisSoA>(maxFedWords + 1, queue), nDigis_h(maxFedWords) {}
 
   void setNModulesDigis(uint32_t nModules, uint32_t nDigis) {
     nModules_h = nModules;
@@ -23,8 +23,8 @@ class SiPixelDigisHost : public PortableHostCollection<SiPixelDigisSoA> {
   uint32_t nDigis() const { return nDigis_h; }
 
 private:
-  uint32_t nModules_h = 0;
   uint32_t nDigis_h = 0;
+  uint32_t nModules_h = 0;
 };
 
 #endif  // DataFormats_SiPixelDigiSoA_interface_SiPixelDigisHost_h

DataFormats/SiPixelDigiSoA/interface/alpaka/SiPixelDigisSoACollection.h

@@ -23,7 +23,7 @@ namespace cms::alpakatools {
   struct CopyToHost<SiPixelDigisDevice<TDevice>> {
     template <typename TQueue>
     static auto copyAsync(TQueue &queue, SiPixelDigisDevice<TDevice> const &srcData) {
-      SiPixelDigisHost dstData(srcData.view().metadata().size(), queue);
+      SiPixelDigisHost dstData(srcData.nDigis(), queue);
       alpaka::memcpy(queue, dstData.buffer(), srcData.buffer());
       dstData.setNModulesDigis(srcData.nModules(), srcData.nDigis());
       return dstData;

DataFormats/SiPixelDigiSoA/src/classes_def.xml

@@ -2,7 +2,8 @@
   <class name="SiPixelDigisSoA"/>
   <class name="SiPixelDigisSoA::View"/>
   <class name="PortableHostCollection<SiPixelDigisSoA>"/>
-  <class name="SiPixelDigisHost" ClassVersion="3">
+  <class name="SiPixelDigisHost" ClassVersion="4">
+   <version ClassVersion="4" checksum="649262062"/>
     <version ClassVersion="3" checksum="3022474662"/>
   </class>
   <class name="edm::Wrapper<SiPixelDigisHost>" splitLevel="0"/>

Geometry/CommonTopologies/interface/SimplePixelTopology.h

@@ -380,8 +380,10 @@ namespace pixelTopology {
 
     static constexpr uint16_t numberOfModules = 3892;
 
-    // 1024 bins, 10 bits
-    static constexpr uint16_t clusterBinning = 1024;
+    // 1000 bins (<10 bits) must be:
+    // - < 32*32 (warpSize*warpSize for block prefix scan for CUDA)
+    // - > in y of any module (in each y bin only pixels with same integer y)
+    static constexpr uint16_t clusterBinning = 1000;
     static constexpr uint16_t clusterBits = 10;
 
     static constexpr uint16_t numberOfModulesInBarrel = 756;

RecoLocalTracker/SiPixelClusterizer/plugins/alpaka/ClusterChargeCut.h

@@ -27,6 +27,19 @@ namespace pixelClustering {
                                   const uint32_t numElements) const {
       constexpr int32_t maxNumClustersPerModules = TrackerTraits::maxNumClustersPerModules;
 
+#ifdef GPU_DEBUG
+      if (cms::alpakatools::once_per_grid(acc)) {
+        printf("All digis before cut: \n");
+        for (uint32_t i = 0; i < numElements; i++)
+          printf("%d %d %d %d %d \n",
+                 i,
+                 digi_view[i].rawIdArr(),
+                 digi_view[i].clus(),
+                 digi_view[i].pdigi(),
+                 digi_view[i].adc());
+      }
+#endif
+
       auto& charge = alpaka::declareSharedVar<int32_t[maxNumClustersPerModules], __COUNTER__>(acc);
       auto& ok = alpaka::declareSharedVar<uint8_t[maxNumClustersPerModules], __COUNTER__>(acc);
       auto& newclusId = alpaka::declareSharedVar<uint16_t[maxNumClustersPerModules], __COUNTER__>(acc);
@@ -37,6 +50,7 @@ namespace pixelClustering {
       ALPAKA_ASSERT_ACC(startBPIX2 < TrackerTraits::numberOfModules);
 
       auto endModule = clus_view[0].moduleStart();
+
       for (auto module : cms::alpakatools::independent_groups(acc, endModule)) {
         auto firstPixel = clus_view[1 + module].moduleStart();
         auto thisModuleId = digi_view[firstPixel].moduleId();
@@ -111,9 +125,18 @@ namespace pixelClustering {
         for (auto i : cms::alpakatools::independent_group_elements(acc, nclus)) {
           newclusId[i] = ok[i] = (charge[i] >= chargeCut) ? 1 : 0;
           if (0 == ok[i])
-            good = false;
+            good &= false;  //better than simple assignment in case of race?
+#ifdef GPU_DEBUG
+          printf("Cutting pix %d in module %d newId %d ok? %d charge %d cut %d -> good %d \n",
+                 i,
+                 thisModuleId,
+                 newclusId[i],
+                 ok[i],
+                 charge[i],
+                 chargeCut,
+                 good);
+#endif
         }
-
         // if all clusters are above threshold, do nothing
         if (alpaka::syncBlockThreadsPredicate<alpaka::BlockAnd>(acc, good))
           continue;
@@ -136,6 +159,7 @@ namespace pixelClustering {
             alpaka::syncBlockThreads(acc);
           }
         }
+
         ALPAKA_ASSERT_ACC(nclus >= newclusId[nclus - 1]);
 
         clus_view[thisModuleId].clusInModule() = newclusId[nclus - 1];
@@ -154,6 +178,18 @@ namespace pixelClustering {
 
         // done
         alpaka::syncBlockThreads(acc);
+#ifdef GPU_DEBUG
+        if (cms::alpakatools::once_per_grid(acc)) {
+          printf("All digis AFTER cut: \n");
+          for (uint32_t i = 0; i < numElements; i++)
+            printf("%d %d %d %d %d \n",
+                   i,
+                   digi_view[i].rawIdArr(),
+                   digi_view[i].clus(),
+                   digi_view[i].pdigi(),
+                   digi_view[i].adc());
+        }
+#endif
       }
     }
   };

RecoLocalTracker/SiPixelClusterizer/plugins/alpaka/SiPixelPhase2DigiToCluster.cc

@@ -45,12 +45,10 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     const edm::EDGetTokenT<edm::DetSetVector<PixelDigi>> pixelDigiToken_;
 
     device::EDPutToken<SiPixelDigisSoACollection> digiPutToken_;
-    device::EDPutToken<SiPixelDigiErrorsSoACollection> digiErrorPutToken_;
     device::EDPutToken<SiPixelClustersSoACollection> clusterPutToken_;
 
     Algo Algo_;
 
-    const bool includeErrors_;
     const SiPixelClusterThresholds clusterThresholds_;
     uint32_t nDigis_ = 0;
 
@@ -62,17 +60,12 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
         pixelDigiToken_(consumes<edm::DetSetVector<PixelDigi>>(iConfig.getParameter<edm::InputTag>("InputDigis"))),
         digiPutToken_(produces()),
         clusterPutToken_(produces()),
-        includeErrors_(iConfig.getParameter<bool>("IncludeErrors")),
         clusterThresholds_{iConfig.getParameter<int32_t>("clusterThreshold_layer1"),
                            iConfig.getParameter<int32_t>("clusterThreshold_otherLayers"),
                            static_cast<float>(iConfig.getParameter<double>("ElectronPerADCGain")),
                            static_cast<int8_t>(iConfig.getParameter<int>("Phase2ReadoutMode")),
                            static_cast<uint16_t>(iConfig.getParameter<uint32_t>("Phase2DigiBaseline")),
-                           static_cast<uint8_t>(iConfig.getParameter<uint32_t>("Phase2KinkADC"))} {
-    if (includeErrors_) {
-      digiErrorPutToken_ = produces();
-    }
-  }
+                           static_cast<uint8_t>(iConfig.getParameter<uint32_t>("Phase2KinkADC"))} {}
 
   void SiPixelPhase2DigiToCluster::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
     edm::ParameterSetDescription desc;
@@ -100,11 +93,11 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
       nDigis += det.size();
     }
 
-    if (nDigis_ == 0)
+    if (nDigis == 0)
       return;
 
-    SiPixelDigisHost digis_h(nDigis, iEvent.queue());
     nDigis_ = nDigis;
+    SiPixelDigisHost digis_h(nDigis_, iEvent.queue());
 
     nDigis = 0;
     for (const auto& det : input) {
@@ -119,6 +112,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
         digis_h.view()[nDigis].yy() = uint16_t(px.column());
         digis_h.view()[nDigis].adc() = uint16_t(px.adc());
 
+        digis_h.view()[nDigis].clus() = 0;
+
         digis_h.view()[nDigis].pdigi() = uint32_t(px.packedData());
 
         digis_h.view()[nDigis].rawIdArr() = uint32_t(detid);
@@ -130,27 +125,23 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     digis_d = SiPixelDigisSoACollection(nDigis, iEvent.queue());
     alpaka::memcpy(iEvent.queue(), digis_d.buffer(), digis_h.buffer());
 
-    Algo_.makePhase2ClustersAsync(iEvent.queue(), clusterThresholds_, digis_d.view(), nDigis);
+    Algo_.makePhase2ClustersAsync(iEvent.queue(), clusterThresholds_, digis_d.view(), nDigis_);
   }
 
   void SiPixelPhase2DigiToCluster::produce(device::Event& iEvent, device::EventSetup const& iSetup) {
     if (nDigis_ == 0) {
-      SiPixelClustersSoACollection clusters_d{pixelTopology::Phase1::numberOfModules, iEvent.queue()};
-      iEvent.emplace(digiPutToken_, std::move(digis_d));
+      SiPixelClustersSoACollection clusters_d{pixelTopology::Phase2::numberOfModules, iEvent.queue()};
+      SiPixelDigisSoACollection digis_d_zero{nDigis_, iEvent.queue()};
+      iEvent.emplace(digiPutToken_, std::move(digis_d_zero));
       iEvent.emplace(clusterPutToken_, std::move(clusters_d));
-      if (includeErrors_) {
-        iEvent.emplace(digiErrorPutToken_, SiPixelDigiErrorsSoACollection());
-      }
       return;
     }
 
     digis_d.setNModulesDigis(Algo_.nModules(), nDigis_);
-
     iEvent.emplace(digiPutToken_, std::move(digis_d));
+
+    // iEvent.emplace(digiPutToken_, Algo_.getDigis());
     iEvent.emplace(clusterPutToken_, Algo_.getClusters());
-    if (includeErrors_) {
-      iEvent.emplace(digiErrorPutToken_, Algo_.getErrors());
-    }
   }
 
 }  // namespace ALPAKA_ACCELERATOR_NAMESPACE

RecoLocalTracker/SiPixelClusterizer/plugins/alpaka/SiPixelRawToClusterKernel.dev.cc

@@ -432,7 +432,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     struct FillHitsModuleStart {
       template <typename TAcc>
       ALPAKA_FN_ACC void operator()(const TAcc &acc, SiPixelClustersSoAView clus_view) const {
-        ALPAKA_ASSERT_ACC(TrackerTraits::numberOfModules < 2048);  // easy to extend at least till 32*1024
+        ALPAKA_ASSERT_ACC(TrackerTraits::numberOfModules < 4096);  // easy to extend at least till 32*1024
 
         constexpr int numberOfModules = TrackerTraits::numberOfModules;
         constexpr uint32_t maxHitsInModule = TrackerTraits::maxHitsInModule;