Merge pull request #43853 from fwyzard/alpaka_integration_branch

Follow up to the Alpaka integration in CMSSW

CondFormats/SiPixelObjects/BuildFile.xml

@@ -1,20 +1,24 @@
 <use name="alpaka"/>
-<use name="DataFormats/DetId"/>
-<use name="DataFormats/SiPixelDetId"/>
 <use name="CalibFormats/SiPixelObjects"/>
-<use name="FWCore/Utilities"/>
-<use name="DataFormats/FEDRawData"/>
-<use name="CondFormats/Serialization"/>
+<use name="CondCore/DBOutputService"/>
 <use name="CondFormats/DataRecord"/>
-<use name="CondFormats/SiStripObjects"/>
-<use name="Geometry/TrackerGeometryBuilder"/>
 <use name="CondFormats/External"/>
+<use name="CondFormats/Serialization"/>
+<use name="CondFormats/SiStripObjects"/>
+<use name="DataFormats/DetId"/>
+<use name="DataFormats/FEDRawData"/>
 <use name="DataFormats/GeometryVector"/>
+<use name="DataFormats/Portable"/>
+<use name="DataFormats/SiPixelDetId"/>
+<use name="DataFormats/SoATemplate"/>
+<use name="DataFormats/TrackerCommon"/>
 <use name="FWCore/MessageLogger"/>
+<use name="FWCore/Utilities"/>
 <use name="Geometry/CommonDetUnit"/>
 <use name="Geometry/CommonTopologies"/>
-<use name="HeterogeneousCore/AlpakaInterface"/>
+<use name="Geometry/TrackerGeometryBuilder"/>
 <use name="HeterogeneousCore/AlpakaCore"/>
+<use name="HeterogeneousCore/AlpakaInterface"/>
 <flags ALPAKA_BACKENDS="1"/>
 <export>
   <lib name="1"/>

CondFormats/SiPixelObjects/interface/SiPixelMappingHost.h

@@ -1,10 +1,9 @@
-#ifndef CondFormats_SiPixelObjects_SiPixelMappingHost_h
-#define CondFormats_SiPixelObjects_SiPixelMappingHost_h
+#ifndef CondFormats_SiPixelObjects_interface_SiPixelMappingHost_h
+#define CondFormats_SiPixelObjects_interface_SiPixelMappingHost_h
 
-#include <alpaka/alpaka.hpp>
-#include "DataFormats/Portable/interface/PortableHostCollection.h"
 #include "CondFormats/SiPixelObjects/interface/SiPixelMappingLayout.h"
+#include "DataFormats/Portable/interface/PortableHostCollection.h"
 
 using SiPixelMappingHost = PortableHostCollection<SiPixelMappingSoA>;
 
-#endif  // CondFormats_SiPixelObjects_SiPixelMappingHost_h
+#endif  // CondFormats_SiPixelObjects_interface_SiPixelMappingHost_h

CondFormats/SiPixelObjects/interface/alpaka/SiPixelGainCalibrationForHLTDevice.h

@@ -1,13 +1,14 @@
 #ifndef CondFormats_SiPixelObjects_interface_alpaka_SiPixelGainCalibrationForHLTDevice_h
 #define CondFormats_SiPixelObjects_interface_alpaka_SiPixelGainCalibrationForHLTDevice_h
 
-#include <alpaka/alpaka.hpp>
-#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
 #include "CondFormats/SiPixelObjects/interface/SiPixelGainCalibrationForHLTLayout.h"
+#include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 
 namespace ALPAKA_ACCELERATOR_NAMESPACE {
 
   using SiPixelGainCalibrationForHLTDevice = PortableCollection<SiPixelGainCalibrationForHLTSoA>;
 
 }  // namespace ALPAKA_ACCELERATOR_NAMESPACE
+
 #endif  // CondFormats_SiPixelObjects_interface_alpaka_SiPixelGainCalibrationForHLTDevice_h

CondFormats/SiPixelObjects/interface/alpaka/SiPixelMappingDevice.h

@@ -2,11 +2,10 @@
 #define CondFormats_SiPixelObjects_interface_alpaka_SiPixelMappingDevice_h
 
 #include <cstdint>
-#include <alpaka/alpaka.hpp>
+
 #include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
 #include "CondFormats/SiPixelObjects/interface/SiPixelMappingLayout.h"
-#include "HeterogeneousCore/AlpakaCore/interface/alpaka/ESProducer.h"
-#include "DataFormats/Portable/interface/PortableHostCollection.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 
 namespace ALPAKA_ACCELERATOR_NAMESPACE {
 

CondFormats/SiPixelObjects/interface/alpaka/SiPixelMappingUtilities.h

@@ -1,11 +1,18 @@
 #ifndef CondFormats_SiPixelObjects_interface_alpaka_SiPixelMappingUtilities_h
 #define CondFormats_SiPixelObjects_interface_alpaka_SiPixelMappingUtilities_h
 
-#include <cstdint>
+#include <set>
+#include <vector>
+
 #include <alpaka/alpaka.hpp>
-#include "CondFormats/SiPixelObjects/interface/SiPixelMappingLayout.h"
+
+#include "CondFormats/SiPixelObjects/interface/CablingPathToDetUnit.h"
+#include "CondFormats/SiPixelObjects/interface/PixelROC.h"
 #include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingMap.h"
 #include "CondFormats/SiPixelObjects/interface/SiPixelFedCablingTree.h"
+#include "CondFormats/SiPixelObjects/interface/SiPixelMappingLayout.h"
+#include "CondFormats/SiPixelObjects/interface/SiPixelROCsStatusAndMapping.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 #include "HeterogeneousCore/AlpakaInterface/interface/memory.h"
 
 namespace ALPAKA_ACCELERATOR_NAMESPACE {

CondFormats/SiPixelObjects/test/BuildFile.xml

@@ -1,6 +1,10 @@
 <use name="CondCore/DBOutputService"/>
 <use name="CondFormats/DataRecord"/>
 <use name="CondFormats/SiPixelObjects"/>
+<use name="FWCore/Framework"/>
+<use name="FWCore/ParameterSet"/>
+<use name="FWCore/ServiceRegistry"/>
+<use name="Geometry/Records"/>
 <use name="Geometry/TrackerGeometryBuilder"/>
 <library name="SiPixelObjectsTest" file="*.cc">
   <flags EDM_PLUGIN="1"/>

Configuration/PyReleaseValidation/README.md

@@ -30,7 +30,10 @@ The offsets currently in use are:
 * 0.2: Tracking Run-2 era, `Run2_2017_trackingRun2`
 * 0.3: 0.1 + 0.2
 * 0.4: LowPU tracking era, `Run2_2017_trackingLowPU`
-* 0.411: Patatrack, ECAL only, Alpaka
+* 0.402: Alpaka, pixel only quadruplets, portable
+* 0.403: Alpaka, pixel only quadruplets, portable vs. CPU validation
+* 0.404: Alpaka, pixel only quadruplets, portable profiling
+* 0.412: Alpaka, ECAL only, portable
 * 0.5: Pixel tracking only + 0.1
 * 0.501: Patatrack, pixel only quadruplets, on CPU
 * 0.502: Patatrack, pixel only quadruplets, with automatic offload to GPU if available

Configuration/PyReleaseValidation/python/relval_gpu.py

@@ -13,7 +13,11 @@
 
 # mc WFs to run in IB:
 
-# mc 2023   Patatrack pixel-only quadruplets:                   ZMM - on GPU (optional), GPU-vs-CPU validation, profiling
+# mc 2023   Alpaka pixel-only quadruplets:                      ZMM: any backend, any backend vs cpu validation, profiling
+#           Alpaka pixel-only quadruplets:                      TTbar: any backend, any backend vs cpu validation, profiling
+#           Alpaka ECAL-only:                                   TTbar: any backend
+#           Alpaka pixel-only quadruplets:                      Single Nu E10: any backend
+#           Patatrack pixel-only quadruplets:                   ZMM - on GPU (optional), GPU-vs-CPU validation, profiling
 #           Patatrack pixel-only triplets:                      ZMM - on GPU (optional), GPU-vs-CPU validation, profiling
 #           Patatrack pixel-only quadruplets:                   TTbar - on GPU (optional), GPU-vs-CPU validation, profiling
 #           Patatrack pixel-only triplets:                      TTbar - on GPU (optional), GPU-vs-CPU validation, profiling
@@ -23,10 +27,17 @@
 #           Patatrack pixel-only triplets, ECAL, HCAL:          TTbar - on GPU (optional), GPU-vs-CPU validation, profiling (to be implemented)
 #           full reco with Patatrack pixel-only quadruplets:    TTbar - on GPU (optional), GPU-vs-CPU validation
 #           full reco with Patatrack pixel-only triplets:       TTbar - on GPU (optional), GPU-vs-CPU validation
-#           Patatrack Single Nu E10 on GPU (optional)
-# mc 2026   Patatrack Single Nu E10 on GPU (optional)
+#           Patatrack pixel-only quadruplets:                   Single Nu E10 on GPU (optional)
+# mc 2026   Patatrack pixel-only quadruplets:                   Single Nu E10 on GPU (optional)
 numWFIB = [
-           # 2023
+           # 2023, Alpaka-based
+           12450.402, 12450.403, 12450.404,
+           #12450.406, 12450.407, 12450.408,
+           12434.402, 12434.403, 12434.404,
+           #12434.406, 12434.407, 12434.408,
+           12434.412, #12434.413, 12434.414,
+           12461.402,
+           # 2023, CUDA-based
            12450.502, 12450.503, 12450.504,
            12450.506, 12450.507, 12450.508,
            12434.502, 12434.503, 12434.504,
@@ -37,7 +48,8 @@
            12434.586, 12434.587, # 12434.588,
            12434.592, 12434.593,
            12434.596, 12434.597,
-           12461.502, 
+           12461.502,
+           # 2026, CUDA-based
            24861.502
         ]
 

Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py

@@ -1079,7 +1079,7 @@ def setup_(self, step, stepName, stepDict, k, properties):
         '-s': 'HARVESTING:@ecalOnlyValidation+@ecal'
     },
     suffix = 'Patatrack_ECALOnlyAlpaka',
-    offset = 0.411,
+    offset = 0.412,
 )
 
 # ECAL-only workflow running on CPU

DataFormats/SiPixelClusterSoA/interface/alpaka/SiPixelClustersSoACollection.h

@@ -20,7 +20,8 @@ namespace cms::alpakatools {
   struct CopyToHost<SiPixelClustersDevice<TDevice>> {
     template <typename TQueue>
     static auto copyAsync(TQueue &queue, SiPixelClustersDevice<TDevice> const &srcData) {
-      SiPixelClustersHost dstData(srcData->metadata().size(), queue);
+      // SiPixelClustersHost and SiPixelClustersDevice have a capacity larger than the ctor argument by one
+      SiPixelClustersHost dstData(srcData->metadata().size() - 1, queue);
       alpaka::memcpy(queue, dstData.buffer(), srcData.buffer());
       dstData.setNClusters(srcData.nClusters(), srcData.offsetBPIX2());
 #ifdef GPU_DEBUG  //keeping this untiil copies are in the Tracer

DataFormats/SiPixelClusterSoA/test/alpaka/Clusters_test.dev.cc

@@ -15,7 +15,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     public:
       template <typename TAcc, typename = std::enable_if_t<isAccelerator<TAcc>>>
       ALPAKA_FN_ACC void operator()(TAcc const& acc, SiPixelClustersSoAView clust_view) const {
-        for (int32_t j : elements_with_stride(acc, clust_view.metadata().size())) {
+        for (int32_t j : uniform_elements(acc, clust_view.metadata().size())) {
           clust_view[j].moduleStart() = j;
           clust_view[j].clusInModule() = j * 2;
           clust_view[j].moduleId() = j * 3;
@@ -28,7 +28,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
     public:
       template <typename TAcc, typename = std::enable_if_t<isAccelerator<TAcc>>>
       ALPAKA_FN_ACC void operator()(TAcc const& acc, SiPixelClustersSoAConstView clust_view) const {
-        for (uint32_t j : elements_with_stride(acc, clust_view.metadata().size())) {
+        for (uint32_t j : uniform_elements(acc, clust_view.metadata().size())) {
           assert(clust_view[j].moduleStart() == j);
           assert(clust_view[j].clusInModule() == j * 2);
           assert(clust_view[j].moduleId() == j * 3);

DataFormats/TrackSoA/interface/alpaka/TrackUtilities.h

@@ -1,9 +1,14 @@
-#ifndef DataFormats_Track_interface_alpaka_TrackUtilities_h
-#define DataFormats_Track_interface_alpaka_TrackUtilities_h
+#ifndef DataFormats_TrackSoA_interface_alpaka_TrackUtilities_h
+#define DataFormats_TrackSoA_interface_alpaka_TrackUtilities_h
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+
+#include <alpaka/alpaka.hpp>
 
-#include "Geometry/CommonTopologies/interface/SimplePixelTopology.h"
-#include "DataFormats/TrackSoA/interface/TrackDefinitions.h"
 #include "DataFormats/TrackSoA/interface/TracksSoA.h"
+#include "Geometry/CommonTopologies/interface/SimplePixelTopology.h"
 
 // Methods that operate on View and ConstView of the TrackSoA, and cannot be class methods.
 template <typename TrackerTraits>
@@ -170,4 +175,4 @@ namespace pixelTrack {
 template struct TracksUtilities<pixelTopology::Phase1>;
 template struct TracksUtilities<pixelTopology::Phase2>;
 
-#endif
+#endif  // DataFormats_TrackSoA_interface_alpaka_TrackUtilities_h

DataFormats/TrackSoA/interface/alpaka/TracksSoACollection.h

@@ -1,15 +1,17 @@
-#ifndef DataFormats_Track_interface_alpaka_TracksSoACollection_h
-#define DataFormats_Track_interface_alpaka_TracksSoACollection_h
+#ifndef DataFormats_TrackSoA_interface_alpaka_TracksSoACollection_h
+#define DataFormats_TrackSoA_interface_alpaka_TracksSoACollection_h
+
+#include <type_traits>
 
-#include <cstdint>
 #include <alpaka/alpaka.hpp>
-#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
+
 #include "DataFormats/Portable/interface/alpaka/PortableCollection.h"
-#include "DataFormats/TrackSoA/interface/TracksSoA.h"
-#include "DataFormats/TrackSoA/interface/TrackDefinitions.h"
-#include "DataFormats/TrackSoA/interface/TracksHost.h"
 #include "DataFormats/TrackSoA/interface/TracksDevice.h"
+#include "DataFormats/TrackSoA/interface/TracksHost.h"
+#include "Geometry/CommonTopologies/interface/SimplePixelTopology.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/AssertDeviceMatchesHostCollection.h"
 #include "HeterogeneousCore/AlpakaInterface/interface/CopyToHost.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 
 // TODO: The class is created via inheritance of the PortableCollection.
 // This is generally discouraged, and should be done via composition.
@@ -49,4 +51,4 @@ ASSERT_DEVICE_MATCHES_HOST_COLLECTION(pixelTrack::TracksSoACollectionPhase1, pix
 ASSERT_DEVICE_MATCHES_HOST_COLLECTION(pixelTrack::TracksSoACollectionPhase2, pixelTrack::TracksHostPhase2);
 ASSERT_DEVICE_MATCHES_HOST_COLLECTION(pixelTrack::TracksSoACollectionHIonPhase1, pixelTrack::TracksHostHIonPhase1);
 
-#endif  // DataFormats_Track_interface_alpaka_TracksSoACollection_h
+#endif  // DataFormats_TrackSoA_interface_alpaka_TracksSoACollection_h

DataFormats/TrackSoA/src/alpaka/classes_cuda.h

@@ -3,9 +3,9 @@
 
 #include "DataFormats/Common/interface/DeviceProduct.h"
 #include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/TrackSoA/interface/TracksDevice.h"
 #include "DataFormats/TrackSoA/interface/TracksSoA.h"
 #include "DataFormats/TrackSoA/interface/alpaka/TracksSoACollection.h"
-#include "DataFormats/TrackSoA/interface/TracksDevice.h"
 #include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 
 using namespace pixelTopology;

DataFormats/TrackSoA/src/alpaka/classes_rocm.h

@@ -3,9 +3,9 @@
 
 #include "DataFormats/Common/interface/DeviceProduct.h"
 #include "DataFormats/Common/interface/Wrapper.h"
+#include "DataFormats/TrackSoA/interface/TracksDevice.h"
 #include "DataFormats/TrackSoA/interface/TracksSoA.h"
 #include "DataFormats/TrackSoA/interface/alpaka/TracksSoACollection.h"
-#include "DataFormats/TrackSoA/interface/TracksDevice.h"
 #include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 
 using namespace pixelTopology;

DataFormats/TrackSoA/test/alpaka/TrackSoAHeterogeneous_test.dev.cc

@@ -1,8 +1,13 @@
-#include "Geometry/CommonTopologies/interface/SimplePixelTopology.h"
+#include <type_traits>
+
+#include <alpaka/alpaka.hpp>
+
 #include "DataFormats/TrackSoA/interface/TrackDefinitions.h"
-#include "DataFormats/TrackSoA/interface/alpaka/TracksSoACollection.h"
 #include "DataFormats/TrackSoA/interface/TracksDevice.h"
 #include "DataFormats/TrackSoA/interface/TracksHost.h"
+#include "DataFormats/TrackSoA/interface/alpaka/TracksSoACollection.h"
+#include "Geometry/CommonTopologies/interface/SimplePixelTopology.h"
+#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 
 using namespace reco;
 
@@ -22,7 +27,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
           tracks_view.nTracks() = nTracks;
         }
 
-        for (int32_t j : elements_with_stride(acc, nTracks)) {
+        for (int32_t j : uniform_elements(acc, nTracks)) {
           tracks_view[j].pt() = (float)j;
           tracks_view[j].eta() = (float)j;
           tracks_view[j].chi2() = (float)j;
@@ -45,7 +50,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
         if (cms::alpakatools::once_per_grid(acc)) {
           ALPAKA_ASSERT(tracks_view.nTracks() == nTracks);
         }
-        for (int32_t j : elements_with_stride(acc, tracks_view.nTracks())) {
+        for (int32_t j : uniform_elements(acc, tracks_view.nTracks())) {
           ALPAKA_ASSERT(abs(tracks_view[j].pt() - (float)j) < .0001);
           ALPAKA_ASSERT(abs(tracks_view[j].eta() - (float)j) < .0001);
           ALPAKA_ASSERT(abs(tracks_view[j].chi2() - (float)j) < .0001);

DataFormats/TrackingRecHitSoA/interface/TrackingRecHitsDevice.h

@@ -8,23 +8,23 @@
 #include "DataFormats/Portable/interface/PortableDeviceCollection.h"
 #include "DataFormats/TrackingRecHitSoA/interface/TrackingRecHitsHost.h"
 #include "DataFormats/TrackingRecHitSoA/interface/TrackingRecHitsSoA.h"
-#include "HeterogeneousCore/AlpakaInterface/interface/config.h"
 
 template <typename TrackerTraits, typename TDev>
 class TrackingRecHitDevice : public PortableDeviceCollection<TrackingRecHitLayout<TrackerTraits>, TDev> {
 public:
   using hitSoA = TrackingRecHitSoA<TrackerTraits>;
-  //Need to decorate the class with the inherited portable accessors being now a template
+
+  // Need to decorate the class with the inherited portable accessors being now a template
   using PortableDeviceCollection<TrackingRecHitLayout<TrackerTraits>, TDev>::view;
   using PortableDeviceCollection<TrackingRecHitLayout<TrackerTraits>, TDev>::const_view;
   using PortableDeviceCollection<TrackingRecHitLayout<TrackerTraits>, TDev>::buffer;
 
   TrackingRecHitDevice() = default;
 
-  // Constructor which specifies the SoA size
+  // Constructor which specifies the SoA size, number of BPIX1 hits, and the modules entry points
   template <typename TQueue>
-  explicit TrackingRecHitDevice(uint32_t nHits, int32_t offsetBPIX2, uint32_t const* hitsModuleStart, TQueue queue)
-      : PortableDeviceCollection<TrackingRecHitLayout<TrackerTraits>, TDev>(nHits, queue) {
+  explicit TrackingRecHitDevice(TQueue queue, uint32_t nHits, int32_t offsetBPIX2, uint32_t const* hitsModuleStart)
+      : PortableDeviceCollection<TrackingRecHitLayout<TrackerTraits>, TDev>(nHits, queue), offsetBPIX2_{offsetBPIX2} {
     const auto device = alpaka::getDev(queue);
 
     auto start_h = cms::alpakatools::make_host_view(hitsModuleStart, TrackerTraits::numberOfModules + 1);
@@ -39,6 +39,22 @@ class TrackingRecHitDevice : public PortableDeviceCollection<TrackingRecHitLayou
   }
 
   uint32_t nHits() const { return view().metadata().size(); }
+
+  int32_t offsetBPIX2() const { return offsetBPIX2_; }
+
   uint32_t const* hitsModuleStart() const { return view().hitsModuleStart().data(); }
+
+  // asynchronously update the information cached within the class itself from the information on the device
+  template <typename TQueue>
+  void updateFromDevice(TQueue queue) {
+    auto off_h = cms::alpakatools::make_host_view(offsetBPIX2_);
+    auto off_d = cms::alpakatools::make_device_view(alpaka::getDev(queue), view().offsetBPIX2());
+    alpaka::memcpy(queue, off_h, off_d);
+  }
+
+private:
+  // offsetBPIX2 is used on host functions so is useful to have it also stored in the class and not only in the layout
+  int32_t offsetBPIX2_ = 0;
 };
+
 #endif  // DataFormats_RecHits_interface_TrackingRecHitSoADevice_h