Merge branch 'AliceO2Group:dev' into new-detector4

Detectors/TPC/qc/include/TPCQC/PID.h

@@ -24,6 +24,7 @@
 
 // root includes
 #include "TH1.h"
+// Decapricated to be removed in next PR
 #include "TCanvas.h"
 
 // o2 includes
@@ -70,7 +71,7 @@ class PID
 
   // To set the elementary track cuts
   void setPIDCuts(int minnCls = 60, float absTgl = 1., float mindEdxTot = 10.0,
-                  float maxdEdxTot = 70., float minpTPC = 0.05, float maxpTPC = 20., float minpTPCMIPs = 0.45, float maxpTPCMIPs = 0.55, bool turnOffHistosForAsync = false)
+                  float maxdEdxTot = 70., float minpTPC = 0.05, float maxpTPC = 20., float minpTPCMIPs = 0.45, float maxpTPCMIPs = 0.55, bool turnOffHistosForAsync = false, bool getdEdxVspHypoHist = false)
   {
     mCutMinnCls = minnCls;
     mCutAbsTgl = absTgl;
@@ -81,15 +82,21 @@ class PID
     mCutMinpTPCMIPs = minpTPCMIPs;
     mCutMaxpTPCMIPs = maxpTPCMIPs;
     mTurnOffHistosForAsync = turnOffHistosForAsync;
+    mGetdEdxVspHypoHist = getdEdxVspHypoHist;
   }
+
+  // Decapricated to be removed in next PR
   void setCreateCanvas(int createCanvas = 1)
   {
     mCreateCanvas = createCanvas;
   }
+
   std::unordered_map<std::string_view, std::vector<std::unique_ptr<TH1>>>& getMapOfHisto() { return mMapHist; }
+  const std::unordered_map<std::string_view, std::vector<std::unique_ptr<TH1>>>& getMapOfHisto() const { return mMapHist; }
+
+  // Decapricated to be removed in next PR
   std::unordered_map<std::string_view, std::vector<std::unique_ptr<TCanvas>>>& getMapOfCanvas() { return mMapCanvas; }
   TCanvas* getSeparationPowerCanvas() { return mSeparationPowerCanvas.get(); }
-  const std::unordered_map<std::string_view, std::vector<std::unique_ptr<TH1>>>& getMapOfHisto() const { return mMapHist; }
   const std::unordered_map<std::string_view, std::vector<std::unique_ptr<TCanvas>>>& getMapOfCanvas() const { return mMapCanvas; }
 
  private:
@@ -101,15 +108,19 @@ class PID
   float mCutMaxpTPC = 20.f;            // pTPC max value
   float mCutMinpTPCMIPs = 0.45f;       // pTPC min value for MIPs
   float mCutMaxpTPCMIPs = 0.55f;       // pTPC max value for MIPs
-  bool mCreateCanvas = true;           // Decide whether to create the TCanvas Object as it cannot be merged
   bool mTurnOffHistosForAsync = false; // Decide whether to turn off some histograms for async to reduce memory
+  bool mGetdEdxVspHypoHist = false;    // Decide whether to generate the EdxVspHypo histograms
+  // Decapricated to be removed in next PR
+  bool mCreateCanvas = true; // Decide whether to create the TCanvas Object as it cannot be merged
+
   std::unordered_map<std::string_view, std::vector<std::unique_ptr<TH1>>> mMapHist;
+  // Decapricated to be removed in next PR
   // Map for Canvases to be published
   std::unordered_map<std::string_view, std::vector<std::unique_ptr<TCanvas>>> mMapCanvas;
   // Map for Histograms which will be put onto the canvases, and not published separately
   std::unordered_map<std::string_view, std::vector<std::unique_ptr<TH1>>> mMapHistCanvas;
-  // Canvas for Trending Separation Power
   std::unique_ptr<TCanvas> mSeparationPowerCanvas;
+
   ClassDefNV(PID, 1)
 };
 } // namespace qc

Detectors/TPC/qc/src/PID.cxx

@@ -18,7 +18,6 @@
 // root includes
 #include "TStyle.h"
 #include "TFile.h"
-#include "TCanvas.h"
 #include "TMathBase.h"
 #include "TObjArray.h"
 
@@ -93,16 +92,11 @@ void PID::initializeHistograms()
     mMapHist["hdEdxMaxMIPVsSec"].emplace_back(std::make_unique<TH2F>(fmt::format("hdEdxMaxMIPVsSec_{}", name).data(), (fmt::format("MIP Q_{{Max}} {}", name) + ";sector;d#it{E}/d#it{x}_{Max} (arb. unit)").data(), binsSec.bins, binsSec.min, binsSec.max, binsdEdxMIPMax.bins, binsdEdxMIPMax.min, binsdEdxMIPMax.max));
     mMapHist["hMIPNclVsTgl"].emplace_back(std::make_unique<TH2F>(fmt::format("hMIPNclVsTgl_{}", name).data(), (fmt::format("rec. clusters {}", name) + ";#tan(#lambda); rec clusters").data(), 50, -2, 2, nclMax[idEdxType] - nclCuts[idEdxType], nclCuts[idEdxType], nclMax[idEdxType]));
     mMapHist["hMIPNclVsTglSub"].emplace_back(std::make_unique<TH2F>(fmt::format("hMIPNclVsTglSub_{}", name).data(), (fmt::format("sub-thrs. clusters {}", name) + ";#tan(#lambda);sub-thrs. clusters").data(), 50, -2, 2, 20, 0, 20));
-    if (mCreateCanvas) {
-      mMapHistCanvas["hdEdxVspHypoPos"].emplace_back(std::make_unique<TH2F>(fmt::format("hdEdxVspHypoPos_{}", name).data(), (fmt::format("Q_{{Tot}} Pos {}", name) + ";#it{p} (GeV/#it{c});d#it{E}/d#it{x}_{Tot} (arb. unit)").data(), 200, bins.data(), binNumber, binsdEdxTot_Log.data()));
-      mMapHistCanvas["hdEdxVspHypoNeg"].emplace_back(std::make_unique<TH2F>(fmt::format("hdEdxVspHypoNeg_{}", name).data(), (fmt::format("Q_{{Tot}} Neg {}", name) + ";#it{p} (GeV/#it{c});d#it{E}/d#it{x}_{Tot} (arb. unit)").data(), 200, bins.data(), binNumber, binsdEdxTot_Log.data()));
+    if (mGetdEdxVspHypoHist) {
+      mMapHist["hdEdxVspHypoPos"].emplace_back(std::make_unique<TH2F>(fmt::format("hdEdxVspHypoPos_{}", name).data(), (fmt::format("Q_{{Tot}} Pos {}", name) + ";#it{p} (GeV/#it{c});d#it{E}/d#it{x}_{Tot} (arb. unit)").data(), 200, bins.data(), binNumber, binsdEdxTot_Log.data()));
+      mMapHist["hdEdxVspHypoNeg"].emplace_back(std::make_unique<TH2F>(fmt::format("hdEdxVspHypoNeg_{}", name).data(), (fmt::format("Q_{{Tot}} Neg {}", name) + ";#it{p} (GeV/#it{c});d#it{E}/d#it{x}_{Tot} (arb. unit)").data(), 200, bins.data(), binNumber, binsdEdxTot_Log.data()));
     }
   }
-  if (mCreateCanvas) {
-    mMapCanvas["CdEdxPIDHypothesisVsp"].emplace_back(std::make_unique<TCanvas>("CdEdxPIDHypothesisVsp", "PID Hypothesis Ratio"));
-    mMapCanvas["CdEdxPIDHypothesisVsp"].at(0)->Divide(5, 2);
-  }
-  mSeparationPowerCanvas.reset(new TCanvas("CSeparationPower", "Separation Power"));
 }
 
 //______________________________________________________________________________
@@ -113,11 +107,6 @@ void PID::resetHistograms()
       hist->Reset();
     }
   }
-  for (const auto& pair : mMapHistCanvas) {
-    for (auto& hist : pair.second) {
-      hist->Reset();
-    }
-  }
 }
 
 //______________________________________________________________________________
@@ -185,10 +174,10 @@ bool PID::processTrack(const o2::tpc::TrackTPC& track, size_t nTracks)
     if (std::abs(tgl) < mCutAbsTgl) {
       mMapHist["hdEdxTotVsp"][idEdxType]->Fill(pTPC, dEdxTot[idEdxType]);
       mMapHist["hdEdxMaxVsp"][idEdxType]->Fill(pTPC, dEdxMax[idEdxType]);
-      if (mCreateCanvas) {
+      if (mGetdEdxVspHypoHist) {
         const auto pidHypothesis = track.getPID().getID();
         if (pidHypothesis <= o2::track::PID::NIDs) {
-          auto pidHist = mMapHistCanvas[(track.getCharge() > 0) ? "hdEdxVspHypoPos" : "hdEdxVspHypoNeg"][idEdxType].get();
+          auto pidHist = mMapHist[(track.getCharge() > 0) ? "hdEdxVspHypoPos" : "hdEdxVspHypoNeg"][idEdxType].get();
           pidHist->SetBinContent(pidHist->GetXaxis()->FindBin(pTPC), pidHist->GetYaxis()->FindBin(dEdxTot[idEdxType]), pidHypothesis + 1);
         }
       }
@@ -227,21 +216,6 @@ bool PID::processTrack(const o2::tpc::TrackTPC& track, size_t nTracks)
       }
     }
   }
-
-  if (mCreateCanvas) {
-    for (auto const& pairC : mMapCanvas) {
-      for (auto& canv : pairC.second) {
-        int h = 1;
-        for (auto const& pairH : mMapHistCanvas) {
-          for (auto& hist : pairH.second) {
-            canv->cd(h);
-            hist->Draw();
-            h++;
-          }
-        }
-      }
-    }
-  }
   return true;
 }
 

Framework/Core/src/CompletionPolicy.cxx

@@ -26,7 +26,7 @@ std::vector<CompletionPolicy>
 {
   return {
     CompletionPolicyHelpers::consumeWhenAllOrdered("internal-dpl-aod-writer"),
-    CompletionPolicyHelpers::consumeWhenAny("internal-dpl-injected-dummy-sink", [](DeviceSpec const& s) { return s.name == "internal-dpl-injected-dummy-sink"; }),
+    CompletionPolicyHelpers::consumeWhenAny("internal-dpl-injected-dummy-sink", [](DeviceSpec const& s) { return s.name.find("internal-dpl-injected-dummy-sink") != std::string::npos; }),
     CompletionPolicyHelpers::consumeWhenAll()};
 }
 

Framework/Core/src/DataProcessingDevice.cxx

@@ -1096,7 +1096,7 @@ void DataProcessingDevice::fillContext(DataProcessorContext& context, DeviceCont
   context.balancingInputs = spec.completionPolicy.balanceChannels;
   // This is needed because the internal injected dummy sink should not
   // try to balance inputs unless the rate limiting is requested.
-  if (enableRateLimiting == false && spec.name == "internal-dpl-injected-dummy-sink") {
+  if (enableRateLimiting == false && spec.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
     context.balancingInputs = false;
   }
   if (enableRateLimiting) {

Framework/Core/src/DeviceSpecHelpers.cxx

@@ -631,6 +631,28 @@ void DeviceSpecHelpers::processOutEdgeActions(ConfigContext const& configContext
     }
     DeviceConnectionId id{edge.producer, edge.consumer, edge.timeIndex, edge.producerTimeIndex, channel.port};
     connections.push_back(id);
+
+    auto& source = workflow[edge.producer];
+
+    O2_SIGNPOST_ID_GENERATE(sid, device_spec_helpers);
+    O2_SIGNPOST_START(device_spec_helpers, sid, "new channels", "Channel %{public}s has been created.", channel.name.c_str());
+    O2_SIGNPOST_ID_GENERATE(iid, device_spec_helpers);
+    O2_SIGNPOST_START(device_spec_helpers, iid, "producer outputs", "Producer %{public}s has the following outputs:", source.name.c_str());
+    for (auto& output : source.outputs) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, iid, "producer outputs", "%{public}s", DataSpecUtils::describe(output).c_str());
+    }
+    O2_SIGNPOST_END(device_spec_helpers, iid, "producer outputs", "");
+    O2_SIGNPOST_START(device_spec_helpers, iid, "producer forwards", "Producer %{public}s has the following forwards:", source.name.c_str());
+    for (auto& forwards : device.forwards) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, iid, "producer forwards", "%{public}s", DataSpecUtils::describe(forwards.matcher).c_str());
+    }
+    O2_SIGNPOST_END(device_spec_helpers, iid, "producer forwards", "");
+    O2_SIGNPOST_START(device_spec_helpers, iid, "consumer inputs", "Consumer %{public}s has the following inputs:", consumer.name.c_str());
+    for (auto& input : consumer.inputs) {
+      O2_SIGNPOST_EVENT_EMIT(device_spec_helpers, iid, "consumer inputs", "%{public}s", DataSpecUtils::describe(input).c_str());
+    }
+    O2_SIGNPOST_END(device_spec_helpers, iid, "consumer inputs", "");
+    O2_SIGNPOST_END(device_spec_helpers, sid, "new channels", "");
     return channel;
   };
 

Framework/Core/src/TopologyPolicy.cxx

@@ -162,10 +162,15 @@ TopologyPolicy::DependencyChecker TopologyPolicyHelpers::alwaysDependent()
       O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. %s and %s are the same.", dependent.name.c_str(), ancestor.name.c_str());
       return false;
     }
-    if (ancestor.name == "internal-dpl-injected-dummy-sink") {
-      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. %s is a dummy sink.", ancestor.name.c_str());
+    if (ancestor.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "false. Nothing can depend on %s by policy.", ancestor.name.c_str());
       return false;
     }
+    // We never put anything behind the dummy sink.
+    if (dependent.name.find("internal-dpl-injected-dummy-sink") != std::string::npos) {
+      O2_SIGNPOST_END(topology, sid, "alwaysDependent", "true. %s is always last.", ancestor.name.c_str());
+      return true;
+    }
     const std::regex matcher(".*output-proxy.*");
     // Check if regex applies
     std::cmatch m;

Framework/Core/src/WorkflowHelpers.cxx

@@ -24,6 +24,7 @@
 #include "Framework/PluginManager.h"
 #include "Framework/DataTakingContext.h"
 #include "Framework/DefaultsHelpers.h"
+#include "Framework/Signpost.h"
 
 #include "Headers/DataHeader.h"
 #include <algorithm>
@@ -34,8 +35,7 @@
 #include <climits>
 #include <thread>
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wpedantic"
+O2_DECLARE_DYNAMIC_LOG(workflow_helpers);
 
 namespace o2::framework
 {
@@ -367,8 +367,23 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
     // timeframe data.
     bool timeframeSink = hasTimeframeInputs && !hasTimeframeOutputs;
     if (std::stoi(ctx.options().get<std::string>("timeframes-rate-limit-ipcid")) != -1) {
-      if (timeframeSink && processor.name != "internal-dpl-injected-dummy-sink") {
-        processor.outputs.push_back(OutputSpec{{"dpl-summary"}, ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(runtime_hash(processor.name.c_str()))}});
+      if (timeframeSink && processor.name.find("internal-dpl-injected-dummy-sink") == std::string::npos) {
+        O2_SIGNPOST_ID_GENERATE(sid, workflow_helpers);
+        uint32_t hash = runtime_hash(processor.name.c_str());
+        bool hasMatch = false;
+        ConcreteDataMatcher summaryMatcher = ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(hash)};
+        for (auto& output : processor.outputs) {
+          if (DataSpecUtils::match(output, summaryMatcher)) {
+            O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "%{public}s already there in %{public}s",
+                                   DataSpecUtils::describe(output).c_str(), processor.name.c_str());
+            hasMatch = true;
+            break;
+          }
+        }
+        if (!hasMatch) {
+          O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "Adding DPL/SUMMARY/%d to %{public}s", hash, processor.name.c_str());
+          processor.outputs.push_back(OutputSpec{{"dpl-summary"}, ConcreteDataMatcher{"DPL", "SUMMARY", static_cast<DataAllocator::SubSpecificationType>(hash)}});
+        }
       }
     }
     bool hasConditionOption = false;
@@ -838,15 +853,23 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
   // Notice that availableOutputsInfo MUST be updated first, since it relies on
   // the size of outputs to be the one before the update.
   auto enumerateAvailableOutputs = [&workflow, &outputs, &availableOutputsInfo]() {
+    O2_SIGNPOST_ID_GENERATE(sid, workflow_helpers);
     for (size_t wi = 0; wi < workflow.size(); ++wi) {
       auto& producer = workflow[wi];
+      if (producer.outputs.empty()) {
+        O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "No outputs for [%zu] %{public}s", wi, producer.name.c_str());
+      }
+      O2_SIGNPOST_START(workflow_helpers, sid, "output enumeration", "Enumerating outputs for producer [%zu] %{}s public", wi, producer.name.c_str());
 
       for (size_t oi = 0; oi < producer.outputs.size(); ++oi) {
         auto& out = producer.outputs[oi];
         auto uniqueOutputId = outputs.size();
         availableOutputsInfo.emplace_back(LogicalOutputInfo{wi, uniqueOutputId, false});
+        O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output enumeration", "- [%zu, %zu] %{public}s",
+                               oi, uniqueOutputId, DataSpecUtils::describe(out).c_str());
         outputs.push_back(out);
       }
+      O2_SIGNPOST_END(workflow_helpers, sid, "output enumeration", "");
     }
   };
 
@@ -878,10 +901,17 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
 
   std::vector<bool> matches(constOutputs.size());
   for (size_t consumer = 0; consumer < workflow.size(); ++consumer) {
+    O2_SIGNPOST_ID_GENERATE(sid, workflow_helpers);
+    O2_SIGNPOST_START(workflow_helpers, sid, "input matching", "Matching inputs of consumer [%zu] %{}s public", consumer, workflow[consumer].name.c_str());
     for (size_t input = 0; input < workflow[consumer].inputs.size(); ++input) {
       forwards.clear();
       for (size_t i = 0; i < constOutputs.size(); i++) {
         matches[i] = DataSpecUtils::match(workflow[consumer].inputs[input], constOutputs[i]);
+        if (matches[i]) {
+          O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output", "Input %{public}s matches %{public}s",
+                                 DataSpecUtils::describe(workflow[consumer].inputs[input]).c_str(),
+                                 DataSpecUtils::describe(constOutputs[i]).c_str());
+        }
       }
 
       for (size_t i = 0; i < availableOutputsInfo.size(); i++) {
@@ -897,6 +927,8 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
         auto uniqueOutputId = oif->outputGlobalIndex;
         for (size_t tpi = 0; tpi < workflow[consumer].maxInputTimeslices; ++tpi) {
           for (size_t ptpi = 0; ptpi < workflow[producer].maxInputTimeslices; ++ptpi) {
+            O2_SIGNPOST_EVENT_EMIT(workflow_helpers, sid, "output", "Adding edge between %{public}s and %{public}s", workflow[consumer].name.c_str(),
+                                   workflow[producer].name.c_str());
             logicalEdges.emplace_back(DeviceConnectionEdge{producer, consumer, tpi, ptpi, uniqueOutputId, input, oif->forward});
           }
         }
@@ -912,6 +944,7 @@ void WorkflowHelpers::constructGraph(const WorkflowSpec& workflow,
         availableOutputsInfo.push_back(forward);
       }
     }
+    O2_SIGNPOST_END(workflow_helpers, sid, "input matching", "");
   }
 }