From 3a0b16fdb494d87d009812b6b64e5213f33fc405 Mon Sep 17 00:00:00 2001
From: francescobrivio <francesco.brivio@cern.ch>
Date: Wed, 14 Feb 2024 22:19:50 +0100
Subject: [PATCH 1/4] fix LHCInfoPer* PopCons to produce lumi type IOVs when
 running in duringFill mode

---
 CondTools/RunInfo/interface/LHCInfoHelper.h   | 21 ++++++++++
 .../plugins/LHCInfoPerFillPopConAnalyzer.cc   | 42 +++++++++++--------
 .../plugins/LHCInfoPerLSPopConAnalyzer.cc     | 36 +++++++++-------
 .../python/LHCInfoPerFillPopConAnalyzer.py    |  8 +++-
 .../python/LHCInfoPerLSPopConAnalyzer.py      |  8 +++-
 CondTools/RunInfo/src/LHCInfoHelper.cc        | 30 +++++++++++++
 6 files changed, 112 insertions(+), 33 deletions(-)
 create mode 100644 CondTools/RunInfo/interface/LHCInfoHelper.h
 create mode 100644 CondTools/RunInfo/src/LHCInfoHelper.cc

diff --git a/CondTools/RunInfo/interface/LHCInfoHelper.h b/CondTools/RunInfo/interface/LHCInfoHelper.h
new file mode 100644
index 0000000000000..61d005c287a17
--- /dev/null
+++ b/CondTools/RunInfo/interface/LHCInfoHelper.h
@@ -0,0 +1,21 @@
+#ifndef CondTools_RunInfo_LHCInfoHelper_h
+#define CondTools_RunInfo_LHCInfoHelper_h
+
+#include "CondCore/CondDB/interface/Time.h"
+#include "CondTools/RunInfo/interface/OMSAccess.h"
+
+namespace cond {
+
+  namespace lhcInfoHelper {
+
+    // Large number of LS for the OMS query, covering around 25 hours
+    static constexpr unsigned int kLumisectionsQueryLimit = 4000;
+
+    // Returns lumi-type IOV from last LS of last Run of the specified Fill
+    cond::Time_t getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId);
+
+  }  // namespace lhcInfoHelper
+
+}  // namespace cond
+
+#endif
\ No newline at end of file
diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
index 4bf7fe697e923..45b40b6c3b87c 100644
--- a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
+++ b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
@@ -4,6 +4,7 @@
 #include "CondFormats/Common/interface/TimeConversions.h"
 #include "CondFormats/RunInfo/interface/LHCInfoPerFill.h"
 #include "CondTools/RunInfo/interface/LumiSectionFilter.h"
+#include "CondTools/RunInfo/interface/LHCInfoHelper.h"
 #include "CondTools/RunInfo/interface/OMSAccess.h"
 #include "CoralBase/Attribute.h"
 #include "CoralBase/AttributeList.h"
@@ -17,13 +18,6 @@
 #include "RelationalAccess/IQuery.h"
 #include "RelationalAccess/ISchema.h"
 #include "RelationalAccess/ISessionProxy.h"
-#include <cmath>
-#include <iostream>
-#include <memory>
-#include <sstream>
-#include <string>
-#include <utility>
-#include <vector>
 
 class LHCInfoPerFillPopConSourceHandler;
 
@@ -255,8 +249,13 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
     cond::Time_t lastSince = tagInfo().lastInterval.since;
     if (tagInfo().isEmpty()) {
       // for a new or empty tag, an empty payload should be added on top with since=1
-      addEmptyPayload(1);
-      lastSince = 1;
+      if (m_endFillMode) {
+        addEmptyPayload(1);
+        lastSince = 1;
+      } else {
+        addEmptyPayload(cond::time::lumiTime(1, 1));
+        lastSince = cond::time::lumiTime(1, 1);
+      }
     } else {
       edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from "
                            << m_name << "::getNewObjects";
@@ -295,7 +294,6 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
       session3.transaction().commit();
     }
 
-    // bool iovAdded = false;
     while (true) {
       if (targetSince >= executionTimeIov) {
         edm::LogInfo(m_name) << "Sampling ended at the time "
@@ -327,8 +325,6 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
         foundFill = theLHCInfoPerFillImpl::makeFillPayload(m_fillPayload, query->result());
       if (!foundFill) {
         edm::LogInfo(m_name) << "No fill found - END of job.";
-        // if (iovAdded)
-        //   addEmptyPayload(targetSince);
         break;
       }
 
@@ -367,9 +363,13 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
       size_t niovs = theLHCInfoPerFillImpl::transferPayloads(m_tmpBuffer, m_iovs, m_prevPayload);
       edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time";
       m_tmpBuffer.clear();
-      // iovAdded = true;
-      if (m_prevPayload->fillNumber() and !ongoingFill)
-        addEmptyPayload(endFillTime);
+      if (m_prevPayload->fillNumber() and !ongoingFill) {
+        if (m_endFillMode) {
+          addEmptyPayload(endFillTime);
+        } else {
+          addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill));
+        }
+      }
     }
   }
 
@@ -403,7 +403,14 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
     auto delivLumi = row.get<float>("delivered_lumi");
     auto recLumi = row.get<float>("recorded_lumi");
     LHCInfoPerFill* thisLumiSectionInfo = m_fillPayload->cloneFill();
-    m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo));
+    if (m_endFillMode) {
+      m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo));
+    } else {
+      m_tmpBuffer.emplace_back(
+          std::make_pair(cond::time::lumiTime(std::stoull(row.get<std::string>("run_number")),
+                                              std::stoul(row.get<std::string>("lumisection_number"))),
+                         thisLumiSectionInfo));
+    }
     LHCInfoPerFill& payload = *thisLumiSectionInfo;
     payload.setDelivLumi(delivLumi);
     payload.setRecLumi(recLumi);
@@ -414,7 +421,8 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
                      const boost::posix_time::ptime& beginFillTime,
                      const boost::posix_time::ptime& endFillTime) {
     auto query = oms.query("lumisections");
-    query->addOutputVars({"start_time", "delivered_lumi", "recorded_lumi", "beams_stable"});
+    query->addOutputVars(
+        {"start_time", "delivered_lumi", "recorded_lumi", "beams_stable", "run_number", "lumisection_number"});
     query->filterEQ("fill_number", fillId);
     query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime);
     query->filterEQ("beams_stable", "true");
diff --git a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc
index c8c3c3e8c0069..e14284e642026 100644
--- a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc
+++ b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc
@@ -4,6 +4,7 @@
 #include "CondCore/PopCon/interface/PopConSourceHandler.h"
 #include "CondFormats/Common/interface/TimeConversions.h"
 #include "CondFormats/RunInfo/interface/LHCInfoPerLS.h"
+#include "CondTools/RunInfo/interface/LHCInfoHelper.h"
 #include "CondTools/RunInfo/interface/OMSAccess.h"
 #include "CoralBase/Attribute.h"
 #include "CoralBase/AttributeList.h"
@@ -17,15 +18,6 @@
 #include "RelationalAccess/IQuery.h"
 #include "RelationalAccess/ISchema.h"
 #include "RelationalAccess/ISessionProxy.h"
-#include <cmath>
-#include <iostream>
-#include <map>
-#include <memory>
-#include <sstream>
-#include <sstream>
-#include <string>
-#include <utility>
-#include <vector>
 
 using std::make_pair;
 using std::pair;
@@ -173,8 +165,13 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
     cond::Time_t lastSince = tagInfo().lastInterval.since;
     if (tagInfo().isEmpty()) {
       // for a new or empty tag, an empty payload should be added on top with since=1
-      addEmptyPayload(1);
-      lastSince = 1;
+      if (m_endFillMode) {
+        addEmptyPayload(1);
+        lastSince = 1;
+      } else {
+        addEmptyPayload(cond::time::lumiTime(1, 1));
+        lastSince = cond::time::lumiTime(1, 1);
+      }
     } else {
       edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from "
                            << m_name << "::getNewObjects";
@@ -329,8 +326,13 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
       }
       m_tmpBuffer.clear();
       m_lsIdMap.clear();
-      if (m_prevPayload->fillNumber() and !ongoingFill)
-        addEmptyPayload(m_endFillTime);
+      if (m_prevPayload->fillNumber() and !ongoingFill) {
+        if (m_endFillMode) {
+          addEmptyPayload(m_endFillTime);
+        } else {
+          addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill));
+        }
+      }
     }
   }
   std::string id() const override { return m_name; }
@@ -384,7 +386,13 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
     thisLumiSectionInfo->setLumiSection(std::stoul(row.get<std::string>("lumisection_number")));
     thisLumiSectionInfo->setRunNumber(std::stoull(row.get<std::string>("run_number")));
     m_lsIdMap[make_pair(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection())] = make_pair(-1, -1);
-    m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo));
+    if (m_endFillMode) {
+      m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo));
+    } else {
+      m_tmpBuffer.emplace_back(
+          make_pair(cond::time::lumiTime(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection()),
+                    thisLumiSectionInfo));
+    }
   }
 
   size_t bufferAllLS(const cond::OMSServiceResult& queryResult) {
diff --git a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py
index e4b6dd762df21..3cd88075c7e8c 100644
--- a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py
+++ b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py
@@ -81,9 +81,15 @@
                             interval = cms.uint64(1)
                             )
 
+# Write different time-types tags depending on the O2O mode
+if options.mode == 'endFill':
+  timetype = 'timestamp'
+else:
+  timetype = 'lumiid'
+
 process.PoolDBOutputService = cms.Service("PoolDBOutputService",
                                           CondDBConnection,
-                                          timetype = cms.untracked.string('timestamp'),
+                                          timetype = cms.untracked.string(timetype),
                                           toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerFillRcd'),
                                                                      tag = cms.string( options.tag )
                                                                      )
diff --git a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py
index 522aedae4ec1c..c015f9bf2198a 100644
--- a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py
+++ b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py
@@ -81,9 +81,15 @@
                             interval = cms.uint64(1)
                             )
 
+# Write different time-types tags depending on the O2O mode
+if options.mode == 'endFill':
+  timetype = 'timestamp'
+else:
+  timetype = 'lumiid'
+
 process.PoolDBOutputService = cms.Service("PoolDBOutputService",
                                           CondDBConnection,
-                                          timetype = cms.untracked.string('timestamp'),
+                                          timetype = cms.untracked.string(timetype),
                                           toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerLSRcd'),
                                                                      tag = cms.string( options.tag )
                                                                      )
diff --git a/CondTools/RunInfo/src/LHCInfoHelper.cc b/CondTools/RunInfo/src/LHCInfoHelper.cc
new file mode 100644
index 0000000000000..5ce44e21981ba
--- /dev/null
+++ b/CondTools/RunInfo/src/LHCInfoHelper.cc
@@ -0,0 +1,30 @@
+#include "CondTools/RunInfo/interface/LHCInfoHelper.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+
+//*****************************************************************
+// Returns lumi-type IOV (packed using cond::time::lumiTime) from
+// last LS of last Run of the specified Fill
+//*****************************************************************
+cond::Time_t cond::lhcInfoHelper::getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId) {
+  // Define query
+  auto query = oms.query("lumisections");
+  query->addOutputVars({"lumisection_number", "run_number"});
+  query->filterEQ("fill_number", fillId);
+  query->limit(cond::lhcInfoHelper::kLumisectionsQueryLimit);
+
+  // Execute query
+  if (!query->execute()) {
+    throw cms::Exception("OMSQueryFailure")
+        << "OMS query of fill " << fillId << " failed, status:" << query->status() << "\n";
+  }
+
+  // Get query result
+  auto queryResult = query->result();
+  if (queryResult.empty())
+    throw cms::Exception("OMSQueryFailure") << "OMS query of fill " << fillId << " returned empty result!\n";
+
+  // Return the final IOV
+  auto lastRun = queryResult.back().get<int>("run_number");
+  auto lastLumi = queryResult.back().get<unsigned short>("lumisection_number");
+  return cond::time::lumiTime(lastRun, lastLumi);
+}

From e74ddb739d1e0a1e22dc1aa3f4c1076e2352e3ae Mon Sep 17 00:00:00 2001
From: francescobrivio <francesco.brivio@cern.ch>
Date: Thu, 22 Feb 2024 14:31:37 +0100
Subject: [PATCH 2/4] add conversion from timestamp to lumiid in
 LHCInfoPerFillPopConAnalyzer

---
 .../plugins/LHCInfoPerFillPopConAnalyzer.cc   | 38 +++++++++++++++----
 CondTools/RunInfo/src/LHCInfoHelper.cc        |  3 +-
 2 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
index 45b40b6c3b87c..cce7fd953dfc9 100644
--- a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
+++ b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
@@ -360,9 +360,16 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
         }
       }
 
+      // In duringFill mode, convert the timestamp-type IOVs to lumiid-type IOVs
+      // before transferring the payloads from the buffer to the final collection
+      if (!m_endFillMode) {
+        convertBufferedIovsToLumiid(m_timestampToLumiid);
+      }
+
       size_t niovs = theLHCInfoPerFillImpl::transferPayloads(m_tmpBuffer, m_iovs, m_prevPayload);
       edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time";
       m_tmpBuffer.clear();
+      m_timestampToLumiid.clear();
       if (m_prevPayload->fillNumber() and !ongoingFill) {
         if (m_endFillMode) {
           addEmptyPayload(endFillTime);
@@ -398,24 +405,37 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
     }
   }
 
+  // Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map
   void addPayloadToBuffer(cond::OMSServiceResultRef& row) {
-    auto lumiTime = row.get<boost::posix_time::ptime>("start_time");
+    auto startTime = row.get<boost::posix_time::ptime>("start_time");
     auto delivLumi = row.get<float>("delivered_lumi");
     auto recLumi = row.get<float>("recorded_lumi");
+    auto runNumber = std::stoull(row.get<std::string>("run_number"));
+    auto lsNumber = std::stoul(row.get<std::string>("lumisection_number"));
+    auto lumiid = cond::time::lumiTime(runNumber, lsNumber);
+
     LHCInfoPerFill* thisLumiSectionInfo = m_fillPayload->cloneFill();
-    if (m_endFillMode) {
-      m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo));
-    } else {
-      m_tmpBuffer.emplace_back(
-          std::make_pair(cond::time::lumiTime(std::stoull(row.get<std::string>("run_number")),
-                                              std::stoul(row.get<std::string>("lumisection_number"))),
-                         thisLumiSectionInfo));
+    m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(startTime), thisLumiSectionInfo));
+    if (!m_endFillMode) {
+      m_timestampToLumiid.insert(std::make_pair(cond::time::from_boost(startTime), lumiid));
     }
     LHCInfoPerFill& payload = *thisLumiSectionInfo;
     payload.setDelivLumi(delivLumi);
     payload.setRecLumi(recLumi);
   }
 
+  void convertBufferedIovsToLumiid(std::map<cond::Time_t, cond::Time_t> timestampToLumiid) {
+    for (auto& item : m_tmpBuffer) {
+      // Check if the lumiid IOV corresponding to the timestamp is present in the map
+      if (timestampToLumiid.find(item.first) == timestampToLumiid.end()) {
+        throw cms::Exception("LHCInfoPerFillPopConSourceHandler")
+            << "Can't find corresponding lumiid IOV for timestamp " << item.first << "\n";
+      }
+      // Update the buffer with the lumiid-type IOV
+      item.first = timestampToLumiid.at(item.first);
+    }
+  }
+
   size_t getLumiData(const cond::OMSService& oms,
                      unsigned short fillId,
                      const boost::posix_time::ptime& beginFillTime,
@@ -740,4 +760,6 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
   std::shared_ptr<LHCInfoPerFill> m_prevPayload;
   std::vector<std::pair<cond::Time_t, std::shared_ptr<LHCInfoPerFill>>> m_tmpBuffer;
   bool m_lastPayloadEmpty = false;
+  // to hold correspondance between timestamp-type IOVs and lumiid-type IOVs
+  std::map<cond::Time_t, cond::Time_t> m_timestampToLumiid;
 };
diff --git a/CondTools/RunInfo/src/LHCInfoHelper.cc b/CondTools/RunInfo/src/LHCInfoHelper.cc
index 5ce44e21981ba..54503068b8924 100644
--- a/CondTools/RunInfo/src/LHCInfoHelper.cc
+++ b/CondTools/RunInfo/src/LHCInfoHelper.cc
@@ -20,8 +20,9 @@ cond::Time_t cond::lhcInfoHelper::getFillLastLumiIOV(const cond::OMSService& oms
 
   // Get query result
   auto queryResult = query->result();
-  if (queryResult.empty())
+  if (queryResult.empty()) {
     throw cms::Exception("OMSQueryFailure") << "OMS query of fill " << fillId << " returned empty result!\n";
+  }
 
   // Return the final IOV
   auto lastRun = queryResult.back().get<int>("run_number");

From 99900e3b948dde5874adee7e3e10a16e756e9f58 Mon Sep 17 00:00:00 2001
From: francescobrivio <francesco.brivio@cern.ch>
Date: Fri, 1 Mar 2024 18:29:12 +0100
Subject: [PATCH 3/4] use stoul instead of stoull in LHCInfoPer* PopCons

---
 CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc | 2 +-
 CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
index cce7fd953dfc9..9fa6e7642d681 100644
--- a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
+++ b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
@@ -410,7 +410,7 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
     auto startTime = row.get<boost::posix_time::ptime>("start_time");
     auto delivLumi = row.get<float>("delivered_lumi");
     auto recLumi = row.get<float>("recorded_lumi");
-    auto runNumber = std::stoull(row.get<std::string>("run_number"));
+    auto runNumber = std::stoul(row.get<std::string>("run_number"));
     auto lsNumber = std::stoul(row.get<std::string>("lumisection_number"));
     auto lumiid = cond::time::lumiTime(runNumber, lsNumber);
 
diff --git a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc
index e14284e642026..992a6183b53b5 100644
--- a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc
+++ b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc
@@ -384,7 +384,7 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
     auto lumiTime = row.get<boost::posix_time::ptime>("start_time");
     LHCInfoPerLS* thisLumiSectionInfo = new LHCInfoPerLS(*m_fillPayload);
     thisLumiSectionInfo->setLumiSection(std::stoul(row.get<std::string>("lumisection_number")));
-    thisLumiSectionInfo->setRunNumber(std::stoull(row.get<std::string>("run_number")));
+    thisLumiSectionInfo->setRunNumber(std::stoul(row.get<std::string>("run_number")));
     m_lsIdMap[make_pair(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection())] = make_pair(-1, -1);
     if (m_endFillMode) {
       m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo));

From 7aa6331b15a047fb0d49dd3814460caa0fc2e472 Mon Sep 17 00:00:00 2001
From: francescobrivio <francesco.brivio@cern.ch>
Date: Fri, 1 Mar 2024 19:07:47 +0100
Subject: [PATCH 4/4] fix assigning lumiid value to targetTime

---
 .../plugins/LHCInfoPerFillPopConAnalyzer.cc   | 60 ++++++++-----------
 .../plugins/LHCInfoPerLSPopConAnalyzer.cc     | 50 +++++++---------
 .../python/LHCInfoPerFillPopConAnalyzer.py    |  2 -
 .../python/LHCInfoPerLSPopConAnalyzer.py      |  2 -
 4 files changed, 46 insertions(+), 68 deletions(-)

diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
index 9fa6e7642d681..c87e36815e557 100644
--- a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
+++ b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
@@ -205,12 +205,10 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
       : m_debug(pset.getUntrackedParameter<bool>("debug", false)),
         m_startTime(),
         m_endTime(),
-        m_samplingInterval((unsigned int)pset.getUntrackedParameter<unsigned int>("samplingInterval", 300)),
         m_endFillMode(pset.getUntrackedParameter<bool>("endFill", true)),
         m_name(pset.getUntrackedParameter<std::string>("name", "LHCInfoPerFillPopConSourceHandler")),
         m_connectionString(pset.getUntrackedParameter<std::string>("connectionString", "")),
         m_ecalConnectionString(pset.getUntrackedParameter<std::string>("ecalConnectionString", "")),
-        m_dipSchema(pset.getUntrackedParameter<std::string>("DIPSchema", "")),
         m_authpath(pset.getUntrackedParameter<std::string>("authenticationPath", "")),
         m_omsBaseUrl(pset.getUntrackedParameter<std::string>("omsBaseUrl", "")),
         m_fillPayload(),
@@ -227,13 +225,10 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
         m_endTime = now;
     }
   }
-  //L1: try with different m_dipSchema
-  //L2: try with different m_name
+
   ~LHCInfoPerFillPopConSourceHandler() override = default;
-  void getNewObjects() override {
-    //reference to the last payload in the tag
-    Ref previousFill;
 
+  void getNewObjects() override {
     //if a new tag is created, transfer fake fill from 1 to the first fill for the first time
     if (tagInfo().size == 0) {
       edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects";
@@ -261,18 +256,6 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
                            << m_name << "::getNewObjects";
     }
 
-    boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time();
-    cond::Time_t targetSince = 0;
-    cond::Time_t executionTimeIov = cond::time::from_boost(executionTime);
-    if (!m_startTime.is_not_a_date_time()) {
-      targetSince = cond::time::from_boost(m_startTime);
-    }
-    if (lastSince > targetSince)
-      targetSince = lastSince;
-
-    edm::LogInfo(m_name) << "Starting sampling at "
-                         << boost::posix_time::to_simple_string(cond::time::to_boost(targetSince));
-
     //retrieve the data from the relational database source
     cond::persistency::ConnectionPool connection;
     //configure the connection
@@ -294,14 +277,23 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
       session3.transaction().commit();
     }
 
+    boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time();
+    cond::Time_t executionTimeIov = cond::time::from_boost(executionTime);
+
+    cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime);
+    cond::Time_t nextFillSearchTimestamp =
+        std::max(startTimestamp, m_endFillMode ? lastSince : m_prevPayload->createTime());
+
+    edm::LogInfo(m_name) << "Starting sampling at "
+                         << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp));
+
     while (true) {
-      if (targetSince >= executionTimeIov) {
+      if (nextFillSearchTimestamp >= executionTimeIov) {
         edm::LogInfo(m_name) << "Sampling ended at the time "
                              << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov));
         break;
       }
-      bool updateEcal = false;
-      boost::posix_time::ptime targetTime = cond::time::to_boost(targetSince);
+      boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp);
       boost::posix_time::ptime startSampleTime;
       boost::posix_time::ptime endSampleTime;
 
@@ -309,12 +301,12 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
       oms.connect(m_omsBaseUrl);
       auto query = oms.query("fills");
 
-      edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(targetTime);
+      edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime);
       query->filterNotNull("start_stable_beam").filterNotNull("fill_number");
-      if (targetTime > cond::time::to_boost(m_prevPayload->createTime())) {
-        query->filterGE("start_time", targetTime);
+      if (nextFillSearchTime > cond::time::to_boost(m_prevPayload->createTime())) {
+        query->filterGE("start_time", nextFillSearchTime);
       } else {
-        query->filterGT("start_time", targetTime);
+        query->filterGT("start_time", nextFillSearchTime);
       }
 
       query->filterLT("start_time", m_endTime);
@@ -337,12 +329,12 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
         edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at "
                              << cond::time::to_boost(startFillTime);
         endSampleTime = executionTime;
-        targetSince = executionTimeIov;
+        nextFillSearchTimestamp = executionTimeIov;
       } else {
         edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(startFillTime)
                              << " ending at " << cond::time::to_boost(endFillTime);
         endSampleTime = cond::time::to_boost(endFillTime);
-        targetSince = endFillTime;
+        nextFillSearchTimestamp = endFillTime;
       }
       if (m_endFillMode || ongoingFill) {
         getDipData(oms, startSampleTime, endSampleTime);
@@ -355,7 +347,7 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
           getCTPPSData(session, startSampleTime, endSampleTime);
           session.transaction().commit();
           session2.transaction().start(true);
-          getEcalData(session2, startSampleTime, endSampleTime, updateEcal);
+          getEcalData(session2, startSampleTime, endSampleTime);
           session2.transaction().commit();
         }
       }
@@ -636,8 +628,7 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
 
   bool getEcalData(cond::persistency::Session& session,
                    const boost::posix_time::ptime& lowerTime,
-                   const boost::posix_time::ptime& upperTime,
-                   bool update) {
+                   const boost::posix_time::ptime& upperTime) {
     //run the sixth query against the CMS_DCS_ENV_PVSS_COND schema
     //Initializing the CMS_DCS_ENV_PVSS_COND schema.
     coral::ISchema& ECAL = session.nominalSchema();
@@ -712,7 +703,7 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
         }
         changeTime = cond::time::from_boost(chTime);
         cond::Time_t iovTime = changeTime;
-        if (!update and changeTime == firstTime)
+        if (changeTime == firstTime)
           iovTime = lowerLumi;
         coral::Attribute const& dipValAttribute = ECALDataCursor.currentRow()[std::string("DIP_value")];
         coral::Attribute const& valueNumberAttribute = ECALDataCursor.currentRow()[std::string("VALUE_NUMBER")];
@@ -730,7 +721,6 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
               theLHCInfoPerFillImpl::setElementData(it->first, dipVal, elementNr, value, payload, initializedVectors);
             }
           }
-          //}
         }
       }
     }
@@ -748,13 +738,11 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
   // starting date for sampling
   boost::posix_time::ptime m_startTime;
   boost::posix_time::ptime m_endTime;
-  // sampling interval in seconds
-  unsigned int m_samplingInterval;
   bool m_endFillMode = true;
   std::string m_name;
   //for reading from relational database source
   std::string m_connectionString, m_ecalConnectionString;
-  std::string m_dipSchema, m_authpath;
+  std::string m_authpath;
   std::string m_omsBaseUrl;
   std::unique_ptr<LHCInfoPerFill> m_fillPayload;
   std::shared_ptr<LHCInfoPerFill> m_prevPayload;
diff --git a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc
index 992a6183b53b5..dc5fe9b298c8e 100644
--- a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc
+++ b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc
@@ -119,17 +119,16 @@ namespace theLHCInfoPerLSImpl {
   }
 
 }  // namespace theLHCInfoPerLSImpl
+
 class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCInfoPerLS> {
 public:
   LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset)
       : m_debug(pset.getUntrackedParameter<bool>("debug", false)),
         m_startTime(),
         m_endTime(),
-        m_samplingInterval((unsigned int)pset.getUntrackedParameter<unsigned int>("samplingInterval", 300)),
         m_endFillMode(pset.getUntrackedParameter<bool>("endFill", true)),
         m_name(pset.getUntrackedParameter<std::string>("name", "LHCInfoPerLSPopConSourceHandler")),
         m_connectionString(pset.getUntrackedParameter<std::string>("connectionString", "")),
-        m_dipSchema(pset.getUntrackedParameter<std::string>("DIPSchema", "")),
         m_authpath(pset.getUntrackedParameter<std::string>("authenticationPath", "")),
         m_omsBaseUrl(pset.getUntrackedParameter<std::string>("omsBaseUrl", "")),
         m_fillPayload(),
@@ -146,9 +145,9 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
         m_endTime = now;
     }
   }
-  //L1: try with different m_dipSchema
-  //L2: try with different m_name
+
   ~LHCInfoPerLSPopConSourceHandler() override = default;
+
   void getNewObjects() override {
     //if a new tag is created, transfer fake fill from 1 to the first fill for the first time
     if (tagInfo().size == 0) {
@@ -177,18 +176,6 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
                            << m_name << "::getNewObjects";
     }
 
-    boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time();
-    cond::Time_t targetSince = 0;
-    cond::Time_t executionTimeIov = cond::time::from_boost(executionTime);
-    if (!m_startTime.is_not_a_date_time()) {
-      targetSince = cond::time::from_boost(m_startTime);
-    }
-    if (lastSince > targetSince)
-      targetSince = lastSince;
-
-    edm::LogInfo(m_name) << "Starting sampling at "
-                         << boost::posix_time::to_simple_string(cond::time::to_boost(targetSince));
-
     //retrieve the data from the relational database source
     cond::persistency::ConnectionPool connection;
     //configure the connection
@@ -241,13 +228,22 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
       }
     }
 
+    boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time();
+    cond::Time_t executionTimeIov = cond::time::from_boost(executionTime);
+
+    cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime);
+    cond::Time_t nextFillSearchTimestamp = std::max(startTimestamp, m_endFillMode ? lastSince : m_prevEndFillTime);
+
+    edm::LogInfo(m_name) << "Starting sampling at "
+                         << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp));
+
     while (true) {
-      if (targetSince >= executionTimeIov) {
+      if (nextFillSearchTimestamp >= executionTimeIov) {
         edm::LogInfo(m_name) << "Sampling ended at the time "
                              << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov));
         break;
       }
-      boost::posix_time::ptime targetTime = cond::time::to_boost(targetSince);
+      boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp);
       boost::posix_time::ptime startSampleTime;
       boost::posix_time::ptime endSampleTime;
 
@@ -268,12 +264,12 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
         }
         startSampleTime = cond::time::to_boost(lastSince);
       } else {
-        edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(targetTime);
+        edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime);
         query->filterNotNull("start_stable_beam").filterNotNull("fill_number");
-        if (targetTime > cond::time::to_boost(m_prevStartFillTime)) {
-          query->filterGE("start_time", targetTime);
+        if (nextFillSearchTime > cond::time::to_boost(m_prevStartFillTime)) {
+          query->filterGE("start_time", nextFillSearchTime);
         } else {
-          query->filterGT("start_time", targetTime);
+          query->filterGT("start_time", nextFillSearchTime);
         }
 
         query->filterLT("start_time", m_endTime);
@@ -295,12 +291,12 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
         edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at "
                              << cond::time::to_boost(m_startFillTime);
         endSampleTime = executionTime;
-        targetSince = executionTimeIov;
+        nextFillSearchTimestamp = executionTimeIov;
       } else {
         edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(m_startFillTime)
                              << " ending at " << cond::time::to_boost(m_endFillTime);
         endSampleTime = cond::time::to_boost(m_endFillTime);
-        targetSince = m_endFillTime;
+        nextFillSearchTimestamp = m_endFillTime;
       }
 
       if (m_endFillMode || ongoingFill) {
@@ -571,13 +567,11 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler<LHCIn
   // starting date for sampling
   boost::posix_time::ptime m_startTime;
   boost::posix_time::ptime m_endTime;
-  // sampling interval in seconds
-  unsigned int m_samplingInterval;
   bool m_endFillMode = true;
   std::string m_name;
   //for reading from relational database source
-  std::string m_connectionString, m_ecalConnectionString;
-  std::string m_dipSchema, m_authpath;
+  std::string m_connectionString;
+  std::string m_authpath;
   std::string m_omsBaseUrl;
   std::unique_ptr<LHCInfoPerLS> m_fillPayload;
   std::shared_ptr<LHCInfoPerLS> m_prevPayload;
diff --git a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py
index 3cd88075c7e8c..d71362ecebfa8 100644
--- a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py
+++ b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py
@@ -103,12 +103,10 @@
                                Source = cms.PSet(fill = cms.untracked.uint32(6417),
                                    startTime = cms.untracked.string(options.startTime),
                                    endTime = cms.untracked.string(options.endTime),
-                                   samplingInterval = cms.untracked.uint32( 600 ),
                                    endFill = cms.untracked.bool(True if options.mode == "endFill" else False),
                                    name = cms.untracked.string("LHCInfoPerFillPopConSourceHandler"),
                                    connectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER"),
                                    ecalConnectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_DCS_ENV_PVSS_COND"),
-                                   DIPSchema = cms.untracked.string("CMS_BEAM_COND"),
                                    omsBaseUrl = cms.untracked.string("http://vocms0184.cern.ch/agg/api/v1"),
                                    authenticationPath = cms.untracked.string(""),
                                    debug=cms.untracked.bool(False)
diff --git a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py
index c015f9bf2198a..603c7f514d4ee 100644
--- a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py
+++ b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py
@@ -103,11 +103,9 @@
                                Source = cms.PSet(fill = cms.untracked.uint32(6417),
                                    startTime = cms.untracked.string(options.startTime),
                                    endTime = cms.untracked.string(options.endTime),
-                                   samplingInterval = cms.untracked.uint32( 600 ),
                                    endFill = cms.untracked.bool(True if options.mode == "endFill" else False),
                                    name = cms.untracked.string("LHCInfoPerLSPopConSourceHandler"),
                                    connectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER"),
-                                   DIPSchema = cms.untracked.string("CMS_BEAM_COND"),
                                    omsBaseUrl = cms.untracked.string("http://vocms0184.cern.ch/agg/api/v1"),
                                    authenticationPath = cms.untracked.string(""),
                                    debug=cms.untracked.bool(False)