Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[14_0_X] Fix LHCInfoPer* PopCons to produce lumi-type IOVs when running in duringFill mode #43972

Merged
merged 4 commits into from
Mar 5, 2024
Merged

[14_0_X] Fix LHCInfoPer* PopCons to produce lumi-type IOVs when running in duringFill mode #43972

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3a0b16f
fix LHCInfoPer* PopCons to produce lumi type IOVs when running in dur…
francescobrivio Feb 14, 2024
e74ddb7
add conversion from timestamp to lumiid in LHCInfoPerFillPopConAnalyzer
francescobrivio Feb 22, 2024
99900e3
use stoul instead of stoull in LHCInfoPer* PopCons
francescobrivio Mar 1, 2024
7aa6331
fix assigning lumiid value to targetTime
francescobrivio Mar 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions CondTools/RunInfo/interface/LHCInfoHelper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#ifndef CondTools_RunInfo_LHCInfoHelper_h
#define CondTools_RunInfo_LHCInfoHelper_h

#include "CondCore/CondDB/interface/Time.h"
#include "CondTools/RunInfo/interface/OMSAccess.h"

namespace cond {

namespace lhcInfoHelper {

// Large number of LS for the OMS query, covering around 25 hours
static constexpr unsigned int kLumisectionsQueryLimit = 4000;

// Returns lumi-type IOV from last LS of last Run of the specified Fill
cond::Time_t getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId);

} // namespace lhcInfoHelper

} // namespace cond

#endif
126 changes: 72 additions & 54 deletions CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "CondFormats/Common/interface/TimeConversions.h"
#include "CondFormats/RunInfo/interface/LHCInfoPerFill.h"
#include "CondTools/RunInfo/interface/LumiSectionFilter.h"
#include "CondTools/RunInfo/interface/LHCInfoHelper.h"
#include "CondTools/RunInfo/interface/OMSAccess.h"
#include "CoralBase/Attribute.h"
#include "CoralBase/AttributeList.h"
Expand All @@ -17,13 +18,6 @@
#include "RelationalAccess/IQuery.h"
#include "RelationalAccess/ISchema.h"
#include "RelationalAccess/ISessionProxy.h"
#include <cmath>
#include <iostream>
#include <memory>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

class LHCInfoPerFillPopConSourceHandler;

Expand Down Expand Up @@ -211,12 +205,10 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
: m_debug(pset.getUntrackedParameter<bool>("debug", false)),
m_startTime(),
m_endTime(),
m_samplingInterval((unsigned int)pset.getUntrackedParameter<unsigned int>("samplingInterval", 300)),
m_endFillMode(pset.getUntrackedParameter<bool>("endFill", true)),
m_name(pset.getUntrackedParameter<std::string>("name", "LHCInfoPerFillPopConSourceHandler")),
m_connectionString(pset.getUntrackedParameter<std::string>("connectionString", "")),
m_ecalConnectionString(pset.getUntrackedParameter<std::string>("ecalConnectionString", "")),
m_dipSchema(pset.getUntrackedParameter<std::string>("DIPSchema", "")),
m_authpath(pset.getUntrackedParameter<std::string>("authenticationPath", "")),
m_omsBaseUrl(pset.getUntrackedParameter<std::string>("omsBaseUrl", "")),
m_fillPayload(),
Expand All @@ -233,13 +225,10 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
m_endTime = now;
}
}
//L1: try with different m_dipSchema
//L2: try with different m_name

~LHCInfoPerFillPopConSourceHandler() override = default;
void getNewObjects() override {
//reference to the last payload in the tag
Ref previousFill;

void getNewObjects() override {
//if a new tag is created, transfer fake fill from 1 to the first fill for the first time
if (tagInfo().size == 0) {
edm::LogInfo(m_name) << "New tag " << tagInfo().name << "; from " << m_name << "::getNewObjects";
Expand All @@ -255,25 +244,18 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
cond::Time_t lastSince = tagInfo().lastInterval.since;
if (tagInfo().isEmpty()) {
// for a new or empty tag, an empty payload should be added on top with since=1
addEmptyPayload(1);
lastSince = 1;
if (m_endFillMode) {
addEmptyPayload(1);
lastSince = 1;
} else {
addEmptyPayload(cond::time::lumiTime(1, 1));
lastSince = cond::time::lumiTime(1, 1);
}
} else {
edm::LogInfo(m_name) << "The last Iov in tag " << tagInfo().name << " valid since " << lastSince << "from "
<< m_name << "::getNewObjects";
}

boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time();
cond::Time_t targetSince = 0;
cond::Time_t executionTimeIov = cond::time::from_boost(executionTime);
if (!m_startTime.is_not_a_date_time()) {
targetSince = cond::time::from_boost(m_startTime);
}
if (lastSince > targetSince)
targetSince = lastSince;

edm::LogInfo(m_name) << "Starting sampling at "
<< boost::posix_time::to_simple_string(cond::time::to_boost(targetSince));

//retrieve the data from the relational database source
cond::persistency::ConnectionPool connection;
//configure the connection
Expand All @@ -295,28 +277,36 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
session3.transaction().commit();
}

// bool iovAdded = false;
boost::posix_time::ptime executionTime = boost::posix_time::second_clock::local_time();
cond::Time_t executionTimeIov = cond::time::from_boost(executionTime);

cond::Time_t startTimestamp = m_startTime.is_not_a_date_time() ? 0 : cond::time::from_boost(m_startTime);
cond::Time_t nextFillSearchTimestamp =
std::max(startTimestamp, m_endFillMode ? lastSince : m_prevPayload->createTime());

edm::LogInfo(m_name) << "Starting sampling at "
<< boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp));

while (true) {
if (targetSince >= executionTimeIov) {
if (nextFillSearchTimestamp >= executionTimeIov) {
edm::LogInfo(m_name) << "Sampling ended at the time "
<< boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov));
break;
}
bool updateEcal = false;
boost::posix_time::ptime targetTime = cond::time::to_boost(targetSince);
boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp);
boost::posix_time::ptime startSampleTime;
boost::posix_time::ptime endSampleTime;

cond::OMSService oms;
oms.connect(m_omsBaseUrl);
auto query = oms.query("fills");

edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(targetTime);
edm::LogInfo(m_name) << "Searching new fill after " << boost::posix_time::to_simple_string(nextFillSearchTime);
query->filterNotNull("start_stable_beam").filterNotNull("fill_number");
if (targetTime > cond::time::to_boost(m_prevPayload->createTime())) {
query->filterGE("start_time", targetTime);
if (nextFillSearchTime > cond::time::to_boost(m_prevPayload->createTime())) {
query->filterGE("start_time", nextFillSearchTime);
} else {
query->filterGT("start_time", targetTime);
query->filterGT("start_time", nextFillSearchTime);
}

query->filterLT("start_time", m_endTime);
Expand All @@ -327,8 +317,6 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
foundFill = theLHCInfoPerFillImpl::makeFillPayload(m_fillPayload, query->result());
if (!foundFill) {
edm::LogInfo(m_name) << "No fill found - END of job.";
// if (iovAdded)
// addEmptyPayload(targetSince);
break;
}

Expand All @@ -341,12 +329,12 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
edm::LogInfo(m_name) << "Found ongoing fill " << lhcFill << " created at "
<< cond::time::to_boost(startFillTime);
endSampleTime = executionTime;
targetSince = executionTimeIov;
nextFillSearchTimestamp = executionTimeIov;
} else {
edm::LogInfo(m_name) << "Found fill " << lhcFill << " created at " << cond::time::to_boost(startFillTime)
<< " ending at " << cond::time::to_boost(endFillTime);
endSampleTime = cond::time::to_boost(endFillTime);
targetSince = endFillTime;
nextFillSearchTimestamp = endFillTime;
}
if (m_endFillMode || ongoingFill) {
getDipData(oms, startSampleTime, endSampleTime);
Expand All @@ -359,17 +347,28 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
getCTPPSData(session, startSampleTime, endSampleTime);
session.transaction().commit();
session2.transaction().start(true);
getEcalData(session2, startSampleTime, endSampleTime, updateEcal);
getEcalData(session2, startSampleTime, endSampleTime);
session2.transaction().commit();
}
}

// In duringFill mode, convert the timestamp-type IOVs to lumiid-type IOVs
// before transferring the payloads from the buffer to the final collection
if (!m_endFillMode) {
convertBufferedIovsToLumiid(m_timestampToLumiid);
}

size_t niovs = theLHCInfoPerFillImpl::transferPayloads(m_tmpBuffer, m_iovs, m_prevPayload);
edm::LogInfo(m_name) << "Added " << niovs << " iovs within the Fill time";
m_tmpBuffer.clear();
// iovAdded = true;
if (m_prevPayload->fillNumber() and !ongoingFill)
addEmptyPayload(endFillTime);
m_timestampToLumiid.clear();
if (m_prevPayload->fillNumber() and !ongoingFill) {
if (m_endFillMode) {
addEmptyPayload(endFillTime);
} else {
addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill));
}
}
}
}

Expand Down Expand Up @@ -398,23 +397,44 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
}
}

// Add payload to buffer and store corresponding lumiid IOV in m_timestampToLumiid map
void addPayloadToBuffer(cond::OMSServiceResultRef& row) {
auto lumiTime = row.get<boost::posix_time::ptime>("start_time");
auto startTime = row.get<boost::posix_time::ptime>("start_time");
auto delivLumi = row.get<float>("delivered_lumi");
auto recLumi = row.get<float>("recorded_lumi");
auto runNumber = std::stoul(row.get<std::string>("run_number"));
auto lsNumber = std::stoul(row.get<std::string>("lumisection_number"));
auto lumiid = cond::time::lumiTime(runNumber, lsNumber);

LHCInfoPerFill* thisLumiSectionInfo = m_fillPayload->cloneFill();
m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo));
m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(startTime), thisLumiSectionInfo));
if (!m_endFillMode) {
m_timestampToLumiid.insert(std::make_pair(cond::time::from_boost(startTime), lumiid));
}
LHCInfoPerFill& payload = *thisLumiSectionInfo;
payload.setDelivLumi(delivLumi);
payload.setRecLumi(recLumi);
}

void convertBufferedIovsToLumiid(std::map<cond::Time_t, cond::Time_t> timestampToLumiid) {
for (auto& item : m_tmpBuffer) {
// Check if the lumiid IOV corresponding to the timestamp is present in the map
if (timestampToLumiid.find(item.first) == timestampToLumiid.end()) {
throw cms::Exception("LHCInfoPerFillPopConSourceHandler")
<< "Can't find corresponding lumiid IOV for timestamp " << item.first << "\n";
}
// Update the buffer with the lumiid-type IOV
item.first = timestampToLumiid.at(item.first);
}
}

size_t getLumiData(const cond::OMSService& oms,
unsigned short fillId,
const boost::posix_time::ptime& beginFillTime,
const boost::posix_time::ptime& endFillTime) {
auto query = oms.query("lumisections");
query->addOutputVars({"start_time", "delivered_lumi", "recorded_lumi", "beams_stable"});
query->addOutputVars(
{"start_time", "delivered_lumi", "recorded_lumi", "beams_stable", "run_number", "lumisection_number"});
query->filterEQ("fill_number", fillId);
query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime);
query->filterEQ("beams_stable", "true");
Expand Down Expand Up @@ -608,8 +628,7 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC

bool getEcalData(cond::persistency::Session& session,
const boost::posix_time::ptime& lowerTime,
const boost::posix_time::ptime& upperTime,
bool update) {
const boost::posix_time::ptime& upperTime) {
//run the sixth query against the CMS_DCS_ENV_PVSS_COND schema
//Initializing the CMS_DCS_ENV_PVSS_COND schema.
coral::ISchema& ECAL = session.nominalSchema();
Expand Down Expand Up @@ -684,7 +703,7 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
}
changeTime = cond::time::from_boost(chTime);
cond::Time_t iovTime = changeTime;
if (!update and changeTime == firstTime)
if (changeTime == firstTime)
iovTime = lowerLumi;
coral::Attribute const& dipValAttribute = ECALDataCursor.currentRow()[std::string("DIP_value")];
coral::Attribute const& valueNumberAttribute = ECALDataCursor.currentRow()[std::string("VALUE_NUMBER")];
Expand All @@ -702,7 +721,6 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
theLHCInfoPerFillImpl::setElementData(it->first, dipVal, elementNr, value, payload, initializedVectors);
}
}
//}
}
}
}
Expand All @@ -720,16 +738,16 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler<LHC
// starting date for sampling
boost::posix_time::ptime m_startTime;
boost::posix_time::ptime m_endTime;
// sampling interval in seconds
unsigned int m_samplingInterval;
bool m_endFillMode = true;
std::string m_name;
//for reading from relational database source
std::string m_connectionString, m_ecalConnectionString;
std::string m_dipSchema, m_authpath;
std::string m_authpath;
std::string m_omsBaseUrl;
std::unique_ptr<LHCInfoPerFill> m_fillPayload;
std::shared_ptr<LHCInfoPerFill> m_prevPayload;
std::vector<std::pair<cond::Time_t, std::shared_ptr<LHCInfoPerFill>>> m_tmpBuffer;
bool m_lastPayloadEmpty = false;
// to hold correspondance between timestamp-type IOVs and lumiid-type IOVs
std::map<cond::Time_t, cond::Time_t> m_timestampToLumiid;
};
Loading