diff --git a/CondTools/RunInfo/interface/LHCInfoHelper.h b/CondTools/RunInfo/interface/LHCInfoHelper.h new file mode 100644 index 0000000000000..61d005c287a17 --- /dev/null +++ b/CondTools/RunInfo/interface/LHCInfoHelper.h @@ -0,0 +1,21 @@ +#ifndef CondTools_RunInfo_LHCInfoHelper_h +#define CondTools_RunInfo_LHCInfoHelper_h + +#include "CondCore/CondDB/interface/Time.h" +#include "CondTools/RunInfo/interface/OMSAccess.h" + +namespace cond { + + namespace lhcInfoHelper { + + // Large number of LS for the OMS query, covering around 25 hours + static constexpr unsigned int kLumisectionsQueryLimit = 4000; + + // Returns lumi-type IOV from last LS of last Run of the specified Fill + cond::Time_t getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId); + + } // namespace lhcInfoHelper + +} // namespace cond + +#endif \ No newline at end of file diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc index 4bf7fe697e923..c87e36815e557 100644 --- a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc +++ b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc @@ -4,6 +4,7 @@ #include "CondFormats/Common/interface/TimeConversions.h" #include "CondFormats/RunInfo/interface/LHCInfoPerFill.h" #include "CondTools/RunInfo/interface/LumiSectionFilter.h" +#include "CondTools/RunInfo/interface/LHCInfoHelper.h" #include "CondTools/RunInfo/interface/OMSAccess.h" #include "CoralBase/Attribute.h" #include "CoralBase/AttributeList.h" @@ -17,13 +18,6 @@ #include "RelationalAccess/IQuery.h" #include "RelationalAccess/ISchema.h" #include "RelationalAccess/ISessionProxy.h" -#include -#include -#include -#include -#include -#include -#include class LHCInfoPerFillPopConSourceHandler; @@ -211,12 +205,10 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler("debug", false)), m_startTime(), m_endTime(), - m_samplingInterval((unsigned int)pset.getUntrackedParameter("samplingInterval", 300)), m_endFillMode(pset.getUntrackedParameter("endFill", true)), m_name(pset.getUntrackedParameter("name", "LHCInfoPerFillPopConSourceHandler")), m_connectionString(pset.getUntrackedParameter("connectionString", "")), m_ecalConnectionString(pset.getUntrackedParameter("ecalConnectionString", "")), - m_dipSchema(pset.getUntrackedParameter("DIPSchema", "")), m_authpath(pset.getUntrackedParameter("authenticationPath", "")), m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), m_fillPayload(), @@ -233,13 +225,10 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler targetSince) - targetSince = lastSince; - - edm::LogInfo(m_name) << "Starting sampling at " - << boost::posix_time::to_simple_string(cond::time::to_boost(targetSince)); - //retrieve the data from the relational database source cond::persistency::ConnectionPool connection; //configure the connection @@ -295,15 +277,23 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandlercreateTime()); + + edm::LogInfo(m_name) << "Starting sampling at " + << boost::posix_time::to_simple_string(cond::time::to_boost(nextFillSearchTimestamp)); + while (true) { - if (targetSince >= executionTimeIov) { + if (nextFillSearchTimestamp >= executionTimeIov) { edm::LogInfo(m_name) << "Sampling ended at the time " << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); break; } - bool updateEcal = false; - boost::posix_time::ptime targetTime = cond::time::to_boost(targetSince); + boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); boost::posix_time::ptime startSampleTime; boost::posix_time::ptime endSampleTime; @@ -311,12 +301,12 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandlerfilterNotNull("start_stable_beam").filterNotNull("fill_number"); - if (targetTime > cond::time::to_boost(m_prevPayload->createTime())) { - query->filterGE("start_time", targetTime); + if (nextFillSearchTime > cond::time::to_boost(m_prevPayload->createTime())) { + query->filterGE("start_time", nextFillSearchTime); } else { - query->filterGT("start_time", targetTime); + query->filterGT("start_time", nextFillSearchTime); } query->filterLT("start_time", m_endTime); @@ -327,8 +317,6 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandlerresult()); if (!foundFill) { edm::LogInfo(m_name) << "No fill found - END of job."; - // if (iovAdded) - // addEmptyPayload(targetSince); break; } @@ -341,12 +329,12 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandlerfillNumber() and !ongoingFill) - addEmptyPayload(endFillTime); + m_timestampToLumiid.clear(); + if (m_prevPayload->fillNumber() and !ongoingFill) { + if (m_endFillMode) { + addEmptyPayload(endFillTime); + } else { + addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); + } + } } } @@ -398,23 +397,44 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler("start_time"); + auto startTime = row.get("start_time"); auto delivLumi = row.get("delivered_lumi"); auto recLumi = row.get("recorded_lumi"); + auto runNumber = std::stoul(row.get("run_number")); + auto lsNumber = std::stoul(row.get("lumisection_number")); + auto lumiid = cond::time::lumiTime(runNumber, lsNumber); + LHCInfoPerFill* thisLumiSectionInfo = m_fillPayload->cloneFill(); - m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo)); + m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(startTime), thisLumiSectionInfo)); + if (!m_endFillMode) { + m_timestampToLumiid.insert(std::make_pair(cond::time::from_boost(startTime), lumiid)); + } LHCInfoPerFill& payload = *thisLumiSectionInfo; payload.setDelivLumi(delivLumi); payload.setRecLumi(recLumi); } + void convertBufferedIovsToLumiid(std::map timestampToLumiid) { + for (auto& item : m_tmpBuffer) { + // Check if the lumiid IOV corresponding to the timestamp is present in the map + if (timestampToLumiid.find(item.first) == timestampToLumiid.end()) { + throw cms::Exception("LHCInfoPerFillPopConSourceHandler") + << "Can't find corresponding lumiid IOV for timestamp " << item.first << "\n"; + } + // Update the buffer with the lumiid-type IOV + item.first = timestampToLumiid.at(item.first); + } + } + size_t getLumiData(const cond::OMSService& oms, unsigned short fillId, const boost::posix_time::ptime& beginFillTime, const boost::posix_time::ptime& endFillTime) { auto query = oms.query("lumisections"); - query->addOutputVars({"start_time", "delivered_lumi", "recorded_lumi", "beams_stable"}); + query->addOutputVars( + {"start_time", "delivered_lumi", "recorded_lumi", "beams_stable", "run_number", "lumisection_number"}); query->filterEQ("fill_number", fillId); query->filterGT("start_time", beginFillTime).filterLT("start_time", endFillTime); query->filterEQ("beams_stable", "true"); @@ -608,8 +628,7 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandlerfirst, dipVal, elementNr, value, payload, initializedVectors); } } - //} } } } @@ -720,16 +738,16 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler m_fillPayload; std::shared_ptr m_prevPayload; std::vector>> m_tmpBuffer; bool m_lastPayloadEmpty = false; + // to hold correspondance between timestamp-type IOVs and lumiid-type IOVs + std::map m_timestampToLumiid; }; diff --git a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc index c8c3c3e8c0069..dc5fe9b298c8e 100644 --- a/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc +++ b/CondTools/RunInfo/plugins/LHCInfoPerLSPopConAnalyzer.cc @@ -4,6 +4,7 @@ #include "CondCore/PopCon/interface/PopConSourceHandler.h" #include "CondFormats/Common/interface/TimeConversions.h" #include "CondFormats/RunInfo/interface/LHCInfoPerLS.h" +#include "CondTools/RunInfo/interface/LHCInfoHelper.h" #include "CondTools/RunInfo/interface/OMSAccess.h" #include "CoralBase/Attribute.h" #include "CoralBase/AttributeList.h" @@ -17,15 +18,6 @@ #include "RelationalAccess/IQuery.h" #include "RelationalAccess/ISchema.h" #include "RelationalAccess/ISessionProxy.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include using std::make_pair; using std::pair; @@ -127,17 +119,16 @@ namespace theLHCInfoPerLSImpl { } } // namespace theLHCInfoPerLSImpl + class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler { public: LHCInfoPerLSPopConSourceHandler(edm::ParameterSet const& pset) : m_debug(pset.getUntrackedParameter("debug", false)), m_startTime(), m_endTime(), - m_samplingInterval((unsigned int)pset.getUntrackedParameter("samplingInterval", 300)), m_endFillMode(pset.getUntrackedParameter("endFill", true)), m_name(pset.getUntrackedParameter("name", "LHCInfoPerLSPopConSourceHandler")), m_connectionString(pset.getUntrackedParameter("connectionString", "")), - m_dipSchema(pset.getUntrackedParameter("DIPSchema", "")), m_authpath(pset.getUntrackedParameter("authenticationPath", "")), m_omsBaseUrl(pset.getUntrackedParameter("omsBaseUrl", "")), m_fillPayload(), @@ -154,9 +145,9 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler targetSince) - targetSince = lastSince; - - edm::LogInfo(m_name) << "Starting sampling at " - << boost::posix_time::to_simple_string(cond::time::to_boost(targetSince)); - //retrieve the data from the relational database source cond::persistency::ConnectionPool connection; //configure the connection @@ -244,13 +228,22 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler= executionTimeIov) { + if (nextFillSearchTimestamp >= executionTimeIov) { edm::LogInfo(m_name) << "Sampling ended at the time " << boost::posix_time::to_simple_string(cond::time::to_boost(executionTimeIov)); break; } - boost::posix_time::ptime targetTime = cond::time::to_boost(targetSince); + boost::posix_time::ptime nextFillSearchTime = cond::time::to_boost(nextFillSearchTimestamp); boost::posix_time::ptime startSampleTime; boost::posix_time::ptime endSampleTime; @@ -271,12 +264,12 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandlerfilterNotNull("start_stable_beam").filterNotNull("fill_number"); - if (targetTime > cond::time::to_boost(m_prevStartFillTime)) { - query->filterGE("start_time", targetTime); + if (nextFillSearchTime > cond::time::to_boost(m_prevStartFillTime)) { + query->filterGE("start_time", nextFillSearchTime); } else { - query->filterGT("start_time", targetTime); + query->filterGT("start_time", nextFillSearchTime); } query->filterLT("start_time", m_endTime); @@ -298,12 +291,12 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandlerfillNumber() and !ongoingFill) - addEmptyPayload(m_endFillTime); + if (m_prevPayload->fillNumber() and !ongoingFill) { + if (m_endFillMode) { + addEmptyPayload(m_endFillTime); + } else { + addEmptyPayload(cond::lhcInfoHelper::getFillLastLumiIOV(oms, lhcFill)); + } + } } } std::string id() const override { return m_name; } @@ -382,9 +380,15 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler("start_time"); LHCInfoPerLS* thisLumiSectionInfo = new LHCInfoPerLS(*m_fillPayload); thisLumiSectionInfo->setLumiSection(std::stoul(row.get("lumisection_number"))); - thisLumiSectionInfo->setRunNumber(std::stoull(row.get("run_number"))); + thisLumiSectionInfo->setRunNumber(std::stoul(row.get("run_number"))); m_lsIdMap[make_pair(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection())] = make_pair(-1, -1); - m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo)); + if (m_endFillMode) { + m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo)); + } else { + m_tmpBuffer.emplace_back( + make_pair(cond::time::lumiTime(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection()), + thisLumiSectionInfo)); + } } size_t bufferAllLS(const cond::OMSServiceResult& queryResult) { @@ -563,13 +567,11 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler m_fillPayload; std::shared_ptr m_prevPayload; diff --git a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py index e4b6dd762df21..d71362ecebfa8 100644 --- a/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py +++ b/CondTools/RunInfo/python/LHCInfoPerFillPopConAnalyzer.py @@ -81,9 +81,15 @@ interval = cms.uint64(1) ) +# Write different time-types tags depending on the O2O mode +if options.mode == 'endFill': + timetype = 'timestamp' +else: + timetype = 'lumiid' + process.PoolDBOutputService = cms.Service("PoolDBOutputService", CondDBConnection, - timetype = cms.untracked.string('timestamp'), + timetype = cms.untracked.string(timetype), toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerFillRcd'), tag = cms.string( options.tag ) ) @@ -97,12 +103,10 @@ Source = cms.PSet(fill = cms.untracked.uint32(6417), startTime = cms.untracked.string(options.startTime), endTime = cms.untracked.string(options.endTime), - samplingInterval = cms.untracked.uint32( 600 ), endFill = cms.untracked.bool(True if options.mode == "endFill" else False), name = cms.untracked.string("LHCInfoPerFillPopConSourceHandler"), connectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER"), ecalConnectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_DCS_ENV_PVSS_COND"), - DIPSchema = cms.untracked.string("CMS_BEAM_COND"), omsBaseUrl = cms.untracked.string("http://vocms0184.cern.ch/agg/api/v1"), authenticationPath = cms.untracked.string(""), debug=cms.untracked.bool(False) diff --git a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py index 522aedae4ec1c..603c7f514d4ee 100644 --- a/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py +++ b/CondTools/RunInfo/python/LHCInfoPerLSPopConAnalyzer.py @@ -81,9 +81,15 @@ interval = cms.uint64(1) ) +# Write different time-types tags depending on the O2O mode +if options.mode == 'endFill': + timetype = 'timestamp' +else: + timetype = 'lumiid' + process.PoolDBOutputService = cms.Service("PoolDBOutputService", CondDBConnection, - timetype = cms.untracked.string('timestamp'), + timetype = cms.untracked.string(timetype), toPut = cms.VPSet(cms.PSet(record = cms.string('LHCInfoPerLSRcd'), tag = cms.string( options.tag ) ) @@ -97,11 +103,9 @@ Source = cms.PSet(fill = cms.untracked.uint32(6417), startTime = cms.untracked.string(options.startTime), endTime = cms.untracked.string(options.endTime), - samplingInterval = cms.untracked.uint32( 600 ), endFill = cms.untracked.bool(True if options.mode == "endFill" else False), name = cms.untracked.string("LHCInfoPerLSPopConSourceHandler"), connectionString = cms.untracked.string("oracle://cms_orcon_adg/CMS_RUNTIME_LOGGER"), - DIPSchema = cms.untracked.string("CMS_BEAM_COND"), omsBaseUrl = cms.untracked.string("http://vocms0184.cern.ch/agg/api/v1"), authenticationPath = cms.untracked.string(""), debug=cms.untracked.bool(False) diff --git a/CondTools/RunInfo/src/LHCInfoHelper.cc b/CondTools/RunInfo/src/LHCInfoHelper.cc new file mode 100644 index 0000000000000..54503068b8924 --- /dev/null +++ b/CondTools/RunInfo/src/LHCInfoHelper.cc @@ -0,0 +1,31 @@ +#include "CondTools/RunInfo/interface/LHCInfoHelper.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" + +//***************************************************************** +// Returns lumi-type IOV (packed using cond::time::lumiTime) from +// last LS of last Run of the specified Fill +//***************************************************************** +cond::Time_t cond::lhcInfoHelper::getFillLastLumiIOV(const cond::OMSService& oms, unsigned short fillId) { + // Define query + auto query = oms.query("lumisections"); + query->addOutputVars({"lumisection_number", "run_number"}); + query->filterEQ("fill_number", fillId); + query->limit(cond::lhcInfoHelper::kLumisectionsQueryLimit); + + // Execute query + if (!query->execute()) { + throw cms::Exception("OMSQueryFailure") + << "OMS query of fill " << fillId << " failed, status:" << query->status() << "\n"; + } + + // Get query result + auto queryResult = query->result(); + if (queryResult.empty()) { + throw cms::Exception("OMSQueryFailure") << "OMS query of fill " << fillId << " returned empty result!\n"; + } + + // Return the final IOV + auto lastRun = queryResult.back().get("run_number"); + auto lastLumi = queryResult.back().get("lumisection_number"); + return cond::time::lumiTime(lastRun, lastLumi); +}