From dbe2cd10401c3313923b294e84ebc2e29efea89b Mon Sep 17 00:00:00 2001 From: Jan Chyczynski Date: Mon, 7 Aug 2023 22:29:39 +0200 Subject: [PATCH] Fix LHCInfoPerLS PopCon incorrectly matching LS between OMS and PPS db --- .../interface/LHCInfoPopConSourceHandler.h | 2 +- .../plugins/LHCInfoPerFillPopConAnalyzer.cc | 4 +- .../plugins/LHCInfoPerLSPopConAnalyzer.cc | 189 +++++++++++++----- .../RunInfo/src/LHCInfoPopConSourceHandler.cc | 4 +- .../test/summarizePerLSPopConValidityLogs.sh | 10 + .../RunInfo/test/test_lhcInfoNewPopCon.sh | 21 +- 6 files changed, 168 insertions(+), 62 deletions(-) create mode 100755 CondTools/RunInfo/test/summarizePerLSPopConValidityLogs.sh diff --git a/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h b/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h index ae543618d9168..6139ba4becd26 100644 --- a/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h +++ b/CondTools/RunInfo/interface/LHCInfoPopConSourceHandler.h @@ -30,7 +30,7 @@ class LHCInfoPopConSourceHandler : public popcon::PopConSourceHandler { void getDipData(const cond::OMSService& service, const boost::posix_time::ptime& beginFillTime, const boost::posix_time::ptime& endFillTime); - bool getCTTPSData(cond::persistency::Session& session, + bool getCTPPSData(cond::persistency::Session& session, const boost::posix_time::ptime& beginFillTime, const boost::posix_time::ptime& endFillTime); bool getEcalData(cond::persistency::Session& session, diff --git a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc index 4d5a47a69c09b..e4b935290664c 100644 --- a/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc +++ b/CondTools/RunInfo/plugins/LHCInfoPerFillPopConAnalyzer.cc @@ -354,7 +354,7 @@ class LHCInfoPerFillPopConSourceHandler : public popcon::PopConSourceHandler #include +#include #include #include #include @@ -27,6 +27,9 @@ #include #include +using std::make_pair; +using std::pair; + class LHCInfoPerLSPopConSourceHandler; typedef popcon::PopConAnalyzer LHCInfoPerLSPopConAnalyzer; @@ -43,11 +46,18 @@ namespace theLHCInfoPerLSImpl { return true; } - size_t transferPayloads(const std::vector>>& buffer, + size_t transferPayloads(const std::vector>>& buffer, std::map>& iovsToTransfer, - std::shared_ptr& prevPayload) { + std::shared_ptr& prevPayload, + const std::map, pair>& lsIdMap, + cond::Time_t startStableBeamTime, + cond::Time_t endStableBeamTime) { + int lsMissingInPPS = 0; + int xAngleBothZero = 0, xAngleBothNonZero = 0, xAngleNegative = 0; + int betaNegative = 0; size_t niovs = 0; std::stringstream condIovs; + std::stringstream missingLsList; for (auto& iov : buffer) { bool add = false; auto payload = iov.second; @@ -60,13 +70,58 @@ namespace theLHCInfoPerLSImpl { add = true; } } - if (add) { + auto id = make_pair(payload->runNumber(), payload->lumiSection()); + bool stableBeam = since >= startStableBeamTime && since <= endStableBeamTime; + bool isMissing = lsIdMap.find(id) != lsIdMap.end() && id != lsIdMap.at(id); + if (stableBeam && isMissing) { + missingLsList << id.first << "_" << id.second << " "; + lsMissingInPPS += isMissing; + } + if (add && !isMissing) { niovs++; + if (stableBeam) { + if (payload->crossingAngleX() == 0 && payload->crossingAngleY() == 0) + xAngleBothZero++; + if (payload->crossingAngleX() != 0 && payload->crossingAngleY() != 0) + xAngleBothNonZero++; + if (payload->crossingAngleX() < 0 || payload->crossingAngleY() < 0) + xAngleNegative++; + if (payload->betaStarX() < 0 || payload->betaStarY() < 0) + betaNegative++; + } + condIovs << since << " "; - iovsToTransfer.insert(std::make_pair(since, payload)); + iovsToTransfer.insert(make_pair(since, payload)); prevPayload = iov.second; } } + unsigned short fillNumber = (!buffer.empty()) ? buffer.front().second->fillNumber() : 0; + if (lsMissingInPPS > 0) { + edm::LogWarning("transferPayloads") + << "Number of stable beam LS in OMS without corresponding record in PPS DB for fill " << fillNumber << ": " + << lsMissingInPPS; + edm::LogWarning("transferPayloads") + << "Stable beam LS in OMS without corresponding record in PPS DB (run_LS): " << missingLsList.str(); + } + if (xAngleBothZero > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with crossingAngle == 0 for both X and Y for fill " << fillNumber << ": " + << xAngleBothZero; + } + if (xAngleBothNonZero > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with crossingAngle != 0 for both X and Y for fill " << fillNumber << ": " + << xAngleBothNonZero; + } + if (xAngleNegative > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with negative crossingAngle for fill " << fillNumber << ": " << xAngleNegative; + } + if (betaNegative > 0) { + edm::LogWarning("transferPayloads") + << "Number of payloads written with negative betaSta for fill " << fillNumber << ": " << betaNegative; + } + edm::LogInfo("transferPayloads") << "TRANSFERED COND IOVS: " << condIovs.str(); return niovs; } @@ -256,18 +311,20 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandlerfillNumber() and !ongoingFill) addEmptyPayload(m_endFillTime); } @@ -290,7 +347,7 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler(); - m_iovs.insert(std::make_pair(iov, newPayload)); + m_iovs.insert(make_pair(iov, newPayload)); m_prevPayload = newPayload; m_prevEndFillTime = 0; m_prevStartFillTime = 0; @@ -306,6 +363,8 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler("fill_number"); m_startFillTime = cond::time::from_boost(row.get("start_time")); m_endFillTime = cond::time::from_boost(row.get("end_time")); + m_startStableBeamTime = cond::time::from_boost(row.get("start_stable_beam")); + m_endStableBeamTime = cond::time::from_boost(row.get("end_stable_beam")); targetPayload = std::make_unique(); targetPayload->setFillNumber(currentFill); ret = true; @@ -316,7 +375,10 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler("start_time"); LHCInfoPerLS* thisLumiSectionInfo = new LHCInfoPerLS(*m_fillPayload); - m_tmpBuffer.emplace_back(std::make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo)); + thisLumiSectionInfo->setLumiSection(std::stoul(row.get("lumisection_number"))); + thisLumiSectionInfo->setRunNumber(std::stoull(row.get("run_number"))); + m_lsIdMap[make_pair(thisLumiSectionInfo->runNumber(), thisLumiSectionInfo->lumiSection())] = make_pair(-1, -1); + m_tmpBuffer.emplace_back(make_pair(cond::time::from_boost(lumiTime), thisLumiSectionInfo)); } size_t bufferAllLS(const cond::OMSServiceResult& queryResult) { @@ -369,7 +431,7 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandleraddToOutputList(std::string("DIP_UPDATE_TIME")); CTPPSDataQuery->addToOutputList(std::string("LUMI_SECTION")); CTPPSDataQuery->addToOutputList(std::string("RUN_NUMBER")); + CTPPSDataQuery->addToOutputList(std::string("FILL_NUMBER")); CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_X_URAD")); CTPPSDataQuery->addToOutputList(std::string("XING_ANGLE_P5_Y_URAD")); CTPPSDataQuery->addToOutputList(std::string("BETA_STAR_P5_X_M")); @@ -402,6 +465,7 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler(std::string("DIP_UPDATE_TIME")); CTPPSDataOutput.extend(std::string("LUMI_SECTION")); CTPPSDataOutput.extend(std::string("RUN_NUMBER")); + CTPPSDataOutput.extend(std::string("FILL_NUMBER")); CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_X_URAD")); CTPPSDataOutput.extend(std::string("XING_ANGLE_P5_Y_URAD")); CTPPSDataOutput.extend(std::string("BETA_STAR_P5_X_M")); @@ -409,14 +473,16 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandlerdefineOutput(CTPPSDataOutput); //execute the query coral::ICursor& CTPPSDataCursor = CTPPSDataQuery->execute(); - cond::Time_t dipTime = 0; unsigned int lumiSection = 0; cond::Time_t runNumber = 0; + int fillNumber = 0; float crossingAngleX = 0., betaStarX = 0.; float crossingAngleY = 0., betaStarY = 0.; bool ret = false; - LumiSectionFilter filter(m_tmpBuffer); + int wrongFillNumbers = 0; + std::stringstream wrongFills; + std::vector>>::iterator current = m_tmpBuffer.begin(); while (CTPPSDataCursor.next()) { if (m_debug) { std::ostringstream CTPPS; @@ -424,48 +490,64 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler().time()); - if (filter.process(dipTime)) { - ret = true; - coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; - if (!lumiSectionAttribute.isNull()) { - lumiSection = lumiSectionAttribute.data(); - } - coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; - if (!runNumberAttribute.isNull()) { - runNumber = runNumberAttribute.data(); - } - coral::Attribute const& crossingAngleXAttribute = - CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_X_URAD")]; - if (!crossingAngleXAttribute.isNull()) { - crossingAngleX = crossingAngleXAttribute.data(); - } - coral::Attribute const& crossingAngleYAttribute = - CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_Y_URAD")]; - if (!crossingAngleYAttribute.isNull()) { - crossingAngleY = crossingAngleYAttribute.data(); - } - coral::Attribute const& betaStarXAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_X_M")]; - if (!betaStarXAttribute.isNull()) { - betaStarX = betaStarXAttribute.data(); - } - coral::Attribute const& betaStarYAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_Y_M")]; - if (!betaStarYAttribute.isNull()) { - betaStarY = betaStarYAttribute.data(); - } - for (auto it = filter.current(); it != m_tmpBuffer.end(); it++) { - // set the current values to all of the payloads of the lumi section samples after the current since - LHCInfoPerLS& payload = *(it->second); - payload.setCrossingAngleX(crossingAngleX); - payload.setCrossingAngleY(crossingAngleY); - payload.setBetaStarX(betaStarX); - payload.setBetaStarY(betaStarY); - payload.setLumiSection(lumiSection); - payload.setRunNumber(runNumber); + ret = true; + coral::Attribute const& lumiSectionAttribute = CTPPSDataCursor.currentRow()[std::string("LUMI_SECTION")]; + if (!lumiSectionAttribute.isNull()) { + lumiSection = lumiSectionAttribute.data(); + } + coral::Attribute const& runNumberAttribute = CTPPSDataCursor.currentRow()[std::string("RUN_NUMBER")]; + if (!runNumberAttribute.isNull()) { + runNumber = runNumberAttribute.data(); + } + coral::Attribute const& fillNumberAttribute = CTPPSDataCursor.currentRow()[std::string("FILL_NUMBER")]; + if (!fillNumberAttribute.isNull()) { + fillNumber = fillNumberAttribute.data(); + } + coral::Attribute const& crossingAngleXAttribute = + CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_X_URAD")]; + if (!crossingAngleXAttribute.isNull()) { + crossingAngleX = crossingAngleXAttribute.data(); + } + coral::Attribute const& crossingAngleYAttribute = + CTPPSDataCursor.currentRow()[std::string("XING_ANGLE_P5_Y_URAD")]; + if (!crossingAngleYAttribute.isNull()) { + crossingAngleY = crossingAngleYAttribute.data(); + } + coral::Attribute const& betaStarXAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_X_M")]; + if (!betaStarXAttribute.isNull()) { + betaStarX = betaStarXAttribute.data(); + } + coral::Attribute const& betaStarYAttribute = CTPPSDataCursor.currentRow()[std::string("BETA_STAR_P5_Y_M")]; + if (!betaStarYAttribute.isNull()) { + betaStarY = betaStarYAttribute.data(); + } + if (current != m_tmpBuffer.end() && current->second->fillNumber() != fillNumber) { + wrongFills << "( " << runNumber << "_" << lumiSection << " fill: OMS: " << current->second->fillNumber() + << " PPSdb: " << fillNumber << " ) "; + wrongFillNumbers++; + } + for (; + current != m_tmpBuffer.end() && make_pair(current->second->runNumber(), current->second->lumiSection()) <= + make_pair(runNumber, lumiSection); + current++) { + LHCInfoPerLS& payload = *(current->second); + payload.setCrossingAngleX(crossingAngleX); + payload.setCrossingAngleY(crossingAngleY); + payload.setBetaStarX(betaStarX); + payload.setBetaStarY(betaStarY); + payload.setLumiSection(lumiSection); + payload.setRunNumber(runNumber); + if (m_lsIdMap.find(make_pair(payload.runNumber(), payload.lumiSection())) != m_lsIdMap.end()) { + m_lsIdMap[make_pair(payload.runNumber(), payload.lumiSection())] = make_pair(runNumber, lumiSection); } } } } + if (wrongFillNumbers) { + edm::LogWarning("getCTPPSData") << "Number of records from PPS DB with fillNumber different from OMS: " + << wrongFillNumbers; + edm::LogWarning("getCTPPSData") << "Records from PPS DB with fillNumber different from OMS: " << wrongFills.str(); + } return ret; } @@ -488,6 +570,11 @@ class LHCInfoPerLSPopConSourceHandler : public popcon::PopConSourceHandler>> m_tmpBuffer; + cond::Time_t m_startStableBeamTime; + cond::Time_t m_endStableBeamTime; + std::vector>> m_tmpBuffer; bool m_lastPayloadEmpty = false; + //mapping of lumisections IDs (pairs of runnumber an LS number) found in OMS to the IDs they've been assignd from PPS DB + //value pair(-1, -1) means lumisection corresponding to the key exists in OMS but no lumisection was matched from PPS + std::map, pair> m_lsIdMap; }; diff --git a/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc b/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc index 4b40d5a12074f..0a344bd479711 100644 --- a/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc +++ b/CondTools/RunInfo/src/LHCInfoPopConSourceHandler.cc @@ -240,7 +240,7 @@ void LHCInfoPopConSourceHandler::getDipData(const cond::OMSService& oms, } } -bool LHCInfoPopConSourceHandler::getCTTPSData(cond::persistency::Session& session, +bool LHCInfoPopConSourceHandler::getCTPPSData(cond::persistency::Session& session, const boost::posix_time::ptime& beginFillTime, const boost::posix_time::ptime& endFillTime) { //run the fifth query against the CTPPS schema @@ -665,7 +665,7 @@ void LHCInfoPopConSourceHandler::getNewObjects() { boost::posix_time::ptime flumiStop = cond::time::to_boost(m_tmpBuffer.back().first); edm::LogInfo(m_name) << "First lumi starts at " << flumiStart << " last lumi starts at " << flumiStop; session.transaction().start(true); - getCTTPSData(session, startSampleTime, endSampleTime); + getCTPPSData(session, startSampleTime, endSampleTime); session.transaction().commit(); session2.transaction().start(true); getEcalData(session2, startSampleTime, endSampleTime, updateEcal); diff --git a/CondTools/RunInfo/test/summarizePerLSPopConValidityLogs.sh b/CondTools/RunInfo/test/summarizePerLSPopConValidityLogs.sh new file mode 100755 index 0000000000000..69b61dd517401 --- /dev/null +++ b/CondTools/RunInfo/test/summarizePerLSPopConValidityLogs.sh @@ -0,0 +1,10 @@ +LOG_FILE=$1 +echo "Number of invalid payloads found in the logs" +for CONDITION in "crossingAngle == 0 for both X and Y" "crossingAngle != 0 for both X and Y" \ + "negative crossingAngle: " "negative betaStar" "Number of records from PPS DB with fillNumber different from OMS" \ + "Number of stable beam LS in OMS without corresponding record in PPS DB" ; do + echo -n "$CONDITION: max in one fill: " + (cat $LOG_FILE | grep -E "$CONDITION" | awk '{print $NF}' ; echo 0) | sort -gr | head -n 1 + echo -n "$CONDITION: total: " + (cat $LOG_FILE | grep -E "$CONDITION" | awk '{print $NF}'; echo 0) | paste -sd+ | bc +done diff --git a/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh b/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh index e503999330647..d642b824b3a50 100755 --- a/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh +++ b/CondTools/RunInfo/test/test_lhcInfoNewPopCon.sh @@ -36,14 +36,23 @@ assert_equal 7 `cat fill_end_test.log | grep -E '^Since ' | \ wc -l` "LHCInfoPerFillPopConAnalyzer in EndFill mode written wrong number of payloads" assert_found_fills fill_end_test.log "LHCInfoPerFillPopConAnalyzer in EndFill mode" 8307 8309 -echo "testing LHCInfoPerLSPopConAnalyzerEndFill in endFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" +echo "testing LHCInfoPerLSPopConAnalyzer in endFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerLSPopConAnalyzer.py mode=endFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ tag=ls_end_test > ls_end_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer.py mode=endFill" $? assert_equal 169 `cat ls_end_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerLSPopConAnalyzerEndFill in endFill mode written wrong number of payloads" -assert_found_fills ls_end_test.log "LHCInfoPerLSPopConAnalyzerEndFill in endFill mode" 8307 8309 + wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" +assert_found_fills ls_end_test.log "LHCInfoPerLSPopConAnalyzer in endFill mode" 8307 8309 + +echo "testing LHCInfoPerLSPopConAnalyzer in endFill mode for startTime=\"2022-07-11 22:00:00.000\" endTime=\"2022-07-12 18:10:10.000\"" +cmsRun ${SCRIPTS_DIR}/LHCInfoPerLSPopConAnalyzer.py mode=endFill \ + destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ + startTime="2022-07-11 22:00:00.000" endTime="2022-07-12 18:10:10.000" \ + tag=ls_end_test2 > ls_end_test2.log || die "cmsRun LHCInfoPerLSPopConAnalyzer.py mode=endFill" $? +assert_equal 69 `cat ls_end_test2.log | grep -E '^Since ' | \ + wc -l` "LHCInfoPerLSPopConAnalyzer in endFill mode written wrong number of payloads" +assert_found_fills ls_end_test2.log "LHCInfoPerLSPopConAnalyzer in endFill mode" 7967 echo "testing LHCInfoPerFillPopConAnalyzer in DuringFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerFillPopConAnalyzer.py mode=duringFill \ @@ -54,11 +63,11 @@ assert_equal 1 `cat fill_during_test.log | grep -E '^Since ' | \ wc -l` "LHCInfoPerFillPopConAnalyzer in DuringFill written wrong number of payloads" assert_found_fills fill_during_test.log "LHCInfoPerFillPopConAnalyzer in DuringFill" 8307 8309 -echo "testing LHCInfoPerLSPopConAnalyzerEndFill in duringFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" +echo "testing LHCInfoPerLSPopConAnalyzer in duringFill mode for startTime=\"2022-10-24 01:00:00.000\" endTime=\"2022-10-24 20:00:00.000\"" cmsRun ${SCRIPTS_DIR}/LHCInfoPerLSPopConAnalyzer.py mode=duringFill \ destinationConnection="sqlite_file:lhcinfo_pop_unit_test.db" \ startTime="2022-10-24 01:00:00.000" endTime="2022-10-24 20:00:00.000" \ tag=ls_during_test > ls_during_test.log || die "cmsRun LHCInfoPerLSPopConAnalyzer.py mode=duringFill" $? assert_equal 1 `cat ls_during_test.log | grep -E '^Since ' | \ - wc -l` "LHCInfoPerLSPopConAnalyzerEndFill in duringFill mode written wrong number of payloads" -assert_found_fills ls_during_test.log "LHCInfoPerLSPopConAnalyzerEndFill in duringFill mode" 8307 8309 \ No newline at end of file + wc -l` "LHCInfoPerLSPopConAnalyzer in duringFill mode written wrong number of payloads" +assert_found_fills ls_during_test.log "LHCInfoPerLSPopConAnalyzer in duringFill mode" 8307 8309 \ No newline at end of file