diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index 792f64c459f2f..868b1952a2edc 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -14,25 +14,16 @@ #include #include #include -#include -#include -#include #include "TFile.h" #include "TTree.h" #include "TString.h" -#include "TH1.h" -#include "TH2.h" -#include "TProfile.h" // user include files #include "FWCore/Framework/interface/InputSource.h" +#include "FWCore/Sources/interface/PuttableSourceBase.h" #include "FWCore/Catalog/interface/InputFileCatalog.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "DQMServices/Core/interface/DQMStore.h" -#include "FWCore/ServiceRegistry/interface/Service.h" -#include "FWCore/MessageLogger/interface/JobReport.h" -//#include "FWCore/Utilities/interface/GlobalIdentifier.h" -#include "FWCore/Utilities/interface/EDMException.h" #include "FWCore/Utilities/interface/ExceptionPropagate.h" #include "FWCore/Framework/interface/RunPrincipal.h" @@ -48,198 +39,171 @@ #include "FWCore/Framework/interface/InputSourceMacros.h" #include "FWCore/Framework/interface/FileBlock.h" -#include "DataFormats/Provenance/interface/EventRange.h" -#include "DataFormats/Provenance/interface/EventID.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/Utilities/interface/TimeOfDay.h" -#include "DataFormats/Provenance/interface/ProcessHistory.h" -#include "DataFormats/Provenance/interface/ProcessHistoryID.h" -#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" -#include "FWCore/ParameterSet/interface/Registry.h" - -#include "FWCore/Utilities/interface/Digest.h" -#include "FWCore/Utilities/interface/InputType.h" - #include "format.h" namespace { - typedef dqm::legacy::MonitorElement MonitorElement; - typedef dqm::legacy::DQMStore DQMStore; - - //adapter functions - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1F* iHist) { - //std::cout <<"create: hist size "<GetEffectiveEntries()<CanExtendAllAxes() && iToAdd->CanExtendAllAxes()) { - TList list; - list.Add(iToAdd); - if (-1 == iOriginal->Merge(&list)) { - edm::LogError("MergeFailure") << "Failed to merge DQM element " << iOriginal->GetName(); + typedef dqm::harvesting::MonitorElement MonitorElement; + + // TODO: this should probably be moved somewhere else + class DQMMergeHelper { + public: + // Utility function to check the consistency of the axis labels + // Taken from TH1::CheckBinLabels which is not public + static bool CheckBinLabels(const TAxis* a1, const TAxis * a2) { + // Check that axis have same labels + THashList *l1 = (const_cast(a1))->GetLabels(); + THashList *l2 = (const_cast(a2))->GetLabels(); + + if (!l1 && !l2 ) + return true; + if (!l1 || !l2 ) { + return false; } - } else { - if (iOriginal->GetNbinsX() == iToAdd->GetNbinsX() && - iOriginal->GetXaxis()->GetXmin() == iToAdd->GetXaxis()->GetXmin() && - iOriginal->GetXaxis()->GetXmax() == iToAdd->GetXaxis()->GetXmax() && - iOriginal->GetNbinsY() == iToAdd->GetNbinsY() && - iOriginal->GetYaxis()->GetXmin() == iToAdd->GetYaxis()->GetXmin() && - iOriginal->GetYaxis()->GetXmax() == iToAdd->GetYaxis()->GetXmax() && - iOriginal->GetNbinsZ() == iToAdd->GetNbinsZ() && - iOriginal->GetZaxis()->GetXmin() == iToAdd->GetZaxis()->GetXmin() && - iOriginal->GetZaxis()->GetXmax() == iToAdd->GetZaxis()->GetXmax() && - MonitorElement::CheckBinLabels(iOriginal->GetXaxis(), iToAdd->GetXaxis()) && - MonitorElement::CheckBinLabels(iOriginal->GetYaxis(), iToAdd->GetYaxis()) && - MonitorElement::CheckBinLabels(iOriginal->GetZaxis(), iToAdd->GetZaxis())) { - iOriginal->Add(iToAdd); - } else { - edm::LogError("MergeFailure") << "Found histograms with different axis limits or different labels'" - << iOriginal->GetName() << "' not merged."; + + // Check now labels sizes are the same + if (l1->GetSize() != l2->GetSize() ) { + return false; } - } - } - void mergeWithElement(MonitorElement* iElement, TH1F* iHist) { - //std::cout <<"merge: hist size "<getName() <<" "<GetEffectiveEntries()<getTH1F(), iHist); - } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1S* iHist) { - return iStore.book1S(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH1S* iHist) { mergeTogether(iElement->getTH1S(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1D* iHist) { - return iStore.book1DD(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH1D* iHist) { mergeTogether(iElement->getTH1D(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2F* iHist) { - return iStore.book2D(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH2F* iHist) { mergeTogether(iElement->getTH2F(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2S* iHist) { - return iStore.book2S(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH2S* iHist) { mergeTogether(iElement->getTH2S(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2D* iHist) { - return iStore.book2DD(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH2D* iHist) { mergeTogether(iElement->getTH2D(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH3F* iHist) { - return iStore.book3D(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH3F* iHist) { mergeTogether(iElement->getTH3F(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TProfile* iHist) { - return iStore.bookProfile(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TProfile* iHist) { mergeTogether(iElement->getTProfile(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TProfile2D* iHist) { - return iStore.bookProfile2D(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TProfile2D* iHist) { - mergeTogether(iElement->getTProfile2D(), iHist); - } + for (int i = 1; i <= a1->GetNbins(); ++i) { + TString label1 = a1->GetBinLabel(i); + TString label2 = a2->GetBinLabel(i); + if (label1 != label2) { + return false; + } + } - MonitorElement* createElement(DQMStore& iStore, const char* iName, Long64_t& iValue) { - MonitorElement* e = iStore.bookInt(iName); - e->Fill(iValue); - return e; - } + return true; + } - //NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h - void mergeWithElement(MonitorElement* iElement, Long64_t& iValue) { - const std::string& name = iElement->getFullname(); - if (name.find("EventInfo/processedEvents") != std::string::npos) { - iElement->Fill(iValue + iElement->getIntValue()); - } else if (name.find("EventInfo/iEvent") != std::string::npos || - name.find("EventInfo/iLumiSection") != std::string::npos) { - if (iValue > iElement->getIntValue()) { - iElement->Fill(iValue); + // NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h + static void mergeTogether(TH1* original, TH1* toAdd) { + if (original->CanExtendAllAxes() && toAdd->CanExtendAllAxes()) { + TList list; + list.Add(toAdd); + if (original->Merge(&list) == -1) { + edm::LogError("MergeFailure") << "Failed to merge DQM element " << original->GetName(); + } + } + else { + // TODO: Redo. What's wrong with this implementation? + if (original->GetNbinsX() == toAdd->GetNbinsX() && + original->GetXaxis()->GetXmin() == toAdd->GetXaxis()->GetXmin() && + original->GetXaxis()->GetXmax() == toAdd->GetXaxis()->GetXmax() && + original->GetNbinsY() == toAdd->GetNbinsY() && + original->GetYaxis()->GetXmin() == toAdd->GetYaxis()->GetXmin() && + original->GetYaxis()->GetXmax() == toAdd->GetYaxis()->GetXmax() && + original->GetNbinsZ() == toAdd->GetNbinsZ() && + original->GetZaxis()->GetXmin() == toAdd->GetZaxis()->GetXmin() && + original->GetZaxis()->GetXmax() == toAdd->GetZaxis()->GetXmax() && + CheckBinLabels(original->GetXaxis(), toAdd->GetXaxis()) && + CheckBinLabels(original->GetYaxis(), toAdd->GetYaxis()) && + CheckBinLabels(original->GetZaxis(), toAdd->GetZaxis())) { + original->Add(toAdd); + } + else { + edm::LogError("MergeFailure") << "Found histograms with different axis limits or different labels '" + << original->GetName() << "' not merged."; + } } - } else { - iElement->Fill(iValue); } - } - - MonitorElement* createElement(DQMStore& iStore, const char* iName, double& iValue) { - MonitorElement* e = iStore.bookFloat(iName); - e->Fill(iValue); - return e; - } - void mergeWithElement(MonitorElement* iElement, double& iValue) { - //no merging, take the last one - iElement->Fill(iValue); - } - MonitorElement* createElement(DQMStore& iStore, const char* iName, std::string* iValue) { - return iStore.bookString(iName, *iValue); - } - void mergeWithElement(MonitorElement* iElement, std::string* iValue) { - //no merging, take the last one - iElement->Fill(*iValue); - } + }; - void splitName(const std::string& iFullName, std::string& oPath, const char*& oName) { - oPath = iFullName; - size_t index = oPath.find_last_of('/'); - if (index == std::string::npos) { - oPath = std::string(); - oName = iFullName.c_str(); - } else { - oPath.resize(index); - oName = iFullName.c_str() + index + 1; - } - } + using MonitorElementsFromFile = std::map, std::vector>; - struct RunLumiToRange { - unsigned int m_run, m_lumi, m_historyIDIndex; + // This struct allows to find all MEs belonging to a run-lumi pair + // All files will be open at once so m_file property indicates the file where data is saved. + struct FileMetadata { + unsigned int m_run; + unsigned int m_lumi; ULong64_t m_beginTime; ULong64_t m_endTime; - ULong64_t m_firstIndex, m_lastIndex; //last is inclusive - unsigned int m_type; //A value in TypeIndex + ULong64_t m_firstIndex; + ULong64_t m_lastIndex; // Last is inclusive + unsigned int m_type; + TFile* m_file; + + // This will be used when sorting a vector + bool operator < (const FileMetadata& obj) const { + if(m_run == obj.m_run) + return m_lumi < obj.m_lumi; + else + return m_run < obj.m_run; + } + + void describe() { + std::cout << "read r:" << m_run + << " l:" << m_lumi + << " bt:" << m_beginTime + << " et:" << m_endTime + << " fi:" << m_firstIndex + << " li:" << m_lastIndex + << " type:" << m_type + << " file: " << m_file << std::endl; + } }; class TreeReaderBase { public: - TreeReaderBase() {} + TreeReaderBase(MonitorElementData::Kind kind) : m_kind(kind) {} virtual ~TreeReaderBase() {} - MonitorElement* read(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) { return doRead(iIndex, iStore, iIsLumi); } + virtual void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) = 0; virtual void setTree(TTree* iTree) = 0; protected: + MonitorElementData::Kind m_kind; TTree* m_tree; - - private: - virtual MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) = 0; }; template class TreeObjectReader : public TreeReaderBase { public: - TreeObjectReader() : m_tree(nullptr), m_fullName(nullptr), m_buffer(nullptr), m_tag(0) {} - MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) override { + TreeObjectReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(nullptr), m_tag(0) { + assert(m_kind != MonitorElementData::Kind::INT); + assert(m_kind != MonitorElementData::Kind::REAL); + assert(m_kind != MonitorElementData::Kind::STRING); + } + + void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { + // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); - MonitorElement* element = iStore.get(*m_fullName); - try { - if (nullptr == element) { - std::string path; - const char* name; - splitName(*m_fullName, path, name); - iStore.setCurrentFolder(path); - element = createElement(iStore, name, m_buffer); - if (iIsLumi) { - element->setLumiFlag(); - } - } else { - mergeWithElement(element, m_buffer); + + MonitorElementData::Key key; + key.kind_ = m_kind; + key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); + key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; + // TODO: What should the range be for per run MEs? Now lumi will be 0 and not the max lumi for that run. + key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi); + + std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; + bool merged = false; + for (MonitorElementData* meData : runLumiMEs) { + if(meData->key_ == key) { + // Merge with already existing ME! + MonitorElementData::Value::Access value(meData->value_); + DQMMergeHelper::mergeTogether(value.object.get(), m_buffer); + merged = true; + break; } - } catch (cms::Exception& e) { - e.addContext(std::string("While reading element ") + *m_fullName); - e.raise(); } - return element; + + if(!merged) { + MonitorElementData* meData = new MonitorElementData(); + meData->key_ = key; + { + MonitorElementData::Value::Access value(meData->value_); + value.object = std::unique_ptr((T*)(m_buffer->Clone())); + } + + mesFromFile[std::make_tuple(run, lumi)].push_back(meData); + } } + void setTree(TTree* iTree) override { m_tree = iTree; m_tree->SetBranchAddress(kFullNameBranch, &m_fullName); @@ -254,27 +218,107 @@ namespace { uint32_t m_tag; }; + class TreeStringReader : public TreeReaderBase { + public: + TreeStringReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_value(nullptr), m_tag(0) { + assert(m_kind == MonitorElementData::Kind::STRING); + } + + void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { + // This will populate the fields as defined in setTree method + m_tree->GetEntry(iIndex); + + MonitorElementData::Key key; + key.kind_ = m_kind; + key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); + key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; + key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi); + + std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; + bool merged = false; + for (MonitorElementData* meData : runLumiMEs) { + if(meData->key_ == key) { + // Keep the latest one + MonitorElementData::Value::Access value(meData->value_); + value.scalar.str = *m_value; + merged = true; + break; + } + } + + if(!merged) { + MonitorElementData* meData = new MonitorElementData(); + meData->key_ = key; + { + MonitorElementData::Value::Access value(meData->value_); + value.scalar.str = *m_value; + } + + mesFromFile[std::make_tuple(run, lumi)].push_back(std::move(meData)); + } + } + + void setTree(TTree* iTree) override { + m_tree = iTree; + m_tree->SetBranchAddress(kFullNameBranch, &m_fullName); + m_tree->SetBranchAddress(kFlagBranch, &m_tag); + m_tree->SetBranchAddress(kValueBranch, &m_value); + } + + private: + TTree* m_tree; + std::string* m_fullName; + std::string* m_value; + uint32_t m_tag; + }; + template class TreeSimpleReader : public TreeReaderBase { public: - TreeSimpleReader() : m_tree(nullptr), m_fullName(nullptr), m_buffer(0), m_tag(0) {} - MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) override { + TreeSimpleReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(0), m_tag(0) { + assert(m_kind == MonitorElementData::Kind::INT || m_kind == MonitorElementData::Kind::REAL); + } + + void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { + // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); - MonitorElement* element = iStore.get(*m_fullName); - if (nullptr == element) { - std::string path; - const char* name; - splitName(*m_fullName, path, name); - iStore.setCurrentFolder(path); - element = createElement(iStore, name, m_buffer); - if (iIsLumi) { - element->setLumiFlag(); + + MonitorElementData::Key key; + key.kind_ = m_kind; + key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); + key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; + key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi); + + std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; + bool merged = false; + for (MonitorElementData* meData : runLumiMEs) { + if(meData->key_ == key) { + // Keep the latest one + MonitorElementData::Value::Access value(meData->value_); + if(m_kind == MonitorElementData::Kind::INT) + value.scalar.num = m_buffer; + else if(m_kind == MonitorElementData::Kind::REAL) + value.scalar.real = m_buffer; + merged = true; + break; } - } else { - mergeWithElement(element, m_buffer); } - return element; + + if(!merged) { + MonitorElementData* meData = new MonitorElementData(); + meData->key_ = key; + { + MonitorElementData::Value::Access value(meData->value_); + if(m_kind == MonitorElementData::Kind::INT) + value.scalar.num = m_buffer; + else if(m_kind == MonitorElementData::Kind::REAL) + value.scalar.real = m_buffer; + } + + mesFromFile[std::make_tuple(run, lumi)].push_back(std::move(meData)); + } } + void setTree(TTree* iTree) override { m_tree = iTree; m_tree->SetBranchAddress(kFullNameBranch, &m_fullName); @@ -289,9 +333,9 @@ namespace { uint32_t m_tag; }; -} // namespace +} // namespace -class DQMRootSource : public edm::InputSource { +class DQMRootSource : public edm::PuttableSourceBase { public: DQMRootSource(edm::ParameterSet const&, const edm::InputSourceDescription&); ~DQMRootSource() override; @@ -304,102 +348,60 @@ class DQMRootSource : public edm::InputSource { static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); private: - DQMRootSource(const DQMRootSource&) = delete; // stop default - - class RunPHIDKey { - public: - RunPHIDKey(edm::ProcessHistoryID const& phid, unsigned int run) : processHistoryID_(phid), run_(run) {} - edm::ProcessHistoryID const& processHistoryID() const { return processHistoryID_; } - unsigned int run() const { return run_; } - bool operator<(RunPHIDKey const& right) const { - if (processHistoryID_ == right.processHistoryID()) { - return run_ < right.run(); - } - return processHistoryID_ < right.processHistoryID(); - } - - private: - edm::ProcessHistoryID processHistoryID_; - unsigned int run_; - }; - - class RunLumiPHIDKey { - public: - RunLumiPHIDKey(edm::ProcessHistoryID const& phid, unsigned int run, unsigned int lumi) - : processHistoryID_(phid), run_(run), lumi_(lumi) {} - edm::ProcessHistoryID const& processHistoryID() const { return processHistoryID_; } - unsigned int run() const { return run_; } - unsigned int lumi() const { return lumi_; } - bool operator<(RunLumiPHIDKey const& right) const { - if (processHistoryID_ == right.processHistoryID()) { - if (run_ == right.run()) { - return lumi_ < right.lumi(); - } - return run_ < right.run(); - } - return processHistoryID_ < right.processHistoryID(); - } - - private: - edm::ProcessHistoryID processHistoryID_; - unsigned int run_; - unsigned int lumi_; - }; + DQMRootSource(const DQMRootSource&) = delete; edm::InputSource::ItemType getNextItemType() override; - //NOTE: the following is really read next run auxiliary + + std::unique_ptr readFile_() override; std::shared_ptr readRunAuxiliary_() override; std::shared_ptr readLuminosityBlockAuxiliary_() override; void readRun_(edm::RunPrincipal& rpCache) override; void readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) override; void readEvent_(edm::EventPrincipal&) override; - std::unique_ptr readFile_() override; - void closeFile_() override; + // Read MEs from m_fileMetadatas to m_MEsFromFile till run or lumi transition + void readElements(); + // True if m_currentIndex points to an element that has a different + // run or lumi than the previous element (a transition needs to happen). + // False otherwise. + bool isRunOrLumiTransition() const; + void readNextItemType(); + // These methods will be called by the framework. + // MEs in m_MEsFromFile will be put to products. + void beginRun(edm::Run& run); + void beginLuminosityBlock(edm::LuminosityBlock& lumi); + + // If the run matches the filterOnRun configuration parameter, the run + // (and all its lumis) will be kept. + // Otherwise, check if a run and a lumi are in the range that needs to be processed. + // Range is retrieved from lumisToProcess configuration parameter. + // If at least one lumi of a run needs to be kept, per run MEs of that run will also be kept. + bool keepIt(edm::RunNumber_t, edm::LuminosityBlockNumber_t) const; void logFileAction(char const* msg, char const* fileName) const; - void readNextItemType(); - bool setupFile(unsigned int iIndex); - void readElements(); - bool skipIt(edm::RunNumber_t, edm::LuminosityBlockNumber_t) const; - const DQMRootSource& operator=(const DQMRootSource&) = delete; // stop default // ---------- member data -------------------------------- - edm::InputFileCatalog m_catalog; - edm::RunAuxiliary m_runAux; - edm::LuminosityBlockAuxiliary m_lumiAux; - edm::InputSource::ItemType m_nextItemType; - size_t m_fileIndex; - std::string m_presentlyOpenFileName; - std::list::iterator m_nextIndexItr; - std::list::iterator m_presentIndexItr; - std::vector m_runlumiToRange; - std::unique_ptr m_file; - std::vector m_trees; - std::vector > m_treeReaders; - - std::list m_orderedIndices; - edm::ProcessHistoryID m_lastSeenReducedPHID; - unsigned int m_lastSeenRun; - edm::ProcessHistoryID m_lastSeenReducedPHID2; - unsigned int m_lastSeenRun2; - unsigned int m_lastSeenLumi2; - unsigned int m_filterOnRun; + // Properties from python config bool m_skipBadFiles; + unsigned int m_filterOnRun; + edm::InputFileCatalog m_catalog; std::vector m_lumisToProcess; - std::vector m_runsToProcess; - - bool m_justOpenedFileSoNeedToGenerateRunTransition; - bool m_shouldReadMEs; - std::set m_lumiElements; - std::set m_runElements; - std::vector m_historyIDs; - std::vector m_reducedHistoryIDs; - edm::JobReport::Token m_jrToken; + edm::InputSource::ItemType m_nextItemType; + // Each ME type gets its own reader + std::vector> m_treeReaders; + + // Index of currenlty processed row in m_fileMetadatas + unsigned int m_currentIndex; + // All open DQMIO files + std::vector m_openFiles; + // MEs read from files and merged if needed. Ready to be put into products + MonitorElementsFromFile m_MEsFromFile; + // An item here is a row read from DQMIO indices (metadata) table + std::vector m_fileMetadatas; }; // @@ -423,633 +425,301 @@ void DQMRootSource::fillDescriptions(edm::ConfigurationDescriptions& description descriptions.addDefault(desc); } + // // constructors and destructor // + DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSourceDescription& iDesc) - : edm::InputSource(iPSet, iDesc), + : edm::PuttableSourceBase(iPSet, iDesc), + m_skipBadFiles(iPSet.getUntrackedParameter("skipBadFiles", false)), + m_filterOnRun(iPSet.getUntrackedParameter("filterOnRun", 0)), m_catalog(iPSet.getUntrackedParameter >("fileNames"), iPSet.getUntrackedParameter("overrideCatalog")), - m_nextItemType(edm::InputSource::IsFile), - m_fileIndex(0), - m_trees(kNIndicies, static_cast(nullptr)), - m_treeReaders(kNIndicies, std::shared_ptr()), - m_lastSeenReducedPHID(), - m_lastSeenRun(0), - m_lastSeenReducedPHID2(), - m_lastSeenRun2(0), - m_lastSeenLumi2(0), - m_filterOnRun(iPSet.getUntrackedParameter("filterOnRun", 0)), - m_skipBadFiles(iPSet.getUntrackedParameter("skipBadFiles", false)), m_lumisToProcess(iPSet.getUntrackedParameter >( "lumisToProcess", std::vector())), - m_justOpenedFileSoNeedToGenerateRunTransition(false), - m_shouldReadMEs(true) { + m_nextItemType(edm::InputSource::IsFile), + m_treeReaders(kNIndicies, std::shared_ptr()), + m_currentIndex(0), + m_openFiles (std::vector()), + m_MEsFromFile(MonitorElementsFromFile()), + m_fileMetadatas(std::vector()) { edm::sortAndRemoveOverlaps(m_lumisToProcess); - for (std::vector::const_iterator itr = m_lumisToProcess.begin(); - itr != m_lumisToProcess.end(); - ++itr) - m_runsToProcess.push_back(itr->startRun()); - if (m_fileIndex == m_catalog.fileNames().size()) { + if (m_catalog.fileNames().size() == 0) { m_nextItemType = edm::InputSource::IsStop; - } else { - m_treeReaders[kIntIndex].reset(new TreeSimpleReader()); - m_treeReaders[kFloatIndex].reset(new TreeSimpleReader()); - m_treeReaders[kStringIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH1FIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH1SIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH1DIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH2FIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH2SIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH2DIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH3FIndex].reset(new TreeObjectReader()); - m_treeReaders[kTProfileIndex].reset(new TreeObjectReader()); - m_treeReaders[kTProfile2DIndex].reset(new TreeObjectReader()); } + else { + m_treeReaders[kIntIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::INT)); + m_treeReaders[kFloatIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::REAL)); + m_treeReaders[kStringIndex].reset(new TreeStringReader(MonitorElementData::Kind::STRING)); + m_treeReaders[kTH1FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1F)); + m_treeReaders[kTH1SIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1S)); + m_treeReaders[kTH1DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1D)); + m_treeReaders[kTH2FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2F)); + m_treeReaders[kTH2SIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2S)); + m_treeReaders[kTH2DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2D)); + m_treeReaders[kTH3FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH3F)); + m_treeReaders[kTProfileIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TPROFILE)); + m_treeReaders[kTProfile2DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TPROFILE2D)); + } + + produces("DQMGenerationRecoRun"); + produces("DQMGenerationRecoLumi"); } -// DQMRootSource::DQMRootSource(const DQMRootSource& rhs) -// { -// // do actual copying here; -// } - DQMRootSource::~DQMRootSource() { - if (m_file.get() != nullptr && m_file->IsOpen()) { - m_file->Close(); - logFileAction(" Closed file ", m_presentlyOpenFileName.c_str()); + for(auto &file : m_openFiles) { + if (file != nullptr && file->IsOpen()) { + file->Close(); + logFileAction("Closed file", ""); + } } } -// -// assignment operators -// -// const DQMRootSource& DQMRootSource::operator=(const DQMRootSource& rhs) -// { -// //An exception safe implementation is -// DQMRootSource temp(rhs); -// swap(rhs); -// -// return *this; -// } - // // member functions // -void DQMRootSource::readEvent_(edm::EventPrincipal&) { - //std::cout << "readEvent_" << std::endl; -} edm::InputSource::ItemType DQMRootSource::getNextItemType() { - //std::cout <<"getNextItemType "< DQMRootSource::readRunAuxiliary_() { - //std::cout <<"readRunAuxiliary_"< runLumiRange.m_historyIDIndex); - //std::cout <<"readRunAuxiliary_ "<(m_runAux); -} +// We will read the metadata of all files and fill m_fileMetadatas vector +std::unique_ptr DQMRootSource::readFile_() { + const int numFiles = m_catalog.fileNames().size(); + m_openFiles.reserve(numFiles); -std::shared_ptr DQMRootSource::readLuminosityBlockAuxiliary_() { - //std::cout <<"readLuminosityBlockAuxiliary_"< runLumiRange.m_historyIDIndex); - //std::cout <<"lumi "<(m_lumiAux); -} + // TODO: add support to fallback files: https://github.com/cms-sw/cmssw/pull/28064/files + for (auto &filename : m_catalog.fileNames()) { + TFile* file; -void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) { - assert(m_presentIndexItr != m_orderedIndices.end()); - RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr]; + // Try to open a file + try { + file = TFile::Open(filename.c_str()); - m_justOpenedFileSoNeedToGenerateRunTransition = false; - unsigned int runID = rpCache.id().run(); - assert(runID == runLumiRange.m_run); + // Exception will be trapped so we pull it out ourselves + std::exception_ptr e = edm::threadLocalException::getException(); + if (e != std::exception_ptr()) { + edm::threadLocalException::setException(std::exception_ptr()); + std::rethrow_exception(e); + } + + m_openFiles.insert(m_openFiles.begin(), file); + } catch (cms::Exception const& e) { + if (!m_skipBadFiles) { + edm::Exception ex(edm::errors::FileOpenError, "", e); + ex.addContext("Opening DQM Root file"); + ex << "\nInput file " << filename + << " was not found, could not be opened, or is corrupted.\n"; + throw ex; + } + } - m_shouldReadMEs = (m_filterOnRun == 0 || (m_filterOnRun != 0 && m_filterOnRun == runID)); - if (m_lastSeenRun != runID || m_lastSeenReducedPHID != m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex)) { - if (m_shouldReadMEs) { - // TODO: Worry about resetting depending on Collate option here. + // Check if a file is usable + if (!file->IsZombie()) { + logFileAction("Successfully opened file ", filename.c_str()); + } else { + if (!m_skipBadFiles) { + edm::Exception ex(edm::errors::FileOpenError); + ex << "Input file " << filename.c_str() << " could not be opened.\n"; + ex.addContext("Opening DQM Root file"); + throw ex; + } } - m_lastSeenReducedPHID = m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex); - m_lastSeenRun = runID; - } - readNextItemType(); + // Check file format version, which is encoded in the Title of the TFile + if (strcmp(file->GetTitle(), "1") != 0) { + edm::Exception ex(edm::errors::FileReadError); + ex << "Input file " << filename.c_str() << " does not appear to be a DQM Root file.\n"; + } - //NOTE: it is possible to have a Run when all we have stored is lumis - if (runLumiRange.m_lumi == 0) { - readElements(); + // Read metadata from the file + TTree* indicesTree = dynamic_cast(file->Get(kIndicesTree)); + assert(indicesTree != nullptr); + + FileMetadata temp; + // Each line of metadata will be read into the coresponding fields of temp. + indicesTree->SetBranchAddress(kRunBranch, &temp.m_run); + indicesTree->SetBranchAddress(kLumiBranch, &temp.m_lumi); + indicesTree->SetBranchAddress(kBeginTimeBranch, &temp.m_beginTime); + indicesTree->SetBranchAddress(kEndTimeBranch, &temp.m_endTime); + indicesTree->SetBranchAddress(kTypeBranch, &temp.m_type); + indicesTree->SetBranchAddress(kFirstIndex, &temp.m_firstIndex); + indicesTree->SetBranchAddress(kLastIndex, &temp.m_lastIndex); + + for (Long64_t index = 0; index != indicesTree->GetEntries(); ++index) { + indicesTree->GetEntry(index); + temp.m_file = file; + + if(keepIt(temp.m_run, temp.m_lumi)) { + m_fileMetadatas.push_back(temp); + } + } } - edm::Service jr; - jr->reportInputRunNumber(rpCache.id().run()); - - rpCache.fillRunPrincipal(processHistoryRegistryForUpdate()); -} + // Sort to make sure runs and lumis appear in sequential order + std::sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); -void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) { - assert(m_presentIndexItr != m_orderedIndices.end()); - RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr]; - assert(runLumiRange.m_run == lbCache.id().run()); - assert(runLumiRange.m_lumi == lbCache.id().luminosityBlock()); - - //NOTE: need to reset all lumi block elements at this point - if ((m_lastSeenLumi2 != runLumiRange.m_lumi || m_lastSeenRun2 != runLumiRange.m_run || - m_lastSeenReducedPHID2 != m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex)) && - m_shouldReadMEs) { - edm::Service store; - std::vector allMEs = (*store).getAllContents(""); - //for(auto const& ME : allMEs) { - // // We do not want to reset Run Products here! - // if (ME->getLumiFlag()) { - // ME->Reset(); - // } - //} - m_lastSeenReducedPHID2 = m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex); - m_lastSeenRun2 = runLumiRange.m_run; - m_lastSeenLumi2 = runLumiRange.m_lumi; - } + for(auto &metadata : m_fileMetadatas) + metadata.describe(); - readNextItemType(); - readElements(); + // Stop if there's nothing to process. Otherwise start the run. + if(m_fileMetadatas.size() == 0) + m_nextItemType = edm::InputSource::IsStop; + else + m_nextItemType = edm::InputSource::IsRun; - edm::Service jr; - jr->reportInputLumiSection(lbCache.id().run(), lbCache.id().luminosityBlock()); + // We have to return something but not sure why + return std::unique_ptr(new edm::FileBlock); +} - lbCache.fillLuminosityBlockPrincipal(processHistoryRegistry().getMapped(lbCache.aux().processHistoryID())); +std::shared_ptr DQMRootSource::readRunAuxiliary_() { + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; + auto runAux = edm::RunAuxiliary(metadata.m_run, edm::Timestamp(metadata.m_beginTime), edm::Timestamp(metadata.m_endTime)); + return std::make_shared(runAux); } -std::unique_ptr DQMRootSource::readFile_() { - auto const numFiles = m_catalog.fileNames().size(); - while (m_fileIndex < numFiles && not setupFile(m_fileIndex++)) { - } +std::shared_ptr DQMRootSource::readLuminosityBlockAuxiliary_() { + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; + auto lumiAux = edm::LuminosityBlockAuxiliary(edm::LuminosityBlockID(metadata.m_run, metadata.m_lumi), + edm::Timestamp(metadata.m_beginTime), + edm::Timestamp(metadata.m_endTime)); + return std::make_shared(lumiAux); +} - if (m_file.get() == nullptr) { - //last file in list was bad - m_nextItemType = edm::InputSource::IsStop; - return std::unique_ptr(new edm::FileBlock); - } +void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) { + // Read elements of a current run. + do { + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; + if (metadata.m_lumi == 0) { + readElements(); + } + m_currentIndex++; + } while(!isRunOrLumiTransition()); readNextItemType(); - while (m_presentIndexItr != m_orderedIndices.end() && - skipIt(m_runlumiToRange[*m_presentIndexItr].m_run, m_runlumiToRange[*m_presentIndexItr].m_lumi)) - ++m_presentIndexItr; - - edm::Service jr; - std::string guid{m_file->GetUUID().AsString()}; - std::transform(guid.begin(), guid.end(), guid.begin(), (int (*)(int))std::toupper); - m_jrToken = jr->inputFileOpened(m_presentlyOpenFileName, - m_catalog.logicalFileNames()[m_fileIndex - 1], - std::string(), - std::string(), - "DQMRootSource", - "source", - std::move(guid), - std::vector()); - - return std::unique_ptr(new edm::FileBlock); } -void DQMRootSource::closeFile_() { - if (m_file.get() == nullptr) { - return; - } - edm::Service jr; - jr->inputFileClosed(edm::InputType::Primary, m_jrToken); -} - -void DQMRootSource::readElements() { - edm::Service store; - RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr]; - bool shouldContinue = false; +void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) { + // Read elements of a current lumi. do { - shouldContinue = false; - ++m_presentIndexItr; - while (m_presentIndexItr != m_orderedIndices.end() && - skipIt(m_runlumiToRange[*m_presentIndexItr].m_run, m_runlumiToRange[*m_presentIndexItr].m_lumi)) - ++m_presentIndexItr; - - if (runLumiRange.m_type != kNoTypesStored) { - std::shared_ptr reader = m_treeReaders[runLumiRange.m_type]; - ULong64_t index = runLumiRange.m_firstIndex; - ULong64_t endIndex = runLumiRange.m_lastIndex + 1; - for (; index != endIndex; ++index) { - bool isLumi = runLumiRange.m_lumi != 0; - if (m_shouldReadMEs) - reader->read(index, *store, isLumi); - - //std::cout << runLumiRange.m_run << " " << runLumiRange.m_lumi <<" "< reader = m_treeReaders[metadata.m_type]; + TTree* tree = dynamic_cast(metadata.m_file->Get(kTypeNames[metadata.m_type])); + reader->setTree(tree); - bool shouldContinue = false; - do { - shouldContinue = false; - while (m_nextIndexItr != m_orderedIndices.end() && - skipIt(m_runlumiToRange[*m_nextIndexItr].m_run, m_runlumiToRange[*m_nextIndexItr].m_lumi)) - ++m_nextIndexItr; - - if (m_nextIndexItr == m_orderedIndices.end()) { - //go to next file - m_nextItemType = edm::InputSource::IsFile; - //std::cout <<"going to next file"<read(index, m_MEsFromFile, metadata.m_run, metadata.m_lumi); } } } -bool DQMRootSource::setupFile(unsigned int iIndex) { - if (m_file.get() != nullptr && iIndex > 0) { - m_file->Close(); - logFileAction(" Closed file ", m_presentlyOpenFileName.c_str()); +bool DQMRootSource::isRunOrLumiTransition() const { + if(m_currentIndex == 0) { + return false; } - logFileAction(" Initiating request to open file ", m_catalog.fileNames()[iIndex].c_str()); - - auto fallbackFileName = m_catalog.fallbackFileNames()[iIndex]; - bool hasFallback = !fallbackFileName.empty() && fallbackFileName != m_catalog.fileNames()[iIndex]; - - m_presentlyOpenFileName.clear(); - m_file.reset(); - - std::unique_ptr newFile; - std::list originalInfo; - try { - // ROOT's context management implicitly assumes that a file is opened and - // closed on the same thread. To avoid the problem, we declare a local - // TContext object; when it goes out of scope, its destructor unregisters - // the context, guaranteeing the context is unregistered in the same thread - // it was registered in. - { - TDirectory::TContext contextEraser; - newFile = std::unique_ptr(TFile::Open(m_catalog.fileNames()[iIndex].c_str())); - } - //Since ROOT6, we can not propagate an exception through ROOT's plugin - // system so we trap them and then pull from this function - std::exception_ptr e = edm::threadLocalException::getException(); - if (e != std::exception_ptr()) { - edm::threadLocalException::setException(std::exception_ptr()); - std::rethrow_exception(e); - } - } catch (cms::Exception const& e) { - if (!hasFallback) { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileOpenError, "", e); - ex.addContext("Opening DQM Root file"); - ex << "\nInput file " << m_catalog.fileNames()[iIndex] - << " was not found, could not be opened, or is corrupted.\n"; - throw ex; - } - } - originalInfo = e.additionalInfo(); // save in case of fallback error - newFile.reset(); - } - if (newFile && not newFile->IsZombie()) { - m_presentlyOpenFileName = m_catalog.fileNames()[iIndex]; - logFileAction(" Successfully opened file ", m_presentlyOpenFileName.c_str()); - } else { - if (!hasFallback) { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileOpenError); - ex << "Input file " << m_catalog.fileNames()[iIndex].c_str() << " could not be opened.\n"; - ex.addContext("Opening DQM Root file"); - throw ex; - } - } - newFile.reset(); + if(m_currentIndex > m_fileMetadatas.size() - 1) { + // We reached the end + return true; } - if (!newFile && hasFallback) { - logFileAction(" Initiating request to open fallback file ", fallbackFileName.c_str()); - try { - { - TDirectory::TContext contextEraser; - newFile = std::unique_ptr(TFile::Open(fallbackFileName.c_str())); - } - std::exception_ptr e = edm::threadLocalException::getException(); - if (e != std::exception_ptr()) { - edm::threadLocalException::setException(std::exception_ptr()); - std::rethrow_exception(e); - } - } catch (cms::Exception const& e) { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileOpenError, "", e); - ex.addContext("Opening DQM Root file"); - ex << "\nInput file " << m_catalog.fileNames()[iIndex] << " and fallback input file " << fallbackFileName - << " were not found, could not be opened, or are corrupted.\n"; - for (auto const& s : originalInfo) { - ex.addAdditionalInfo(s); - } - throw ex; - } - } - if (not newFile->IsZombie()) { - m_presentlyOpenFileName = fallbackFileName; - logFileAction(" Successfully opened fallback file ", m_presentlyOpenFileName.c_str()); - } else { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileOpenError); - ex << "Input file " << m_catalog.fileNames()[iIndex] << " and fallback input file " << fallbackFileName - << " could not be opened.\n"; - ex.addContext("Opening DQM Root file"); - for (auto const& s : originalInfo) { - ex.addAdditionalInfo(s); - } - throw ex; - } - } - } + FileMetadata previousMetadata = m_fileMetadatas[m_currentIndex - 1]; + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; - //Check file format version, which is encoded in the Title of the TFile - if (0 != strcmp(newFile->GetTitle(), "1")) { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileReadError); - ex << "Input file " << m_presentlyOpenFileName << " does not appear to be a DQM Root file.\n"; - throw ex; - } - } + return previousMetadata.m_run != metadata.m_run || previousMetadata.m_lumi != metadata.m_lumi; +} - //Get meta Data - TDirectory* metaDir = newFile->GetDirectory(kMetaDataDirectoryAbsolute); - if (nullptr == metaDir) { - if (!m_skipBadFiles) { - edm::Exception ex(edm::errors::FileReadError); - ex << "Input file " << m_presentlyOpenFileName - << " appears to be corrupted since it does not have the proper internal structure.\n" - " Check to see if the file was closed properly.\n"; - ex.addContext("Opening DQM Root file"); - throw ex; - } else { - return false; - } +void DQMRootSource::readNextItemType() { + if(m_currentIndex == 0) { + m_nextItemType = edm::InputSource::IsRun; } - m_file = std::move(newFile); //passed all tests so now we want to use this file - TTree* parameterSetTree = dynamic_cast(metaDir->Get(kParameterSetTree)); - assert(nullptr != parameterSetTree); - - edm::pset::Registry* psr = edm::pset::Registry::instance(); - assert(nullptr != psr); - { - std::string blob; - std::string* pBlob = &blob; - parameterSetTree->SetBranchAddress(kParameterSetBranch, &pBlob); - for (unsigned int index = 0; index != parameterSetTree->GetEntries(); ++index) { - parameterSetTree->GetEntry(index); - edm::ParameterSet::registerFromString(blob); - } + else if(m_currentIndex > m_fileMetadatas.size() - 1) { + // We reached the end + m_nextItemType = edm::InputSource::IsStop; } + else { + FileMetadata previousMetadata = m_fileMetadatas[m_currentIndex - 1]; + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; - { - TTree* processHistoryTree = dynamic_cast(metaDir->Get(kProcessHistoryTree)); - assert(nullptr != processHistoryTree); - unsigned int phIndex = 0; - processHistoryTree->SetBranchAddress(kPHIndexBranch, &phIndex); - std::string processName; - std::string* pProcessName = &processName; - processHistoryTree->SetBranchAddress(kProcessConfigurationProcessNameBranch, &pProcessName); - std::string parameterSetIDBlob; - std::string* pParameterSetIDBlob = ¶meterSetIDBlob; - processHistoryTree->SetBranchAddress(kProcessConfigurationParameterSetIDBranch, &pParameterSetIDBlob); - std::string releaseVersion; - std::string* pReleaseVersion = &releaseVersion; - processHistoryTree->SetBranchAddress(kProcessConfigurationReleaseVersion, &pReleaseVersion); - std::string passID; - std::string* pPassID = &passID; - processHistoryTree->SetBranchAddress(kProcessConfigurationPassID, &pPassID); - - edm::ProcessHistoryRegistry& phr = processHistoryRegistryForUpdate(); - std::vector configs; - configs.reserve(5); - m_historyIDs.clear(); - m_reducedHistoryIDs.clear(); - for (unsigned int i = 0; i != processHistoryTree->GetEntries(); ++i) { - processHistoryTree->GetEntry(i); - if (phIndex == 0) { - if (not configs.empty()) { - edm::ProcessHistory ph(configs); - m_historyIDs.push_back(ph.id()); - phr.registerProcessHistory(ph); - m_reducedHistoryIDs.push_back(phr.reducedProcessHistoryID(ph.id())); - } - configs.clear(); - } - edm::ParameterSetID psetID(parameterSetIDBlob); - edm::ProcessConfiguration pc(processName, psetID, releaseVersion, passID); - configs.push_back(pc); + if(previousMetadata.m_run != metadata.m_run) { + m_nextItemType = edm::InputSource::IsRun; } - if (not configs.empty()) { - edm::ProcessHistory ph(configs); - m_historyIDs.push_back(ph.id()); - phr.registerProcessHistory(ph); - m_reducedHistoryIDs.push_back(phr.reducedProcessHistoryID(ph.id())); - //std::cout <<"inserted "<(m_file->Get(kIndicesTree)); - assert(nullptr != indicesTree); - - m_runlumiToRange.clear(); - m_runlumiToRange.reserve(indicesTree->GetEntries()); - m_orderedIndices.clear(); - - RunLumiToRange temp; - indicesTree->SetBranchAddress(kRunBranch, &temp.m_run); - indicesTree->SetBranchAddress(kLumiBranch, &temp.m_lumi); - indicesTree->SetBranchAddress(kBeginTimeBranch, &temp.m_beginTime); - indicesTree->SetBranchAddress(kEndTimeBranch, &temp.m_endTime); - indicesTree->SetBranchAddress(kProcessHistoryIndexBranch, &temp.m_historyIDIndex); - indicesTree->SetBranchAddress(kTypeBranch, &temp.m_type); - indicesTree->SetBranchAddress(kFirstIndex, &temp.m_firstIndex); - indicesTree->SetBranchAddress(kLastIndex, &temp.m_lastIndex); - - //Need to reorder items since if there was a merge done the same Run - //and/or Lumi can appear multiple times but we want to process them - //all at once - - //We use a std::list for m_orderedIndices since inserting into the - //middle of a std::list does not disrupt the iterators to already - //existing entries - - //The Map is used to see if a Run/Lumi pair has appeared before - typedef std::map::iterator> RunLumiToLastEntryMap; - RunLumiToLastEntryMap runLumiToLastEntryMap; - - //Need to group all lumis for the same run together and move the run - //entry to the beginning - typedef std::map::iterator, std::list::iterator> > - RunToFirstLastEntryMap; - RunToFirstLastEntryMap runToFirstLastEntryMap; - - for (Long64_t index = 0; index != indicesTree->GetEntries(); ++index) { - indicesTree->GetEntry(index); - // std::cout <<"read r:"<::iterator itLastOfRun = m_orderedIndices.end(); - - RunToFirstLastEntryMap::iterator itRunFirstLastEntryFind = runToFirstLastEntryMap.find(runKey); - bool needNewEntryInRunFirstLastEntryMap = true; - if (itRunFirstLastEntryFind != runToFirstLastEntryMap.end()) { - needNewEntryInRunFirstLastEntryMap = false; - if (temp.m_lumi != 0) { - //lumis go to the end - itLastOfRun = itRunFirstLastEntryFind->second.second; - //we want to insert after this one so must advance the iterator - ++itLastOfRun; - } else { - //runs go at the beginning - itLastOfRun = itRunFirstLastEntryFind->second.first; - } - } - std::list::iterator iter = m_orderedIndices.insert(itLastOfRun, index); - runLumiToLastEntryMap[runLumi] = iter; - if (needNewEntryInRunFirstLastEntryMap) - runToFirstLastEntryMap[runKey] = std::make_pair(iter, iter); - else { - if (temp.m_lumi != 0) { - //lumis go at end - runToFirstLastEntryMap[runKey].second = iter; - } else { - //since we haven't yet seen this run/lumi combination it means we haven't yet seen - // a run so we can put this first - runToFirstLastEntryMap[runKey].first = iter; - } - } - } else { - //We need to do a merge since the run/lumi already appeared. Put it after the existing entry - //std::cout <<" found a second instance of "<::iterator itNext = itFind->second; - ++itNext; - std::list::iterator iter = m_orderedIndices.insert(itNext, index); - RunToFirstLastEntryMap::iterator itRunFirstLastEntryFind = runToFirstLastEntryMap.find(runKey); - if (itRunFirstLastEntryFind->second.second == itFind->second) { - //if the previous one was the last in the run, we need to update to make this one the last - itRunFirstLastEntryFind->second.second = iter; - } - itFind->second = iter; - } - } - m_nextIndexItr = m_orderedIndices.begin(); - m_presentIndexItr = m_orderedIndices.begin(); - - if (m_nextIndexItr != m_orderedIndices.end()) { - for (size_t index = 0; index < kNIndicies; ++index) { - m_trees[index] = dynamic_cast(m_file->Get(kTypeNames[index])); - assert(nullptr != m_trees[index]); - m_treeReaders[index]->setTree(m_trees[index]); - } +void DQMRootSource::beginRun(edm::Run& run) { + TRACE("Begin run: " + std::to_string(run.run())) + + std::unique_ptr product = std::make_unique(); + + auto mes = m_MEsFromFile[std::make_tuple(run.run(), 0)]; + + TRACE("Found MEs: " + std::to_string(mes.size())) + + for(MonitorElementData* meData_ptr : mes) { + product->push_back(meData_ptr); } - //After a file open, the framework expects to see a new 'IsRun' - m_justOpenedFileSoNeedToGenerateRunTransition = true; - return true; + run.put(std::move(product), "DQMGenerationRecoRun"); + + // Remove already processed MEs + m_MEsFromFile[std::make_tuple(run.run(), 0)] = std::vector(); } -bool DQMRootSource::skipIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const { - if (!m_runsToProcess.empty() && edm::search_all(m_runsToProcess, run) && lumi == 0) { - return false; +void DQMRootSource::beginLuminosityBlock(edm::LuminosityBlock& lumi) { + TRACE("Begin lumi: " + std::to_string(lumi.luminosityBlock())) + + std::unique_ptr product = std::make_unique(); + + auto mes = m_MEsFromFile[std::make_tuple(lumi.run(), lumi.luminosityBlock())]; + + TRACE("Found MEs: " + std::to_string(mes.size())) + + for(MonitorElementData* meData_ptr : mes) { + assert(meData_ptr != nullptr); + product->push_back(meData_ptr); } + + lumi.put(std::move(product), "DQMGenerationRecoLumi"); - edm::LuminosityBlockID lumiID = edm::LuminosityBlockID(run, lumi); - edm::LuminosityBlockRange lumiRange = edm::LuminosityBlockRange(lumiID, lumiID); - bool (*lt)(edm::LuminosityBlockRange const&, edm::LuminosityBlockRange const&) = &edm::lessThan; - if (!m_lumisToProcess.empty() && !binary_search_all(m_lumisToProcess, lumiRange, lt)) { + // Remove already processed MEs + m_MEsFromFile[std::make_tuple(lumi.run(), lumi.luminosityBlock())] = std::vector(); +} + +bool DQMRootSource::keepIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const { + if(run == m_filterOnRun) return true; + + for (edm::LuminosityBlockRange const& lumiToProcess : m_lumisToProcess) { + if(run >= lumiToProcess.startRun() && run <= lumiToProcess.endRun()) { + if(lumi >= lumiToProcess.startLumi() && lumi <= lumiToProcess.endLumi()) { + return true; + } + else if (lumi == 0) { + return true; + } + } } return false; }