Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Import DQMRootSource from @andrius-k.
Browse files Browse the repository at this point in the history
Using commit 70fc23c on his "new
DQMstore" branch.

This might revert changes from cms-sw#28064, needs to be fixed later.
schneiml committed Dec 12, 2019
1 parent 102087a commit 24ece8a
Showing 1 changed file with 478 additions and 808 deletions.
1,286 changes: 478 additions & 808 deletions DQMServices/FwkIO/plugins/DQMRootSource.cc
Original file line number Diff line number Diff line change
@@ -14,25 +14,16 @@
#include <vector>
#include <string>
#include <map>
#include <memory>
#include <list>
#include <set>
#include "TFile.h"
#include "TTree.h"
#include "TString.h"
#include "TH1.h"
#include "TH2.h"
#include "TProfile.h"

// user include files
#include "FWCore/Framework/interface/InputSource.h"
#include "FWCore/Sources/interface/PuttableSourceBase.h"
#include "FWCore/Catalog/interface/InputFileCatalog.h"
#include "FWCore/ParameterSet/interface/ParameterSet.h"
#include "DQMServices/Core/interface/DQMStore.h"
#include "FWCore/ServiceRegistry/interface/Service.h"
#include "FWCore/MessageLogger/interface/JobReport.h"
//#include "FWCore/Utilities/interface/GlobalIdentifier.h"
#include "FWCore/Utilities/interface/EDMException.h"
#include "FWCore/Utilities/interface/ExceptionPropagate.h"

#include "FWCore/Framework/interface/RunPrincipal.h"
@@ -48,198 +39,171 @@

#include "FWCore/Framework/interface/InputSourceMacros.h"
#include "FWCore/Framework/interface/FileBlock.h"
#include "DataFormats/Provenance/interface/EventRange.h"
#include "DataFormats/Provenance/interface/EventID.h"

#include "FWCore/MessageLogger/interface/MessageLogger.h"
#include "FWCore/Utilities/interface/TimeOfDay.h"

#include "DataFormats/Provenance/interface/ProcessHistory.h"
#include "DataFormats/Provenance/interface/ProcessHistoryID.h"
#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h"
#include "FWCore/ParameterSet/interface/Registry.h"

#include "FWCore/Utilities/interface/Digest.h"
#include "FWCore/Utilities/interface/InputType.h"

#include "format.h"

namespace {
typedef dqm::legacy::MonitorElement MonitorElement;
typedef dqm::legacy::DQMStore DQMStore;

//adapter functions
MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1F* iHist) {
//std::cout <<"create: hist size "<<iName <<" "<<iHist->GetEffectiveEntries()<<std::endl;
return iStore.book1D(iName, iHist);
}
//NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h
void mergeTogether(TH1* iOriginal, TH1* iToAdd) {
if (iOriginal->CanExtendAllAxes() && iToAdd->CanExtendAllAxes()) {
TList list;
list.Add(iToAdd);
if (-1 == iOriginal->Merge(&list)) {
edm::LogError("MergeFailure") << "Failed to merge DQM element " << iOriginal->GetName();
typedef dqm::harvesting::MonitorElement MonitorElement;

// TODO: this should probably be moved somewhere else
class DQMMergeHelper {
public:
// Utility function to check the consistency of the axis labels
// Taken from TH1::CheckBinLabels which is not public
static bool CheckBinLabels(const TAxis* a1, const TAxis * a2) {
// Check that axis have same labels
THashList *l1 = (const_cast<TAxis*>(a1))->GetLabels();
THashList *l2 = (const_cast<TAxis*>(a2))->GetLabels();

if (!l1 && !l2 )
return true;
if (!l1 || !l2 ) {
return false;
}
} else {
if (iOriginal->GetNbinsX() == iToAdd->GetNbinsX() &&
iOriginal->GetXaxis()->GetXmin() == iToAdd->GetXaxis()->GetXmin() &&
iOriginal->GetXaxis()->GetXmax() == iToAdd->GetXaxis()->GetXmax() &&
iOriginal->GetNbinsY() == iToAdd->GetNbinsY() &&
iOriginal->GetYaxis()->GetXmin() == iToAdd->GetYaxis()->GetXmin() &&
iOriginal->GetYaxis()->GetXmax() == iToAdd->GetYaxis()->GetXmax() &&
iOriginal->GetNbinsZ() == iToAdd->GetNbinsZ() &&
iOriginal->GetZaxis()->GetXmin() == iToAdd->GetZaxis()->GetXmin() &&
iOriginal->GetZaxis()->GetXmax() == iToAdd->GetZaxis()->GetXmax() &&
MonitorElement::CheckBinLabels(iOriginal->GetXaxis(), iToAdd->GetXaxis()) &&
MonitorElement::CheckBinLabels(iOriginal->GetYaxis(), iToAdd->GetYaxis()) &&
MonitorElement::CheckBinLabels(iOriginal->GetZaxis(), iToAdd->GetZaxis())) {
iOriginal->Add(iToAdd);
} else {
edm::LogError("MergeFailure") << "Found histograms with different axis limits or different labels'"
<< iOriginal->GetName() << "' not merged.";

// Check now labels sizes are the same
if (l1->GetSize() != l2->GetSize() ) {
return false;
}
}
}

void mergeWithElement(MonitorElement* iElement, TH1F* iHist) {
//std::cout <<"merge: hist size "<<iElement->getName() <<" "<<iHist->GetEffectiveEntries()<<std::endl;
mergeTogether(iElement->getTH1F(), iHist);
}
MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1S* iHist) {
return iStore.book1S(iName, iHist);
}
void mergeWithElement(MonitorElement* iElement, TH1S* iHist) { mergeTogether(iElement->getTH1S(), iHist); }
MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1D* iHist) {
return iStore.book1DD(iName, iHist);
}
void mergeWithElement(MonitorElement* iElement, TH1D* iHist) { mergeTogether(iElement->getTH1D(), iHist); }
MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2F* iHist) {
return iStore.book2D(iName, iHist);
}
void mergeWithElement(MonitorElement* iElement, TH2F* iHist) { mergeTogether(iElement->getTH2F(), iHist); }
MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2S* iHist) {
return iStore.book2S(iName, iHist);
}
void mergeWithElement(MonitorElement* iElement, TH2S* iHist) { mergeTogether(iElement->getTH2S(), iHist); }
MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2D* iHist) {
return iStore.book2DD(iName, iHist);
}
void mergeWithElement(MonitorElement* iElement, TH2D* iHist) { mergeTogether(iElement->getTH2D(), iHist); }
MonitorElement* createElement(DQMStore& iStore, const char* iName, TH3F* iHist) {
return iStore.book3D(iName, iHist);
}
void mergeWithElement(MonitorElement* iElement, TH3F* iHist) { mergeTogether(iElement->getTH3F(), iHist); }
MonitorElement* createElement(DQMStore& iStore, const char* iName, TProfile* iHist) {
return iStore.bookProfile(iName, iHist);
}
void mergeWithElement(MonitorElement* iElement, TProfile* iHist) { mergeTogether(iElement->getTProfile(), iHist); }
MonitorElement* createElement(DQMStore& iStore, const char* iName, TProfile2D* iHist) {
return iStore.bookProfile2D(iName, iHist);
}
void mergeWithElement(MonitorElement* iElement, TProfile2D* iHist) {
mergeTogether(iElement->getTProfile2D(), iHist);
}
for (int i = 1; i <= a1->GetNbins(); ++i) {
TString label1 = a1->GetBinLabel(i);
TString label2 = a2->GetBinLabel(i);
if (label1 != label2) {
return false;
}
}

MonitorElement* createElement(DQMStore& iStore, const char* iName, Long64_t& iValue) {
MonitorElement* e = iStore.bookInt(iName);
e->Fill(iValue);
return e;
}
return true;
}

//NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h
void mergeWithElement(MonitorElement* iElement, Long64_t& iValue) {
const std::string& name = iElement->getFullname();
if (name.find("EventInfo/processedEvents") != std::string::npos) {
iElement->Fill(iValue + iElement->getIntValue());
} else if (name.find("EventInfo/iEvent") != std::string::npos ||
name.find("EventInfo/iLumiSection") != std::string::npos) {
if (iValue > iElement->getIntValue()) {
iElement->Fill(iValue);
// NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h
static void mergeTogether(TH1* original, TH1* toAdd) {
if (original->CanExtendAllAxes() && toAdd->CanExtendAllAxes()) {
TList list;
list.Add(toAdd);
if (original->Merge(&list) == -1) {
edm::LogError("MergeFailure") << "Failed to merge DQM element " << original->GetName();
}
}
else {
// TODO: Redo. What's wrong with this implementation?
if (original->GetNbinsX() == toAdd->GetNbinsX() &&
original->GetXaxis()->GetXmin() == toAdd->GetXaxis()->GetXmin() &&
original->GetXaxis()->GetXmax() == toAdd->GetXaxis()->GetXmax() &&
original->GetNbinsY() == toAdd->GetNbinsY() &&
original->GetYaxis()->GetXmin() == toAdd->GetYaxis()->GetXmin() &&
original->GetYaxis()->GetXmax() == toAdd->GetYaxis()->GetXmax() &&
original->GetNbinsZ() == toAdd->GetNbinsZ() &&
original->GetZaxis()->GetXmin() == toAdd->GetZaxis()->GetXmin() &&
original->GetZaxis()->GetXmax() == toAdd->GetZaxis()->GetXmax() &&
CheckBinLabels(original->GetXaxis(), toAdd->GetXaxis()) &&
CheckBinLabels(original->GetYaxis(), toAdd->GetYaxis()) &&
CheckBinLabels(original->GetZaxis(), toAdd->GetZaxis())) {
original->Add(toAdd);
}
else {
edm::LogError("MergeFailure") << "Found histograms with different axis limits or different labels '"
<< original->GetName() << "' not merged.";
}
}
} else {
iElement->Fill(iValue);
}
}

MonitorElement* createElement(DQMStore& iStore, const char* iName, double& iValue) {
MonitorElement* e = iStore.bookFloat(iName);
e->Fill(iValue);
return e;
}
void mergeWithElement(MonitorElement* iElement, double& iValue) {
//no merging, take the last one
iElement->Fill(iValue);
}
MonitorElement* createElement(DQMStore& iStore, const char* iName, std::string* iValue) {
return iStore.bookString(iName, *iValue);
}
void mergeWithElement(MonitorElement* iElement, std::string* iValue) {
//no merging, take the last one
iElement->Fill(*iValue);
}
};

void splitName(const std::string& iFullName, std::string& oPath, const char*& oName) {
oPath = iFullName;
size_t index = oPath.find_last_of('/');
if (index == std::string::npos) {
oPath = std::string();
oName = iFullName.c_str();
} else {
oPath.resize(index);
oName = iFullName.c_str() + index + 1;
}
}
using MonitorElementsFromFile = std::map<std::tuple<int, int>, std::vector<MonitorElementData*>>;

struct RunLumiToRange {
unsigned int m_run, m_lumi, m_historyIDIndex;
// This struct allows to find all MEs belonging to a run-lumi pair
// All files will be open at once so m_file property indicates the file where data is saved.
struct FileMetadata {
unsigned int m_run;
unsigned int m_lumi;
ULong64_t m_beginTime;
ULong64_t m_endTime;
ULong64_t m_firstIndex, m_lastIndex; //last is inclusive
unsigned int m_type; //A value in TypeIndex
ULong64_t m_firstIndex;
ULong64_t m_lastIndex; // Last is inclusive
unsigned int m_type;
TFile* m_file;

// This will be used when sorting a vector
bool operator < (const FileMetadata& obj) const {
if(m_run == obj.m_run)
return m_lumi < obj.m_lumi;
else
return m_run < obj.m_run;
}

void describe() {
std::cout << "read r:" << m_run
<< " l:" << m_lumi
<< " bt:" << m_beginTime
<< " et:" << m_endTime
<< " fi:" << m_firstIndex
<< " li:" << m_lastIndex
<< " type:" << m_type
<< " file: " << m_file << std::endl;
}
};

class TreeReaderBase {
public:
TreeReaderBase() {}
TreeReaderBase(MonitorElementData::Kind kind) : m_kind(kind) {}
virtual ~TreeReaderBase() {}

MonitorElement* read(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) { return doRead(iIndex, iStore, iIsLumi); }
virtual void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) = 0;
virtual void setTree(TTree* iTree) = 0;

protected:
MonitorElementData::Kind m_kind;
TTree* m_tree;

private:
virtual MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) = 0;
};

template <class T>
class TreeObjectReader : public TreeReaderBase {
public:
TreeObjectReader() : m_tree(nullptr), m_fullName(nullptr), m_buffer(nullptr), m_tag(0) {}
MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) override {
TreeObjectReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(nullptr), m_tag(0) {
assert(m_kind != MonitorElementData::Kind::INT);
assert(m_kind != MonitorElementData::Kind::REAL);
assert(m_kind != MonitorElementData::Kind::STRING);
}

void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override {
// This will populate the fields as defined in setTree method
m_tree->GetEntry(iIndex);
MonitorElement* element = iStore.get(*m_fullName);
try {
if (nullptr == element) {
std::string path;
const char* name;
splitName(*m_fullName, path, name);
iStore.setCurrentFolder(path);
element = createElement(iStore, name, m_buffer);
if (iIsLumi) {
element->setLumiFlag();
}
} else {
mergeWithElement(element, m_buffer);

MonitorElementData::Key key;
key.kind_ = m_kind;
key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME);
key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI;
// TODO: What should the range be for per run MEs? Now lumi will be 0 and not the max lumi for that run.
key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi);

std::vector<MonitorElementData*> runLumiMEs = mesFromFile[std::make_tuple(run, lumi)];
bool merged = false;
for (MonitorElementData* meData : runLumiMEs) {
if(meData->key_ == key) {
// Merge with already existing ME!
MonitorElementData::Value::Access value(meData->value_);
DQMMergeHelper::mergeTogether(value.object.get(), m_buffer);
merged = true;
break;
}
} catch (cms::Exception& e) {
e.addContext(std::string("While reading element ") + *m_fullName);
e.raise();
}
return element;

if(!merged) {
MonitorElementData* meData = new MonitorElementData();
meData->key_ = key;
{
MonitorElementData::Value::Access value(meData->value_);
value.object = std::unique_ptr<T>((T*)(m_buffer->Clone()));
}

mesFromFile[std::make_tuple(run, lumi)].push_back(meData);
}
}

void setTree(TTree* iTree) override {
m_tree = iTree;
m_tree->SetBranchAddress(kFullNameBranch, &m_fullName);
@@ -254,27 +218,107 @@ namespace {
uint32_t m_tag;
};

class TreeStringReader : public TreeReaderBase {
public:
TreeStringReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_value(nullptr), m_tag(0) {
assert(m_kind == MonitorElementData::Kind::STRING);
}

void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override {
// This will populate the fields as defined in setTree method
m_tree->GetEntry(iIndex);

MonitorElementData::Key key;
key.kind_ = m_kind;
key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME);
key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI;
key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi);

std::vector<MonitorElementData*> runLumiMEs = mesFromFile[std::make_tuple(run, lumi)];
bool merged = false;
for (MonitorElementData* meData : runLumiMEs) {
if(meData->key_ == key) {
// Keep the latest one
MonitorElementData::Value::Access value(meData->value_);
value.scalar.str = *m_value;
merged = true;
break;
}
}

if(!merged) {
MonitorElementData* meData = new MonitorElementData();
meData->key_ = key;
{
MonitorElementData::Value::Access value(meData->value_);
value.scalar.str = *m_value;
}

mesFromFile[std::make_tuple(run, lumi)].push_back(std::move(meData));
}
}

void setTree(TTree* iTree) override {
m_tree = iTree;
m_tree->SetBranchAddress(kFullNameBranch, &m_fullName);
m_tree->SetBranchAddress(kFlagBranch, &m_tag);
m_tree->SetBranchAddress(kValueBranch, &m_value);
}

private:
TTree* m_tree;
std::string* m_fullName;
std::string* m_value;
uint32_t m_tag;
};

template <class T>
class TreeSimpleReader : public TreeReaderBase {
public:
TreeSimpleReader() : m_tree(nullptr), m_fullName(nullptr), m_buffer(0), m_tag(0) {}
MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) override {
TreeSimpleReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(0), m_tag(0) {
assert(m_kind == MonitorElementData::Kind::INT || m_kind == MonitorElementData::Kind::REAL);
}

void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override {
// This will populate the fields as defined in setTree method
m_tree->GetEntry(iIndex);
MonitorElement* element = iStore.get(*m_fullName);
if (nullptr == element) {
std::string path;
const char* name;
splitName(*m_fullName, path, name);
iStore.setCurrentFolder(path);
element = createElement(iStore, name, m_buffer);
if (iIsLumi) {
element->setLumiFlag();

MonitorElementData::Key key;
key.kind_ = m_kind;
key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME);
key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI;
key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi);

std::vector<MonitorElementData*> runLumiMEs = mesFromFile[std::make_tuple(run, lumi)];
bool merged = false;
for (MonitorElementData* meData : runLumiMEs) {
if(meData->key_ == key) {
// Keep the latest one
MonitorElementData::Value::Access value(meData->value_);
if(m_kind == MonitorElementData::Kind::INT)
value.scalar.num = m_buffer;
else if(m_kind == MonitorElementData::Kind::REAL)
value.scalar.real = m_buffer;
merged = true;
break;
}
} else {
mergeWithElement(element, m_buffer);
}
return element;

if(!merged) {
MonitorElementData* meData = new MonitorElementData();
meData->key_ = key;
{
MonitorElementData::Value::Access value(meData->value_);
if(m_kind == MonitorElementData::Kind::INT)
value.scalar.num = m_buffer;
else if(m_kind == MonitorElementData::Kind::REAL)
value.scalar.real = m_buffer;
}

mesFromFile[std::make_tuple(run, lumi)].push_back(std::move(meData));
}
}

void setTree(TTree* iTree) override {
m_tree = iTree;
m_tree->SetBranchAddress(kFullNameBranch, &m_fullName);
@@ -289,9 +333,9 @@ namespace {
uint32_t m_tag;
};

} // namespace
} // namespace

class DQMRootSource : public edm::InputSource {
class DQMRootSource : public edm::PuttableSourceBase {
public:
DQMRootSource(edm::ParameterSet const&, const edm::InputSourceDescription&);
~DQMRootSource() override;
@@ -304,102 +348,60 @@ class DQMRootSource : public edm::InputSource {
static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);

private:
DQMRootSource(const DQMRootSource&) = delete; // stop default

class RunPHIDKey {
public:
RunPHIDKey(edm::ProcessHistoryID const& phid, unsigned int run) : processHistoryID_(phid), run_(run) {}
edm::ProcessHistoryID const& processHistoryID() const { return processHistoryID_; }
unsigned int run() const { return run_; }
bool operator<(RunPHIDKey const& right) const {
if (processHistoryID_ == right.processHistoryID()) {
return run_ < right.run();
}
return processHistoryID_ < right.processHistoryID();
}

private:
edm::ProcessHistoryID processHistoryID_;
unsigned int run_;
};

class RunLumiPHIDKey {
public:
RunLumiPHIDKey(edm::ProcessHistoryID const& phid, unsigned int run, unsigned int lumi)
: processHistoryID_(phid), run_(run), lumi_(lumi) {}
edm::ProcessHistoryID const& processHistoryID() const { return processHistoryID_; }
unsigned int run() const { return run_; }
unsigned int lumi() const { return lumi_; }
bool operator<(RunLumiPHIDKey const& right) const {
if (processHistoryID_ == right.processHistoryID()) {
if (run_ == right.run()) {
return lumi_ < right.lumi();
}
return run_ < right.run();
}
return processHistoryID_ < right.processHistoryID();
}

private:
edm::ProcessHistoryID processHistoryID_;
unsigned int run_;
unsigned int lumi_;
};
DQMRootSource(const DQMRootSource&) = delete;

edm::InputSource::ItemType getNextItemType() override;
//NOTE: the following is really read next run auxiliary

std::unique_ptr<edm::FileBlock> readFile_() override;
std::shared_ptr<edm::RunAuxiliary> readRunAuxiliary_() override;
std::shared_ptr<edm::LuminosityBlockAuxiliary> readLuminosityBlockAuxiliary_() override;
void readRun_(edm::RunPrincipal& rpCache) override;
void readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) override;
void readEvent_(edm::EventPrincipal&) override;

std::unique_ptr<edm::FileBlock> readFile_() override;
void closeFile_() override;
// Read MEs from m_fileMetadatas to m_MEsFromFile till run or lumi transition
void readElements();
// True if m_currentIndex points to an element that has a different
// run or lumi than the previous element (a transition needs to happen).
// False otherwise.
bool isRunOrLumiTransition() const;
void readNextItemType();

// These methods will be called by the framework.
// MEs in m_MEsFromFile will be put to products.
void beginRun(edm::Run& run);
void beginLuminosityBlock(edm::LuminosityBlock& lumi);

// If the run matches the filterOnRun configuration parameter, the run
// (and all its lumis) will be kept.
// Otherwise, check if a run and a lumi are in the range that needs to be processed.
// Range is retrieved from lumisToProcess configuration parameter.
// If at least one lumi of a run needs to be kept, per run MEs of that run will also be kept.
bool keepIt(edm::RunNumber_t, edm::LuminosityBlockNumber_t) const;
void logFileAction(char const* msg, char const* fileName) const;

void readNextItemType();
bool setupFile(unsigned int iIndex);
void readElements();
bool skipIt(edm::RunNumber_t, edm::LuminosityBlockNumber_t) const;

const DQMRootSource& operator=(const DQMRootSource&) = delete; // stop default

// ---------- member data --------------------------------
edm::InputFileCatalog m_catalog;
edm::RunAuxiliary m_runAux;
edm::LuminosityBlockAuxiliary m_lumiAux;
edm::InputSource::ItemType m_nextItemType;

size_t m_fileIndex;
std::string m_presentlyOpenFileName;
std::list<unsigned int>::iterator m_nextIndexItr;
std::list<unsigned int>::iterator m_presentIndexItr;
std::vector<RunLumiToRange> m_runlumiToRange;
std::unique_ptr<TFile> m_file;
std::vector<TTree*> m_trees;
std::vector<std::shared_ptr<TreeReaderBase> > m_treeReaders;

std::list<unsigned int> m_orderedIndices;
edm::ProcessHistoryID m_lastSeenReducedPHID;
unsigned int m_lastSeenRun;
edm::ProcessHistoryID m_lastSeenReducedPHID2;
unsigned int m_lastSeenRun2;
unsigned int m_lastSeenLumi2;
unsigned int m_filterOnRun;
// Properties from python config
bool m_skipBadFiles;
unsigned int m_filterOnRun;
edm::InputFileCatalog m_catalog;
std::vector<edm::LuminosityBlockRange> m_lumisToProcess;
std::vector<edm::RunNumber_t> m_runsToProcess;

bool m_justOpenedFileSoNeedToGenerateRunTransition;
bool m_shouldReadMEs;
std::set<MonitorElement*> m_lumiElements;
std::set<MonitorElement*> m_runElements;
std::vector<edm::ProcessHistoryID> m_historyIDs;
std::vector<edm::ProcessHistoryID> m_reducedHistoryIDs;

edm::JobReport::Token m_jrToken;
edm::InputSource::ItemType m_nextItemType;
// Each ME type gets its own reader
std::vector<std::shared_ptr<TreeReaderBase>> m_treeReaders;

// Index of currenlty processed row in m_fileMetadatas
unsigned int m_currentIndex;
// All open DQMIO files
std::vector<TFile*> m_openFiles;
// MEs read from files and merged if needed. Ready to be put into products
MonitorElementsFromFile m_MEsFromFile;
// An item here is a row read from DQMIO indices (metadata) table
std::vector<FileMetadata> m_fileMetadatas;
};

//
@@ -423,633 +425,301 @@ void DQMRootSource::fillDescriptions(edm::ConfigurationDescriptions& description

descriptions.addDefault(desc);
}

//
// constructors and destructor
//

DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSourceDescription& iDesc)
: edm::InputSource(iPSet, iDesc),
: edm::PuttableSourceBase(iPSet, iDesc),
m_skipBadFiles(iPSet.getUntrackedParameter<bool>("skipBadFiles", false)),
m_filterOnRun(iPSet.getUntrackedParameter<unsigned int>("filterOnRun", 0)),
m_catalog(iPSet.getUntrackedParameter<std::vector<std::string> >("fileNames"),
iPSet.getUntrackedParameter<std::string>("overrideCatalog")),
m_nextItemType(edm::InputSource::IsFile),
m_fileIndex(0),
m_trees(kNIndicies, static_cast<TTree*>(nullptr)),
m_treeReaders(kNIndicies, std::shared_ptr<TreeReaderBase>()),
m_lastSeenReducedPHID(),
m_lastSeenRun(0),
m_lastSeenReducedPHID2(),
m_lastSeenRun2(0),
m_lastSeenLumi2(0),
m_filterOnRun(iPSet.getUntrackedParameter<unsigned int>("filterOnRun", 0)),
m_skipBadFiles(iPSet.getUntrackedParameter<bool>("skipBadFiles", false)),
m_lumisToProcess(iPSet.getUntrackedParameter<std::vector<edm::LuminosityBlockRange> >(
"lumisToProcess", std::vector<edm::LuminosityBlockRange>())),
m_justOpenedFileSoNeedToGenerateRunTransition(false),
m_shouldReadMEs(true) {
m_nextItemType(edm::InputSource::IsFile),
m_treeReaders(kNIndicies, std::shared_ptr<TreeReaderBase>()),
m_currentIndex(0),
m_openFiles (std::vector<TFile*>()),
m_MEsFromFile(MonitorElementsFromFile()),
m_fileMetadatas(std::vector<FileMetadata>()) {
edm::sortAndRemoveOverlaps(m_lumisToProcess);
for (std::vector<edm::LuminosityBlockRange>::const_iterator itr = m_lumisToProcess.begin();
itr != m_lumisToProcess.end();
++itr)
m_runsToProcess.push_back(itr->startRun());

if (m_fileIndex == m_catalog.fileNames().size()) {
if (m_catalog.fileNames().size() == 0) {
m_nextItemType = edm::InputSource::IsStop;
} else {
m_treeReaders[kIntIndex].reset(new TreeSimpleReader<Long64_t>());
m_treeReaders[kFloatIndex].reset(new TreeSimpleReader<double>());
m_treeReaders[kStringIndex].reset(new TreeObjectReader<std::string>());
m_treeReaders[kTH1FIndex].reset(new TreeObjectReader<TH1F>());
m_treeReaders[kTH1SIndex].reset(new TreeObjectReader<TH1S>());
m_treeReaders[kTH1DIndex].reset(new TreeObjectReader<TH1D>());
m_treeReaders[kTH2FIndex].reset(new TreeObjectReader<TH2F>());
m_treeReaders[kTH2SIndex].reset(new TreeObjectReader<TH2S>());
m_treeReaders[kTH2DIndex].reset(new TreeObjectReader<TH2D>());
m_treeReaders[kTH3FIndex].reset(new TreeObjectReader<TH3F>());
m_treeReaders[kTProfileIndex].reset(new TreeObjectReader<TProfile>());
m_treeReaders[kTProfile2DIndex].reset(new TreeObjectReader<TProfile2D>());
}
else {
m_treeReaders[kIntIndex].reset(new TreeSimpleReader<Long64_t>(MonitorElementData::Kind::INT));
m_treeReaders[kFloatIndex].reset(new TreeSimpleReader<double>(MonitorElementData::Kind::REAL));
m_treeReaders[kStringIndex].reset(new TreeStringReader(MonitorElementData::Kind::STRING));
m_treeReaders[kTH1FIndex].reset(new TreeObjectReader<TH1F>(MonitorElementData::Kind::TH1F));
m_treeReaders[kTH1SIndex].reset(new TreeObjectReader<TH1S>(MonitorElementData::Kind::TH1S));
m_treeReaders[kTH1DIndex].reset(new TreeObjectReader<TH1D>(MonitorElementData::Kind::TH1D));
m_treeReaders[kTH2FIndex].reset(new TreeObjectReader<TH2F>(MonitorElementData::Kind::TH2F));
m_treeReaders[kTH2SIndex].reset(new TreeObjectReader<TH2S>(MonitorElementData::Kind::TH2S));
m_treeReaders[kTH2DIndex].reset(new TreeObjectReader<TH2D>(MonitorElementData::Kind::TH2D));
m_treeReaders[kTH3FIndex].reset(new TreeObjectReader<TH3F>(MonitorElementData::Kind::TH3F));
m_treeReaders[kTProfileIndex].reset(new TreeObjectReader<TProfile>(MonitorElementData::Kind::TPROFILE));
m_treeReaders[kTProfile2DIndex].reset(new TreeObjectReader<TProfile2D>(MonitorElementData::Kind::TPROFILE2D));
}

produces<MonitorElementCollection, edm::Transition::BeginRun>("DQMGenerationRecoRun");
produces<MonitorElementCollection, edm::Transition::BeginLuminosityBlock>("DQMGenerationRecoLumi");
}

// DQMRootSource::DQMRootSource(const DQMRootSource& rhs)
// {
// // do actual copying here;
// }

DQMRootSource::~DQMRootSource() {
if (m_file.get() != nullptr && m_file->IsOpen()) {
m_file->Close();
logFileAction(" Closed file ", m_presentlyOpenFileName.c_str());
for(auto &file : m_openFiles) {
if (file != nullptr && file->IsOpen()) {
file->Close();
logFileAction("Closed file", "");
}
}
}

//
// assignment operators
//
// const DQMRootSource& DQMRootSource::operator=(const DQMRootSource& rhs)
// {
// //An exception safe implementation is
// DQMRootSource temp(rhs);
// swap(rhs);
//
// return *this;
// }

//
// member functions
//
void DQMRootSource::readEvent_(edm::EventPrincipal&) {
//std::cout << "readEvent_" << std::endl;
}

edm::InputSource::ItemType DQMRootSource::getNextItemType() {
//std::cout <<"getNextItemType "<<m_nextItemType<<std::endl;
return m_nextItemType;
}

std::shared_ptr<edm::RunAuxiliary> DQMRootSource::readRunAuxiliary_() {
//std::cout <<"readRunAuxiliary_"<<std::endl;
assert(m_nextIndexItr != m_orderedIndices.end());
RunLumiToRange runLumiRange = m_runlumiToRange[*m_nextIndexItr];

//NOTE: the setBeginTime and setEndTime functions of RunAuxiliary only work if the previous value was invalid
// therefore we must copy
m_runAux = edm::RunAuxiliary(
runLumiRange.m_run, edm::Timestamp(runLumiRange.m_beginTime), edm::Timestamp(runLumiRange.m_endTime));
assert(m_historyIDs.size() > runLumiRange.m_historyIDIndex);
//std::cout <<"readRunAuxiliary_ "<<m_historyIDs[runLumiRange.m_historyIDIndex]<<std::endl;
m_runAux.setProcessHistoryID(m_historyIDs[runLumiRange.m_historyIDIndex]);
return std::make_shared<edm::RunAuxiliary>(m_runAux);
}
// We will read the metadata of all files and fill m_fileMetadatas vector
std::unique_ptr<edm::FileBlock> DQMRootSource::readFile_() {
const int numFiles = m_catalog.fileNames().size();
m_openFiles.reserve(numFiles);

std::shared_ptr<edm::LuminosityBlockAuxiliary> DQMRootSource::readLuminosityBlockAuxiliary_() {
//std::cout <<"readLuminosityBlockAuxiliary_"<<std::endl;
assert(m_nextIndexItr != m_orderedIndices.end());
const RunLumiToRange runLumiRange = m_runlumiToRange[*m_nextIndexItr];
m_lumiAux = edm::LuminosityBlockAuxiliary(edm::LuminosityBlockID(runLumiRange.m_run, runLumiRange.m_lumi),
edm::Timestamp(runLumiRange.m_beginTime),
edm::Timestamp(runLumiRange.m_endTime));
assert(m_historyIDs.size() > runLumiRange.m_historyIDIndex);
//std::cout <<"lumi "<<m_lumiAux.beginTime().value()<<" "<<runLumiRange.m_beginTime<<std::endl;
m_lumiAux.setProcessHistoryID(m_historyIDs[runLumiRange.m_historyIDIndex]);

return std::make_shared<edm::LuminosityBlockAuxiliary>(m_lumiAux);
}
// TODO: add support to fallback files: https://github.com/cms-sw/cmssw/pull/28064/files
for (auto &filename : m_catalog.fileNames()) {
TFile* file;

void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) {
assert(m_presentIndexItr != m_orderedIndices.end());
RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr];
// Try to open a file
try {
file = TFile::Open(filename.c_str());

m_justOpenedFileSoNeedToGenerateRunTransition = false;
unsigned int runID = rpCache.id().run();
assert(runID == runLumiRange.m_run);
// Exception will be trapped so we pull it out ourselves
std::exception_ptr e = edm::threadLocalException::getException();
if (e != std::exception_ptr()) {
edm::threadLocalException::setException(std::exception_ptr());
std::rethrow_exception(e);
}

m_openFiles.insert(m_openFiles.begin(), file);
} catch (cms::Exception const& e) {
if (!m_skipBadFiles) {
edm::Exception ex(edm::errors::FileOpenError, "", e);
ex.addContext("Opening DQM Root file");
ex << "\nInput file " << filename
<< " was not found, could not be opened, or is corrupted.\n";
throw ex;
}
}

m_shouldReadMEs = (m_filterOnRun == 0 || (m_filterOnRun != 0 && m_filterOnRun == runID));
if (m_lastSeenRun != runID || m_lastSeenReducedPHID != m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex)) {
if (m_shouldReadMEs) {
// TODO: Worry about resetting depending on Collate option here.
// Check if a file is usable
if (!file->IsZombie()) {
logFileAction("Successfully opened file ", filename.c_str());
} else {
if (!m_skipBadFiles) {
edm::Exception ex(edm::errors::FileOpenError);
ex << "Input file " << filename.c_str() << " could not be opened.\n";
ex.addContext("Opening DQM Root file");
throw ex;
}
}
m_lastSeenReducedPHID = m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex);
m_lastSeenRun = runID;
}

readNextItemType();
// Check file format version, which is encoded in the Title of the TFile
if (strcmp(file->GetTitle(), "1") != 0) {
edm::Exception ex(edm::errors::FileReadError);
ex << "Input file " << filename.c_str() << " does not appear to be a DQM Root file.\n";
}

//NOTE: it is possible to have a Run when all we have stored is lumis
if (runLumiRange.m_lumi == 0) {
readElements();
// Read metadata from the file
TTree* indicesTree = dynamic_cast<TTree*>(file->Get(kIndicesTree));
assert(indicesTree != nullptr);

FileMetadata temp;
// Each line of metadata will be read into the coresponding fields of temp.
indicesTree->SetBranchAddress(kRunBranch, &temp.m_run);
indicesTree->SetBranchAddress(kLumiBranch, &temp.m_lumi);
indicesTree->SetBranchAddress(kBeginTimeBranch, &temp.m_beginTime);
indicesTree->SetBranchAddress(kEndTimeBranch, &temp.m_endTime);
indicesTree->SetBranchAddress(kTypeBranch, &temp.m_type);
indicesTree->SetBranchAddress(kFirstIndex, &temp.m_firstIndex);
indicesTree->SetBranchAddress(kLastIndex, &temp.m_lastIndex);

for (Long64_t index = 0; index != indicesTree->GetEntries(); ++index) {
indicesTree->GetEntry(index);
temp.m_file = file;

if(keepIt(temp.m_run, temp.m_lumi)) {
m_fileMetadatas.push_back(temp);
}
}
}

edm::Service<edm::JobReport> jr;
jr->reportInputRunNumber(rpCache.id().run());

rpCache.fillRunPrincipal(processHistoryRegistryForUpdate());
}
// Sort to make sure runs and lumis appear in sequential order
std::sort(m_fileMetadatas.begin(), m_fileMetadatas.end());

void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) {
assert(m_presentIndexItr != m_orderedIndices.end());
RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr];
assert(runLumiRange.m_run == lbCache.id().run());
assert(runLumiRange.m_lumi == lbCache.id().luminosityBlock());

//NOTE: need to reset all lumi block elements at this point
if ((m_lastSeenLumi2 != runLumiRange.m_lumi || m_lastSeenRun2 != runLumiRange.m_run ||
m_lastSeenReducedPHID2 != m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex)) &&
m_shouldReadMEs) {
edm::Service<DQMStore> store;
std::vector<MonitorElement*> allMEs = (*store).getAllContents("");
//for(auto const& ME : allMEs) {
// // We do not want to reset Run Products here!
// if (ME->getLumiFlag()) {
// ME->Reset();
// }
//}
m_lastSeenReducedPHID2 = m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex);
m_lastSeenRun2 = runLumiRange.m_run;
m_lastSeenLumi2 = runLumiRange.m_lumi;
}
for(auto &metadata : m_fileMetadatas)
metadata.describe();

readNextItemType();
readElements();
// Stop if there's nothing to process. Otherwise start the run.
if(m_fileMetadatas.size() == 0)
m_nextItemType = edm::InputSource::IsStop;
else
m_nextItemType = edm::InputSource::IsRun;

edm::Service<edm::JobReport> jr;
jr->reportInputLumiSection(lbCache.id().run(), lbCache.id().luminosityBlock());
// We have to return something but not sure why
return std::unique_ptr<edm::FileBlock>(new edm::FileBlock);
}

lbCache.fillLuminosityBlockPrincipal(processHistoryRegistry().getMapped(lbCache.aux().processHistoryID()));
std::shared_ptr<edm::RunAuxiliary> DQMRootSource::readRunAuxiliary_() {
FileMetadata metadata = m_fileMetadatas[m_currentIndex];
auto runAux = edm::RunAuxiliary(metadata.m_run, edm::Timestamp(metadata.m_beginTime), edm::Timestamp(metadata.m_endTime));
return std::make_shared<edm::RunAuxiliary>(runAux);
}

std::unique_ptr<edm::FileBlock> DQMRootSource::readFile_() {
auto const numFiles = m_catalog.fileNames().size();
while (m_fileIndex < numFiles && not setupFile(m_fileIndex++)) {
}
std::shared_ptr<edm::LuminosityBlockAuxiliary> DQMRootSource::readLuminosityBlockAuxiliary_() {
FileMetadata metadata = m_fileMetadatas[m_currentIndex];
auto lumiAux = edm::LuminosityBlockAuxiliary(edm::LuminosityBlockID(metadata.m_run, metadata.m_lumi),
edm::Timestamp(metadata.m_beginTime),
edm::Timestamp(metadata.m_endTime));
return std::make_shared<edm::LuminosityBlockAuxiliary>(lumiAux);
}

if (m_file.get() == nullptr) {
//last file in list was bad
m_nextItemType = edm::InputSource::IsStop;
return std::unique_ptr<edm::FileBlock>(new edm::FileBlock);
}
void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) {
// Read elements of a current run.
do {
FileMetadata metadata = m_fileMetadatas[m_currentIndex];
if (metadata.m_lumi == 0) {
readElements();
}
m_currentIndex++;
} while(!isRunOrLumiTransition());

readNextItemType();
while (m_presentIndexItr != m_orderedIndices.end() &&
skipIt(m_runlumiToRange[*m_presentIndexItr].m_run, m_runlumiToRange[*m_presentIndexItr].m_lumi))
++m_presentIndexItr;

edm::Service<edm::JobReport> jr;
std::string guid{m_file->GetUUID().AsString()};
std::transform(guid.begin(), guid.end(), guid.begin(), (int (*)(int))std::toupper);
m_jrToken = jr->inputFileOpened(m_presentlyOpenFileName,
m_catalog.logicalFileNames()[m_fileIndex - 1],
std::string(),
std::string(),
"DQMRootSource",
"source",
std::move(guid),
std::vector<std::string>());

return std::unique_ptr<edm::FileBlock>(new edm::FileBlock);
}

void DQMRootSource::closeFile_() {
if (m_file.get() == nullptr) {
return;
}
edm::Service<edm::JobReport> jr;
jr->inputFileClosed(edm::InputType::Primary, m_jrToken);
}

void DQMRootSource::readElements() {
edm::Service<DQMStore> store;
RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr];
bool shouldContinue = false;
void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) {
// Read elements of a current lumi.
do {
shouldContinue = false;
++m_presentIndexItr;
while (m_presentIndexItr != m_orderedIndices.end() &&
skipIt(m_runlumiToRange[*m_presentIndexItr].m_run, m_runlumiToRange[*m_presentIndexItr].m_lumi))
++m_presentIndexItr;

if (runLumiRange.m_type != kNoTypesStored) {
std::shared_ptr<TreeReaderBase> reader = m_treeReaders[runLumiRange.m_type];
ULong64_t index = runLumiRange.m_firstIndex;
ULong64_t endIndex = runLumiRange.m_lastIndex + 1;
for (; index != endIndex; ++index) {
bool isLumi = runLumiRange.m_lumi != 0;
if (m_shouldReadMEs)
reader->read(index, *store, isLumi);

//std::cout << runLumiRange.m_run << " " << runLumiRange.m_lumi <<" "<<index<< " " << runLumiRange.m_type << std::endl;
}
}
readElements();
m_currentIndex++;
} while(!isRunOrLumiTransition());

if (m_presentIndexItr != m_orderedIndices.end()) {
//are there more parts to this same run/lumi?
const RunLumiToRange nextRunLumiRange = m_runlumiToRange[*m_presentIndexItr];
//continue to the next item if that item is either
if ((m_reducedHistoryIDs.at(nextRunLumiRange.m_historyIDIndex) ==
m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex)) &&
(nextRunLumiRange.m_run == runLumiRange.m_run) && (nextRunLumiRange.m_lumi == runLumiRange.m_lumi)) {
shouldContinue = true;
runLumiRange = nextRunLumiRange;
}
}
} while (shouldContinue);
readNextItemType();
}

void DQMRootSource::readNextItemType() {
//Do the work of actually figuring out where next to go
void DQMRootSource::readEvent_(edm::EventPrincipal&) {
}

assert(m_nextIndexItr != m_orderedIndices.end());
RunLumiToRange runLumiRange = m_runlumiToRange[*m_nextIndexItr];
void DQMRootSource::readElements() {
FileMetadata metadata = m_fileMetadatas[m_currentIndex];

if (m_nextItemType != edm::InputSource::IsFile) {
if (runLumiRange.m_lumi != 0 && m_nextItemType == edm::InputSource::IsRun) {
m_nextItemType = edm::InputSource::IsLumi;
return;
}
++m_nextIndexItr;
} else {
//NOTE: the following makes the iterator not be advanced in the
//do while loop below.
runLumiRange.m_run = 0;
}
if (metadata.m_type != kNoTypesStored) {
std::shared_ptr<TreeReaderBase> reader = m_treeReaders[metadata.m_type];
TTree* tree = dynamic_cast<TTree*>(metadata.m_file->Get(kTypeNames[metadata.m_type]));
reader->setTree(tree);

bool shouldContinue = false;
do {
shouldContinue = false;
while (m_nextIndexItr != m_orderedIndices.end() &&
skipIt(m_runlumiToRange[*m_nextIndexItr].m_run, m_runlumiToRange[*m_nextIndexItr].m_lumi))
++m_nextIndexItr;

if (m_nextIndexItr == m_orderedIndices.end()) {
//go to next file
m_nextItemType = edm::InputSource::IsFile;
//std::cout <<"going to next file"<<std::endl;
if (m_fileIndex == m_catalog.fileNames().size()) {
m_nextItemType = edm::InputSource::IsStop;
}
break;
}
const RunLumiToRange nextRunLumiRange = m_runlumiToRange[*m_nextIndexItr];
//continue to the next item if that item is the same run or lumi as we just did
if ((m_reducedHistoryIDs.at(nextRunLumiRange.m_historyIDIndex) ==
m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex)) &&
(nextRunLumiRange.m_run == runLumiRange.m_run) && (nextRunLumiRange.m_lumi == runLumiRange.m_lumi)) {
shouldContinue = true;
++m_nextIndexItr;
//std::cout <<"advancing " <<nextRunLumiRange.m_run<<" "<<nextRunLumiRange.m_lumi<<std::endl;
}
} while (shouldContinue);
ULong64_t index = metadata.m_firstIndex;
ULong64_t endIndex = metadata.m_lastIndex + 1;

if (m_nextIndexItr != m_orderedIndices.end()) {
if (m_justOpenedFileSoNeedToGenerateRunTransition || m_lastSeenRun != m_runlumiToRange[*m_nextIndexItr].m_run ||
m_lastSeenReducedPHID != m_reducedHistoryIDs.at(m_runlumiToRange[*m_nextIndexItr].m_historyIDIndex)) {
m_nextItemType = edm::InputSource::IsRun;
} else {
m_nextItemType = edm::InputSource::IsLumi;
for (; index != endIndex; ++index) {
reader->read(index, m_MEsFromFile, metadata.m_run, metadata.m_lumi);
}
}
}

bool DQMRootSource::setupFile(unsigned int iIndex) {
if (m_file.get() != nullptr && iIndex > 0) {
m_file->Close();
logFileAction(" Closed file ", m_presentlyOpenFileName.c_str());
bool DQMRootSource::isRunOrLumiTransition() const {
if(m_currentIndex == 0) {
return false;
}
logFileAction(" Initiating request to open file ", m_catalog.fileNames()[iIndex].c_str());

auto fallbackFileName = m_catalog.fallbackFileNames()[iIndex];
bool hasFallback = !fallbackFileName.empty() && fallbackFileName != m_catalog.fileNames()[iIndex];

m_presentlyOpenFileName.clear();
m_file.reset();

std::unique_ptr<TFile> newFile;
std::list<std::string> originalInfo;
try {
// ROOT's context management implicitly assumes that a file is opened and
// closed on the same thread. To avoid the problem, we declare a local
// TContext object; when it goes out of scope, its destructor unregisters
// the context, guaranteeing the context is unregistered in the same thread
// it was registered in.
{
TDirectory::TContext contextEraser;
newFile = std::unique_ptr<TFile>(TFile::Open(m_catalog.fileNames()[iIndex].c_str()));
}
//Since ROOT6, we can not propagate an exception through ROOT's plugin
// system so we trap them and then pull from this function
std::exception_ptr e = edm::threadLocalException::getException();
if (e != std::exception_ptr()) {
edm::threadLocalException::setException(std::exception_ptr());
std::rethrow_exception(e);
}

} catch (cms::Exception const& e) {
if (!hasFallback) {
if (m_skipBadFiles) {
return false;
} else {
edm::Exception ex(edm::errors::FileOpenError, "", e);
ex.addContext("Opening DQM Root file");
ex << "\nInput file " << m_catalog.fileNames()[iIndex]
<< " was not found, could not be opened, or is corrupted.\n";
throw ex;
}
}
originalInfo = e.additionalInfo(); // save in case of fallback error
newFile.reset();
}
if (newFile && not newFile->IsZombie()) {
m_presentlyOpenFileName = m_catalog.fileNames()[iIndex];
logFileAction(" Successfully opened file ", m_presentlyOpenFileName.c_str());
} else {
if (!hasFallback) {
if (m_skipBadFiles) {
return false;
} else {
edm::Exception ex(edm::errors::FileOpenError);
ex << "Input file " << m_catalog.fileNames()[iIndex].c_str() << " could not be opened.\n";
ex.addContext("Opening DQM Root file");
throw ex;
}
}
newFile.reset();
if(m_currentIndex > m_fileMetadatas.size() - 1) {
// We reached the end
return true;
}

if (!newFile && hasFallback) {
logFileAction(" Initiating request to open fallback file ", fallbackFileName.c_str());
try {
{
TDirectory::TContext contextEraser;
newFile = std::unique_ptr<TFile>(TFile::Open(fallbackFileName.c_str()));
}
std::exception_ptr e = edm::threadLocalException::getException();
if (e != std::exception_ptr()) {
edm::threadLocalException::setException(std::exception_ptr());
std::rethrow_exception(e);
}
} catch (cms::Exception const& e) {
if (m_skipBadFiles) {
return false;
} else {
edm::Exception ex(edm::errors::FileOpenError, "", e);
ex.addContext("Opening DQM Root file");
ex << "\nInput file " << m_catalog.fileNames()[iIndex] << " and fallback input file " << fallbackFileName
<< " were not found, could not be opened, or are corrupted.\n";
for (auto const& s : originalInfo) {
ex.addAdditionalInfo(s);
}
throw ex;
}
}
if (not newFile->IsZombie()) {
m_presentlyOpenFileName = fallbackFileName;
logFileAction(" Successfully opened fallback file ", m_presentlyOpenFileName.c_str());
} else {
if (m_skipBadFiles) {
return false;
} else {
edm::Exception ex(edm::errors::FileOpenError);
ex << "Input file " << m_catalog.fileNames()[iIndex] << " and fallback input file " << fallbackFileName
<< " could not be opened.\n";
ex.addContext("Opening DQM Root file");
for (auto const& s : originalInfo) {
ex.addAdditionalInfo(s);
}
throw ex;
}
}
}
FileMetadata previousMetadata = m_fileMetadatas[m_currentIndex - 1];
FileMetadata metadata = m_fileMetadatas[m_currentIndex];

//Check file format version, which is encoded in the Title of the TFile
if (0 != strcmp(newFile->GetTitle(), "1")) {
if (m_skipBadFiles) {
return false;
} else {
edm::Exception ex(edm::errors::FileReadError);
ex << "Input file " << m_presentlyOpenFileName << " does not appear to be a DQM Root file.\n";
throw ex;
}
}
return previousMetadata.m_run != metadata.m_run || previousMetadata.m_lumi != metadata.m_lumi;
}

//Get meta Data
TDirectory* metaDir = newFile->GetDirectory(kMetaDataDirectoryAbsolute);
if (nullptr == metaDir) {
if (!m_skipBadFiles) {
edm::Exception ex(edm::errors::FileReadError);
ex << "Input file " << m_presentlyOpenFileName
<< " appears to be corrupted since it does not have the proper internal structure.\n"
" Check to see if the file was closed properly.\n";
ex.addContext("Opening DQM Root file");
throw ex;
} else {
return false;
}
void DQMRootSource::readNextItemType() {
if(m_currentIndex == 0) {
m_nextItemType = edm::InputSource::IsRun;
}
m_file = std::move(newFile); //passed all tests so now we want to use this file
TTree* parameterSetTree = dynamic_cast<TTree*>(metaDir->Get(kParameterSetTree));
assert(nullptr != parameterSetTree);

edm::pset::Registry* psr = edm::pset::Registry::instance();
assert(nullptr != psr);
{
std::string blob;
std::string* pBlob = &blob;
parameterSetTree->SetBranchAddress(kParameterSetBranch, &pBlob);
for (unsigned int index = 0; index != parameterSetTree->GetEntries(); ++index) {
parameterSetTree->GetEntry(index);
edm::ParameterSet::registerFromString(blob);
}
else if(m_currentIndex > m_fileMetadatas.size() - 1) {
// We reached the end
m_nextItemType = edm::InputSource::IsStop;
}
else {
FileMetadata previousMetadata = m_fileMetadatas[m_currentIndex - 1];
FileMetadata metadata = m_fileMetadatas[m_currentIndex];

{
TTree* processHistoryTree = dynamic_cast<TTree*>(metaDir->Get(kProcessHistoryTree));
assert(nullptr != processHistoryTree);
unsigned int phIndex = 0;
processHistoryTree->SetBranchAddress(kPHIndexBranch, &phIndex);
std::string processName;
std::string* pProcessName = &processName;
processHistoryTree->SetBranchAddress(kProcessConfigurationProcessNameBranch, &pProcessName);
std::string parameterSetIDBlob;
std::string* pParameterSetIDBlob = &parameterSetIDBlob;
processHistoryTree->SetBranchAddress(kProcessConfigurationParameterSetIDBranch, &pParameterSetIDBlob);
std::string releaseVersion;
std::string* pReleaseVersion = &releaseVersion;
processHistoryTree->SetBranchAddress(kProcessConfigurationReleaseVersion, &pReleaseVersion);
std::string passID;
std::string* pPassID = &passID;
processHistoryTree->SetBranchAddress(kProcessConfigurationPassID, &pPassID);

edm::ProcessHistoryRegistry& phr = processHistoryRegistryForUpdate();
std::vector<edm::ProcessConfiguration> configs;
configs.reserve(5);
m_historyIDs.clear();
m_reducedHistoryIDs.clear();
for (unsigned int i = 0; i != processHistoryTree->GetEntries(); ++i) {
processHistoryTree->GetEntry(i);
if (phIndex == 0) {
if (not configs.empty()) {
edm::ProcessHistory ph(configs);
m_historyIDs.push_back(ph.id());
phr.registerProcessHistory(ph);
m_reducedHistoryIDs.push_back(phr.reducedProcessHistoryID(ph.id()));
}
configs.clear();
}
edm::ParameterSetID psetID(parameterSetIDBlob);
edm::ProcessConfiguration pc(processName, psetID, releaseVersion, passID);
configs.push_back(pc);
if(previousMetadata.m_run != metadata.m_run) {
m_nextItemType = edm::InputSource::IsRun;
}
if (not configs.empty()) {
edm::ProcessHistory ph(configs);
m_historyIDs.push_back(ph.id());
phr.registerProcessHistory(ph);
m_reducedHistoryIDs.push_back(phr.reducedProcessHistoryID(ph.id()));
//std::cout <<"inserted "<<ph.id()<<std::endl;
else if(previousMetadata.m_lumi != metadata.m_lumi) {
m_nextItemType = edm::InputSource::IsLumi;
}
}
}

//Setup the indices
TTree* indicesTree = dynamic_cast<TTree*>(m_file->Get(kIndicesTree));
assert(nullptr != indicesTree);

m_runlumiToRange.clear();
m_runlumiToRange.reserve(indicesTree->GetEntries());
m_orderedIndices.clear();

RunLumiToRange temp;
indicesTree->SetBranchAddress(kRunBranch, &temp.m_run);
indicesTree->SetBranchAddress(kLumiBranch, &temp.m_lumi);
indicesTree->SetBranchAddress(kBeginTimeBranch, &temp.m_beginTime);
indicesTree->SetBranchAddress(kEndTimeBranch, &temp.m_endTime);
indicesTree->SetBranchAddress(kProcessHistoryIndexBranch, &temp.m_historyIDIndex);
indicesTree->SetBranchAddress(kTypeBranch, &temp.m_type);
indicesTree->SetBranchAddress(kFirstIndex, &temp.m_firstIndex);
indicesTree->SetBranchAddress(kLastIndex, &temp.m_lastIndex);

//Need to reorder items since if there was a merge done the same Run
//and/or Lumi can appear multiple times but we want to process them
//all at once

//We use a std::list for m_orderedIndices since inserting into the
//middle of a std::list does not disrupt the iterators to already
//existing entries

//The Map is used to see if a Run/Lumi pair has appeared before
typedef std::map<RunLumiPHIDKey, std::list<unsigned int>::iterator> RunLumiToLastEntryMap;
RunLumiToLastEntryMap runLumiToLastEntryMap;

//Need to group all lumis for the same run together and move the run
//entry to the beginning
typedef std::map<RunPHIDKey, std::pair<std::list<unsigned int>::iterator, std::list<unsigned int>::iterator> >
RunToFirstLastEntryMap;
RunToFirstLastEntryMap runToFirstLastEntryMap;

for (Long64_t index = 0; index != indicesTree->GetEntries(); ++index) {
indicesTree->GetEntry(index);
// std::cout <<"read r:"<<temp.m_run
// <<" l:"<<temp.m_lumi
// <<" b:"<<temp.m_beginTime
// <<" e:"<<temp.m_endTime
// <<" fi:" << temp.m_firstIndex
// <<" li:" << temp.m_lastIndex
// <<" type:" << temp.m_type << std::endl;
m_runlumiToRange.push_back(temp);

RunLumiPHIDKey runLumi(m_reducedHistoryIDs.at(temp.m_historyIDIndex), temp.m_run, temp.m_lumi);
RunPHIDKey runKey(m_reducedHistoryIDs.at(temp.m_historyIDIndex), temp.m_run);

RunLumiToLastEntryMap::iterator itFind = runLumiToLastEntryMap.find(runLumi);
if (itFind == runLumiToLastEntryMap.end()) {
//does not already exist
//does the run for this already exist?
std::list<unsigned int>::iterator itLastOfRun = m_orderedIndices.end();

RunToFirstLastEntryMap::iterator itRunFirstLastEntryFind = runToFirstLastEntryMap.find(runKey);
bool needNewEntryInRunFirstLastEntryMap = true;
if (itRunFirstLastEntryFind != runToFirstLastEntryMap.end()) {
needNewEntryInRunFirstLastEntryMap = false;
if (temp.m_lumi != 0) {
//lumis go to the end
itLastOfRun = itRunFirstLastEntryFind->second.second;
//we want to insert after this one so must advance the iterator
++itLastOfRun;
} else {
//runs go at the beginning
itLastOfRun = itRunFirstLastEntryFind->second.first;
}
}
std::list<unsigned int>::iterator iter = m_orderedIndices.insert(itLastOfRun, index);
runLumiToLastEntryMap[runLumi] = iter;
if (needNewEntryInRunFirstLastEntryMap)
runToFirstLastEntryMap[runKey] = std::make_pair(iter, iter);
else {
if (temp.m_lumi != 0) {
//lumis go at end
runToFirstLastEntryMap[runKey].second = iter;
} else {
//since we haven't yet seen this run/lumi combination it means we haven't yet seen
// a run so we can put this first
runToFirstLastEntryMap[runKey].first = iter;
}
}
} else {
//We need to do a merge since the run/lumi already appeared. Put it after the existing entry
//std::cout <<" found a second instance of "<<runLumi.first<<" "<<runLumi.second<<" at "<<index<<std::endl;
std::list<unsigned int>::iterator itNext = itFind->second;
++itNext;
std::list<unsigned int>::iterator iter = m_orderedIndices.insert(itNext, index);
RunToFirstLastEntryMap::iterator itRunFirstLastEntryFind = runToFirstLastEntryMap.find(runKey);
if (itRunFirstLastEntryFind->second.second == itFind->second) {
//if the previous one was the last in the run, we need to update to make this one the last
itRunFirstLastEntryFind->second.second = iter;
}
itFind->second = iter;
}
}
m_nextIndexItr = m_orderedIndices.begin();
m_presentIndexItr = m_orderedIndices.begin();

if (m_nextIndexItr != m_orderedIndices.end()) {
for (size_t index = 0; index < kNIndicies; ++index) {
m_trees[index] = dynamic_cast<TTree*>(m_file->Get(kTypeNames[index]));
assert(nullptr != m_trees[index]);
m_treeReaders[index]->setTree(m_trees[index]);
}
void DQMRootSource::beginRun(edm::Run& run) {
TRACE("Begin run: " + std::to_string(run.run()))

std::unique_ptr<MonitorElementCollection> product = std::make_unique<MonitorElementCollection>();

auto mes = m_MEsFromFile[std::make_tuple(run.run(), 0)];

TRACE("Found MEs: " + std::to_string(mes.size()))

for(MonitorElementData* meData_ptr : mes) {
product->push_back(meData_ptr);
}
//After a file open, the framework expects to see a new 'IsRun'
m_justOpenedFileSoNeedToGenerateRunTransition = true;

return true;
run.put(std::move(product), "DQMGenerationRecoRun");

// Remove already processed MEs
m_MEsFromFile[std::make_tuple(run.run(), 0)] = std::vector<MonitorElementData*>();
}

bool DQMRootSource::skipIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const {
if (!m_runsToProcess.empty() && edm::search_all(m_runsToProcess, run) && lumi == 0) {
return false;
void DQMRootSource::beginLuminosityBlock(edm::LuminosityBlock& lumi) {
TRACE("Begin lumi: " + std::to_string(lumi.luminosityBlock()))

std::unique_ptr<MonitorElementCollection> product = std::make_unique<MonitorElementCollection>();

auto mes = m_MEsFromFile[std::make_tuple(lumi.run(), lumi.luminosityBlock())];

TRACE("Found MEs: " + std::to_string(mes.size()))

for(MonitorElementData* meData_ptr : mes) {
assert(meData_ptr != nullptr);
product->push_back(meData_ptr);
}

lumi.put(std::move(product), "DQMGenerationRecoLumi");

edm::LuminosityBlockID lumiID = edm::LuminosityBlockID(run, lumi);
edm::LuminosityBlockRange lumiRange = edm::LuminosityBlockRange(lumiID, lumiID);
bool (*lt)(edm::LuminosityBlockRange const&, edm::LuminosityBlockRange const&) = &edm::lessThan;
if (!m_lumisToProcess.empty() && !binary_search_all(m_lumisToProcess, lumiRange, lt)) {
// Remove already processed MEs
m_MEsFromFile[std::make_tuple(lumi.run(), lumi.luminosityBlock())] = std::vector<MonitorElementData*>();
}

bool DQMRootSource::keepIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const {
if(run == m_filterOnRun)
return true;

for (edm::LuminosityBlockRange const& lumiToProcess : m_lumisToProcess) {
if(run >= lumiToProcess.startRun() && run <= lumiToProcess.endRun()) {
if(lumi >= lumiToProcess.startLumi() && lumi <= lumiToProcess.endLumi()) {
return true;
}
else if (lumi == 0) {
return true;
}
}
}
return false;
}

0 comments on commit 24ece8a

Please sign in to comment.