From 7b15a4b87a947f7ad8f6d8befb33a160e859fe9b Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 9 Dec 2019 17:45:52 +0100 Subject: [PATCH 01/11] Import Legacy IO code from @andrius-k. Using commit 70fc23c56872f2ee43330ac2ab6cb6b72ddb1682 from andrius-k:dqm-new-dqmstore-on-CMSSW_11_0_0_pre5. Includes changes to make the coe compile and run with the old DQMStore. --- .../Components/plugins/DQMFileSaver.cc | 152 +++++++++++++++++- DQMServices/Components/plugins/DQMFileSaver.h | 4 +- 2 files changed, 150 insertions(+), 6 deletions(-) diff --git a/DQMServices/Components/plugins/DQMFileSaver.cc b/DQMServices/Components/plugins/DQMFileSaver.cc index 3236e88e41917..89ff0c0288e0c 100644 --- a/DQMServices/Components/plugins/DQMFileSaver.cc +++ b/DQMServices/Components/plugins/DQMFileSaver.cc @@ -1,5 +1,6 @@ #include "DQMFileSaver.h" #include "DQMServices/Components/interface/fillJson.h" +#include "DQMServices/Core/src/DQMError.h" #include "DQMServices/Core/interface/DQMStore.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/Run.h" @@ -10,6 +11,8 @@ #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/MessageLogger/interface/JobReport.h" +#include "DataFormats/Histograms/interface/DQMToken.h" + #include "EventFilter/Utilities/interface/EvFDaqDirector.h" #include "EventFilter/Utilities/interface/FastMonitoringService.h" @@ -24,6 +27,8 @@ #include #include +#include "TFile.h" + #include #include #include @@ -67,6 +72,7 @@ static std::string onlineOfflineFileName(const std::string &fileBaseName, } void DQMFileSaver::saveForOfflinePB(const std::string &workflow, int run) const { + return; char suffix[64]; sprintf(suffix, "R%09d", run); std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_, PB); @@ -74,6 +80,8 @@ void DQMFileSaver::saveForOfflinePB(const std::string &workflow, int run) const } void DQMFileSaver::saveForOffline(const std::string &workflow, int run, int lumi) const { + return; + std::cout << "DQMFileSaver::saveForOffline" << std::endl; char suffix[64]; sprintf(suffix, "R%09d", run); @@ -160,6 +168,7 @@ static void doSaveForOnline(DQMFileSaver::DQMStore *store, int saveRefQMin, const std::string &filterName, DQMFileSaver::FileFormat fileFormat) { + return; // TODO(rovere): fix the online case. so far we simply rely on the // fact that we assume we will not run multithreaded in online. if (fileFormat == DQMFileSaver::ROOT) @@ -169,12 +178,13 @@ static void doSaveForOnline(DQMFileSaver::DQMStore *store, } void DQMFileSaver::saveForOnlinePB(int run, const std::string &suffix) const { + return; // The file name contains the Online workflow name, // as we do not want to look inside the DQMStore, // and the @a suffix, defined in the run/lumi transitions. // TODO(diguida): add the possibility to change the dir structure with rewrite. std::string filename = onlineOfflineFileName(fileBaseName_, suffix, workflow_, child_, PB); - doSaveForOnline(dbe_, + doSaveForOnline(&*dbe_, run, enableMultiThread_, filename, @@ -188,13 +198,14 @@ void DQMFileSaver::saveForOnlinePB(int run, const std::string &suffix) const { } void DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std::string &rewrite) const { +#if 0 std::vector systems = (dbe_->cd(), dbe_->getSubdirs()); for (size_t i = 0, e = systems.size(); i != e; ++i) { if (systems[i] != "Reference") { dbe_->cd(); if (MonitorElement *me = dbe_->get(systems[i] + "/EventInfo/processName")) { - doSaveForOnline(dbe_, + doSaveForOnline(&*dbe_, run, enableMultiThread_, fileBaseName_ + me->getStringValue() + suffix + child_ + ".root", @@ -217,7 +228,7 @@ void DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std:: std::vector pNamesVector = dbe_->getMatchingContents("^" + systems[i] + "/.*/EventInfo/processName", lat::Regexp::Perl); if (!pNamesVector.empty()) { - doSaveForOnline(dbe_, + doSaveForOnline(&*dbe_, run, enableMultiThread_, fileBaseName_ + systems[i] + suffix + child_ + ".root", @@ -236,7 +247,7 @@ void DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std:: // if no EventInfo Folder is found, then store subsystem wise for (size_t i = 0, e = systems.size(); i != e; ++i) if (systems[i] != "Reference") - doSaveForOnline(dbe_, + doSaveForOnline(&*dbe_, run, enableMultiThread_, fileBaseName_ + systems[i] + suffix + child_ + ".root", @@ -247,6 +258,7 @@ void DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std:: saveReferenceQMin_, "", ROOT); +#endif } boost::property_tree::ptree DQMFileSaver::fillJson(int run, @@ -262,6 +274,7 @@ void DQMFileSaver::saveForFilterUnit(const std::string &rewrite, int run, int lumi, const DQMFileSaver::FileFormat fileFormat) const { + return; // get from DAQ2 services where to store the files according to their format namespace bpt = boost::property_tree; @@ -329,6 +342,7 @@ void DQMFileSaver::saveForFilterUnit(const std::string &rewrite, } void DQMFileSaver::saveJobReport(const std::string &filename) const { + return; // Report the file to job report service. edm::Service jr; if (jr.isAvailable()) { @@ -365,6 +379,7 @@ DQMFileSaver::DQMFileSaver(const edm::ParameterSet &ps) nlumi_(0), irun_(0), fms_(nullptr) { + consumesMany(); // Determine the file saving convention, and adjust defaults accordingly. std::string convention = ps.getUntrackedParameter("convention", "Offline"); fakeFilterUnitMode_ = ps.getUntrackedParameter("fakeFilterUnitMode", false); @@ -508,6 +523,7 @@ DQMFileSaver::DQMFileSaver(const edm::ParameterSet &ps) //-------------------------------------------------------- void DQMFileSaver::beginJob() { +#if 0 nrun_ = nlumi_ = irun_ = 0; // Determine if we are running multithreading asking to the DQMStore. Not to be moved in the ctor @@ -517,9 +533,11 @@ void DQMFileSaver::beginJob() { transferDestination_ = edm::Service()->getStreamDestinations(stream_label_); mergeType_ = edm::Service()->getStreamMergeType(stream_label_, evf::MergeTypePB); } +#endif } std::shared_ptr DQMFileSaver::globalBeginRun(const edm::Run &r, const edm::EventSetup &) const { + return nullptr; ++nrun_; // For Filter Unit, create an empty ini file: @@ -549,6 +567,7 @@ void DQMFileSaver::analyze(edm::StreamID, const edm::Event &e, const edm::EventS } void DQMFileSaver::globalEndLuminosityBlock(const edm::LuminosityBlock &iLS, const edm::EventSetup &) const { + return; int ilumi = iLS.id().luminosityBlock(); int irun = iLS.id().run(); if (ilumi > 0 && saveByLumiSection_ > 0) { @@ -595,11 +614,89 @@ void DQMFileSaver::globalEndLuminosityBlock(const edm::LuminosityBlock &iLS, con } // after saving per LS, delete the old LS global histograms. - dbe_->deleteUnusedLumiHistograms(enableMultiThread_ ? irun : 0, ilumi); + // TODO: revise + //dbe_->deleteUnusedLumiHistograms(enableMultiThread_ ? irun : 0, ilumi); } } void DQMFileSaver::globalEndRun(const edm::Run &iRun, const edm::EventSetup &) const { + // TFile flushes to disk with fsync() on every TDirectory written to + // the file. This makes DQM file saving painfully slow, and + // ironically makes it _more_ likely the file saving gets + // interrupted and corrupts the file. The utility class below + // simply ignores the flush synchronisation. + class TFileNoSync : public TFile { + public: + TFileNoSync(char const* file, char const* opt) : TFile{file, opt} {} + Int_t SysSync(Int_t) override { return 0; } + }; + + std::cout << "DQMFileSaver::globalEndRun()" << std::endl; + + //std::string filename = "legacy.root"; //onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_, ROOT); + char suffix[64]; + sprintf(suffix, "R%09d", iRun.run()); + // workflow_ = "TestingLegacyOutputModule1"; + // child_ = "DQMIO"; + std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), "/Legacy/Output/Module1", "DQMIO", ROOT); + TFileNoSync *file = new TFileNoSync(filename.c_str(), "RECREATE"); // open file + + // Traverse all MEs + auto mes = dbe_->getAllContents(""); // TODO run/lumi and/or job? + for (auto me : mes) { + // Modify dirname to comply with DQM GUI format. Change: + // A/B/C/plot + // into: + // DQMData/Run X/A/Run summary/B/C/plot + std::string dirName = me->getPathname(); + uint64_t firstSlashPos = dirName.find("/"); + if (firstSlashPos == std::string::npos) { + firstSlashPos = dirName.length(); + } + dirName = dirName.substr(0, firstSlashPos) + "/Run summary" + dirName.substr(firstSlashPos, dirName.size()); + dirName = "DQMData/Run " + std::to_string(iRun.run()) + "/" + dirName; + + std::string objectName = me->getName(); + + // Create dir if it doesn't exist and cd into it + createDirectoryIfNeededAndCd(dirName); + + // INTs are saved as strings in this format: i=value + // REALs are saved as strings in this format: f=value + // STRINGs are saved as strings in this format: s="value" + if(me->kind() == MonitorElement::Kind::INT) { + int value = me->getIntValue(); + std::string content = "<" + objectName + ">i=" + std::to_string(value) + ""; + TObjString str(content.c_str()); + str.Write(); + } + else if(me->kind() == MonitorElement::Kind::REAL) { + double value = me->getFloatValue(); + std::string content = "<" + objectName + ">f=" + std::to_string(value) + ""; + TObjString str(content.c_str()); + str.Write(); + } + else if(me->kind() == MonitorElement::Kind::STRING) { + std::string value = me->getStringValue(); + std::string content = "<" + objectName + ">s=\"" + value.c_str() + "\""; + TObjString str(content.c_str()); + str.Write(); + } + else { + // Write a histogram + TH1* value = me->getTH1(); + value->Write(); + } + + // Go back to the root directory + gDirectory->cd("/"); + } + + file->Close(); + + std::cout << "DQMFileSaver::globalEndRun() after" << std::endl; + return; + int irun = iRun.id().run(); irun_ = irun; if (irun > 0 && saveByRun_ > 0 && (nrun_ % saveByRun_) == 0) { @@ -661,6 +758,8 @@ void DQMFileSaver::globalEndRun(const edm::Run &iRun, const edm::EventSetup &) c } void DQMFileSaver::endJob() { + return; + std::cout << "DQMFileSaver::endJob()" << std::endl; if (saveAtJobEnd_) { if (convention_ == Offline && forceRunNumber_ > 0) saveForOffline(workflow_, forceRunNumber_, 0); @@ -671,3 +770,46 @@ void DQMFileSaver::endJob() { << " job in Offline mode."; } } + +// Use this for saving monitoring objects in ROOT files with dir structure; +// cds into directory (creates it first if it doesn't exist); +// returns a success flag +bool DQMFileSaver::createDirectoryIfNeededAndCd(const std::string &path) const { + assert(!path.empty()); + + // Find the first path component. + size_t start = 0; + size_t end = path.find('/', start); + if (end == std::string::npos) + end = path.size(); + + while (true) { + // Check if this subdirectory component exists. If yes, make sure + // it is actually a subdirectory. Otherwise create or cd into it. + std::string part(path, start, end-start); + TObject* o = gDirectory->Get(part.c_str()); + if (o && ! dynamic_cast(o)) + raiseDQMError("DQMStore", "Attempt to create directory '%s' in a file" + " fails because the part '%s' already exists and is not" + " directory", path.c_str(), part.c_str()); + else if (!o) + gDirectory->mkdir(part.c_str()); + + if (!gDirectory->cd(part.c_str())) + raiseDQMError("DQMStore", "Attempt to create directory '%s' in a file" + " fails because could not cd into subdirectory '%s'", + path.c_str(), part.c_str()); + + // Stop if we reached the end, ignoring any trailing '/'. + if (end+1 >= path.size()) + break; + + // Find the next path component. + start = end+1; + end = path.find('/', start); + if (end == std::string::npos) + end = path.size(); + } + + return true; +} diff --git a/DQMServices/Components/plugins/DQMFileSaver.h b/DQMServices/Components/plugins/DQMFileSaver.h index 22cf913065dff..32a4c47d963e0 100644 --- a/DQMServices/Components/plugins/DQMFileSaver.h +++ b/DQMServices/Components/plugins/DQMFileSaver.h @@ -58,6 +58,8 @@ class DQMFileSaver : public edm::global::EDAnalyzer fileUpdate_; - DQMStore *dbe_; + DQMStore* dbe_; mutable std::atomic nrun_; mutable std::atomic nlumi_; From 94a8b090c1c414f6b7690f58875fc301b26ca693 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 9 Dec 2019 18:16:58 +0100 Subject: [PATCH 02/11] WIP: make DQMFileSaver a single file, and save() a function in the module. --- .../Components/plugins/DQMFileSaver.cc | 190 ++++++++++++++---- DQMServices/Components/plugins/DQMFileSaver.h | 102 ---------- DQMServices/Components/plugins/SealModule.cc | 2 - 3 files changed, 153 insertions(+), 141 deletions(-) delete mode 100644 DQMServices/Components/plugins/DQMFileSaver.h diff --git a/DQMServices/Components/plugins/DQMFileSaver.cc b/DQMServices/Components/plugins/DQMFileSaver.cc index 89ff0c0288e0c..07d872e8c67ca 100644 --- a/DQMServices/Components/plugins/DQMFileSaver.cc +++ b/DQMServices/Components/plugins/DQMFileSaver.cc @@ -1,7 +1,6 @@ -#include "DQMFileSaver.h" #include "DQMServices/Components/interface/fillJson.h" -#include "DQMServices/Core/src/DQMError.h" #include "DQMServices/Core/interface/DQMStore.h" +#include "DQMServices/Core/src/DQMError.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" @@ -34,6 +33,118 @@ #include #include +#include "FWCore/Framework/interface/global/EDAnalyzer.h" +#include "DQMServices/Core/interface/DQMStore.h" + +#include +#include +#include + +#include "FWCore/Framework/interface/global/EDAnalyzer.h" +#include "DQMServices/Core/interface/DQMStore.h" + +#include +#include +#include + +namespace evf { + class FastMonitoringService; +} +namespace saverDetails { + struct NoCache {}; +} // namespace saverDetails + +class DQMFileSaver : public edm::global::EDAnalyzer, + edm::LuminosityBlockCache > { +public: + typedef dqm::legacy::DQMStore DQMStore; + typedef dqm::legacy::MonitorElement MonitorElement; + DQMFileSaver(const edm::ParameterSet &ps); + + static boost::property_tree::ptree fillJson(int run, + int lumi, + const std::string &dataFilePathName, + const std::string &transferDestinationStr, + const std::string &mergeTypeStr, + evf::FastMonitoringService *fms); + +protected: + void save(std::string const &filename, + std::string const &path = "", + std::string const &pattern = "", + std::string const &rewrite = "", + uint32_t const run = 0, + uint32_t const lumi = 0); + + void beginJob() override; + std::shared_ptr globalBeginRun(const edm::Run &, const edm::EventSetup &) const override; + std::shared_ptr globalBeginLuminosityBlock(const edm::LuminosityBlock &, + const edm::EventSetup &) const override; + void analyze(edm::StreamID, const edm::Event &e, const edm::EventSetup &) const override; + void globalEndLuminosityBlock(const edm::LuminosityBlock &, const edm::EventSetup &) const override; + void globalEndRun(const edm::Run &, const edm::EventSetup &) const override; + void endJob() override; + +public: + enum Convention { + Online, + Offline, + FilterUnit, + }; + + enum FileFormat { ROOT, PB }; + +private: + void saveForOfflinePB(const std::string &workflow, int run) const; + void saveForOffline(const std::string &workflow, int run, int lumi) const; + + void saveForOnlinePB(int run, const std::string &suffix) const; + void saveForOnline(int run, const std::string &suffix, const std::string &rewrite) const; + + void saveForFilterUnit(const std::string &rewrite, int run, int lumi, const FileFormat fileFormat) const; + void saveJobReport(const std::string &filename) const; + + bool createDirectoryIfNeededAndCd(const std::string &path) const; + + Convention convention_; + FileFormat fileFormat_; + std::string workflow_; + std::string producer_; + std::string stream_label_; + std::string dirName_; + std::string child_; + std::string filterName_; + int version_; + bool runIsComplete_; + bool enableMultiThread_; + bool fakeFilterUnitMode_; + + int saveByLumiSection_; + int saveByRun_; + bool saveAtJobEnd_; + int saveReference_; + int saveReferenceQMin_; + int forceRunNumber_; + + std::string fileBaseName_; + mutable std::atomic fileUpdate_; + + DQMStore *dbe_; + mutable std::atomic nrun_; + mutable std::atomic nlumi_; + + // needed only for the harvesting step when saving in the endJob + mutable std::atomic irun_; + + // Services used in DAQ2 (so for FilterUnit case only) + evf::FastMonitoringService *fms_; + + static const std::string streamPrefix_; + static const std::string streamSuffix_; + std::string transferDestination_; + std::string mergeType_; +}; + //-------------------------------------------------------- const std::string DQMFileSaver::streamPrefix_("stream"); const std::string DQMFileSaver::streamSuffix_("Histograms"); @@ -342,7 +453,7 @@ void DQMFileSaver::saveForFilterUnit(const std::string &rewrite, } void DQMFileSaver::saveJobReport(const std::string &filename) const { - return; + return; // Report the file to job report service. edm::Service jr; if (jr.isAvailable()) { @@ -619,7 +730,12 @@ void DQMFileSaver::globalEndLuminosityBlock(const edm::LuminosityBlock &iLS, con } } -void DQMFileSaver::globalEndRun(const edm::Run &iRun, const edm::EventSetup &) const { +void DQMFileSaver::save(std::string const &filename, + std::string const &path /* = "" */, + std::string const &pattern /* = "" */, + std::string const &rewrite /* = "" */, + uint32_t const run /* = 0 */, + uint32_t const lumi /* = 0 */) { // TFile flushes to disk with fsync() on every TDirectory written to // the file. This makes DQM file saving painfully slow, and // ironically makes it _more_ likely the file saving gets @@ -627,76 +743,68 @@ void DQMFileSaver::globalEndRun(const edm::Run &iRun, const edm::EventSetup &) c // simply ignores the flush synchronisation. class TFileNoSync : public TFile { public: - TFileNoSync(char const* file, char const* opt) : TFile{file, opt} {} + TFileNoSync(char const *file, char const *opt) : TFile{file, opt} {} Int_t SysSync(Int_t) override { return 0; } }; std::cout << "DQMFileSaver::globalEndRun()" << std::endl; - //std::string filename = "legacy.root"; //onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_, ROOT); char suffix[64]; - sprintf(suffix, "R%09d", iRun.run()); - // workflow_ = "TestingLegacyOutputModule1"; - // child_ = "DQMIO"; - std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), "/Legacy/Output/Module1", "DQMIO", ROOT); - TFileNoSync *file = new TFileNoSync(filename.c_str(), "RECREATE"); // open file + sprintf(suffix, "R%09d", run); + TFileNoSync *file = new TFileNoSync(filename.c_str(), "RECREATE"); // open file // Traverse all MEs - auto mes = dbe_->getAllContents(""); // TODO run/lumi and/or job? + auto mes = dbe_->getAllContents(""); // TODO run/lumi and/or job? for (auto me : mes) { // Modify dirname to comply with DQM GUI format. Change: // A/B/C/plot // into: // DQMData/Run X/A/Run summary/B/C/plot std::string dirName = me->getPathname(); - uint64_t firstSlashPos = dirName.find("/"); + uint64_t firstSlashPos = dirName.find("/"); if (firstSlashPos == std::string::npos) { firstSlashPos = dirName.length(); } dirName = dirName.substr(0, firstSlashPos) + "/Run summary" + dirName.substr(firstSlashPos, dirName.size()); - dirName = "DQMData/Run " + std::to_string(iRun.run()) + "/" + dirName; - + dirName = "DQMData/Run " + std::to_string(run) + "/" + dirName; + std::string objectName = me->getName(); // Create dir if it doesn't exist and cd into it createDirectoryIfNeededAndCd(dirName); - + // INTs are saved as strings in this format: i=value // REALs are saved as strings in this format: f=value // STRINGs are saved as strings in this format: s="value" - if(me->kind() == MonitorElement::Kind::INT) { + if (me->kind() == MonitorElement::Kind::INT) { int value = me->getIntValue(); std::string content = "<" + objectName + ">i=" + std::to_string(value) + ""; TObjString str(content.c_str()); str.Write(); - } - else if(me->kind() == MonitorElement::Kind::REAL) { + } else if (me->kind() == MonitorElement::Kind::REAL) { double value = me->getFloatValue(); std::string content = "<" + objectName + ">f=" + std::to_string(value) + ""; TObjString str(content.c_str()); str.Write(); - } - else if(me->kind() == MonitorElement::Kind::STRING) { + } else if (me->kind() == MonitorElement::Kind::STRING) { std::string value = me->getStringValue(); std::string content = "<" + objectName + ">s=\"" + value.c_str() + "\""; TObjString str(content.c_str()); str.Write(); - } - else { + } else { // Write a histogram - TH1* value = me->getTH1(); + TH1 *value = me->getTH1(); value->Write(); } // Go back to the root directory gDirectory->cd("/"); } - + file->Close(); +} - std::cout << "DQMFileSaver::globalEndRun() after" << std::endl; - return; - +void DQMFileSaver::globalEndRun(const edm::Run &iRun, const edm::EventSetup &) const { int irun = iRun.id().run(); irun_ = irun; if (irun > 0 && saveByRun_ > 0 && (nrun_ % saveByRun_) == 0) { @@ -786,26 +894,31 @@ bool DQMFileSaver::createDirectoryIfNeededAndCd(const std::string &path) const { while (true) { // Check if this subdirectory component exists. If yes, make sure // it is actually a subdirectory. Otherwise create or cd into it. - std::string part(path, start, end-start); - TObject* o = gDirectory->Get(part.c_str()); - if (o && ! dynamic_cast(o)) - raiseDQMError("DQMStore", "Attempt to create directory '%s' in a file" + std::string part(path, start, end - start); + TObject *o = gDirectory->Get(part.c_str()); + if (o && !dynamic_cast(o)) + raiseDQMError("DQMStore", + "Attempt to create directory '%s' in a file" " fails because the part '%s' already exists and is not" - " directory", path.c_str(), part.c_str()); + " directory", + path.c_str(), + part.c_str()); else if (!o) gDirectory->mkdir(part.c_str()); if (!gDirectory->cd(part.c_str())) - raiseDQMError("DQMStore", "Attempt to create directory '%s' in a file" + raiseDQMError("DQMStore", + "Attempt to create directory '%s' in a file" " fails because could not cd into subdirectory '%s'", - path.c_str(), part.c_str()); + path.c_str(), + part.c_str()); // Stop if we reached the end, ignoring any trailing '/'. - if (end+1 >= path.size()) + if (end + 1 >= path.size()) break; // Find the next path component. - start = end+1; + start = end + 1; end = path.find('/', start); if (end == std::string::npos) end = path.size(); @@ -813,3 +926,6 @@ bool DQMFileSaver::createDirectoryIfNeededAndCd(const std::string &path) const { return true; } + +#include "FWCore/Framework/interface/MakerMacros.h" +DEFINE_FWK_MODULE(DQMFileSaver); diff --git a/DQMServices/Components/plugins/DQMFileSaver.h b/DQMServices/Components/plugins/DQMFileSaver.h deleted file mode 100644 index 32a4c47d963e0..0000000000000 --- a/DQMServices/Components/plugins/DQMFileSaver.h +++ /dev/null @@ -1,102 +0,0 @@ -#ifndef DQMSERVICES_COMPONENTS_DQMFILESAVER_H -#define DQMSERVICES_COMPONENTS_DQMFILESAVER_H - -#include "FWCore/Framework/interface/global/EDAnalyzer.h" -#include "DQMServices/Core/interface/DQMStore.h" - -#include -#include -#include - -namespace evf { - class FastMonitoringService; -} -namespace saverDetails { - struct NoCache {}; -} // namespace saverDetails - -class DQMFileSaver : public edm::global::EDAnalyzer, - edm::LuminosityBlockCache > { -public: - typedef dqm::legacy::DQMStore DQMStore; - typedef dqm::legacy::MonitorElement MonitorElement; - DQMFileSaver(const edm::ParameterSet &ps); - - static boost::property_tree::ptree fillJson(int run, - int lumi, - const std::string &dataFilePathName, - const std::string &transferDestinationStr, - const std::string &mergeTypeStr, - evf::FastMonitoringService *fms); - -protected: - void beginJob() override; - std::shared_ptr globalBeginRun(const edm::Run &, const edm::EventSetup &) const override; - std::shared_ptr globalBeginLuminosityBlock(const edm::LuminosityBlock &, - const edm::EventSetup &) const override; - void analyze(edm::StreamID, const edm::Event &e, const edm::EventSetup &) const override; - void globalEndLuminosityBlock(const edm::LuminosityBlock &, const edm::EventSetup &) const override; - void globalEndRun(const edm::Run &, const edm::EventSetup &) const override; - void endJob() override; - -public: - enum Convention { - Online, - Offline, - FilterUnit, - }; - - enum FileFormat { ROOT, PB }; - -private: - void saveForOfflinePB(const std::string &workflow, int run) const; - void saveForOffline(const std::string &workflow, int run, int lumi) const; - - void saveForOnlinePB(int run, const std::string &suffix) const; - void saveForOnline(int run, const std::string &suffix, const std::string &rewrite) const; - - void saveForFilterUnit(const std::string &rewrite, int run, int lumi, const FileFormat fileFormat) const; - void saveJobReport(const std::string &filename) const; - - bool createDirectoryIfNeededAndCd(const std::string &path) const; - - Convention convention_; - FileFormat fileFormat_; - std::string workflow_; - std::string producer_; - std::string stream_label_; - std::string dirName_; - std::string child_; - std::string filterName_; - int version_; - bool runIsComplete_; - bool enableMultiThread_; - bool fakeFilterUnitMode_; - - int saveByLumiSection_; - int saveByRun_; - bool saveAtJobEnd_; - int saveReference_; - int saveReferenceQMin_; - int forceRunNumber_; - - std::string fileBaseName_; - mutable std::atomic fileUpdate_; - - DQMStore* dbe_; - mutable std::atomic nrun_; - mutable std::atomic nlumi_; - - // needed only for the harvesting step when saving in the endJob - mutable std::atomic irun_; - - // Services used in DAQ2 (so for FilterUnit case only) - evf::FastMonitoringService *fms_; - - static const std::string streamPrefix_; - static const std::string streamSuffix_; - std::string transferDestination_; - std::string mergeType_; -}; - -#endif // DQMSERVICES_COMPONEntS_DQMFILESAVER_H diff --git a/DQMServices/Components/plugins/SealModule.cc b/DQMServices/Components/plugins/SealModule.cc index 48d036fa2dee7..17659829db534 100644 --- a/DQMServices/Components/plugins/SealModule.cc +++ b/DQMServices/Components/plugins/SealModule.cc @@ -4,8 +4,6 @@ // The module providing event information #include "DQMEventInfo.h" DEFINE_FWK_MODULE(DQMEventInfo); -#include "DQMFileSaver.h" -DEFINE_FWK_MODULE(DQMFileSaver); #include "DQMFEDIntegrityClient.h" DEFINE_FWK_MODULE(DQMFEDIntegrityClient); #include "DQMStoreStats.h" From 9c425b0b385de88efab9d5764a60bc8a68084dbb Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 9 Dec 2019 18:19:57 +0100 Subject: [PATCH 03/11] Revert to old logic in DQMFileSaver --- .../Components/plugins/DQMFileSaver.cc | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/DQMServices/Components/plugins/DQMFileSaver.cc b/DQMServices/Components/plugins/DQMFileSaver.cc index 07d872e8c67ca..d99c237ccc5ab 100644 --- a/DQMServices/Components/plugins/DQMFileSaver.cc +++ b/DQMServices/Components/plugins/DQMFileSaver.cc @@ -183,7 +183,6 @@ static std::string onlineOfflineFileName(const std::string &fileBaseName, } void DQMFileSaver::saveForOfflinePB(const std::string &workflow, int run) const { - return; char suffix[64]; sprintf(suffix, "R%09d", run); std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_, PB); @@ -191,8 +190,6 @@ void DQMFileSaver::saveForOfflinePB(const std::string &workflow, int run) const } void DQMFileSaver::saveForOffline(const std::string &workflow, int run, int lumi) const { - return; - std::cout << "DQMFileSaver::saveForOffline" << std::endl; char suffix[64]; sprintf(suffix, "R%09d", run); @@ -279,7 +276,6 @@ static void doSaveForOnline(DQMFileSaver::DQMStore *store, int saveRefQMin, const std::string &filterName, DQMFileSaver::FileFormat fileFormat) { - return; // TODO(rovere): fix the online case. so far we simply rely on the // fact that we assume we will not run multithreaded in online. if (fileFormat == DQMFileSaver::ROOT) @@ -289,13 +285,12 @@ static void doSaveForOnline(DQMFileSaver::DQMStore *store, } void DQMFileSaver::saveForOnlinePB(int run, const std::string &suffix) const { - return; // The file name contains the Online workflow name, // as we do not want to look inside the DQMStore, // and the @a suffix, defined in the run/lumi transitions. // TODO(diguida): add the possibility to change the dir structure with rewrite. std::string filename = onlineOfflineFileName(fileBaseName_, suffix, workflow_, child_, PB); - doSaveForOnline(&*dbe_, + doSaveForOnline(dbe_, run, enableMultiThread_, filename, @@ -309,14 +304,13 @@ void DQMFileSaver::saveForOnlinePB(int run, const std::string &suffix) const { } void DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std::string &rewrite) const { -#if 0 std::vector systems = (dbe_->cd(), dbe_->getSubdirs()); for (size_t i = 0, e = systems.size(); i != e; ++i) { if (systems[i] != "Reference") { dbe_->cd(); if (MonitorElement *me = dbe_->get(systems[i] + "/EventInfo/processName")) { - doSaveForOnline(&*dbe_, + doSaveForOnline(dbe_, run, enableMultiThread_, fileBaseName_ + me->getStringValue() + suffix + child_ + ".root", @@ -339,7 +333,7 @@ void DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std:: std::vector pNamesVector = dbe_->getMatchingContents("^" + systems[i] + "/.*/EventInfo/processName", lat::Regexp::Perl); if (!pNamesVector.empty()) { - doSaveForOnline(&*dbe_, + doSaveForOnline(dbe_, run, enableMultiThread_, fileBaseName_ + systems[i] + suffix + child_ + ".root", @@ -358,7 +352,7 @@ void DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std:: // if no EventInfo Folder is found, then store subsystem wise for (size_t i = 0, e = systems.size(); i != e; ++i) if (systems[i] != "Reference") - doSaveForOnline(&*dbe_, + doSaveForOnline(dbe_, run, enableMultiThread_, fileBaseName_ + systems[i] + suffix + child_ + ".root", @@ -369,7 +363,6 @@ void DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std:: saveReferenceQMin_, "", ROOT); -#endif } boost::property_tree::ptree DQMFileSaver::fillJson(int run, @@ -385,7 +378,6 @@ void DQMFileSaver::saveForFilterUnit(const std::string &rewrite, int run, int lumi, const DQMFileSaver::FileFormat fileFormat) const { - return; // get from DAQ2 services where to store the files according to their format namespace bpt = boost::property_tree; @@ -453,7 +445,6 @@ void DQMFileSaver::saveForFilterUnit(const std::string &rewrite, } void DQMFileSaver::saveJobReport(const std::string &filename) const { - return; // Report the file to job report service. edm::Service jr; if (jr.isAvailable()) { @@ -634,7 +625,6 @@ DQMFileSaver::DQMFileSaver(const edm::ParameterSet &ps) //-------------------------------------------------------- void DQMFileSaver::beginJob() { -#if 0 nrun_ = nlumi_ = irun_ = 0; // Determine if we are running multithreading asking to the DQMStore. Not to be moved in the ctor @@ -644,11 +634,9 @@ void DQMFileSaver::beginJob() { transferDestination_ = edm::Service()->getStreamDestinations(stream_label_); mergeType_ = edm::Service()->getStreamMergeType(stream_label_, evf::MergeTypePB); } -#endif } std::shared_ptr DQMFileSaver::globalBeginRun(const edm::Run &r, const edm::EventSetup &) const { - return nullptr; ++nrun_; // For Filter Unit, create an empty ini file: @@ -678,7 +666,6 @@ void DQMFileSaver::analyze(edm::StreamID, const edm::Event &e, const edm::EventS } void DQMFileSaver::globalEndLuminosityBlock(const edm::LuminosityBlock &iLS, const edm::EventSetup &) const { - return; int ilumi = iLS.id().luminosityBlock(); int irun = iLS.id().run(); if (ilumi > 0 && saveByLumiSection_ > 0) { @@ -866,8 +853,6 @@ void DQMFileSaver::globalEndRun(const edm::Run &iRun, const edm::EventSetup &) c } void DQMFileSaver::endJob() { - return; - std::cout << "DQMFileSaver::endJob()" << std::endl; if (saveAtJobEnd_) { if (convention_ == Offline && forceRunNumber_ > 0) saveForOffline(workflow_, forceRunNumber_, 0); From eddafb9837b575d0a5f212010240b3f77e1932cc Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 9 Dec 2019 19:50:10 +0100 Subject: [PATCH 04/11] Strip out old code from DQMFileSaver It looks like this was never actually used in online DQM or HLT in run2. There are plenty of threading-related code paths, but DQMServices/FileIO seems to have taken over at some point. --- .../Components/plugins/DQMFileSaver.cc | 745 +++--------------- 1 file changed, 91 insertions(+), 654 deletions(-) diff --git a/DQMServices/Components/plugins/DQMFileSaver.cc b/DQMServices/Components/plugins/DQMFileSaver.cc index d99c237ccc5ab..fb61579423ef0 100644 --- a/DQMServices/Components/plugins/DQMFileSaver.cc +++ b/DQMServices/Components/plugins/DQMFileSaver.cc @@ -1,6 +1,4 @@ -#include "DQMServices/Components/interface/fillJson.h" #include "DQMServices/Core/interface/DQMStore.h" -#include "DQMServices/Core/src/DQMError.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" @@ -12,9 +10,6 @@ #include "DataFormats/Histograms/interface/DQMToken.h" -#include "EventFilter/Utilities/interface/EvFDaqDirector.h" -#include "EventFilter/Utilities/interface/FastMonitoringService.h" - #include #include #include @@ -28,127 +23,62 @@ #include "TFile.h" -#include -#include -#include #include -#include "FWCore/Framework/interface/global/EDAnalyzer.h" -#include "DQMServices/Core/interface/DQMStore.h" +#include "FWCore/Framework/interface/one/EDAnalyzer.h" -#include -#include #include -#include "FWCore/Framework/interface/global/EDAnalyzer.h" -#include "DQMServices/Core/interface/DQMStore.h" - -#include -#include -#include - -namespace evf { - class FastMonitoringService; -} namespace saverDetails { struct NoCache {}; } // namespace saverDetails -class DQMFileSaver : public edm::global::EDAnalyzer, - edm::LuminosityBlockCache > { +// NOTE: This module is only save to use in a very restricted set of circumstances: +// - In offline HARVESTING jobs, running single-threaded. RUN and JOB histograms are saved at end of JOB. +// - In multi-run harvesting. JOB histograms are save at end of job. +// - This includes ALCAHARVEST. TODO: check if the data written there is needed for the PCL. +// This module is not used in online. This module is (hopefully?) not used at HLT. +// Online and HLT use modules in DQMServices/FileIO. +class DQMFileSaver : public edm::one::EDAnalyzer { public: typedef dqm::legacy::DQMStore DQMStore; typedef dqm::legacy::MonitorElement MonitorElement; DQMFileSaver(const edm::ParameterSet &ps); - static boost::property_tree::ptree fillJson(int run, - int lumi, - const std::string &dataFilePathName, - const std::string &transferDestinationStr, - const std::string &mergeTypeStr, - evf::FastMonitoringService *fms); - protected: - void save(std::string const &filename, - std::string const &path = "", - std::string const &pattern = "", - std::string const &rewrite = "", - uint32_t const run = 0, - uint32_t const lumi = 0); - - void beginJob() override; - std::shared_ptr globalBeginRun(const edm::Run &, const edm::EventSetup &) const override; - std::shared_ptr globalBeginLuminosityBlock(const edm::LuminosityBlock &, - const edm::EventSetup &) const override; - void analyze(edm::StreamID, const edm::Event &e, const edm::EventSetup &) const override; - void globalEndLuminosityBlock(const edm::LuminosityBlock &, const edm::EventSetup &) const override; - void globalEndRun(const edm::Run &, const edm::EventSetup &) const override; + void beginRun(edm::Run const &, edm::EventSetup const &) override{}; + void analyze(const edm::Event &e, const edm::EventSetup &) override; + void endRun(const edm::Run &, const edm::EventSetup &) override; void endJob() override; -public: - enum Convention { - Online, - Offline, - FilterUnit, - }; - - enum FileFormat { ROOT, PB }; - private: - void saveForOfflinePB(const std::string &workflow, int run) const; - void saveForOffline(const std::string &workflow, int run, int lumi) const; - - void saveForOnlinePB(int run, const std::string &suffix) const; - void saveForOnline(int run, const std::string &suffix, const std::string &rewrite) const; - - void saveForFilterUnit(const std::string &rewrite, int run, int lumi, const FileFormat fileFormat) const; - void saveJobReport(const std::string &filename) const; + void saveForOffline(const std::string &workflow, int run, int lumi); + void saveJobReport(const std::string &filename); - bool createDirectoryIfNeededAndCd(const std::string &path) const; + void save(std::string const &filename, uint32_t const run = 0); + bool createDirectoryIfNeededAndCd(const std::string &path); - Convention convention_; - FileFormat fileFormat_; std::string workflow_; std::string producer_; std::string stream_label_; std::string dirName_; std::string child_; - std::string filterName_; int version_; bool runIsComplete_; - bool enableMultiThread_; - bool fakeFilterUnitMode_; - int saveByLumiSection_; int saveByRun_; bool saveAtJobEnd_; - int saveReference_; - int saveReferenceQMin_; int forceRunNumber_; std::string fileBaseName_; - mutable std::atomic fileUpdate_; DQMStore *dbe_; - mutable std::atomic nrun_; - mutable std::atomic nlumi_; + int nrun_; // needed only for the harvesting step when saving in the endJob - mutable std::atomic irun_; - - // Services used in DAQ2 (so for FilterUnit case only) - evf::FastMonitoringService *fms_; - - static const std::string streamPrefix_; - static const std::string streamSuffix_; - std::string transferDestination_; - std::string mergeType_; + int irun_; }; -//-------------------------------------------------------- -const std::string DQMFileSaver::streamPrefix_("stream"); -const std::string DQMFileSaver::streamSuffix_("Histograms"); - //-------------------------------------------------------- static void getAnInt(const edm::ParameterSet &ps, int &value, const std::string &name) { value = ps.getUntrackedParameter(name, value); @@ -157,20 +87,10 @@ static void getAnInt(const edm::ParameterSet &ps, int &value, const std::string << "'. Must be -1 or >= 1."; } -static std::string dataFileExtension(DQMFileSaver::FileFormat fileFormat) { - std::string extension; - if (fileFormat == DQMFileSaver::ROOT) - extension = ".root"; - else if (fileFormat == DQMFileSaver::PB) - extension = ".pb"; - return extension; -} - static std::string onlineOfflineFileName(const std::string &fileBaseName, const std::string &suffix, const std::string &workflow, - const std::string &child, - DQMFileSaver::FileFormat fileFormat) { + const std::string &child) { size_t pos = 0; std::string wflow; wflow.reserve(workflow.size() + 3); @@ -178,273 +98,44 @@ static std::string onlineOfflineFileName(const std::string &fileBaseName, while ((pos = wflow.find('/', pos)) != std::string::npos) wflow.replace(pos++, 1, "__"); - std::string filename = fileBaseName + suffix + wflow + child + dataFileExtension(fileFormat); + std::string filename = fileBaseName + suffix + wflow + child + ".root"; return filename; } -void DQMFileSaver::saveForOfflinePB(const std::string &workflow, int run) const { +void DQMFileSaver::saveForOffline(const std::string &workflow, int run, int lumi) { char suffix[64]; sprintf(suffix, "R%09d", run); - std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_, PB); - dbe_->savePB(filename, filterName_); -} -void DQMFileSaver::saveForOffline(const std::string &workflow, int run, int lumi) const { - char suffix[64]; - sprintf(suffix, "R%09d", run); + std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_); + assert(lumi == 0); - char rewrite[128]; - if (lumi == 0) // save for run - sprintf(rewrite, "\\1Run %d/\\2/Run summary", run); - else - sprintf(rewrite, "\\1Run %d/\\2/By Lumi Section %d-%d", run, lumi, lumi); - - std::string filename = onlineOfflineFileName(fileBaseName_, std::string(suffix), workflow, child_, ROOT); - - if (lumi == 0) // save for run - { - // set run end flag - dbe_->cd(); - dbe_->setCurrentFolder("Info/ProvInfo"); - - // do this, because ProvInfo is not yet run in offline DQM - MonitorElement *me = dbe_->get("Info/ProvInfo/CMSSW"); - if (!me) - me = dbe_->bookString("CMSSW", edm::getReleaseVersion().c_str()); - - me = dbe_->get("Info/ProvInfo/runIsComplete"); - if (!me) - me = dbe_->bookFloat("runIsComplete"); - - if (me) { - if (runIsComplete_) - me->Fill(1.); - else - me->Fill(0.); - } + // set run end flag + dbe_->cd(); + dbe_->setCurrentFolder("Info/ProvInfo"); - dbe_->save(filename, - "", - "^(Reference/)?([^/]+)", - rewrite, - enableMultiThread_ ? run : 0, - lumi, - (DQMStore::SaveReferenceTag)saveReference_, - saveReferenceQMin_, - fileUpdate_ ? "UPDATE" : "RECREATE"); - - // save the JobReport - saveJobReport(filename); - } else // save EventInfo folders for luminosity sections - { - std::vector systems = (dbe_->cd(), dbe_->getSubdirs()); - - edm::LogAbsolute msg("fileAction"); - msg << "DQMFileSaver: storing EventInfo folders for Run: " << run << ", Lumi Section: " << lumi << ", Subsystems: "; - - for (size_t i = 0, e = systems.size(); i != e; ++i) { - if (systems[i] != "Reference") { - dbe_->cd(); - msg << systems[i] << " "; - - dbe_->save(filename, - systems[i] + "/EventInfo", - "^(Reference/)?([^/]+)", - rewrite, - enableMultiThread_ ? run : 0, - lumi, - DQMStore::SaveWithoutReference, - dqm::qstatus::STATUS_OK, - fileUpdate_ ? "UPDATE" : "RECREATE"); - - // from now on update newly created file - if (fileUpdate_.load() == 0) - fileUpdate_ = 1; - } - } - } -} + // do this, because ProvInfo is not yet run in offline DQM + MonitorElement *me = dbe_->get("Info/ProvInfo/CMSSW"); + if (!me) + me = dbe_->bookString("CMSSW", edm::getReleaseVersion().c_str()); -static void doSaveForOnline(DQMFileSaver::DQMStore *store, - int run, - bool enableMultiThread, - const std::string &filename, - const std::string &directory, - const std::string &rxpat, - const std::string &rewrite, - DQMFileSaver::DQMStore::SaveReferenceTag saveref, - int saveRefQMin, - const std::string &filterName, - DQMFileSaver::FileFormat fileFormat) { - // TODO(rovere): fix the online case. so far we simply rely on the - // fact that we assume we will not run multithreaded in online. - if (fileFormat == DQMFileSaver::ROOT) - store->save(filename, directory, rxpat, rewrite, enableMultiThread ? run : 0, 0, saveref, saveRefQMin); - else if (fileFormat == DQMFileSaver::PB) - store->savePB(filename, filterName, enableMultiThread ? run : 0); -} - -void DQMFileSaver::saveForOnlinePB(int run, const std::string &suffix) const { - // The file name contains the Online workflow name, - // as we do not want to look inside the DQMStore, - // and the @a suffix, defined in the run/lumi transitions. - // TODO(diguida): add the possibility to change the dir structure with rewrite. - std::string filename = onlineOfflineFileName(fileBaseName_, suffix, workflow_, child_, PB); - doSaveForOnline(dbe_, - run, - enableMultiThread_, - filename, - "", - "^(Reference/)?([^/]+)", - "\\1\\2", - (DQMStore::SaveReferenceTag)saveReference_, - saveReferenceQMin_, - filterName_, - PB); -} + me = dbe_->get("Info/ProvInfo/runIsComplete"); + if (!me) + me = dbe_->bookFloat("runIsComplete"); -void DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std::string &rewrite) const { - std::vector systems = (dbe_->cd(), dbe_->getSubdirs()); - - for (size_t i = 0, e = systems.size(); i != e; ++i) { - if (systems[i] != "Reference") { - dbe_->cd(); - if (MonitorElement *me = dbe_->get(systems[i] + "/EventInfo/processName")) { - doSaveForOnline(dbe_, - run, - enableMultiThread_, - fileBaseName_ + me->getStringValue() + suffix + child_ + ".root", - "", - "^(Reference/)?([^/]+)", - rewrite, - (DQMStore::SaveReferenceTag)saveReference_, - saveReferenceQMin_, - "", - ROOT); - return; - } - } - } - - // look for EventInfo folder in an unorthodox location - for (size_t i = 0, e = systems.size(); i != e; ++i) - if (systems[i] != "Reference") { - dbe_->cd(); - std::vector pNamesVector = - dbe_->getMatchingContents("^" + systems[i] + "/.*/EventInfo/processName", lat::Regexp::Perl); - if (!pNamesVector.empty()) { - doSaveForOnline(dbe_, - run, - enableMultiThread_, - fileBaseName_ + systems[i] + suffix + child_ + ".root", - "", - "^(Reference/)?([^/]+)", - rewrite, - (DQMStore::SaveReferenceTag)saveReference_, - saveReferenceQMin_, - "", - ROOT); - pNamesVector.clear(); - return; - } - } - - // if no EventInfo Folder is found, then store subsystem wise - for (size_t i = 0, e = systems.size(); i != e; ++i) - if (systems[i] != "Reference") - doSaveForOnline(dbe_, - run, - enableMultiThread_, - fileBaseName_ + systems[i] + suffix + child_ + ".root", - systems[i], - "^(Reference/)?([^/]+)", - rewrite, - (DQMStore::SaveReferenceTag)saveReference_, - saveReferenceQMin_, - "", - ROOT); -} - -boost::property_tree::ptree DQMFileSaver::fillJson(int run, - int lumi, - const std::string &dataFilePathName, - const std::string &transferDestinationStr, - const std::string &mergeTypeStr, - evf::FastMonitoringService *fms) { - return dqmfilesaver::fillJson(run, lumi, dataFilePathName, transferDestinationStr, mergeTypeStr, fms); -} - -void DQMFileSaver::saveForFilterUnit(const std::string &rewrite, - int run, - int lumi, - const DQMFileSaver::FileFormat fileFormat) const { - // get from DAQ2 services where to store the files according to their format - namespace bpt = boost::property_tree; - - std::string openJsonFilePathName; - std::string jsonFilePathName; - std::string openHistoFilePathName; - std::string histoFilePathName; - - // create the files names - if (fakeFilterUnitMode_) { - std::string runDir = str(boost::format("%s/run%06d") % dirName_ % run); - std::string baseName = str(boost::format("%s/run%06d_ls%04d_%s") % runDir % run % lumi % stream_label_); - - boost::filesystem::create_directories(runDir); - - jsonFilePathName = baseName + ".jsn"; - openJsonFilePathName = jsonFilePathName + ".open"; - - histoFilePathName = baseName + dataFileExtension(fileFormat); - openHistoFilePathName = histoFilePathName + ".open"; - } else { - openJsonFilePathName = edm::Service()->getOpenOutputJsonFilePath(lumi, stream_label_); - jsonFilePathName = edm::Service()->getOutputJsonFilePath(lumi, stream_label_); - - if (fileFormat == ROOT) { - openHistoFilePathName = edm::Service()->getOpenRootHistogramFilePath(lumi, stream_label_); - histoFilePathName = edm::Service()->getRootHistogramFilePath(lumi, stream_label_); - } else if (fileFormat == PB) { - openHistoFilePathName = - edm::Service()->getOpenProtocolBufferHistogramFilePath(lumi, stream_label_); - histoFilePathName = edm::Service()->getProtocolBufferHistogramFilePath(lumi, stream_label_); - } + if (me) { + if (runIsComplete_) + me->Fill(1.); + else + me->Fill(0.); } - if (fms_ ? fms_->getEventsProcessedForLumi(lumi) : true) { - if (fileFormat == ROOT) { - // Save the file with the full directory tree, - // modifying it according to @a rewrite, - // but not looking for MEs inside the DQMStore, as in the online case, - // nor filling new MEs, as in the offline case. - dbe_->save(openHistoFilePathName, - "", - "^(Reference/)?([^/]+)", - rewrite, - enableMultiThread_ ? run : 0, - lumi, - (DQMStore::SaveReferenceTag)saveReference_, - saveReferenceQMin_, - fileUpdate_ ? "UPDATE" : "RECREATE"); - } else if (fileFormat == PB) { - // Save the file in the open directory. - dbe_->savePB(openHistoFilePathName, filterName_, enableMultiThread_ ? run : 0, lumi); - } else - throw cms::Exception("DQMFileSaver") << "Internal error, can save files" - << " only in ROOT or ProtocolBuffer format."; - - // Now move the the data and json files into the output directory. - rename(openHistoFilePathName.c_str(), histoFilePathName.c_str()); - } + this->save(filename, run); - // Write the json file in the open directory. - bpt::ptree pt = fillJson(run, lumi, histoFilePathName, transferDestination_, mergeType_, fms_); - write_json(openJsonFilePathName, pt); - rename(openJsonFilePathName.c_str(), jsonFilePathName.c_str()); + // save the JobReport + saveJobReport(filename); } -void DQMFileSaver::saveJobReport(const std::string &filename) const { +void DQMFileSaver::saveJobReport(const std::string &filename) { // Report the file to job report service. edm::Service jr; if (jr.isAvailable()) { @@ -457,272 +148,92 @@ void DQMFileSaver::saveJobReport(const std::string &filename) const { //-------------------------------------------------------- DQMFileSaver::DQMFileSaver(const edm::ParameterSet &ps) - : convention_(Offline), - fileFormat_(ROOT), + : + workflow_(""), producer_("DQM"), - stream_label_(""), dirName_("."), child_(""), - filterName_(""), version_(1), runIsComplete_(false), - enableMultiThread_(false), - saveByLumiSection_(-1), saveByRun_(-1), saveAtJobEnd_(false), - saveReference_(DQMStore::SaveWithReference), - saveReferenceQMin_(dqm::qstatus::STATUS_OK), forceRunNumber_(-1), fileBaseName_(""), - fileUpdate_(0), dbe_(&*edm::Service()), nrun_(0), - nlumi_(0), - irun_(0), - fms_(nullptr) { + irun_(0) { + // Note: this is insufficient, we also need to enforce running *after* all + // DQMEDAnalyzers (a.k.a. EDProducers) in endJob. + // This is not supported in edm currently. consumesMany(); - // Determine the file saving convention, and adjust defaults accordingly. - std::string convention = ps.getUntrackedParameter("convention", "Offline"); - fakeFilterUnitMode_ = ps.getUntrackedParameter("fakeFilterUnitMode", false); - - if (convention == "Offline") - convention_ = Offline; - else if (convention == "Online") - convention_ = Online; - else if (convention == "FilterUnit") - convention_ = FilterUnit; - else - throw cms::Exception("DQMFileSaver") << "Invalid 'convention' parameter '" << convention << "'." - << " Expected one of 'Online' or 'Offline' or 'FilterUnit'."; - - // If this is neither online nor FU convention, check workflow. - // In this way, FU is treated as online, so we cannot specify a workflow. TBC - if (convention_ != Online && convention_ != FilterUnit) { - workflow_ = ps.getUntrackedParameter("workflow", workflow_); - if (workflow_.empty() || workflow_[0] != '/' || *workflow_.rbegin() == '/' || - std::count(workflow_.begin(), workflow_.end(), '/') != 3 || - workflow_.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789" - "-_/") != std::string::npos) - throw cms::Exception("DQMFileSaver") << "Invalid 'workflow' parameter '" << workflow_ << "'. Expected '/A/B/C'."; - } else if (!ps.getUntrackedParameter("workflow", "").empty()) - throw cms::Exception("DQMFileSaver") - << "The 'workflow' parameter must be empty in 'Online' and 'FilterUnit' conventions."; - else // for online set parameters - { - workflow_ = "/Global/Online/P5"; - } - - // Determine the file format, and adjust defaults accordingly. - std::string fileFormat = ps.getUntrackedParameter("fileFormat", "ROOT"); - if (fileFormat == "ROOT") - fileFormat_ = ROOT; - else if (fileFormat == "PB") - fileFormat_ = PB; - else - throw cms::Exception("DQMFileSaver") << "Invalid 'fileFormat' parameter '" << fileFormat << "'." - << " Expected one of 'ROOT' or 'PB'."; - - // Allow file producer to be set to specific values in certain conditions. - producer_ = ps.getUntrackedParameter("producer", producer_); - // Setting the same constraints on file producer both for online and FilterUnit conventions - // TODO(diguida): limit the producer for FilterUnit to be 'DQM' or 'HLTDQM'? - // TODO(diguida): how to handle histograms produced in the playback for the FU case? - if ((convention_ == Online || convention_ == FilterUnit) && producer_ != "DQM" && producer_ != "HLTDQM" && - producer_ != "Playback") { - throw cms::Exception("DQMFileSaver") << "Invalid 'producer' parameter '" << producer_ - << "'. Expected 'DQM', 'HLTDQM' or 'Playback'."; - } else if (convention_ != Online && convention != FilterUnit && producer_ != "DQM") { - throw cms::Exception("DQMFileSaver") << "Invalid 'producer' parameter '" << producer_ << "'. Expected 'DQM'."; - } - - stream_label_ = streamPrefix_ + producer_ + streamSuffix_; + workflow_ = ps.getUntrackedParameter("workflow", workflow_); + if (workflow_.empty() || workflow_[0] != '/' || *workflow_.rbegin() == '/' || + std::count(workflow_.begin(), workflow_.end(), '/') != 3 || + workflow_.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789" + "-_/") != std::string::npos) + throw cms::Exception("DQMFileSaver") << "Invalid 'workflow' parameter '" << workflow_ << "'. Expected '/A/B/C'."; // version number to be used in filename + // Note that version *must* always be 1 vor DQMGUI upload. version_ = ps.getUntrackedParameter("version", version_); // flag to signal that file contains data from complete run runIsComplete_ = ps.getUntrackedParameter("runIsComplete", runIsComplete_); - // Check how we should save the references. - std::string refsave = ps.getUntrackedParameter("referenceHandling", "default"); - if (refsave == "default") - ; - else if (refsave == "skip") { - saveReference_ = DQMStore::SaveWithoutReference; - // std::cout << "skip saving all references" << std::endl; - } else if (refsave == "all") { - saveReference_ = DQMStore::SaveWithReference; - // std::cout << "saving all references" << std::endl; - } else if (refsave == "qtests") { - saveReference_ = DQMStore::SaveWithReferenceForQTest; - // std::cout << "saving qtest references" << std::endl; - } else - throw cms::Exception("DQMFileSaver") << "Invalid 'referenceHandling' parameter '" << refsave - << "'. Expected 'default', 'skip', 'all' or 'qtests'."; - - // Check minimum required quality test result for which reference is saved. - saveReferenceQMin_ = ps.getUntrackedParameter("referenceRequireStatus", saveReferenceQMin_); - // Get and check the output directory. struct stat s; dirName_ = ps.getUntrackedParameter("dirName", dirName_); if (dirName_.empty() || stat(dirName_.c_str(), &s) == -1) throw cms::Exception("DQMFileSaver") << "Invalid 'dirName' parameter '" << dirName_ << "'."; - filterName_ = ps.getUntrackedParameter("filterName", filterName_); // Find out when and how to save files. The following contraints apply: - // - For online, filter unit, and offline, allow files to be saved per lumi - // - For online, allow files to be saved per run, at event and time intervals. // - For offline allow files to be saved per run, at job end, and run number to be overridden (for mc data). - if (convention_ == Online || convention_ == Offline || convention_ == FilterUnit) { - getAnInt(ps, saveByLumiSection_, "saveByLumiSection"); - } - - if (convention_ == Online) { - getAnInt(ps, saveByRun_, "saveByRun"); - } - if (convention_ == Offline) { - getAnInt(ps, saveByRun_, "saveByRun"); - getAnInt(ps, forceRunNumber_, "forceRunNumber"); - saveAtJobEnd_ = ps.getUntrackedParameter("saveAtJobEnd", saveAtJobEnd_); - } + getAnInt(ps, saveByRun_, "saveByRun"); + getAnInt(ps, forceRunNumber_, "forceRunNumber"); + saveAtJobEnd_ = ps.getUntrackedParameter("saveAtJobEnd", saveAtJobEnd_); // Set up base file name: // - for online and offline, follow the convention /_V<4digits>_ - // - for FilterUnit, we need to follow the DAQ2 convention, so we need the run and lumisection: - // the path is provided by the DAQ director service. - if (convention_ != FilterUnit) { - char version[8]; - sprintf(version, "_V%04d_", int(version_)); - version[7] = '\0'; - fileBaseName_ = dirName_ + "/" + producer_ + version; - } else if (fakeFilterUnitMode_) { - edm::LogInfo("DQMFileSaver") - << "Fake FU mode, files are saved under /runXXXXXX/runXXXXXX_lsXXXX_.pb.\n"; - } else { - // For FU, dirName_ will not be considered at all - edm::LogInfo("DQMFileSaver") << "The base dir provided in the configuration '" << dirName_ << "'\n" - << " will not be considered: for FU, the DAQ2 services will handle directories\n"; - //check that DAQ2 services are available: throw if not - fms_ = (evf::FastMonitoringService *)(edm::Service().operator->()); - evf::EvFDaqDirector *daqDirector = (evf::EvFDaqDirector *)(edm::Service().operator->()); - - if (!(fms_ && daqDirector)) - throw cms::Exception("DQMFileSaver") << "Internal error, cannot initialize DAQ services."; - } + char version[8]; + sprintf(version, "_V%04d_", int(version_)); + version[7] = '\0'; + fileBaseName_ = dirName_ + "/" + producer_ + version; // Log some information what we will do. edm::LogInfo("DQMFileSaver") << "DQM file saving settings:\n" << " using base file name '" << fileBaseName_ << "'\n" << " forcing run number " << forceRunNumber_ << "\n" - << " saving every " << saveByLumiSection_ << " lumi section(s)\n" << " saving every " << saveByRun_ << " run(s)\n" << " saving at job end: " << (saveAtJobEnd_ ? "yes" : "no") << "\n"; } -//-------------------------------------------------------- -void DQMFileSaver::beginJob() { - nrun_ = nlumi_ = irun_ = 0; - - // Determine if we are running multithreading asking to the DQMStore. Not to be moved in the ctor - enableMultiThread_ = dbe_->enableMultiThread_; - - if ((convention_ == FilterUnit) && (!fakeFilterUnitMode_)) { - transferDestination_ = edm::Service()->getStreamDestinations(stream_label_); - mergeType_ = edm::Service()->getStreamMergeType(stream_label_, evf::MergeTypePB); - } -} - -std::shared_ptr DQMFileSaver::globalBeginRun(const edm::Run &r, const edm::EventSetup &) const { - ++nrun_; - - // For Filter Unit, create an empty ini file: - // it is needed by the HLT deamon in order to start merging - // The run number is established in the service - // TODO(diguida): check that they are the same? - if ((convention_ == FilterUnit) && (!fakeFilterUnitMode_)) { - evf::EvFDaqDirector *daqDirector = (evf::EvFDaqDirector *)(edm::Service().operator->()); - const std::string initFileName = daqDirector->getInitFilePath(stream_label_); - std::ofstream file(initFileName); - file.close(); - } - - return nullptr; -} - -std::shared_ptr DQMFileSaver::globalBeginLuminosityBlock(const edm::LuminosityBlock &l, - const edm::EventSetup &) const { - ++nlumi_; - return nullptr; -} - -void DQMFileSaver::analyze(edm::StreamID, const edm::Event &e, const edm::EventSetup &) const { +void DQMFileSaver::analyze(const edm::Event &e, const edm::EventSetup &) { //save by event and save by time are not supported //anymore in the threaded framework. please use //savebyLumiSection instead. } -void DQMFileSaver::globalEndLuminosityBlock(const edm::LuminosityBlock &iLS, const edm::EventSetup &) const { - int ilumi = iLS.id().luminosityBlock(); - int irun = iLS.id().run(); - if (ilumi > 0 && saveByLumiSection_ > 0) { - if (convention_ != Online && convention_ != FilterUnit && convention_ != Offline) - throw cms::Exception("DQMFileSaver") << "Internal error, can save files at end of lumi block" - << " only in Online, FilterUnit or Offline mode."; - - if (convention_ == Online && (nlumi_ % saveByLumiSection_) == 0) // insist on lumi section ordering - { - char suffix[64]; - char rewrite[128]; - sprintf(suffix, "_R%09d_L%06d", irun, ilumi); - sprintf(rewrite, "\\1Run %d/\\2/By Lumi Section %d-%d", irun, ilumi - nlumi_, ilumi); - if (fileFormat_ == ROOT) - saveForOnline(irun, suffix, rewrite); - else if (fileFormat_ == PB) - saveForOnlinePB(irun, suffix); - else - throw cms::Exception("DQMFileSaver") << "Internal error, can save files" - << " only in ROOT or ProtocolBuffer format."; - } - - // Store at every lumi section end only if some events have been processed. - // Caveat: if faking FilterUnit, i.e. not accessing DAQ2 services, - // we cannot ask FastMonitoringService the processed events, so we are forced - // to save the file at every lumisection, even with no statistics. - // Here, we protect the call to get the processed events in a lumi section - // by testing the pointer to FastMonitoringService: if not null, i.e. in real FU mode, - // we check that the events are not 0; otherwise, we skip the test, so we store at every lumi transition. - // TODO(diguida): allow fake FU mode to skip file creation at empty lumi sections. - if (convention_ == FilterUnit && (fms_ ? fms_->shouldWriteFiles(ilumi) : !fms_)) { - char rewrite[128]; - sprintf(rewrite, "\\1Run %d/\\2/By Lumi Section %d-%d", irun, ilumi, ilumi); - saveForFilterUnit(rewrite, irun, ilumi, fileFormat_); - } - if (convention_ == Offline) { - if (fileFormat_ == ROOT) - saveForOffline(workflow_, irun, ilumi); - else - // TODO(diguida): do we need to support lumisection saving in Offline for PB? - // In this case, for ROOT, we only save EventInfo folders: we can filter them... - throw cms::Exception("DQMFileSaver") << "Internal error, can save files" - << " only in ROOT format."; - } +void DQMFileSaver::endRun(const edm::Run &iRun, const edm::EventSetup &) { + int irun = iRun.id().run(); + irun_ = irun; + if (irun > 0 && saveByRun_ > 0 && (nrun_ % saveByRun_) == 0) { + saveForOffline(workflow_, irun, 0); + } +} - // after saving per LS, delete the old LS global histograms. - // TODO: revise - //dbe_->deleteUnusedLumiHistograms(enableMultiThread_ ? irun : 0, ilumi); +void DQMFileSaver::endJob() { + if (saveAtJobEnd_) { + if (forceRunNumber_ > 0) + saveForOffline(workflow_, forceRunNumber_, 0); + else + saveForOffline(workflow_, irun_, 0); } } -void DQMFileSaver::save(std::string const &filename, - std::string const &path /* = "" */, - std::string const &pattern /* = "" */, - std::string const &rewrite /* = "" */, - uint32_t const run /* = 0 */, - uint32_t const lumi /* = 0 */) { +void DQMFileSaver::save(std::string const &filename, uint32_t const run /* = 0 */) { // TFile flushes to disk with fsync() on every TDirectory written to // the file. This makes DQM file saving painfully slow, and // ironically makes it _more_ likely the file saving gets @@ -774,8 +285,8 @@ void DQMFileSaver::save(std::string const &filename, TObjString str(content.c_str()); str.Write(); } else if (me->kind() == MonitorElement::Kind::STRING) { - std::string value = me->getStringValue(); - std::string content = "<" + objectName + ">s=\"" + value.c_str() + "\""; + const std::string &value = me->getStringValue(); + std::string content = "<" + objectName + ">s=\"" + value + "\""; TObjString str(content.c_str()); str.Write(); } else { @@ -791,83 +302,10 @@ void DQMFileSaver::save(std::string const &filename, file->Close(); } -void DQMFileSaver::globalEndRun(const edm::Run &iRun, const edm::EventSetup &) const { - int irun = iRun.id().run(); - irun_ = irun; - if (irun > 0 && saveByRun_ > 0 && (nrun_ % saveByRun_) == 0) { - if (convention_ == Online) { - char suffix[64]; - sprintf(suffix, "_R%09d", irun); - char rewrite[64]; - sprintf(rewrite, "\\1Run %d/\\2/Run summary", irun); - if (fileFormat_ == ROOT) - saveForOnline(irun, suffix, rewrite); - else if (fileFormat_ == PB) - saveForOnlinePB(irun, suffix); - else - throw cms::Exception("DQMFileSaver") << "Internal error, can save files" - << " only in ROOT or ProtocolBuffer format."; - } else if (convention_ == Offline && fileFormat_ == ROOT) - saveForOffline(workflow_, irun, 0); - else if (convention_ == Offline && fileFormat_ == PB) - saveForOfflinePB(workflow_, irun); - else - throw cms::Exception("DQMFileSaver") << "Internal error. Can only save files in endRun()" - << " in Online and Offline modes."; - } - - // create a fake EoR file for testing purposes. - if (fakeFilterUnitMode_) { - edm::LogInfo("DQMFileSaver") << "Producing fake EoR file.\n"; - - std::string runDir = str(boost::format("%s/run%06d") % dirName_ % irun); - std::string jsonFilePathName = str(boost::format("%s/run%06d_ls0000_EoR.jsn") % runDir % irun); - std::string openJsonFilePathName = jsonFilePathName + ".open"; - - boost::filesystem::create_directories(runDir); - - using namespace boost::property_tree; - ptree pt; - ptree data; - - ptree child1, child2, child3; - - child1.put("", -1); // Processed - child2.put("", -1); // Accepted - child3.put("", nlumi_); // number of lumi - - data.push_back(std::make_pair("", child1)); - data.push_back(std::make_pair("", child2)); - data.push_back(std::make_pair("", child3)); - - pt.add_child("data", data); - pt.put("definition", "/non-existant/"); - pt.put("source", "--hostname--"); - - std::ofstream file(jsonFilePathName); - write_json(file, pt, true); - file.close(); - - rename(openJsonFilePathName.c_str(), jsonFilePathName.c_str()); - } -} - -void DQMFileSaver::endJob() { - if (saveAtJobEnd_) { - if (convention_ == Offline && forceRunNumber_ > 0) - saveForOffline(workflow_, forceRunNumber_, 0); - else if (convention_ == Offline) - saveForOffline(workflow_, irun_, 0); - else - throw cms::Exception("DQMFileSaver") << "Internal error. Can only save files at the end of the" - << " job in Offline mode."; - } -} - // Use this for saving monitoring objects in ROOT files with dir structure; // cds into directory (creates it first if it doesn't exist); // returns a success flag -bool DQMFileSaver::createDirectoryIfNeededAndCd(const std::string &path) const { +bool DQMFileSaver::createDirectoryIfNeededAndCd(const std::string &path) { assert(!path.empty()); // Find the first path component. @@ -882,21 +320,20 @@ bool DQMFileSaver::createDirectoryIfNeededAndCd(const std::string &path) const { std::string part(path, start, end - start); TObject *o = gDirectory->Get(part.c_str()); if (o && !dynamic_cast(o)) - raiseDQMError("DQMStore", - "Attempt to create directory '%s' in a file" - " fails because the part '%s' already exists and is not" - " directory", - path.c_str(), - part.c_str()); + throw cms::Exception("DQMFileSaver") << "Attempt to create directory '" << path + << "' in a file" + " fails because the part '" + << part + << "' already exists and is not" + " directory"; else if (!o) gDirectory->mkdir(part.c_str()); if (!gDirectory->cd(part.c_str())) - raiseDQMError("DQMStore", - "Attempt to create directory '%s' in a file" - " fails because could not cd into subdirectory '%s'", - path.c_str(), - part.c_str()); + throw cms::Exception("DQMFileSaver") << "Attempt to create directory '" << path + << "' in a file" + " fails because could not cd into subdirectory '" + << part << "'"; // Stop if we reached the end, ignoring any trailing '/'. if (end + 1 >= path.size()) From b6273b73318fbccc71c3f2225084e608442046da Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 10 Dec 2019 11:41:10 +0100 Subject: [PATCH 05/11] Fix file writing bugs. This includes - No qoutes around strings - Correct (that is, same as before) format for floats - Also write efficiency fag - Also write qtests For the efficiency flag and QT algorithm there were no public getters, so they are also added here. --- .../Components/plugins/DQMFileSaver.cc | 41 +++++++++++++++---- DQMServices/Core/interface/MonitorElement.h | 1 + DQMServices/Core/interface/QReport.h | 3 ++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/DQMServices/Components/plugins/DQMFileSaver.cc b/DQMServices/Components/plugins/DQMFileSaver.cc index fb61579423ef0..2841059aa89f3 100644 --- a/DQMServices/Components/plugins/DQMFileSaver.cc +++ b/DQMServices/Components/plugins/DQMFileSaver.cc @@ -1,4 +1,6 @@ #include "DQMServices/Core/interface/DQMStore.h" + +#include "FWCore/Framework/interface/one/EDAnalyzer.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" @@ -13,21 +15,20 @@ #include #include #include +#include +#include + #include #include #include #include #include -#include -#include +#include "TString.h" +#include "TSystem.h" #include "TFile.h" -#include - -#include "FWCore/Framework/interface/one/EDAnalyzer.h" - -#include +#include "boost/format.hpp" namespace saverDetails { struct NoCache {}; @@ -281,18 +282,40 @@ void DQMFileSaver::save(std::string const &filename, uint32_t const run /* = 0 * str.Write(); } else if (me->kind() == MonitorElement::Kind::REAL) { double value = me->getFloatValue(); - std::string content = "<" + objectName + ">f=" + std::to_string(value) + ""; + char buf[64]; + // use printf here to preserve exactly the classic formatting. + std::snprintf(buf, sizeof(buf), "%.*g", DBL_DIG + 2, value); + std::string content = "<" + objectName + ">f=" + buf + ""; TObjString str(content.c_str()); str.Write(); } else if (me->kind() == MonitorElement::Kind::STRING) { const std::string &value = me->getStringValue(); - std::string content = "<" + objectName + ">s=\"" + value + "\""; + std::string content = "<" + objectName + ">s=" + value + ""; TObjString str(content.c_str()); str.Write(); } else { // Write a histogram TH1 *value = me->getTH1(); value->Write(); + + if (me->getEfficiencyFlag()) { + std::string content = "<" + objectName + ">e=1"; + TObjString str(content.c_str()); + str.Write(); + } + + for (QReport *qr : me->getQReports()) { + std::string result; + // TODO: 64 is likely too short; memory corruption in the old code? + char buf[64]; + std::snprintf(buf, sizeof(buf), "qr=st:%d:%.*g:", qr->getStatus(), DBL_DIG + 2, qr->getQTresult()); + result = '<' + objectName + '.' + qr->getQRName() + '>'; + result += buf; + result += qr->getAlgorithm() + ':' + qr->getMessage(); + result += "getQRName() + '>'; + TObjString str(result.c_str()); + str.Write(); + } } // Go back to the root directory diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index ada0d45ed7222..8a5e21745d7ec 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -226,6 +226,7 @@ namespace dqm::impl { /// this ME is meant to be an efficiency plot that must not be /// normalized when drawn in the DQM GUI. void setEfficiencyFlag() { data_.flags |= DQMNet::DQM_PROP_EFFICIENCY_PLOT; } + bool getEfficiencyFlag() { return data_.flags & DQMNet::DQM_PROP_EFFICIENCY_PLOT; } // A static assert to check that T actually fits in // int64_t. diff --git a/DQMServices/Core/interface/QReport.h b/DQMServices/Core/interface/QReport.h index d45bdce26088e..36b700ac1ccb6 100644 --- a/DQMServices/Core/interface/QReport.h +++ b/DQMServices/Core/interface/QReport.h @@ -32,6 +32,9 @@ class QReport { /// get name of quality test const std::string &getQRName() const { return qvalue_->qtname; } + /// get quality test algorithm + const std::string &getAlgorithm() const { return qvalue_->algorithm; } + /// get vector of channels that failed test /// (not relevant for all quality tests!) const std::vector &getBadChannels() const { return badChannels_; } From 5ecc1c5232c29ce77d3298ef1954536663f25a26 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 10 Dec 2019 15:00:29 +0100 Subject: [PATCH 06/11] Move new TDirectory code to a shared place. --- .../Components/plugins/DQMFileSaver.cc | 161 +----------------- DQMServices/Core/interface/LegacyIOHelper.h | 32 ++++ DQMServices/Core/src/LegacyIOHelper.cc | 156 +++++++++++++++++ 3 files changed, 191 insertions(+), 158 deletions(-) create mode 100644 DQMServices/Core/interface/LegacyIOHelper.h create mode 100644 DQMServices/Core/src/LegacyIOHelper.cc diff --git a/DQMServices/Components/plugins/DQMFileSaver.cc b/DQMServices/Components/plugins/DQMFileSaver.cc index 2841059aa89f3..fabed1d57a147 100644 --- a/DQMServices/Components/plugins/DQMFileSaver.cc +++ b/DQMServices/Components/plugins/DQMFileSaver.cc @@ -1,4 +1,5 @@ #include "DQMServices/Core/interface/DQMStore.h" +#include "DQMServices/Core/interface/LegacyIOHelper.h" #include "FWCore/Framework/interface/one/EDAnalyzer.h" #include "FWCore/Framework/interface/Event.h" @@ -13,22 +14,6 @@ #include "DataFormats/Histograms/interface/DQMToken.h" #include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "TString.h" -#include "TSystem.h" -#include "TFile.h" - -#include "boost/format.hpp" namespace saverDetails { struct NoCache {}; @@ -56,9 +41,6 @@ class DQMFileSaver : public edm::one::EDAnalyzer { void saveForOffline(const std::string &workflow, int run, int lumi); void saveJobReport(const std::string &filename); - void save(std::string const &filename, uint32_t const run = 0); - bool createDirectoryIfNeededAndCd(const std::string &path); - std::string workflow_; std::string producer_; std::string stream_label_; @@ -130,7 +112,8 @@ void DQMFileSaver::saveForOffline(const std::string &workflow, int run, int lumi me->Fill(0.); } - this->save(filename, run); + LegacyIOHelper h(dbe_); + h.save(filename, run); // save the JobReport saveJobReport(filename); @@ -234,143 +217,5 @@ void DQMFileSaver::endJob() { } } -void DQMFileSaver::save(std::string const &filename, uint32_t const run /* = 0 */) { - // TFile flushes to disk with fsync() on every TDirectory written to - // the file. This makes DQM file saving painfully slow, and - // ironically makes it _more_ likely the file saving gets - // interrupted and corrupts the file. The utility class below - // simply ignores the flush synchronisation. - class TFileNoSync : public TFile { - public: - TFileNoSync(char const *file, char const *opt) : TFile{file, opt} {} - Int_t SysSync(Int_t) override { return 0; } - }; - - std::cout << "DQMFileSaver::globalEndRun()" << std::endl; - - char suffix[64]; - sprintf(suffix, "R%09d", run); - TFileNoSync *file = new TFileNoSync(filename.c_str(), "RECREATE"); // open file - - // Traverse all MEs - auto mes = dbe_->getAllContents(""); // TODO run/lumi and/or job? - for (auto me : mes) { - // Modify dirname to comply with DQM GUI format. Change: - // A/B/C/plot - // into: - // DQMData/Run X/A/Run summary/B/C/plot - std::string dirName = me->getPathname(); - uint64_t firstSlashPos = dirName.find("/"); - if (firstSlashPos == std::string::npos) { - firstSlashPos = dirName.length(); - } - dirName = dirName.substr(0, firstSlashPos) + "/Run summary" + dirName.substr(firstSlashPos, dirName.size()); - dirName = "DQMData/Run " + std::to_string(run) + "/" + dirName; - - std::string objectName = me->getName(); - - // Create dir if it doesn't exist and cd into it - createDirectoryIfNeededAndCd(dirName); - - // INTs are saved as strings in this format: i=value - // REALs are saved as strings in this format: f=value - // STRINGs are saved as strings in this format: s="value" - if (me->kind() == MonitorElement::Kind::INT) { - int value = me->getIntValue(); - std::string content = "<" + objectName + ">i=" + std::to_string(value) + ""; - TObjString str(content.c_str()); - str.Write(); - } else if (me->kind() == MonitorElement::Kind::REAL) { - double value = me->getFloatValue(); - char buf[64]; - // use printf here to preserve exactly the classic formatting. - std::snprintf(buf, sizeof(buf), "%.*g", DBL_DIG + 2, value); - std::string content = "<" + objectName + ">f=" + buf + ""; - TObjString str(content.c_str()); - str.Write(); - } else if (me->kind() == MonitorElement::Kind::STRING) { - const std::string &value = me->getStringValue(); - std::string content = "<" + objectName + ">s=" + value + ""; - TObjString str(content.c_str()); - str.Write(); - } else { - // Write a histogram - TH1 *value = me->getTH1(); - value->Write(); - - if (me->getEfficiencyFlag()) { - std::string content = "<" + objectName + ">e=1"; - TObjString str(content.c_str()); - str.Write(); - } - - for (QReport *qr : me->getQReports()) { - std::string result; - // TODO: 64 is likely too short; memory corruption in the old code? - char buf[64]; - std::snprintf(buf, sizeof(buf), "qr=st:%d:%.*g:", qr->getStatus(), DBL_DIG + 2, qr->getQTresult()); - result = '<' + objectName + '.' + qr->getQRName() + '>'; - result += buf; - result += qr->getAlgorithm() + ':' + qr->getMessage(); - result += "getQRName() + '>'; - TObjString str(result.c_str()); - str.Write(); - } - } - - // Go back to the root directory - gDirectory->cd("/"); - } - - file->Close(); -} - -// Use this for saving monitoring objects in ROOT files with dir structure; -// cds into directory (creates it first if it doesn't exist); -// returns a success flag -bool DQMFileSaver::createDirectoryIfNeededAndCd(const std::string &path) { - assert(!path.empty()); - - // Find the first path component. - size_t start = 0; - size_t end = path.find('/', start); - if (end == std::string::npos) - end = path.size(); - - while (true) { - // Check if this subdirectory component exists. If yes, make sure - // it is actually a subdirectory. Otherwise create or cd into it. - std::string part(path, start, end - start); - TObject *o = gDirectory->Get(part.c_str()); - if (o && !dynamic_cast(o)) - throw cms::Exception("DQMFileSaver") << "Attempt to create directory '" << path - << "' in a file" - " fails because the part '" - << part - << "' already exists and is not" - " directory"; - else if (!o) - gDirectory->mkdir(part.c_str()); - - if (!gDirectory->cd(part.c_str())) - throw cms::Exception("DQMFileSaver") << "Attempt to create directory '" << path - << "' in a file" - " fails because could not cd into subdirectory '" - << part << "'"; - - // Stop if we reached the end, ignoring any trailing '/'. - if (end + 1 >= path.size()) - break; - - // Find the next path component. - start = end + 1; - end = path.find('/', start); - if (end == std::string::npos) - end = path.size(); - } - - return true; -} - #include "FWCore/Framework/interface/MakerMacros.h" DEFINE_FWK_MODULE(DQMFileSaver); diff --git a/DQMServices/Core/interface/LegacyIOHelper.h b/DQMServices/Core/interface/LegacyIOHelper.h new file mode 100644 index 0000000000000..89a037e86c5c7 --- /dev/null +++ b/DQMServices/Core/interface/LegacyIOHelper.h @@ -0,0 +1,32 @@ +#ifndef DQMSERVICES_CORE_LEGACYIOHELPER_H +#define DQMSERVICES_CORE_LEGACYIOHELPER_H + +#include "DQMServices/Core/interface/DQMStore.h" + +// This class encapsulates the TDirectory based file format used for DQMGUI +// uploads and many other use cases. +// This should be part of `DQMFileSaver`, however since DQMServices/Components +// DQMFileSaver and DQMServices/FileIO DQMFileSaverOnline both write this +// format, the code is shared here (evnetually, these modules should become one +// again). +// This code is in DQMServices/Core to also allow the legacy DQMStore::save +// interface to use this without adding another dependency. +class LegacyIOHelper { +public: + typedef dqm::legacy::DQMStore DQMStore; + typedef dqm::legacy::MonitorElement MonitorElement; + LegacyIOHelper(DQMStore* dqmstore) : dbe_(dqmstore){}; + + // Replace or append to `filename`, a TDirectory ROOT file. If a run number + // is passed, the paths are rewritten to the "Run Summary" format used by + // DQMGUI. The run number does not affect which MEs are saved; this code only + // supports non-threaded mode. `fileupdate` is passed to ROOT unchanged. + // TODO: allow to select only RUN, LUMI or JOB histograms. + void save(std::string const& filename, uint32_t const run = 0, std::string const& fileupdate = "RECREATE"); + +private: + bool createDirectoryIfNeededAndCd(const std::string& path); + DQMStore* dbe_; +}; + +#endif diff --git a/DQMServices/Core/src/LegacyIOHelper.cc b/DQMServices/Core/src/LegacyIOHelper.cc new file mode 100644 index 0000000000000..8c7c9d1961fc8 --- /dev/null +++ b/DQMServices/Core/src/LegacyIOHelper.cc @@ -0,0 +1,156 @@ +#include "DQMServices/Core/interface/LegacyIOHelper.h" + +#include +#include +#include +#include + +#include "TString.h" +#include "TSystem.h" +#include "TFile.h" +#include + +void LegacyIOHelper::save(std::string const &filename, + uint32_t const run /* = 0 */, + std::string const &fileupdate /* = "RECREATE" */) { + // TFile flushes to disk with fsync() on every TDirectory written to + // the file. This makes DQM file saving painfully slow, and + // ironically makes it _more_ likely the file saving gets + // interrupted and corrupts the file. The utility class below + // simply ignores the flush synchronisation. + class TFileNoSync : public TFile { + public: + TFileNoSync(char const *file, char const *opt) : TFile{file, opt} {} + Int_t SysSync(Int_t) override { return 0; } + }; + + std::cout << "DQMFileSaver::globalEndRun()" << std::endl; + + char suffix[64]; + sprintf(suffix, "R%09d", run); + TFileNoSync *file = new TFileNoSync(filename.c_str(), fileupdate.c_str()); // open file + + // Traverse all MEs + auto mes = dbe_->getAllContents(""); // TODO run/lumi and/or job? + for (auto me : mes) { + // Modify dirname to comply with DQM GUI format. Change: + // A/B/C/plot + // into: + // DQMData/Run X/A/Run summary/B/C/plot + std::string dirName = me->getPathname(); + uint64_t firstSlashPos = dirName.find("/"); + if (firstSlashPos == std::string::npos) { + firstSlashPos = dirName.length(); + } + + if (run) { + // Rewrite paths to "Run Summary" format when given a run number. + // Else, write a simple, flat TDirectory for local usage. + dirName = dirName.substr(0, firstSlashPos) + "/Run summary" + dirName.substr(firstSlashPos, dirName.size()); + dirName = "DQMData/Run " + std::to_string(run) + "/" + dirName; + } + + std::string objectName = me->getName(); + + // Create dir if it doesn't exist and cd into it + createDirectoryIfNeededAndCd(dirName); + + // INTs are saved as strings in this format: i=value + // REALs are saved as strings in this format: f=value + // STRINGs are saved as strings in this format: s="value" + if (me->kind() == MonitorElement::Kind::INT) { + int value = me->getIntValue(); + std::string content = "<" + objectName + ">i=" + std::to_string(value) + ""; + TObjString str(content.c_str()); + str.Write(); + } else if (me->kind() == MonitorElement::Kind::REAL) { + double value = me->getFloatValue(); + char buf[64]; + // use printf here to preserve exactly the classic formatting. + std::snprintf(buf, sizeof(buf), "%.*g", DBL_DIG + 2, value); + std::string content = "<" + objectName + ">f=" + buf + ""; + TObjString str(content.c_str()); + str.Write(); + } else if (me->kind() == MonitorElement::Kind::STRING) { + const std::string &value = me->getStringValue(); + std::string content = "<" + objectName + ">s=" + value + ""; + TObjString str(content.c_str()); + str.Write(); + } else { + // Write a histogram + TH1 *value = me->getTH1(); + value->Write(); + + if (me->getEfficiencyFlag()) { + std::string content = "<" + objectName + ">e=1"; + TObjString str(content.c_str()); + str.Write(); + } + + for (QReport *qr : me->getQReports()) { + std::string result; + // TODO: 64 is likely too short; memory corruption in the old code? + char buf[64]; + std::snprintf(buf, sizeof(buf), "qr=st:%d:%.*g:", qr->getStatus(), DBL_DIG + 2, qr->getQTresult()); + result = '<' + objectName + '.' + qr->getQRName() + '>'; + result += buf; + result += qr->getAlgorithm() + ':' + qr->getMessage(); + result += "getQRName() + '>'; + TObjString str(result.c_str()); + str.Write(); + } + } + + // Go back to the root directory + gDirectory->cd("/"); + } + + file->Close(); +} + +// Use this for saving monitoring objects in ROOT files with dir structure; +// cds into directory (creates it first if it doesn't exist); +// returns a success flag +bool LegacyIOHelper::createDirectoryIfNeededAndCd(const std::string &path) { + assert(!path.empty()); + + // Find the first path component. + size_t start = 0; + size_t end = path.find('/', start); + if (end == std::string::npos) + end = path.size(); + + while (true) { + // Check if this subdirectory component exists. If yes, make sure + // it is actually a subdirectory. Otherwise create or cd into it. + std::string part(path, start, end - start); + TObject *o = gDirectory->Get(part.c_str()); + if (o && !dynamic_cast(o)) + throw cms::Exception("DQMFileSaver") << "Attempt to create directory '" << path + << "' in a file" + " fails because the part '" + << part + << "' already exists and is not" + " directory"; + else if (!o) + gDirectory->mkdir(part.c_str()); + + if (!gDirectory->cd(part.c_str())) + throw cms::Exception("DQMFileSaver") << "Attempt to create directory '" << path + << "' in a file" + " fails because could not cd into subdirectory '" + << part << "'"; + + // Stop if we reached the end, ignoring any trailing '/'. + if (end + 1 >= path.size()) + break; + + // Find the next path component. + start = end + 1; + end = path.find('/', start); + if (end == std::string::npos) + end = path.size(); + } + + return true; +} From 09710107c5c1409b6d817389fb8887e6ce0faa26 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 10 Dec 2019 15:08:02 +0100 Subject: [PATCH 07/11] Convert DQMFileSaverOnline to new IO code. Untested. --- .../FileIO/plugins/DQMFileSaverOnline.cc | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/DQMServices/FileIO/plugins/DQMFileSaverOnline.cc b/DQMServices/FileIO/plugins/DQMFileSaverOnline.cc index a0da7cbfac8b2..85c519bd3bdde 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverOnline.cc +++ b/DQMServices/FileIO/plugins/DQMFileSaverOnline.cc @@ -1,4 +1,5 @@ #include "DQMServices/Core/interface/DQMStore.h" +#include "DQMServices/Core/interface/LegacyIOHelper.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/LuminosityBlock.h" #include "FWCore/Framework/interface/Run.h" @@ -72,18 +73,22 @@ void DQMFileSaverOnline::makeSnapshot(const FileParameters& fp, bool final) cons logFileAction("Writing DQM Root file: ", root_fp); // logFileAction("Writing DQM Origin file: ", meta_fp); - char rewrite[128]; - snprintf(rewrite, 128, "\\1Run %ld/\\2/Run summary", fp.run_); - - store->save(tmp_root_fp, /* filename */ - "", /* path */ - "^(Reference/)?([^/]+)", /* pattern */ - rewrite, /* rewrite */ - store->mtEnabled() ? fp.run_ : 0, /* run */ - 0, /* lumi */ - fp.saveReference_, /* ref */ - fp.saveReferenceQMin_, /* ref minStatus */ - "RECREATE"); /* fileupdate */ + //char rewrite[128]; + //snprintf(rewrite, 128, "\\1Run %ld/\\2/Run summary", fp.run_); + + //store->save(tmp_root_fp, /* filename */ + // "", /* path */ + // "^(Reference/)?([^/]+)", /* pattern */ + // rewrite, /* rewrite */ + // store->mtEnabled() ? fp.run_ : 0, /* run */ + // 0, /* lumi */ + // fp.saveReference_, /* ref */ + // fp.saveReferenceQMin_, /* ref minStatus */ + // "RECREATE"); /* fileupdate */ + // TODO: some parameters prepared here are now unused, and the code should + // eventually be removed. + LegacyIOHelper h(&*store); + h.save(tmp_root_fp, fp.run_, "RECREATE"); // write metadata // format.origin: md5:d566a34b27f48d507150a332b189398b 294835 From 20c497d2e483dca29b79a9fc05c410ccd051be29 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 10 Dec 2019 17:00:12 +0100 Subject: [PATCH 08/11] Move Protobuf writing into DQMFileSaverPB. Untested. --- DQMServices/FileIO/plugins/DQMFileSaverPB.cc | 77 +++++++++++++++++++- DQMServices/FileIO/plugins/DQMFileSaverPB.h | 1 + 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/DQMServices/FileIO/plugins/DQMFileSaverPB.cc b/DQMServices/FileIO/plugins/DQMFileSaverPB.cc index 2ebafead982ec..0aac52f033027 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverPB.cc +++ b/DQMServices/FileIO/plugins/DQMFileSaverPB.cc @@ -13,14 +13,20 @@ #include #include #include -#include -#include +#include "TString.h" +#include "TSystem.h" +#include "TBufferFile.h" #include #include #include #include +#include +#include +#include +#include "DQMServices/Core/src/ROOTFilePB.pb.h" + using namespace dqm; DQMFileSaverPB::DQMFileSaverPB(const edm::ParameterSet& ps) : DQMFileSaverBase(ps) { @@ -77,7 +83,7 @@ void DQMFileSaverPB::saveLumi(const FileParameters& fp) const { if (fms ? fms->getEventsProcessedForLumi(fp.lumi_) : true) { // Save the file in the open directory. - store->savePB(openHistoFilePathName, "", store->mtEnabled() ? fp.run_ : 0, fp.lumi_); + this->savePB(&*store, openHistoFilePathName, store->mtEnabled() ? fp.run_ : 0, fp.lumi_); // Now move the the data and json files into the output directory. ::rename(openHistoFilePathName.c_str(), histoFilePathName.c_str()); @@ -192,5 +198,70 @@ void DQMFileSaverPB::fillDescriptions(edm::ConfigurationDescriptions& descriptio descriptions.addDefault(desc); } +void DQMFileSaverPB::savePB(DQMStore* store, std::string const& filename, int run, int lumi) const { + using google::protobuf::io::FileOutputStream; + using google::protobuf::io::GzipOutputStream; + using google::protobuf::io::StringOutputStream; + + unsigned int nme = 0; + + dqmstorepb::ROOTFilePB dqmstore_message; + + // TODO: while we still have enableMultiThread, maybe this does the wrong thing. + auto mes = store->getAllContents("", run, lumi); + for (auto const me : mes) { + TBufferFile buffer(TBufferFile::kWrite); + if (me->kind() < MonitorElement::Kind::TH1F) { + // TODO: this is almost 100% not what was supposed to be here. + TObjString object(me->tagString().c_str()); + buffer.WriteObject(&object); + } else { + buffer.WriteObject(me->getRootObject()); + } + dqmstorepb::ROOTFilePB::Histo& histo = *dqmstore_message.add_histo(); + // TODO: getPathname returns a name with trailing slash? + histo.set_full_pathname(me->getPathname() + "/" + me->getName()); + uint32_t flags = 0; + flags |= (uint32_t)me->kind(); + if (me->getLumiFlag()) + flags |= DQMNet::DQM_PROP_LUMI; + if (me->getEfficiencyFlag()) + flags |= DQMNet::DQM_PROP_EFFICIENCY_PLOT; + histo.set_flags(flags); + histo.set_size(buffer.Length()); + histo.set_streamed_histo((void const*)buffer.Buffer(), buffer.Length()); + + // Save quality reports if this is not in reference section. + // XXX not supported by protobuf files. + + // Save efficiency tag, if any. + // XXX not supported by protobuf files. + + // Save tag if any. + // XXX not supported by protobuf files. + + // Count saved histograms + ++nme; + } + + int filedescriptor = + ::open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH); + FileOutputStream file_stream(filedescriptor); + GzipOutputStream::Options options; + options.format = GzipOutputStream::GZIP; + options.compression_level = 1; + GzipOutputStream gzip_stream(&file_stream, options); + dqmstore_message.SerializeToZeroCopyStream(&gzip_stream); + + // Flush the internal streams before closing the fd. + gzip_stream.Close(); + file_stream.Close(); + ::close(filedescriptor); + + // Maybe make some noise. + edm::LogInfo("DQMFileSaverPB") << "savePB: successfully wrote " << nme << " objects " + << "into DQM file '" << filename << "'\n"; +} + #include "FWCore/Framework/interface/MakerMacros.h" DEFINE_FWK_MODULE(DQMFileSaverPB); diff --git a/DQMServices/FileIO/plugins/DQMFileSaverPB.h b/DQMServices/FileIO/plugins/DQMFileSaverPB.h index 391207fd1d692..46a98e1319f15 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverPB.h +++ b/DQMServices/FileIO/plugins/DQMFileSaverPB.h @@ -30,6 +30,7 @@ namespace dqm { void initRun() const override; void saveLumi(const FileParameters& fp) const override; void saveRun(const FileParameters& fp) const override; + void savePB(DQMStore* store, std::string const& filename, int run, int lumi) const; bool fakeFilterUnitMode_; std::string streamLabel_; From bfdfb3e0c7647fb7e914404296e7945ff27eae9b Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 10 Dec 2019 17:15:10 +0100 Subject: [PATCH 09/11] Remove save() logic from DQMStore ... and use LegacyIOHelper::save as a drop-in replacement. it is not really feature complete but should work for all usages. --- DQMServices/Core/interface/DQMStore.h | 19 -- DQMServices/Core/src/DQMStore.cc | 337 +------------------------- 2 files changed, 6 insertions(+), 350 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index d37b4bf949079..365220f23de4c 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -510,7 +510,6 @@ namespace dqm::dqmstoreimpl { SaveReferenceTag ref = SaveWithReference, int minStatus = dqm::qstatus::STATUS_OK, std::string const& fileupdate = "RECREATE"); - void savePB(std::string const& filename, std::string const& path = "", uint32_t run = 0, uint32_t lumi = 0); bool open(std::string const& filename, bool overwrite = false, std::string const& path = "", @@ -630,24 +629,6 @@ namespace dqm::dqmstoreimpl { using MEMap = std::set; // ------------------------ private I/O helpers ------------------------------ - void saveMonitorElementToPB(MonitorElement const& me, dqmstorepb::ROOTFilePB& file); - void saveMonitorElementRangeToPB(std::string const& dir, - unsigned int run, - MEMap::const_iterator begin, - MEMap::const_iterator end, - dqmstorepb::ROOTFilePB& file, - unsigned int& counter); - void saveMonitorElementToROOT(MonitorElement const& me, TFile& file); - void saveMonitorElementRangeToROOT(std::string const& dir, - std::string const& refpath, - SaveReferenceTag ref, - int minStatus, - unsigned int run, - MEMap::const_iterator begin, - MEMap::const_iterator end, - TFile& file, - unsigned int& counter); - unsigned verbose_{1}; unsigned verboseQT_{1}; bool reset_{false}; diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 3b025b7f35135..263f207bf8329 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -1,5 +1,6 @@ #include "DQMServices/Core/interface/Standalone.h" #include "DQMServices/Core/interface/DQMStore.h" +#include "DQMServices/Core/interface/LegacyIOHelper.h" #include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/src/ROOTFilePB.pb.h" #include "DQMServices/Core/src/DQMError.h" @@ -2112,103 +2113,6 @@ namespace dqm::dqmstoreimpl { return true; } - void DQMStore::saveMonitorElementToROOT(MonitorElement const& me, TFile& file) { - // Save the object. - if (me.kind() < MonitorElement::Kind::TH1F) { - TObjString(me.tagString().c_str()).Write(); - } else { - me.getRootObject()->Write(); - } - - // Save quality reports if this is not in reference section. - if (not isSubdirectory(s_referenceDirName, *me.data_.dirname)) { - for (auto const& report : me.data_.qreports) { - TObjString(me.qualityTagString(report).c_str()).Write(); - } - } - - // Save efficiency tag, if any. - if (me.data_.flags & DQMNet::DQM_PROP_EFFICIENCY_PLOT) { - TObjString(me.effLabelString().c_str()).Write(); - } - - // Save tag if any. - if (me.data_.flags & DQMNet::DQM_PROP_TAGGED) { - TObjString(me.tagLabelString().c_str()).Write(); - } - } - - void DQMStore::saveMonitorElementRangeToROOT(std::string const& dir, - std::string const& refpath, - SaveReferenceTag const ref, - int const minStatus, - unsigned int const run, - MEMap::const_iterator const begin, - MEMap::const_iterator const end, - TFile& file, - unsigned int& counter) { - for (auto const& me : boost::make_iterator_range(begin, end)) { - if (not isSubdirectory(dir, *me.data_.dirname)) - break; - - if (verbose_ > 1) - std::cout << "DQMStore::save:" - << " run: " << me.run() << " lumi: " << me.lumi() << " lumiFlag: " << me.getLumiFlag() - << " moduleId: " << me.moduleId() << " fullpathname: " << me.getFullname() << " flags: " << std::hex - << me.data_.flags << std::endl; - - // Skip MonitorElements in a subdirectory of the current one. - if (dir != *me.data_.dirname) { - if (verbose_ > 1) { - std::cout << "DQMStore::save: skipping monitor element in a subfolder of " << dir << "/" << std::endl; - } - continue; - } - - // Handle reference histograms, with three distinct cases: - // 1) Skip all references entirely on saving. - // 2) Blanket saving of all references. - // 3) Save only references for monitor elements with qtests. - // The latter two are affected by "path" sub-tree selection, - // i.e. references are saved only in the selected tree part. - if (isSubdirectory(refpath, *me.data_.dirname)) { - if (ref == SaveWithoutReference) - // Skip the reference entirely. - continue; - else if (ref == SaveWithReference) - // Save all references regardless of qtests. - ; - else if (ref == SaveWithReferenceForQTest) { - // Save only references for monitor elements with qtests - // with an optional cut on minimum quality test result. - int status = -1; - std::string mname(me.getFullname(), s_referenceDirName.size() + 1, std::string::npos); - MonitorElement* master = get(mname); - if (master) - for (auto const& qreport : master->data_.qreports) - status = std::max(status, qreport.code); - - if (not master or status < minStatus) { - if (verbose_ > 1) - std::cout << "DQMStore::save: skipping monitor element '" << me.data_.objname - << "' while saving, status is " << status << ", required minimum status is " << minStatus - << std::endl; - continue; - } - } - } - - if (verbose_ > 1) { - std::cout << "DQMStore::save: saving monitor element" << std::endl; - } - - saveMonitorElementToROOT(me, file); - - // Count saved histograms - ++counter; - } - } - /// save directory with monitoring objects into protobuf file ; /// if directory="", save full monitoring structure void DQMStore::save(std::string const& filename, @@ -2220,240 +2124,11 @@ namespace dqm::dqmstoreimpl { SaveReferenceTag const ref /* = SaveWithReference */, int const minStatus /* = dqm::qstatus::STATUS_OK */, std::string const& fileupdate /* = RECREATE */) { - // TFile flushes to disk with fsync() on every TDirectory written to - // the file. This makes DQM file saving painfully slow, and - // ironically makes it _more_ likely the file saving gets - // interrupted and corrupts the file. The utility class below - // simply ignores the flush synchronisation. - class TFileNoSync : public TFile { - public: - TFileNoSync(char const* file, char const* opt) : TFile{file, opt} {} - Int_t SysSync(Int_t) override { return 0; } - }; - - std::lock_guard guard(book_mutex_); - - unsigned int nme = 0; - - // open output file, on 1st save recreate, later update - if (verbose_) { - std::cout << "DQMStore::save: Opening TFile '" << filename << "' with option '" << fileupdate << "'" << std::endl; - } - - TFileNoSync f(filename.c_str(), fileupdate.c_str()); // open file - if (f.IsZombie()) - raiseDQMError("DQMStore", "Failed to create/update file '%s'", filename.c_str()); - f.cd(); - - // Construct a regular expression from the pattern string. - std::unique_ptr rxpat; - if (not pattern.empty()) - rxpat = std::make_unique(pattern); - - // Prepare a path for the reference object selection. - std::string refpath; - refpath.reserve(s_referenceDirName.size() + path.size() + 2); - refpath += s_referenceDirName; - if (not path.empty()) { - refpath += '/'; - refpath += path; - } - - // Loop over the directory structure. - for (auto const& dir : dirs_) { - // Check if we should process this directory. We process the - // requested part of the object tree, including references. - if (not path.empty() and not isSubdirectory(refpath, dir) and not isSubdirectory(path, dir)) - continue; - - if (verbose_ > 1) { - std::cout << "DQMStore::save: DQM folder " << dir << "/" << std::endl; - } - - // Create the directory. - gDirectory->cd("/"); - if (dir.empty()) - cdInto(s_monitorDirName); - else if (rxpat.get()) - cdInto(s_monitorDirName + '/' + lat::StringOps::replace(dir, *rxpat, rewrite)); - else - cdInto(s_monitorDirName + '/' + dir); - - // Loop over monitor elements in this directory. - if (not enableMultiThread_) { - MonitorElement proto(&dir, std::string(), run, 0); - auto begin = data_.lower_bound(proto); - auto end = data_.end(); - saveMonitorElementRangeToROOT(dir, refpath, ref, minStatus, run, begin, end, f, nme); - } else { - // Restrict the loop to the monitor elements for the current lumisection - MonitorElement proto(&dir, std::string(), run, 0); - proto.setLumi(lumi); - auto begin = data_.lower_bound(proto); - proto.setLumi(lumi + 1); - auto end = data_.lower_bound(proto); - saveMonitorElementRangeToROOT(dir, refpath, ref, minStatus, run, begin, end, f, nme); - } - - // In LSbasedMode, loop also over the (run, 0) global histograms; - // these could be the merged global histrograms of their per-stream - // counterparts after the streamEndRun transition - but they are not - // produced in LSbasedMode. - if (enableMultiThread_ and LSbasedMode_ and lumi != 0) { - auto begin = data_.lower_bound(MonitorElement(&dir, std::string(), run, 0)); - auto end = data_.lower_bound(MonitorElement(&dir, std::string(), run, 1)); - saveMonitorElementRangeToROOT(dir, refpath, ref, minStatus, run, begin, end, f, nme); - } - } - - f.Close(); - - // Maybe make some noise. - if (verbose_) { - std::cout << "DQMStore::save: successfully wrote " << nme << " objects from path '" << path << "/" - << "' into DQM file '" << filename << "'\n"; - } - } - - void DQMStore::saveMonitorElementToPB(MonitorElement const& me, dqmstorepb::ROOTFilePB& file) { - // Save the object. - TBufferFile buffer(TBufferFile::kWrite); - if (me.kind() < MonitorElement::Kind::TH1F) { - TObjString object(me.tagString().c_str()); - buffer.WriteObject(&object); - } else { - buffer.WriteObject(me.getRootObject()); - } - dqmstorepb::ROOTFilePB::Histo& histo = *file.add_histo(); - histo.set_full_pathname(*me.data_.dirname + '/' + me.data_.objname); - histo.set_flags(me.data_.flags); - histo.set_size(buffer.Length()); - histo.set_streamed_histo((void const*)buffer.Buffer(), buffer.Length()); - - // Save quality reports if this is not in reference section. - // XXX not supported by protobuf files. - - // Save efficiency tag, if any. - // XXX not supported by protobuf files. - - // Save tag if any. - // XXX not supported by protobuf files. - } - - void DQMStore::saveMonitorElementRangeToPB(std::string const& dir, - unsigned int const run, - MEMap::const_iterator const begin, - MEMap::const_iterator const end, - dqmstorepb::ROOTFilePB& file, - unsigned int& counter) { - for (auto const& me : boost::make_iterator_range(begin, end)) { - if (not isSubdirectory(dir, *me.data_.dirname)) - break; - - if (verbose_ > 1) - std::cout << "DQMStore::savePB:" - << " run: " << me.run() << " lumi: " << me.lumi() << " lumiFlag: " << me.getLumiFlag() - << " moduleId: " << me.moduleId() << " fullpathname: " << me.getFullname() << " flags: " << std::hex - << me.data_.flags << std::endl; - - // Skip MonitorElements in a subdirectory of the current one. - if (dir != *me.data_.dirname) { - if (verbose_ > 1) { - std::cout << "DQMStore::savePB: skipping monitor element in a subfolder of " << dir << "/" << std::endl; - } - continue; - } - - // Handle reference histograms, with three distinct cases: - // XXX not supported by protobuf files. - - if (verbose_ > 1) { - std::cout << "DQMStore::savePB: saving monitor element" << std::endl; - } - - saveMonitorElementToPB(me, file); - - // Count saved histograms - ++counter; - } - } - - /// save directory with monitoring objects into protobuf file ; - /// if directory="", save full monitoring structure - void DQMStore::savePB(std::string const& filename, - std::string const& path /* = "" */, - uint32_t const run /* = 0 */, - uint32_t const lumi /* = 0 */) { - using google::protobuf::io::FileOutputStream; - using google::protobuf::io::GzipOutputStream; - using google::protobuf::io::StringOutputStream; - - std::lock_guard guard(book_mutex_); - - unsigned int nme = 0; - - if (verbose_) { - std::cout << "DQMStore::savePB: Opening PBFile '" << filename << "'" << std::endl; - } - dqmstorepb::ROOTFilePB dqmstore_message; - - // Loop over the directory structure. - for (auto const& dir : dirs_) { - // Check if we should process this directory. We process the - // requested part of the object tree, including references. - if (not path.empty() and not isSubdirectory(path, dir)) - continue; - - if (verbose_ > 1) { - std::cout << "DQMStore::savePB: DQM folder " << dir << "/" << std::endl; - } - - // Loop over monitor elements in this directory. - if (not enableMultiThread_) { - MonitorElement proto(&dir, std::string(), run, 0); - auto begin = data_.lower_bound(proto); - auto end = data_.end(); - saveMonitorElementRangeToPB(dir, run, begin, end, dqmstore_message, nme); - } else { - // Restrict the loop to the monitor elements for the current lumisection - MonitorElement proto(&dir, std::string(), run, 0); - proto.setLumi(lumi); - auto begin = data_.lower_bound(proto); - proto.setLumi(lumi + 1); - auto end = data_.lower_bound(proto); - saveMonitorElementRangeToPB(dir, run, begin, end, dqmstore_message, nme); - } - - // In LSbasedMode, loop also over the (run, 0) global histograms; - // these could be the merged global histrograms of their per-stream - // counterparts after the streamEndRun transition - but they are not - // produced in LSbasedMode. - if (enableMultiThread_ and LSbasedMode_ and lumi != 0) { - auto begin = data_.lower_bound(MonitorElement(&dir, std::string(), run, 0)); - auto end = data_.lower_bound(MonitorElement(&dir, std::string(), run, 1)); - saveMonitorElementRangeToPB(dir, run, begin, end, dqmstore_message, nme); - } - } - - int filedescriptor = - ::open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH); - FileOutputStream file_stream(filedescriptor); - GzipOutputStream::Options options; - options.format = GzipOutputStream::GZIP; - options.compression_level = 1; - GzipOutputStream gzip_stream(&file_stream, options); - dqmstore_message.SerializeToZeroCopyStream(&gzip_stream); - - // Flush the internal streams before closing the fd. - gzip_stream.Close(); - file_stream.Close(); - ::close(filedescriptor); - - // Maybe make some noise. - if (verbose_) { - std::cout << "DQMStore::savePB: successfully wrote " << nme << " objects from path '" << path << "/" - << "' into DQM file '" << filename << "'\n"; - } + // This has slightly different semantics compared to the legacy save(), but + // should work for all relevant cases. + // Most legacy users only pass a filename anyways. + LegacyIOHelper h(this); + h.save(filename, run, fileupdate); } /// read ROOT objects from file in directory ; From f9180fb300d3faee4941b71380286879a5337c30 Mon Sep 17 00:00:00 2001 From: Marcel Schneider Date: Thu, 12 Dec 2019 16:33:47 +0100 Subject: [PATCH 10/11] Fix bug in DQMFileSaverPB. We need to save everything there. The output is still not bit-by-bit identical to the old code, since we drop the `DQM_PROP_NEW` flag (among others). --- DQMServices/FileIO/plugins/DQMFileSaverPB.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DQMServices/FileIO/plugins/DQMFileSaverPB.cc b/DQMServices/FileIO/plugins/DQMFileSaverPB.cc index 0aac52f033027..62b08e8b305da 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverPB.cc +++ b/DQMServices/FileIO/plugins/DQMFileSaverPB.cc @@ -208,11 +208,11 @@ void DQMFileSaverPB::savePB(DQMStore* store, std::string const& filename, int ru dqmstorepb::ROOTFilePB dqmstore_message; // TODO: while we still have enableMultiThread, maybe this does the wrong thing. - auto mes = store->getAllContents("", run, lumi); + // We save all histograms, indifferent of the lumi flag: even tough we save per lumi, this is a *snapshot*. + auto mes = store->getAllContents(""); for (auto const me : mes) { TBufferFile buffer(TBufferFile::kWrite); if (me->kind() < MonitorElement::Kind::TH1F) { - // TODO: this is almost 100% not what was supposed to be here. TObjString object(me->tagString().c_str()); buffer.WriteObject(&object); } else { From b3259d5ce3ec70fe0e4d65fff181cd399566e768 Mon Sep 17 00:00:00 2001 From: Marcel Schneider Date: Fri, 13 Dec 2019 19:15:40 +0100 Subject: [PATCH 11/11] Fix typo in DQMFileSaverPB_cfi.py --- DQMServices/FileIO/python/DQMFileSaverPB_cfi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DQMServices/FileIO/python/DQMFileSaverPB_cfi.py b/DQMServices/FileIO/python/DQMFileSaverPB_cfi.py index 723c720c2707a..ef1428e96eef3 100644 --- a/DQMServices/FileIO/python/DQMFileSaverPB_cfi.py +++ b/DQMServices/FileIO/python/DQMFileSaverPB_cfi.py @@ -16,7 +16,7 @@ referenceRequireStatus = cms.untracked.int32(100), # If set, EvFDaqDirector is emulated and not used - fakeFilterUnitMode = cms.untracked.bool(false), + fakeFilterUnitMode = cms.untracked.bool(False), # Label of the stream streamLabel = cms.untracked.string("streamDQMHistograms"), )