Skip to content

Commit

Permalink
Replace mark-and-delete at next merge algorithm with delete histogram…
Browse files Browse the repository at this point in the history 
…s now.

Additionally, the dqm store will now acquire locks for writing files.

Also add a new file saver for HLT->PB (DQMFileSaverPB).
Which should replace DQMFileSaver in the near future.

(cherry picked from commit f104a00)
  • Loading branch information
Dmitrijus Bugelskis committed Sep 23, 2015
1 parent 3c7bd4e commit 091bfda
Show file tree
Hide file tree
Showing 10 changed files with 336 additions and 158 deletions.
2 changes: 1 addition & 1 deletion DQMServices/Components/src/DQMFileSaver.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -735,7 +735,7 @@ DQMFileSaver::globalEndLuminosityBlock(const edm::LuminosityBlock & iLS, const e
}

// after saving per LS, delete the old LS global histograms.
dbe_->markForDeletion(enableMultiThread_ ? irun : 0, ilumi);
dbe_->deleteUnusedLumiHistograms(enableMultiThread_ ? irun : 0, ilumi);
}
}

Expand Down
2 changes: 1 addition & 1 deletion DQMServices/Core/interface/DQMStore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,7 @@ class DQMStore
uint32_t streamId,
uint32_t moduleId);

void markForDeletion(uint32_t run, uint32_t lumi);
void deleteUnusedLumiHistograms(uint32_t run, uint32_t lumi);
private:

// ---------------- Miscellaneous -----------------------------
Expand Down
44 changes: 20 additions & 24 deletions DQMServices/Core/src/DQMStore.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -476,18 +476,6 @@ void DQMStore::mergeAndResetMEsLuminositySummaryCache(uint32_t run,
// make the ME reusable for the next LS
const_cast<MonitorElement*>(&*i)->Reset();
++i;

// check and remove the global lumi based histo belonging to the previous LSs
// if properly flagged as DQMNet::DQM_PROP_MARKTODELETE
global_me.setLumi(1);
std::set<MonitorElement>::const_iterator i_lumi = data_.lower_bound(global_me);
while (i_lumi->data_.lumi != lumi) {
auto temp = i_lumi++;
if (i_lumi->getName() == i->getName() && i_lumi->getPathname() == i->getPathname() && i_lumi->markedToDelete())
{
data_.erase(temp);
}
}
}
}

Expand Down Expand Up @@ -2098,11 +2086,14 @@ DQMStore::forceReset(void)
//////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////
/** Mark a set of histograms for deletion based on run, lumi and path*/
/** Delete *global* histograms which are no longer in used.
* Such histograms are created at the end of each lumi and should be
* deleted after last globalEndLuminosityBlock.
*/
void
DQMStore::markForDeletion(uint32_t run,
uint32_t lumi)
DQMStore::deleteUnusedLumiHistograms(uint32_t run, uint32_t lumi)
{
std::lock_guard<std::mutex> guard(book_mutex_);

std::string null_str("");
MonitorElement proto(&null_str, null_str, run, 0, 0);
Expand All @@ -2114,25 +2105,26 @@ DQMStore::markForDeletion(uint32_t run,

while (i != e) {
if (i->data_.streamId != 0 ||
i->data_.moduleId != 0)
i->data_.moduleId != 0)
break;
if ((i->data_.lumi != lumi) && enableMultiThread_)
break;
if (i->data_.run != run)
break;

const_cast<MonitorElement*>(&*i)->markToDelete();

if (verbose_ > 1)
std::cout << "DQMStore::markForDeletion: marked monitor element '"
<< *i->data_.dirname << "/" << i->data_.objname << "'"
<< "flags " << i->data_.flags << "\n";

auto temp = i;
++i;

if (verbose_ > 1) {
std::cout << "DQMStore::deleteUnusedLumiHistograms: deleted monitor element '"
<< *i->data_.dirname << "/" << i->data_.objname << "'"
<< "flags " << i->data_.flags << "\n";
}

data_.erase(temp);
}
}


//////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -2487,6 +2479,8 @@ void DQMStore::savePB(const std::string &filename,
using google::protobuf::io::GzipOutputStream;
using google::protobuf::io::StringOutputStream;

std::lock_guard<std::mutex> guard(book_mutex_);

std::set<std::string>::iterator di, de;
MEMap::iterator mi, me = data_.end();
dqmstorepb::ROOTFilePB dqmstore_message;
Expand Down Expand Up @@ -2616,6 +2610,8 @@ DQMStore::save(const std::string &filename,
const std::string &fileupdate /* = RECREATE */,
const bool resetMEsAfterWriting /* = false */)
{
std::lock_guard<std::mutex> guard(book_mutex_);

std::set<std::string>::iterator di, de;
MEMap::iterator mi, me = data_.end();
DQMNet::QReports::const_iterator qi, qe;
Expand Down
110 changes: 5 additions & 105 deletions DQMServices/FileIO/plugins/DQMFileSaverBase.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@
#include <TString.h>
#include <TSystem.h>

#include <openssl/md5.h>
#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/filesystem.hpp>

using namespace dqm;

DQMFileSaverBase::DQMFileSaverBase(const edm::ParameterSet &ps) {
Expand Down Expand Up @@ -72,11 +68,11 @@ DQMFileSaverBase::DQMFileSaverBase(const edm::ParameterSet &ps) {

DQMFileSaverBase::~DQMFileSaverBase() {}

void DQMFileSaverBase::beginJob() {}

std::shared_ptr<NoCache> DQMFileSaverBase::globalBeginRun(
const edm::Run &r, const edm::EventSetup &) const {

this->initRun();

return nullptr;
}

Expand All @@ -103,11 +99,10 @@ void DQMFileSaverBase::globalEndLuminosityBlock(const edm::LuminosityBlock &iLS,
fp.lumi_ = ilumi;
fp.run_ = irun;

edm::Service<DQMStore> store;

this->saveLumi(fp);

store->markForDeletion(store->mtEnabled() ? irun : 0, ilumi);
edm::Service<DQMStore> store;
store->deleteUnusedLumiHistograms(store->mtEnabled() ? irun : 0, ilumi);
}

void DQMFileSaverBase::globalEndRun(const edm::Run &iRun,
Expand All @@ -123,8 +118,6 @@ void DQMFileSaverBase::globalEndRun(const edm::Run &iRun,
this->saveRun(fp);
}

void DQMFileSaverBase::endJob(void) {}

void DQMFileSaverBase::postForkReacquireResources(
unsigned int childIndex, unsigned int numberOfChildren) {
// this is copied from IOPool/Output/src/PoolOutputModule.cc, for consistency
Expand All @@ -145,7 +138,7 @@ void DQMFileSaverBase::postForkReacquireResources(
initial_fp_.child_ = std::string(buffer);
}

const std::string DQMFileSaverBase::filename(FileParameters fp, bool useLumi) {
const std::string DQMFileSaverBase::filename(const FileParameters& fp, bool useLumi) {
char buf[256];
if (useLumi) {
snprintf(buf, 256, "%s_V%04d_%s_R%09ld_L%09ld%s", fp.producer_.c_str(),
Expand All @@ -164,99 +157,6 @@ const std::string DQMFileSaverBase::filename(FileParameters fp, bool useLumi) {
return (path / file).string();
}

// file metadata saving stuff
boost::property_tree::ptree
DQMFileSaverBase::fillJson(int run, int lumi, const std::string& dataFilePathName, const std::string transferDestinationStr, evf::FastMonitoringService *fms)
{
namespace bpt = boost::property_tree;
namespace bfs = boost::filesystem;

bpt::ptree pt;

int hostnameReturn;
char host[32];
hostnameReturn = gethostname(host ,sizeof(host));
if (hostnameReturn == -1)
throw cms::Exception("fillJson")
<< "Internal error, cannot get host name";

int pid = getpid();
std::ostringstream oss_pid;
oss_pid << pid;

// Stat the data file: if not there, throw
struct stat dataFileStat;
if (stat(dataFilePathName.c_str(), &dataFileStat) != 0)
throw cms::Exception("fillJson")
<< "Internal error, cannot get data file: "
<< dataFilePathName;
// Extract only the data file name from the full path
std::string dataFileName = bfs::path(dataFilePathName).filename().string();
// The availability test of the FastMonitoringService was done in the ctor.
bpt::ptree data;
bpt::ptree processedEvents, acceptedEvents, errorEvents, bitmask, fileList, fileSize, inputFiles, fileAdler32, transferDestination;

processedEvents.put("", fms ? (fms->getEventsProcessedForLumi(lumi)) : -1); // Processed events
acceptedEvents.put("", fms ? (fms->getEventsProcessedForLumi(lumi)) : -1); // Accepted events, same as processed for our purposes

errorEvents.put("", 0); // Error events
bitmask.put("", 0); // Bitmask of abs of CMSSW return code
fileList.put("", dataFileName); // Data file the information refers to
fileSize.put("", dataFileStat.st_size); // Size in bytes of the data file
inputFiles.put("", ""); // We do not care about input files!
fileAdler32.put("", -1); // placeholder to match output json definition
transferDestination.put("", transferDestinationStr); // SM Transfer destination field

data.push_back(std::make_pair("", processedEvents));
data.push_back(std::make_pair("", acceptedEvents));
data.push_back(std::make_pair("", errorEvents));
data.push_back(std::make_pair("", bitmask));
data.push_back(std::make_pair("", fileList));
data.push_back(std::make_pair("", fileSize));
data.push_back(std::make_pair("", inputFiles));
data.push_back(std::make_pair("", fileAdler32));
data.push_back(std::make_pair("", transferDestination));

pt.add_child("data", data);

if (fms == nullptr) {
pt.put("definition", "/fakeDefinition.jsn");
} else {
// The availability test of the EvFDaqDirector Service was done in the ctor.
bfs::path outJsonDefName(edm::Service<evf::EvFDaqDirector>()->baseRunDir()); //we assume this file is written bu the EvF Output module
outJsonDefName /= (std::string("output_") + oss_pid.str() + std::string(".jsd"));
pt.put("definition", outJsonDefName.string());
}

char sourceInfo[64]; //host and pid information
sprintf(sourceInfo, "%s_%d", host, pid);
pt.put("source", sourceInfo);

return pt;
}

const std::string DQMFileSaverBase::fillOrigin(const std::string filename,
const std::string final_filename) {

// format.origin (one line):
// md5:d566a34b27f48d507150a332b189398b 294835 final_filename.root

unsigned char md5[MD5_DIGEST_LENGTH];

boost::iostreams::mapped_file_source fp(filename);

MD5((unsigned char *)fp.data(), fp.size(), md5);

std::ostringstream hash;
for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
hash << std::hex << std::setfill('0') << std::setw(2) << (int)(md5[i]);
}

std::ostringstream out;
out << "md5:" << hash.str() << " " << fp.size() << " " << final_filename;
return out.str();
}

void DQMFileSaverBase::saveJobReport(const std::string &filename) const
{
// Report the file to job report service.
Expand Down
40 changes: 18 additions & 22 deletions DQMServices/FileIO/plugins/DQMFileSaverBase.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,38 +45,34 @@ class DQMFileSaverBase
};

protected:
virtual void beginJob(void);
//virtual void beginJob(void) const override final;
//virtual void endJob(void) const override final;

virtual std::shared_ptr<NoCache> globalBeginRun(
const edm::Run &, const edm::EventSetup &) const;
const edm::Run &, const edm::EventSetup &) const override final;

virtual std::shared_ptr<NoCache> globalBeginLuminosityBlock(
const edm::LuminosityBlock &, const edm::EventSetup &) const;
const edm::LuminosityBlock &, const edm::EventSetup &) const override final;

virtual void analyze(edm::StreamID, const edm::Event &e,
const edm::EventSetup &) const;
const edm::EventSetup &) const override final;

virtual void globalEndLuminosityBlock(const edm::LuminosityBlock &,
const edm::EventSetup &) const;
virtual void globalEndRun(const edm::Run &, const edm::EventSetup &) const;
virtual void endJob(void);
const edm::EventSetup &) const override final ;
virtual void globalEndRun(const edm::Run &, const edm::EventSetup &) const override final;

virtual void postForkReacquireResources(unsigned int childIndex,
unsigned int numberOfChildren);

// these two should be overwritten
// in some cases, hsitograms are deleted after saving
// these method (and only these) should be overriden
// so we need to call all file savers
virtual void saveLumi(FileParameters fp) const {};
virtual void saveRun(FileParameters fp) const {};

static const std::string filename(FileParameters fp, bool useLumi = false);

// also used by the JsonWritingTimedPoolOutputModule,
// fms will be nullptr in such case
static boost::property_tree::ptree fillJson(
int run, int lumi, const std::string &dataFilePathName, const std::string transferDestinationStr,
evf::FastMonitoringService *fms);
virtual void initRun(void) const {};
virtual void saveLumi(const FileParameters& fp) const {};
virtual void saveRun(const FileParameters& fp) const {};

static const std::string fillOrigin(const std::string filename,
const std::string final_filename);
static const std::string filename(const FileParameters& fp, bool useLumi = false);

// utilities
// utilities
void logFileAction(const std::string& msg, const std::string& fileName) const;
void saveJobReport(const std::string &filename) const;

Expand Down
34 changes: 31 additions & 3 deletions DQMServices/FileIO/plugins/DQMFileSaverOnline.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@
#include <TString.h>
#include <TSystem.h>

#include <openssl/md5.h>
#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/filesystem.hpp>

using namespace dqm;

DQMFileSaverOnline::DQMFileSaverOnline(const edm::ParameterSet &ps)
Expand All @@ -32,7 +36,7 @@ DQMFileSaverOnline::DQMFileSaverOnline(const edm::ParameterSet &ps)

DQMFileSaverOnline::~DQMFileSaverOnline() {}

void DQMFileSaverOnline::saveLumi(FileParameters fp) const {
void DQMFileSaverOnline::saveLumi(const FileParameters& fp) const {
if (backupLumiCount_ > 0) {
if (fp.lumi_ % backupLumiCount_ == 0) {

Expand All @@ -42,7 +46,7 @@ void DQMFileSaverOnline::saveLumi(FileParameters fp) const {
}
}

void DQMFileSaverOnline::saveRun(FileParameters fp) const {
void DQMFileSaverOnline::saveRun(const FileParameters& fp) const {
makeSnapshot(fp, true);
}

Expand Down Expand Up @@ -91,7 +95,7 @@ void DQMFileSaverOnline::makeSnapshot(const FileParameters& fp, bool final) cons
// write metadata
// format.origin: md5:d566a34b27f48d507150a332b189398b 294835 /home/dqmprolocal/output/DQM_V0001_FED_R000194224.root
std::ofstream meta_fd(tmp_meta_fp);
meta_fd << this->fillOrigin(tmp_root_fp, root_fp);
meta_fd << fillOrigin(tmp_root_fp, root_fp);
meta_fd.close();

checkError("Rename failed: ", root_fp, ::rename(tmp_root_fp.c_str(), root_fp.c_str()));
Expand Down Expand Up @@ -136,6 +140,30 @@ void DQMFileSaverOnline::checkError(const char *msg, const std::string file, int
}
}

const std::string DQMFileSaverOnline::fillOrigin(const std::string filename,
const std::string final_filename) {

// format.origin (one line):
// md5:d566a34b27f48d507150a332b189398b 294835 final_filename.root

unsigned char md5[MD5_DIGEST_LENGTH];

boost::iostreams::mapped_file_source fp(filename);

MD5((unsigned char *)fp.data(), fp.size(), md5);

std::ostringstream hash;
for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
hash << std::hex << std::setfill('0') << std::setw(2) << (int)(md5[i]);
}

std::ostringstream out;
out << "md5:" << hash.str() << " " << fp.size() << " " << final_filename;
return out.str();
}



void DQMFileSaverOnline::fillDescriptions(
edm::ConfigurationDescriptions& descriptions) {

Expand Down
Loading

0 comments on commit 091bfda

Please sign in to comment.