diff --git a/CalibCalorimetry/EcalLaserSorting/interface/WatcherSource.h b/CalibCalorimetry/EcalLaserSorting/interface/WatcherSource.h index 45cbeb8190ef7..791a830305b3f 100644 --- a/CalibCalorimetry/EcalLaserSorting/interface/WatcherSource.h +++ b/CalibCalorimetry/EcalLaserSorting/interface/WatcherSource.h @@ -4,6 +4,6 @@ #include "CalibCalorimetry/EcalLaserSorting/interface/WatcherStreamFileReader.h" #include "IOPool/Streamer/interface/StreamerInputModule.h" -typedef edm::StreamerInputModule WatcherSource; +typedef edm::streamer::StreamerInputModule WatcherSource; #endif //WatcherSourceModule_H not defined diff --git a/CalibCalorimetry/EcalLaserSorting/interface/WatcherStreamFileReader.h b/CalibCalorimetry/EcalLaserSorting/interface/WatcherStreamFileReader.h index ce77f03ec9d82..a4f7870a72c7f 100644 --- a/CalibCalorimetry/EcalLaserSorting/interface/WatcherStreamFileReader.h +++ b/CalibCalorimetry/EcalLaserSorting/interface/WatcherStreamFileReader.h @@ -1,5 +1,5 @@ -#ifndef IOPool_Streamer_StreamerFileReader_h -#define IOPool_Streamer_StreamerFileReader_h +#ifndef CalibCalorimetry_EcalLaserSorting_WatcherStreamFileReader_h +#define CalibCalorimetry_EcalLaserSorting_WatcherStreamFileReader_h #include "IOPool/Streamer/interface/InitMessage.h" #include "IOPool/Streamer/interface/EventMessage.h" @@ -19,7 +19,7 @@ * This protection is obviously not full proof, especially to transfer lag. */ -namespace edm { +namespace edm::streamer { class StreamerInputFile; } @@ -28,15 +28,16 @@ class WatcherStreamFileReader { WatcherStreamFileReader(edm::ParameterSet const& pset); ~WatcherStreamFileReader(); - const InitMsgView* getHeader(); - const EventMsgView* getNextEvent(); + const edm::streamer::InitMsgView* getHeader(); + const edm::streamer::EventMsgView* getNextEvent(); const bool newHeader(); - edm::StreamerInputFile* getInputFile(); + edm::streamer::StreamerInputFile* getInputFile(); void closeFile(); private: + void moveJustReadFile(); /** Directory to look for streamer files */ std::string inputDir_; @@ -59,7 +60,7 @@ class WatcherStreamFileReader { /** Cached input file stream */ - std::unique_ptr streamerInputFile_; + std::unique_ptr streamerInputFile_; std::string fileName_; diff --git a/CalibCalorimetry/EcalLaserSorting/src/WatcherStreamFileReader.cc b/CalibCalorimetry/EcalLaserSorting/src/WatcherStreamFileReader.cc index 262ebe485333e..56b6126931b50 100644 --- a/CalibCalorimetry/EcalLaserSorting/src/WatcherStreamFileReader.cc +++ b/CalibCalorimetry/EcalLaserSorting/src/WatcherStreamFileReader.cc @@ -19,6 +19,7 @@ //using namespace edm; using namespace std; +using namespace edm::streamer; //std::string WatcherStreamFileReader::fileName_; @@ -153,13 +154,10 @@ WatcherStreamFileReader::WatcherStreamFileReader(edm::ParameterSet const& pset) WatcherStreamFileReader::~WatcherStreamFileReader() {} -const bool WatcherStreamFileReader::newHeader() { - edm::StreamerInputFile* inputFile = getInputFile(); - return inputFile ? inputFile->newHeader() : false; -} +const bool WatcherStreamFileReader::newHeader() { return getInputFile() != nullptr; } const InitMsgView* WatcherStreamFileReader::getHeader() { - edm::StreamerInputFile* inputFile = getInputFile(); + StreamerInputFile* inputFile = getInputFile(); //TODO: shall better send an exception... if (inputFile == nullptr) { @@ -177,21 +175,20 @@ const InitMsgView* WatcherStreamFileReader::getHeader() { const EventMsgView* WatcherStreamFileReader::getNextEvent() { if (end_) { - closeFile(); + moveJustReadFile(); return nullptr; } - edm::StreamerInputFile* inputFile; - - //go to next input file, till no new event is found - while ((inputFile = getInputFile()) != nullptr && inputFile->next() != edm::StreamerInputFile::Next::kEvent) { - closeFile(); + StreamerInputFile* inputFile; + if ((inputFile = getInputFile()) != nullptr and inputFile->next() == StreamerInputFile::Next::kStop) { + moveJustReadFile(); + return nullptr; } return inputFile == nullptr ? nullptr : inputFile->currentRecord(); } -edm::StreamerInputFile* WatcherStreamFileReader::getInputFile() { +StreamerInputFile* WatcherStreamFileReader::getInputFile() { char* lineptr = nullptr; size_t n = 0; static stringstream cmd; @@ -379,7 +376,7 @@ edm::StreamerInputFile* WatcherStreamFileReader::getInputFile() { cout << "[WatcherSource " << now() << "]" << " Opening file " << fileName_ << "\n" << flush; - streamerInputFile_ = std::make_unique(fileName_); + streamerInputFile_ = std::make_unique(fileName_); ofstream f(".watcherfile"); f << fileName_; @@ -392,7 +389,9 @@ edm::StreamerInputFile* WatcherStreamFileReader::getInputFile() { return streamerInputFile_.get(); } -void WatcherStreamFileReader::closeFile() { +void WatcherStreamFileReader::closeFile() {} + +void WatcherStreamFileReader::moveJustReadFile() { if (streamerInputFile_.get() == nullptr) return; //delete the streamer input file: diff --git a/CalibCalorimetry/EcalLaserSorting/test/RunStreamer.sh b/CalibCalorimetry/EcalLaserSorting/test/RunStreamer.sh index f404bfdc23298..0dd340c27dbc7 100755 --- a/CalibCalorimetry/EcalLaserSorting/test/RunStreamer.sh +++ b/CalibCalorimetry/EcalLaserSorting/test/RunStreamer.sh @@ -10,30 +10,35 @@ echo "LOCAL_TEST_DIR = $SCRAM_TEST_PATH" RC=0 mkdir inDir +echo "test padding" cmsRun ${SCRAM_TEST_PATH}/streamOutPadding_cfg.py > outp 2>&1 || die "cmsRun streamOutPadding_cfg.py" $? cp teststreamfile.dat teststreamfile.padding mv teststreamfile.dat inDir/ -timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamIn_cfg.py > inp 2>&1 || die "cmsRun streamIn_cfg.py" $? +timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamIn_cfg.py > inp 2>&1 || die "cmsRun streamIn_cfg.py with padding" $? rm -rf inDir mkdir inDir + +echo "test original" cmsRun ${SCRAM_TEST_PATH}/streamOut_cfg.py > out 2>&1 || die "cmsRun streamOut_cfg.py" $? cp teststreamfile.dat teststreamfile.original mv teststreamfile.dat inDir -timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamIn_cfg.py > in 2>&1 || die "cmsRun streamIn_cfg.py" $? +timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamIn_cfg.py > in 2>&1 || die "cmsRun streamIn_cfg.py original" $? +echo "test original and alt" rm watcherSourceToken cp teststreamfile.original inDir/teststreamfile.dat cmsRun ${SCRAM_TEST_PATH}/streamOutAlt_cfg.py > outAlt 2>&1 || die "cmsRun streamOutAlt_cfg.py" $? mv teststreamfile_alt.dat inDir -timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamIn_cfg.py >alt 2>&1 || die "cmsRun streamIn_cfg.py" $? +timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamIn_cfg.py --alt >alt 2>&1 || die "cmsRun streamIn_cfg.py alt" $? #timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamInAlt_cfg.py > alt 2>&1 || die "cmsRun streamInAlt_cfg.py" $? +echo "test ext" rm watcherSourceToken cp teststreamfile.original inDir/teststreamfile.dat -cmsRun ${SCRAM_TEST_PATH}/streamOutExt_cfg.py > outExt 2>&1 || die "cmsRun streamOutExt_cfg.py" $? +cmsRun ${SCRAM_TEST_PATH}/streamOutExt_cfg.py > outExt 2>&1 || die "cmsRun streamOutExt_cfg.py" $? mv teststreamfile_ext.dat inDir -timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamIn_cfg.py > ext 2>&1 || die "cmsRun streamIn_cfg.py" $? +timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamIn_cfg.py --ext > ext 2>&1 || die "cmsRun streamIn_cfg.py ext" $? #timeout --signal SIGTERM 180 cmsRun ${SCRAM_TEST_PATH}/streamInExt_cfg.py > ext 2>&1 || die "cmsRun streamInExt_cfg.py" $? # echo "CHECKSUM = 1" > out diff --git a/CalibCalorimetry/EcalLaserSorting/test/streamIn_cfg.py b/CalibCalorimetry/EcalLaserSorting/test/streamIn_cfg.py index 03a986f7a1052..ec6d8e24671c7 100644 --- a/CalibCalorimetry/EcalLaserSorting/test/streamIn_cfg.py +++ b/CalibCalorimetry/EcalLaserSorting/test/streamIn_cfg.py @@ -1,5 +1,16 @@ import FWCore.ParameterSet.Config as cms +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test streamer input') + +parser.add_argument("--alt", help="Have filter succeed", action="store_true") +parser.add_argument("--ext", help="Switch the order of dependencies", action="store_true") + +args = parser.parse_args() + + process = cms.Process("TRANSFER") import FWCore.Framework.test.cmsExceptionsFatal_cff @@ -26,8 +37,31 @@ product_to_get = cms.string('m1') ) +ids = [cms.EventID(1,0,0), cms.EventID(1,1,0)] +for e in range(10123456789, 10123456839): + ids.append(cms.EventID(1,1,e)) +if args.alt: + for e in range(15123456789, 15123456839): + ids.append(cms.EventID(1,1,e)) + +if args.ext: + ids.append(cms.EventID(1,1,0)) + ids.append(cms.EventID(1,0,0)) + ids.append(cms.EventID(1,0,0)) + ids.append(cms.EventID(1,1,0)) + for e in range(20123456789, 20123456839): + ids.append(cms.EventID(1,1,e)) + +ids.append(cms.EventID(1,1,0)) +ids.append(cms.EventID(1,0,0)) + +process.check = cms.EDAnalyzer("RunLumiEventChecker", + eventSequence = cms.untracked.VEventID(ids) +) + + process.out = cms.OutputModule("PoolOutputModule", fileName = cms.untracked.string('myout.root') ) -process.end = cms.EndPath(process.a1*process.out) +process.end = cms.EndPath(process.a1*process.out*process.check) diff --git a/DQMServices/StreamerIO/plugins/DQMStreamerReader.cc b/DQMServices/StreamerIO/plugins/DQMStreamerReader.cc index 745301e0a6744..906db29d4259e 100644 --- a/DQMServices/StreamerIO/plugins/DQMStreamerReader.cc +++ b/DQMServices/StreamerIO/plugins/DQMStreamerReader.cc @@ -18,6 +18,7 @@ #include namespace dqmservices { + using namespace edm::streamer; DQMStreamerReader::DQMStreamerReader(edm::ParameterSet const& pset, edm::InputSourceDescription const& desc) : StreamerInputSource(pset, desc), @@ -81,17 +82,27 @@ namespace dqmservices { fiterator_.logFileAction("DQMStreamerReader initialised."); } + void DQMStreamerReader::setupMetaData(edm::streamer::InitMsgView const& msg, bool subsequent) { + deserializeAndMergeWithRegistry(msg, subsequent); + auto event = getEventMsg(); + //file might be empty + if (not event) + return; + assert(event->isEventMetaData()); + deserializeEventMetaData(*event); + updateEventMetaData(); + } void DQMStreamerReader::openFileImp_(const DQMFileIterator::LumiEntry& entry) { processedEventPerLs_ = 0; std::string path = entry.get_data_path(); file_.lumi_ = entry; - file_.streamFile_ = std::make_unique(path); + file_.streamFile_ = std::make_unique(path); InitMsgView const* header = getHeaderMsg(); if (isFirstFile_) { - deserializeAndMergeWithRegistry(*header, false); + setupMetaData(*header, false); } // dump the list of HLT trigger name from the header @@ -135,9 +146,14 @@ namespace dqmservices { return; } + if (artificialFileBoundary_) { + updateEventMetaData(); + artificialFileBoundary_ = false; + return; + } //Get header/init from reader InitMsgView const* header = getHeaderMsg(); - deserializeAndMergeWithRegistry(*header, true); + setupMetaData(*header, true); } bool DQMStreamerReader::openNextFileImp_() { @@ -179,11 +195,11 @@ namespace dqmservices { EventMsgView const* DQMStreamerReader::getEventMsg() { auto next = file_.streamFile_->next(); - if (edm::StreamerInputFile::Next::kFile == next) { + if (StreamerInputFile::Next::kFile == next) { return nullptr; } - if (edm::StreamerInputFile::Next::kStop == next) { + if (StreamerInputFile::Next::kStop == next) { return nullptr; } @@ -285,6 +301,25 @@ namespace dqmservices { // this means end of file, so close the file closeFileImp_("eof"); } else { + //NOTE: at this point need to see if meta data checksum changed. If it did + // we need to issue a 'new File' transition + if (eview->isEventMetaData()) { + auto lastEventMetaData = presentEventMetaDataChecksum(); + if (eventMetaDataChecksum(*eview) != lastEventMetaData) { + deserializeEventMetaData(*eview); + artificialFileBoundary_ = true; + return nullptr; + } else { + //skipping + eview = getEventMsg(); + assert((eview == nullptr) or (not eview->isEventMetaData())); + if (eview == nullptr) { + closeFileImp_("eof"); + continue; + } + } + } + if (!acceptEvent(eview)) { continue; } else { @@ -303,7 +338,7 @@ namespace dqmservices { try { EventMsgView const* eview = prepareNextEvent(); if (eview == nullptr) { - if (file_.streamFile_ and file_.streamFile_->newHeader()) { + if (artificialFileBoundary_ or (file_.streamFile_ and file_.streamFile_->newHeader())) { return Next::kFile; } return Next::kStop; @@ -437,7 +472,7 @@ namespace dqmservices { desc.addUntracked("inputFileTransitionsEachEvent", false); DQMFileIterator::fillDescription(desc); - edm::StreamerInputSource::fillDescription(desc); + StreamerInputSource::fillDescription(desc); edm::EventSkipperByID::fillDescription(desc); descriptions.add("source", desc); diff --git a/DQMServices/StreamerIO/plugins/DQMStreamerReader.h b/DQMServices/StreamerIO/plugins/DQMStreamerReader.h index b84dad02816c9..bf13f51784fb1 100644 --- a/DQMServices/StreamerIO/plugins/DQMStreamerReader.h +++ b/DQMServices/StreamerIO/plugins/DQMStreamerReader.h @@ -1,13 +1,11 @@ #ifndef DQMServices_StreamerIO_DQMStreamerReader_h #define DQMServices_StreamerIO_DQMStreamerReader_h -#include "FWCore/ServiceRegistry/interface/Service.h" #include "IOPool/Streamer/interface/StreamerInputSource.h" #include "IOPool/Streamer/interface/StreamerInputFile.h" #include "IOPool/Streamer/interface/MsgTools.h" #include "DQMFileIterator.h" -#include "DQMMonitoringService.h" #include "TriggerSelector.h" #include @@ -16,7 +14,7 @@ namespace dqmservices { - class DQMStreamerReader : public edm::StreamerInputSource { + class DQMStreamerReader : public edm::streamer::StreamerInputSource { public: DQMStreamerReader(edm::ParameterSet const& pset, edm::InputSourceDescription const& desc); ~DQMStreamerReader() override; @@ -41,14 +39,15 @@ namespace dqmservices { bool openNextFileImp_(); - InitMsgView const* getHeaderMsg(); - EventMsgView const* getEventMsg(); + edm::streamer::InitMsgView const* getHeaderMsg(); + edm::streamer::EventMsgView const* getEventMsg(); - EventMsgView const* prepareNextEvent(); + void setupMetaData(edm::streamer::InitMsgView const& msg, bool subsequent); + edm::streamer::EventMsgView const* prepareNextEvent(); bool isFirstFile_ = true; bool prepareNextFile(); - bool acceptEvent(const EventMsgView*); + bool acceptEvent(const edm::streamer::EventMsgView*); DQMFileIterator fiterator_; unsigned int processedEventPerLs_ = 0; @@ -65,8 +64,11 @@ namespace dqmservices { bool matchTriggerSel_ = false; bool setMatchTriggerSel(std::vector const& tnames); + //If the event meta data changes while reading a file, we need to + // cause a file transition to happen to allow synchronous update + bool artificialFileBoundary_ = false; struct OpenFile { - std::unique_ptr streamFile_; + std::unique_ptr streamFile_; DQMFileIterator::LumiEntry lumi_; bool open() { return (streamFile_.get() != nullptr); } @@ -75,9 +77,6 @@ namespace dqmservices { std::shared_ptr eventSkipperByID_; std::shared_ptr triggerSelector_; - - /* this is for monitoring */ - edm::Service mon_; }; } // namespace dqmservices diff --git a/DQMServices/StreamerIO/test/DQMStreamerOutputRepackerTest.cc b/DQMServices/StreamerIO/test/DQMStreamerOutputRepackerTest.cc index 7b83a3c603155..48723ec412ddc 100644 --- a/DQMServices/StreamerIO/test/DQMStreamerOutputRepackerTest.cc +++ b/DQMServices/StreamerIO/test/DQMStreamerOutputRepackerTest.cc @@ -26,8 +26,9 @@ #include "IOPool/Streamer/interface/StreamerOutputModuleBase.h" namespace dqmservices { + using namespace edm::streamer; - class DQMStreamerOutputRepackerTest : public edm::StreamerOutputModuleBase { + class DQMStreamerOutputRepackerTest : public StreamerOutputModuleBase { public: explicit DQMStreamerOutputRepackerTest(edm::ParameterSet const& ps); ~DQMStreamerOutputRepackerTest() override; @@ -64,7 +65,7 @@ namespace dqmservices { }; // end-of-class-def DQMStreamerOutputRepackerTest::DQMStreamerOutputRepackerTest(edm::ParameterSet const& ps) - : edm::one::OutputModuleBase::OutputModuleBase(ps), edm::StreamerOutputModuleBase(ps) { + : edm::one::OutputModuleBase::OutputModuleBase(ps), StreamerOutputModuleBase(ps) { outputPath_ = ps.getUntrackedParameter("outputPath"); streamLabel_ = ps.getUntrackedParameter("streamLabel"); @@ -168,7 +169,7 @@ namespace dqmservices { void DQMStreamerOutputRepackerTest::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { edm::ParameterSetDescription desc; - edm::StreamerOutputModuleBase::fillDescription(desc); + StreamerOutputModuleBase::fillDescription(desc); desc.addUntracked("outputPath", "./output/")->setComment("File output path."); diff --git a/DataFormats/Streamer/interface/StreamedProducts.h b/DataFormats/Streamer/interface/StreamedProducts.h index 2bc6e2a2d64e4..68f78a5431c70 100644 --- a/DataFormats/Streamer/interface/StreamedProducts.h +++ b/DataFormats/Streamer/interface/StreamedProducts.h @@ -67,6 +67,9 @@ namespace edm { typedef std::vector SendProds; // ------------------------------------------ + // Contains either Event data or meta data about an Event. The header of the + // message contains which way an instance of this class is to be interpreted + // via the use of EventMsgView::isEventMetaData() class SendEvent { public: @@ -74,17 +77,29 @@ namespace edm { SendEvent(EventAuxiliary const& aux, ProcessHistory const& processHistory, EventSelectionIDVector const& eventSelectionIDs, - BranchListIndexes const& branchListIndexes) + BranchListIndexes const& branchListIndexes, + BranchIDLists const& branchIDLists, + ThinnedAssociationsHelper const& thinnedAssociationsHelper, + uint32_t metaDataChecksum) : aux_(aux), processHistory_(processHistory), eventSelectionIDs_(eventSelectionIDs), branchListIndexes_(branchListIndexes), - products_() {} + branchIDLists_(branchIDLists), + thinnedAssociationsHelper_(thinnedAssociationsHelper), + products_(), + metaDataChecksum_(metaDataChecksum) {} EventAuxiliary const& aux() const { return aux_; } SendProds const& products() const { return products_; } ProcessHistory const& processHistory() const { return processHistory_; } EventSelectionIDVector const& eventSelectionIDs() const { return eventSelectionIDs_; } BranchListIndexes const& branchListIndexes() const { return branchListIndexes_; } + //This will only hold values for EventMetaData messages + BranchIDLists const& branchIDLists() const { return branchIDLists_; } + //This will only hold values for EventMetaData messages + ThinnedAssociationsHelper const& thinnedAssociationsHelper() const { return thinnedAssociationsHelper_; } + //This is the adler32 checksum of the EventMetaData associated with this Event + uint32_t metaDataChecksum() const { return metaDataChecksum_; } SendProds& products() { return products_; } private: @@ -92,7 +107,10 @@ namespace edm { ProcessHistory processHistory_; EventSelectionIDVector eventSelectionIDs_; BranchListIndexes branchListIndexes_; + BranchIDLists branchIDLists_; + ThinnedAssociationsHelper thinnedAssociationsHelper_; SendProds products_; + uint32_t metaDataChecksum_; // other tables necessary for provenance lookup }; @@ -105,21 +123,13 @@ namespace edm { SendJobHeader() {} SendDescs const& descs() const { return descs_; } ParameterSetMap const& processParameterSet() const { return processParameterSet_; } - BranchIDLists const& branchIDLists() const { return branchIDLists_; } - ThinnedAssociationsHelper const& thinnedAssociationsHelper() const { return thinnedAssociationsHelper_; } void push_back(BranchDescription const& bd) { descs_.push_back(bd); } void setParameterSetMap(ParameterSetMap const& psetMap) { processParameterSet_ = psetMap; } - void setBranchIDLists(BranchIDLists const& bidlists) { branchIDLists_ = bidlists; } - void setThinnedAssociationsHelper(ThinnedAssociationsHelper const& v) { thinnedAssociationsHelper_ = v; } void initializeTransients(); private: SendDescs descs_; ParameterSetMap processParameterSet_; - BranchIDLists branchIDLists_; - ThinnedAssociationsHelper thinnedAssociationsHelper_; - // trigger bit descriptions will be added here and permanent - // provenance values }; } // namespace edm diff --git a/DataFormats/Streamer/src/classes_def.xml b/DataFormats/Streamer/src/classes_def.xml index 9451393a99d33..e0b30db038f98 100644 --- a/DataFormats/Streamer/src/classes_def.xml +++ b/DataFormats/Streamer/src/classes_def.xml @@ -7,10 +7,12 @@ - + + - + + diff --git a/EventFilter/Utilities/interface/DAQSourceModelsFRD.h b/EventFilter/Utilities/interface/DAQSourceModelsFRD.h index c3e1c896ab623..15a45c23c37f4 100644 --- a/EventFilter/Utilities/interface/DAQSourceModelsFRD.h +++ b/EventFilter/Utilities/interface/DAQSourceModelsFRD.h @@ -24,9 +24,9 @@ class DataModeFRD : public DataMode { detectedFRDversion_ = *((uint16_t*)(fileBuf + fileHeaderOffset)); } - uint32_t headerSize() const override { return FRDHeaderVersionSize[detectedFRDversion_]; } + uint32_t headerSize() const override { return edm::streamer::FRDHeaderVersionSize[detectedFRDversion_]; } - bool versionCheck() const override { return detectedFRDversion_ <= FRDHeaderMaxVersion; } + bool versionCheck() const override { return detectedFRDversion_ <= edm::streamer::FRDHeaderMaxVersion; } uint64_t dataBlockSize() const override { return event_->size(); } @@ -76,7 +76,7 @@ class DataModeFRD : public DataMode { std::vector> daqProvenanceHelpers_; uint16_t detectedFRDversion_ = 0; size_t headerSize_ = 0; - std::unique_ptr event_; + std::unique_ptr event_; uint32_t crc_ = 0; unsigned char* dataBlockAddr_ = nullptr; size_t dataBlockMax_ = 0; @@ -106,9 +106,9 @@ class DataModeFRDStriped : public DataMode { detectedFRDversion_ = *((uint16_t*)(fileBuf + fileHeaderOffset)); } - uint32_t headerSize() const override { return FRDHeaderVersionSize[detectedFRDversion_]; } + uint32_t headerSize() const override { return edm::streamer::FRDHeaderVersionSize[detectedFRDversion_]; } - bool versionCheck() const override { return detectedFRDversion_ <= FRDHeaderMaxVersion; } + bool versionCheck() const override { return detectedFRDversion_ <= edm::streamer::FRDHeaderMaxVersion; } uint64_t dataBlockSize() const override { //just get first event size @@ -186,7 +186,7 @@ class DataModeFRDStriped : public DataMode { uint16_t detectedFRDversion_ = 0; size_t fileHeaderSize_ = 0; size_t headerSize_ = 0; - std::vector> events_; + std::vector> events_; std::string crcMsg_; unsigned char* dataBlockAddr_ = nullptr; std::vector dataBlockAddrs_; diff --git a/EventFilter/Utilities/interface/DAQSourceModelsScoutingRun3.h b/EventFilter/Utilities/interface/DAQSourceModelsScoutingRun3.h index a3c8bd43c5fc7..1eff0fdb1bada 100644 --- a/EventFilter/Utilities/interface/DAQSourceModelsScoutingRun3.h +++ b/EventFilter/Utilities/interface/DAQSourceModelsScoutingRun3.h @@ -31,8 +31,8 @@ class DataModeScoutingRun3 : public DataMode { void detectVersion(unsigned char* fileBuf, uint32_t fileHeaderOffset) override { detectedFRDversion_ = *((uint16_t*)(fileBuf + fileHeaderOffset)); } - uint32_t headerSize() const override { return FRDHeaderVersionSize[detectedFRDversion_]; } - bool versionCheck() const override { return detectedFRDversion_ <= FRDHeaderMaxVersion; } + uint32_t headerSize() const override { return edm::streamer::FRDHeaderVersionSize[detectedFRDversion_]; } + bool versionCheck() const override { return detectedFRDversion_ <= edm::streamer::FRDHeaderMaxVersion; } uint64_t dataBlockSize() const override { // get event size from the first data source (main) @@ -113,7 +113,7 @@ class DataModeScoutingRun3 : public DataMode { uint16_t detectedFRDversion_ = 0; size_t fileHeaderSize_ = 0; size_t headerSize_ = 0; - std::vector> events_; + std::vector> events_; unsigned char* dataBlockAddr_ = nullptr; std::vector dataBlockAddrs_; std::vector dataBlockMaxAddrs_; diff --git a/EventFilter/Utilities/interface/FedRawDataInputSource.h b/EventFilter/Utilities/interface/FedRawDataInputSource.h index 9b5189d1e5f37..687ce0e50148c 100644 --- a/EventFilter/Utilities/interface/FedRawDataInputSource.h +++ b/EventFilter/Utilities/interface/FedRawDataInputSource.h @@ -115,7 +115,7 @@ class FedRawDataInputSource : public edm::RawInputSource { const edm::DaqProvenanceHelper daqProvenanceHelper_; - std::unique_ptr event_; + std::unique_ptr event_; edm::EventID eventID_; edm::ProcessHistoryID processHistoryID_; diff --git a/EventFilter/Utilities/plugins/FRDOutputModule.cc b/EventFilter/Utilities/plugins/FRDOutputModule.cc index 65bf2a59ded6a..5b5b9fe43bfc9 100644 --- a/EventFilter/Utilities/plugins/FRDOutputModule.cc +++ b/EventFilter/Utilities/plugins/FRDOutputModule.cc @@ -22,6 +22,8 @@ #include "IOPool/Streamer/interface/FRDFileHeader.h" #include "EventFilter/Utilities/interface/crc32c.h" +using namespace edm::streamer; + FRDOutputModule::FRDOutputModule(edm::ParameterSet const& ps) : edm::one::OutputModuleBase::OutputModuleBase(ps), edm::one::OutputModule(ps), diff --git a/EventFilter/Utilities/plugins/FRDStreamSource.cc b/EventFilter/Utilities/plugins/FRDStreamSource.cc index 884eeb985c523..ff9dca127864b 100644 --- a/EventFilter/Utilities/plugins/FRDStreamSource.cc +++ b/EventFilter/Utilities/plugins/FRDStreamSource.cc @@ -17,6 +17,8 @@ #include "EventFilter/Utilities/plugins/FRDStreamSource.h" #include "EventFilter/Utilities/interface/crc32c.h" +using namespace edm::streamer; + FRDStreamSource::FRDStreamSource(edm::ParameterSet const& pset, edm::InputSourceDescription const& desc) : ProducerSourceFromFiles(pset, desc, true), verifyAdler32_(pset.getUntrackedParameter("verifyAdler32", true)), diff --git a/EventFilter/Utilities/plugins/GlobalEvFOutputModule.cc b/EventFilter/Utilities/plugins/GlobalEvFOutputModule.cc index a9f42ba645172..c499de2780701 100644 --- a/EventFilter/Utilities/plugins/GlobalEvFOutputModule.cc +++ b/EventFilter/Utilities/plugins/GlobalEvFOutputModule.cc @@ -22,7 +22,7 @@ #include "IOPool/Streamer/interface/StreamerOutputFile.h" #include "FWCore/Framework/interface/global/OutputModule.h" -#include "IOPool/Streamer/interface/StreamerOutputModuleCommon.h" +#include "IOPool/Streamer/interface/StreamerOutputMsgBuilders.h" #include "FWCore/Utilities/interface/EDGetToken.h" #include "DataFormats/Streamer/interface/StreamedProducts.h" @@ -38,16 +38,39 @@ typedef edm::detail::TriggerResultsBasedEventSelector::handle_t Trig; namespace evf { + using namespace edm::streamer; class FastMonitoringService; + struct MetaDataCache { + MetaDataCache(StreamerOutputMsgBuilders const& builders, + edm::BranchIDLists const& branchLists, + edm::ThinnedAssociationsHelper const helper) + : buffer_() { + auto ret = builders.serializeEventMetaData(buffer_, branchLists, helper); + builder_ = std::move(ret.first); + checksum_ = ret.second; + } + SerializeDataBuffer buffer_; + std::unique_ptr builder_; + uint32_t checksum_; + }; + class GlobalEvFOutputEventWriter { public: - explicit GlobalEvFOutputEventWriter(std::string const& filePath, unsigned int ls) - : filePath_(filePath), ls_(ls), accepted_(0), stream_writer_events_(new StreamerOutputFile(filePath)) {} + explicit GlobalEvFOutputEventWriter(std::string const& filePath, + unsigned int ls, + std::shared_ptr iMetaData) + : filePath_(filePath), + ls_(ls), + accepted_(0), + stream_writer_events_(new StreamerOutputFile(filePath)), + meta_(std::move(iMetaData)) {} ~GlobalEvFOutputEventWriter() {} + void setMetaCache(std::shared_ptr iMetaData) { meta_ = std::move(iMetaData); } + bool close() { stream_writer_events_->close(); return (discarded_ || edm::Service()->lumisectionDiscarded(ls_)); @@ -68,9 +91,14 @@ namespace evf { return; } auto group = iHolder.group(); - writeQueue_.push(*group, [holder = std::move(iHolder), msg = msg.release(), this]() { + writeQueue_.push(*group, [holder = std::move(iHolder), msg = msg.release(), this]() mutable { try { std::unique_ptr own(msg); + if (meta_) { + auto m = std::move(meta_); + assert(m->builder_); + doOutputEvent(*m->builder_); + } doOutputEvent(*msg); //msg is written and discarded at this point } catch (...) { auto tmp = holder; @@ -116,6 +144,7 @@ namespace evf { const unsigned ls_; std::atomic accepted_; edm::propagate_const> stream_writer_events_; + std::shared_ptr meta_; edm::SerialTaskQueue writeQueue_; bool discarded_ = false; }; @@ -155,7 +184,8 @@ namespace evf { typedef edm::global::OutputModule, edm::LuminosityBlockCache, - edm::StreamCache, + edm::StreamCache, + edm::WatchInputFiles, edm::ExternalWork> GlobalEvFOutputModuleType; @@ -166,7 +196,7 @@ namespace evf { static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); private: - std::unique_ptr beginStream(edm::StreamID) const final; + std::unique_ptr beginStream(edm::StreamID) const final; std::shared_ptr globalBeginRun(edm::RunForOutput const& run) const final; @@ -178,19 +208,31 @@ namespace evf { void writeRun(edm::RunForOutput const&) final {} void globalEndRun(edm::RunForOutput const&) const final {} + void respondToOpenInputFile(edm::FileBlock const&) final; + void respondToCloseInputFile(edm::FileBlock const&) final {} + + void beginJob() final; + void cacheEventMetaData(); + std::shared_ptr globalBeginLuminosityBlock( edm::LuminosityBlockForOutput const& iLB) const final; void globalEndLuminosityBlock(edm::LuminosityBlockForOutput const& iLB) const final; Trig getTriggerResults(edm::EDGetTokenT const& token, edm::EventForOutput const& e) const; - edm::StreamerOutputModuleCommon::Parameters commonParameters_; + StreamerOutputMsgBuilders::Parameters commonParameters_; + std::unique_ptr msgBuilders_; std::string streamLabel_; edm::EDGetTokenT trToken_; edm::EDGetTokenT psetToken_; evf::FastMonitoringService* fms_; + std::shared_ptr metaDataCache_; + //if a new file appears and has different meta data but the same lumi, we need + // to update the writer to write out the new meta data + mutable std::atomic lastWriter_ = nullptr; + unsigned int presentBranchIDListSize_ = 0; }; //end-of-class-def GlobalEvFOutputJSONDef::GlobalEvFOutputJSONDef(std::string const& streamLabel, bool writeJsd) { @@ -286,7 +328,7 @@ namespace evf { GlobalEvFOutputModule::GlobalEvFOutputModule(edm::ParameterSet const& ps) : edm::global::OutputModuleBase(ps), GlobalEvFOutputModuleType(ps), - commonParameters_(edm::StreamerOutputModuleCommon::parameters(ps)), + commonParameters_(StreamerOutputMsgBuilders::parameters(ps)), streamLabel_(ps.getParameter("@module_label")), trToken_(consumes(edm::InputTag("TriggerResults"))), psetToken_(consumes( @@ -333,24 +375,21 @@ namespace evf { void GlobalEvFOutputModule::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { edm::ParameterSetDescription desc; - edm::StreamerOutputModuleCommon::fillDescription(desc); + StreamerOutputMsgBuilders::fillDescription(desc); GlobalEvFOutputModuleType::fillDescription(desc); desc.addUntracked("psetMap", {"hltPSetMap"}) ->setComment("Optionally allow the map of ParameterSets to be calculated externally."); descriptions.add("globalEvfOutputModule", desc); } - std::unique_ptr GlobalEvFOutputModule::beginStream(edm::StreamID) const { - return std::make_unique( - commonParameters_, &keptProducts()[edm::InEvent], description().moduleLabel()); + std::unique_ptr GlobalEvFOutputModule::beginStream(edm::StreamID) const { + return std::make_unique(); } std::shared_ptr GlobalEvFOutputModule::globalBeginRun(edm::RunForOutput const& run) const { //create run Cache holding JSON file writer and variables auto jsonDef = std::make_unique(streamLabel_, false); jsonDef->updateDestination(streamLabel_); - edm::StreamerOutputModuleCommon streamerCommon( - commonParameters_, &keptProducts()[edm::InEvent], description().moduleLabel()); //output INI file (non-const). This doesn't require globalBeginRun to be finished const std::string openIniFileName = edm::Service()->getOpenInitFilePath(streamLabel_); @@ -358,18 +397,16 @@ namespace evf { StreamerOutputFile stream_writer_preamble(openIniFileName); uint32 preamble_adler32 = 1; - edm::BranchIDLists const* bidlPtr = branchIDLists(); auto psetMapHandle = run.getHandle(psetToken_); + SerializeDataBuffer buffer; std::unique_ptr init_message = - streamerCommon.serializeRegistry(*streamerCommon.getSerializerBuffer(), - *bidlPtr, - *thinnedAssociationsHelper(), - OutputModule::processName(), - description().moduleLabel(), - moduleDescription().mainParameterSetID(), - psetMapHandle.isValid() ? psetMapHandle.product() : nullptr); + msgBuilders_->serializeRegistry(buffer, + OutputModule::processName(), + description().moduleLabel(), + moduleDescription().mainParameterSetID(), + psetMapHandle.isValid() ? psetMapHandle.product() : nullptr); //Let us turn it into a View InitMsgView view(init_message->startAddress()); @@ -400,9 +437,6 @@ namespace evf { } src.close(); - //clear serialization buffers - streamerCommon.getSerializerBuffer()->clearHeaderBuffer(); - //free output buffer needed only for the file write outBuf.reset(); @@ -429,7 +463,29 @@ namespace evf { edm::LuminosityBlockForOutput const& iLB) const { auto openDatFilePath = edm::Service()->getOpenDatFilePath(iLB.luminosityBlock(), streamLabel_); - return std::make_shared(openDatFilePath, iLB.luminosityBlock()); + auto ret = std::make_shared(openDatFilePath, iLB.luminosityBlock(), metaDataCache_); + lastWriter_ = ret.get(); + return ret; + } + + void GlobalEvFOutputModule::beginJob() { + msgBuilders_ = std::make_unique( + commonParameters_, &keptProducts()[edm::InEvent], description().moduleLabel()); + + cacheEventMetaData(); + } + + void GlobalEvFOutputModule::respondToOpenInputFile(edm::FileBlock const&) { + if (branchIDLists()->size() != presentBranchIDListSize_) { + cacheEventMetaData(); + if (lastWriter_) { + lastWriter_.load()->setMetaCache(metaDataCache_); + } + } + } + + void GlobalEvFOutputModule::cacheEventMetaData() { + metaDataCache_ = std::make_shared(*msgBuilders_, *branchIDLists(), *thinnedAssociationsHelper()); } void GlobalEvFOutputModule::acquire(edm::StreamID id, @@ -437,9 +493,10 @@ namespace evf { edm::WaitingTaskWithArenaHolder iHolder) const { edm::Handle const& triggerResults = getTriggerResults(trToken_, e); - auto streamerCommon = streamCache(id); + std::cout << " writing Event " << moduleDescription().moduleLabel() << std::endl; + auto buffer = streamCache(id); std::unique_ptr msg = - streamerCommon->serializeEvent(*streamerCommon->getSerializerBuffer(), e, triggerResults, selectorConfig()); + msgBuilders_->serializeEvent(*buffer, e, triggerResults, selectorConfig(), metaDataCache_->checksum_); auto lumiWriter = luminosityBlockCache(e.getLuminosityBlock().index()); const_cast(lumiWriter) diff --git a/EventFilter/Utilities/plugins/RawEventFileWriterForBU.cc b/EventFilter/Utilities/plugins/RawEventFileWriterForBU.cc index 8b4aacc084057..f5745a159efd3 100644 --- a/EventFilter/Utilities/plugins/RawEventFileWriterForBU.cc +++ b/EventFilter/Utilities/plugins/RawEventFileWriterForBU.cc @@ -17,6 +17,7 @@ #include "IOPool/Streamer/interface/FRDFileHeader.h" using namespace jsoncollector; +using namespace edm::streamer; //TODO:get run directory information from DaqDirector diff --git a/EventFilter/Utilities/plugins/RawEventFileWriterForBU.h b/EventFilter/Utilities/plugins/RawEventFileWriterForBU.h index 056082fe36a7c..6bdcefa7d5dd8 100644 --- a/EventFilter/Utilities/plugins/RawEventFileWriterForBU.h +++ b/EventFilter/Utilities/plugins/RawEventFileWriterForBU.h @@ -24,9 +24,9 @@ class RawEventFileWriterForBU { explicit RawEventFileWriterForBU(std::string const& fileName); ~RawEventFileWriterForBU(); - void doOutputEvent(FRDEventMsgView const& msg); + void doOutputEvent(edm::streamer::FRDEventMsgView const& msg); - uint32 adler32() const { return (adlerb_ << 16) | adlera_; } + edm::streamer::uint32 adler32() const { return (adlerb_ << 16) | adlera_; } void start() {} void stop(); @@ -82,8 +82,8 @@ class RawEventFileWriterForBU { int microSleep_; unsigned int frdFileVersion_; - uint32 adlera_; - uint32 adlerb_; + edm::streamer::uint32 adlera_; + edm::streamer::uint32 adlerb_; unsigned int lumiOpen_ = 0; unsigned int lumiClosed_ = 0; diff --git a/EventFilter/Utilities/plugins/RawEventOutputModuleForBU.h b/EventFilter/Utilities/plugins/RawEventOutputModuleForBU.h index 60dd6d01f7e01..d504d922fe28c 100644 --- a/EventFilter/Utilities/plugins/RawEventOutputModuleForBU.h +++ b/EventFilter/Utilities/plugins/RawEventOutputModuleForBU.h @@ -75,6 +75,8 @@ RawEventOutputModuleForBU::~RawEventOutputModuleForBU() {} template void RawEventOutputModuleForBU::write(edm::EventForOutput const& e) { + using namespace edm::streamer; + unsigned int ls = e.luminosityBlock(); if (totevents > 0 && totevents % numEventsPerFile_ == 0) { index_++; @@ -89,8 +91,8 @@ void RawEventOutputModuleForBU::write(edm::EventForOutput const& e) { e.getByToken(token_, fedBuffers); // determine the expected size of the FRDEvent IN BYTES !!!!! - assert(frdVersion_ <= FRDHeaderMaxVersion); - int headerSize = FRDHeaderVersionSize[frdVersion_]; + assert(frdVersion_ <= edm::streamer::FRDHeaderMaxVersion); + int headerSize = edm::streamer::FRDHeaderVersionSize[frdVersion_]; int expectedSize = headerSize; int nFeds = frdVersion_ < 3 ? 1024 : FEDNumbering::lastFEDId() + 1; @@ -156,7 +158,7 @@ void RawEventOutputModuleForBU::write(edm::EventForOutput const& e) { } // create the FRDEventMsgView and use the template consumer to write it out - FRDEventMsgView msg(workBuffer.get()->data()); + edm::streamer::FRDEventMsgView msg(workBuffer.get()->data()); writtensize += msg.size(); templateConsumer_->doOutputEvent(msg); diff --git a/EventFilter/Utilities/src/DAQSourceModelsFRD.cc b/EventFilter/Utilities/src/DAQSourceModelsFRD.cc index 2784cef60ec55..22afae18caa4c 100644 --- a/EventFilter/Utilities/src/DAQSourceModelsFRD.cc +++ b/EventFilter/Utilities/src/DAQSourceModelsFRD.cc @@ -29,6 +29,8 @@ #include "DataFormats/Provenance/interface/Timestamp.h" #include "EventFilter/Utilities/interface/crc32c.h" +using namespace edm::streamer; + void DataModeFRD::readEvent(edm::EventPrincipal& eventPrincipal) { std::unique_ptr rawData(new FEDRawDataCollection); bool tcdsInRange; diff --git a/EventFilter/Utilities/src/DAQSourceModelsScoutingRun3.cc b/EventFilter/Utilities/src/DAQSourceModelsScoutingRun3.cc index f856fdbed66ef..fff4240566716 100644 --- a/EventFilter/Utilities/src/DAQSourceModelsScoutingRun3.cc +++ b/EventFilter/Utilities/src/DAQSourceModelsScoutingRun3.cc @@ -1,5 +1,7 @@ #include "EventFilter//Utilities/interface/DAQSourceModelsScoutingRun3.h" +using namespace edm::streamer; + void DataModeScoutingRun3::makeDirectoryEntries(std::vector const& baseDirs, std::vector const& numSources, std::string const& runDir) { diff --git a/EventFilter/Utilities/src/EvFDaqDirector.cc b/EventFilter/Utilities/src/EvFDaqDirector.cc index 2344195c6efd9..1715780e74bf8 100644 --- a/EventFilter/Utilities/src/EvFDaqDirector.cc +++ b/EventFilter/Utilities/src/EvFDaqDirector.cc @@ -30,6 +30,7 @@ //#define DEBUG using namespace jsoncollector; +using namespace edm::streamer; namespace evf { diff --git a/EventFilter/Utilities/src/FedRawDataInputSource.cc b/EventFilter/Utilities/src/FedRawDataInputSource.cc index ad3b51661cd64..91feced0591d1 100644 --- a/EventFilter/Utilities/src/FedRawDataInputSource.cc +++ b/EventFilter/Utilities/src/FedRawDataInputSource.cc @@ -47,6 +47,7 @@ #include "EventFilter/Utilities/interface/reader.h" using namespace evf::FastMonState; +using namespace edm::streamer; FedRawDataInputSource::FedRawDataInputSource(edm::ParameterSet const& pset, edm::InputSourceDescription const& desc) : edm::RawInputSource(pset, desc), diff --git a/EventFilter/Utilities/test/BuildFile.xml b/EventFilter/Utilities/test/BuildFile.xml index 7ad2f437052ef..f63c46b41a74b 100644 --- a/EventFilter/Utilities/test/BuildFile.xml +++ b/EventFilter/Utilities/test/BuildFile.xml @@ -11,3 +11,4 @@ + diff --git a/EventFilter/Utilities/test/RunBUFU.sh b/EventFilter/Utilities/test/RunBUFU.sh index 18d892f94dfc8..dab4ffb5bb59b 100755 --- a/EventFilter/Utilities/test/RunBUFU.sh +++ b/EventFilter/Utilities/test/RunBUFU.sh @@ -50,11 +50,32 @@ cat data/run${runnumber}/run${runnumber}_ls0001_streamDQM_pid*.dat >> dqmdisk/ru find dqmdisk echo '{"data": [12950, 1620, 0, "run'${runnumber}'_ls0001_streamDQM_test.dat", 40823782, 1999348078, 135, 13150, 0, "Failsafe"]}' > dqmdisk/run${runnumber}/run${runnumber}_ls0001_streamDQM_test.jsn -CMDLINE_STARTDQM="cmsRun test_dqmstream.py runInputDir=./dqmdisk runNumber=100101" +CMDLINE_STARTDQM="cmsRun test_dqmstream.py runInputDir=./dqmdisk runNumber=100101 maxLS=1 eventsPerLS=35" ${CMDLINE_STARTDQM} > out_2_dqm.log 2>&1 || diedqm "${CMDLINE_STARTDQM}" $? $OUTDIR rm -rf $OUTDIR/{ramdisk,data,*.log} +################### +echo "Running test with FRD file header v1 (no index JSONs) and empty files" +CMDLINE_STARTBU="cmsRun startBU.py runNumber=${runnumber} fffBaseDir=${OUTDIR} maxLS=2 fedMeanSize=128 eventsPerFile=20 eventsPerLS=35 frdFileVersion=1" +#CMDLINE_STARTFU="cmsRun startFU.py runNumber=${runnumber} fffBaseDir=${OUTDIR}" +CMDLINE_STARTFU="cmsRun ${FUSCRIPT} runNumber=${runnumber} fffBaseDir=${OUTDIR} numEventsToWrite=0" +${CMDLINE_STARTBU} > out_2_bu.log 2>&1 || diebu "${CMDLINE_STARTBU}" $? $OUTDIR +${CMDLINE_STARTFU} > out_2_fu.log 2>&1 || diefu "${CMDLINE_STARTFU}" $? $OUTDIR + +#prepare DQM files +mkdir dqmdisk/run${runnumber} -p +cat data/run${runnumber}/run${runnumber}_ls0000_streamDQM_pid*.ini > dqmdisk/run${runnumber}/run${runnumber}_ls0001_streamDQM_test.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamDQM_pid*.dat >> dqmdisk/run${runnumber}/run${runnumber}_ls0001_streamDQM_test.dat +find dqmdisk +echo '{"data": [12950, 1620, 0, "run'${runnumber}'_ls0001_streamDQM_test.dat", 40823782, 1999348078, 135, 13150, 0, "Failsafe"]}' > dqmdisk/run${runnumber}/run${runnumber}_ls0001_streamDQM_test.jsn + +CMDLINE_STARTDQM="cmsRun test_dqmstream.py runInputDir=./dqmdisk runNumber=100101 maxLS=1 eventsPerLS=0" +${CMDLINE_STARTDQM} > out_2_dqm.log 2>&1 || diedqm "${CMDLINE_STARTDQM}" $? $OUTDIR + +rm -rf $OUTDIR/{ramdisk,data,*.log} + +################ echo "Running test with FRD file header v2" CMDLINE_STARTBU="cmsRun startBU.py runNumber=${runnumber} fffBaseDir=${OUTDIR} maxLS=2 fedMeanSize=128 eventsPerFile=20 eventsPerLS=35 frdFileVersion=2" CMDLINE_STARTFU="cmsRun unittest_FU.py runNumber=${runnumber} fffBaseDir=${OUTDIR}" diff --git a/EventFilter/Utilities/test/TestReadingStreamerFile.sh b/EventFilter/Utilities/test/TestReadingStreamerFile.sh new file mode 100755 index 0000000000000..e0bf841fa31ae --- /dev/null +++ b/EventFilter/Utilities/test/TestReadingStreamerFile.sh @@ -0,0 +1,258 @@ +#!/bin/bash +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +function diewrite { echo Failure $1: status $2 ; echo "" ; echo "----- Error -----"; echo ""; cat out_2_write.log; rm -rf $3/{ramdisk,data,*.py}; exit $2 ; } +function dieread { echo Failure $1: status $2 ; echo "" ; echo "----- Error -----"; echo ""; cat out_2_read.log; rm -rf $3/{ramdisk,data,*.py}; exit $2 ; } +function diemerge { echo Failure $1: status $2 ; echo "" ; echo "----- Error -----"; echo ""; cat out_2_merge.log; rm -rf $3/{ramdisk,data,*.py}; exit $2 ; } + + + +if [ -z ${SCRAM_TEST_PATH} ]; then +SCRAM_TEST_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +fi +echo "SCRAM_TEST_PATH = ${SCRAM_TEST_PATH}" + +RC=0 +P=$$ +PREFIX=results_${USER}${P} +OUTDIR=${PWD}/${PREFIX} + +echo "OUT_TMP_DIR = $OUTDIR" + +mkdir ${OUTDIR} +cp ${SCRIPTDIR}/writeStreamerFile_cfg.py ${OUTDIR} +cp ${SCRIPTDIR}/readStreamerFile_cfg.py ${OUTDIR} +cp ${SCRIPTDIR}/mergeStreamerFile_cfg.py ${OUTDIR} +cd ${OUTDIR} + +rm -rf $OUTDIR/{ramdisk,data,*.log} + +runnumber="100101" + +############################### +echo "Running test on reading single file" +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test.dat + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test.dat --runNumber=${runnumber} --numEvents=10" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR + +rm -rf data +########################## +echo "Running test on reading two separate files" + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test1.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test1.dat + +rm -rf data + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --startEvent=11 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test2.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test2.dat + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test1.dat --input test2.dat --runNumber=${runnumber} --numEvents=20" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR + +rm -rf data +############################# +echo "Running test on reading two separate files with different BranchIDLists" + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test1.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test1.dat + +rm -rf data + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --startEvent=11 --runNumber=${runnumber} --changeBranchIDLists T" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test2.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test2.dat + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test1.dat --input test2.dat --runNumber=${runnumber} --numEvents=20" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR + +rm -rf data +########################## + +echo "Running test one concatenated file" + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --startEvent=11 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test.dat + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test.dat --runNumber=${runnumber} --numEvents=20" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR +#cat out_2_read.log + +rm -rf data +############################# +echo "Running test on concatenated file from jobs with different BranchIDLists" + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --startEvent=11 --runNumber=${runnumber} --changeBranchIDLists T" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test.dat + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test.dat --runNumber=${runnumber} --numEvents=20" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR +#cat out_2_read.log + +rm -rf data + +########################### +echo "Test merging streamers using cmsRun" + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test1.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test1.dat + +rm -rf data + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --startEvent=11 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test2.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test2.dat + +rm -rf data + + +CMDLINE_MERGE="cmsRun mergeStreamerFile_cfg.py --input test1.dat --input test2.dat --runNumber=${runnumber}" +${CMDLINE_MERGE} > out_2_merge.log 2>&1 || diemerge "${CMDLINE_MERGE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_merge_pid*.ini | head -1 | xargs cat > test.dat +cat data/run${runnumber}/run${runnumber}_ls0001_merge_pid*.dat >> test.dat + + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test.dat --runNumber=${runnumber} --numEvents=20" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR + +rm -rf data +############################# +echo "Test merging files with different BranchIDLists" + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test1.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test1.dat + +rm -rf data + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --startEvent=11 --runNumber=${runnumber} --changeBranchIDLists T" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test2.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test2.dat + +rm -rf data + +CMDLINE_MERGE="cmsRun mergeStreamerFile_cfg.py --input test1.dat --input test2.dat --runNumber=${runnumber}" +${CMDLINE_MERGE} > out_2_merge.log 2>&1 || diemerge "${CMDLINE_MERGE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_merge_pid*.ini | head -1 | xargs cat > test.dat +cat data/run${runnumber}/run${runnumber}_ls0001_merge_pid*.dat >> test.dat + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test.dat --runNumber=${runnumber} --numEvents=20" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR + + +############################### +echo "Running test on reading single empty file" +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=0 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test.dat + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test.dat --runNumber=${runnumber} --numEvents=0" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR + +rm -rf data +########################## +echo "Running test on reading two separate empty files" + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --runNumber=${runnumber} --numEvents=0" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test1.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test1.dat + +rm -rf data + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=10 --startEvent=11 --runNumber=${runnumber} --numEvents=0" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test2.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test2.dat + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test1.dat --input test2.dat --runNumber=${runnumber} --numEvents=0" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR + +rm -rf data +########################## + +echo "Running test one concatenated empty file" + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=0 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +CMDLINE_WRITE="cmsRun writeStreamerFile_cfg.py --numEvents=0 --startEvent=11 --runNumber=${runnumber}" +${CMDLINE_WRITE} > out_2_write.log 2>&1 || diewrite "${CMDLINE_WRITE}" $? $OUTDIR + +#prepare file to read +ls -1 data/run${runnumber}/run${runnumber}_ls0000_streamA_pid*.ini | head -1 | xargs cat > test.dat +cat data/run${runnumber}/run${runnumber}_ls0001_streamA_pid*.dat >> test.dat + +CMDLINE_READ="cmsRun readStreamerFile_cfg.py --input test.dat --runNumber=${runnumber} --numEvents=0" +${CMDLINE_READ} > out_2_read.log 2>&1 || dieread "${CMDLINE_READ}" $? $OUTDIR +#cat out_2_read.log + +rm -rf data +############################ + +#no failures, clean up everything including logs if there are no errors +echo "Completed sucessfully" +#rm -rf $OUTDIR/{ramdisk,data,*.py,*.log} +rm -rf $OUTDIR + +exit ${RC} + diff --git a/EventFilter/Utilities/test/mergeStreamerFile_cfg.py b/EventFilter/Utilities/test/mergeStreamerFile_cfg.py new file mode 100644 index 0000000000000..274e2a4f322f8 --- /dev/null +++ b/EventFilter/Utilities/test/mergeStreamerFile_cfg.py @@ -0,0 +1,59 @@ +import FWCore.ParameterSet.Config as cms +from argparse import ArgumentParser + +parser = ArgumentParser(description='Test merge using GlobalEvFOutputModule') +parser.add_argument("--input", action="append", default=[], help="Input files") +parser.add_argument("--numThreads", help="number of threads to use", type=int, default=3) +parser.add_argument("--numFwkStreams", help="number of cmsRun streams", type=int, default=2) +parser.add_argument("--changeBranchIDLists", help="modify the branchIDLists", type=bool, default=False) +parser.add_argument("--runNumber", help="run number to use", type=int, default=100101) +parser.add_argument("--buBaseDir", help="BU base directory", type=str, default="ramdisk") +parser.add_argument("--fuBaseDir", help="FU base directory", type=str, default="data") +parser.add_argument("--fffBaseDir", help="FFF base directory", type=str, default=".") + +args = parser.parse_args() + +#try to create 'ramdisk' directory +try: + os.makedirs(args.fffBaseDir+"/"+args.buBaseDir+"/run"+str(args.runNumber).zfill(6)) +except:pass +#try to create 'data' directory +try: + os.makedirs(args.fffBaseDir+"/"+args.fuBaseDir+"/run"+str(args.runNumber).zfill(6)) +except Exception as ex: + print(str(ex)) + pass + + +process = cms.Process("MERGE") + +if len(args.input) == 0: + parser.error("No input files") + +process.source = cms.Source("NewEventStreamFileReader", + fileNames = cms.untracked.vstring(["file:"+f for f in args.input]) +) + +process.options = dict(numberOfThreads = args.numThreads, + numberOfStreams = args.numFwkStreams) + +process.merge = cms.OutputModule("GlobalEvFOutputModule", + SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring()), + outputCommands = cms.untracked.vstring("keep *") +) + +process.ep = cms.EndPath(process.merge) + +process.EvFDaqDirector = cms.Service("EvFDaqDirector", + useFileBroker = cms.untracked.bool(False), + fileBrokerHostFromCfg = cms.untracked.bool(True), + fileBrokerHost = cms.untracked.string("htcp40.cern.ch"), + runNumber = cms.untracked.uint32(args.runNumber), + baseDir = cms.untracked.string(args.fffBaseDir+"/"+args.fuBaseDir), + buBaseDir = cms.untracked.string(args.fffBaseDir+"/"+args.buBaseDir), + directorIsBU = cms.untracked.bool(False), +) + +process.FastMonitoringService = cms.Service("FastMonitoringService", + sleepTime = cms.untracked.int32(1) +) diff --git a/EventFilter/Utilities/test/readStreamerFile_cfg.py b/EventFilter/Utilities/test/readStreamerFile_cfg.py new file mode 100644 index 0000000000000..b7dc7b98a0ff4 --- /dev/null +++ b/EventFilter/Utilities/test/readStreamerFile_cfg.py @@ -0,0 +1,44 @@ +import FWCore.ParameterSet.Config as cms +import argparse +import sys + +process = cms.Process("READ") + + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test output of GlobalEvFOutputModule') +parser.add_argument("--input", action="append", default=[], help="Input files") +parser.add_argument("--runNumber", type=int, default=1, help="expected run number") +parser.add_argument("--numEvents", type=int, default=10, help="expected number of events") +args = parser.parse_args() +if len(args.input) == 0: + parser.error("No input files") + +process.source = cms.Source("NewEventStreamFileReader", + fileNames = cms.untracked.vstring(["file:"+f for f in args.input]) +) + +process.tester = cms.EDAnalyzer("OtherThingAnalyzer", + other = cms.untracked.InputTag("otherThing","testUserTag") +) + +rn = args.runNumber +lumi = 1 +transitions = [cms.EventID(rn,0,0),cms.EventID(rn,lumi,0)] +evid = 1 + +for ev in range(0, args.numEvents): + transitions.append(cms.EventID(rn,lumi,evid)) + evid += 1 +transitions.append(cms.EventID(rn,lumi,0)) #end lumi +transitions.append(cms.EventID(rn,0,0)) #end run + +if args.numEvents == 0: + transitions = [] + +process.test = cms.EDAnalyzer("RunLumiEventChecker", + eventSequence = cms.untracked.VEventID(*transitions), + unorderedEvents = cms.untracked.bool(True) +) + + +process.e = cms.EndPath(process.tester+process.test) diff --git a/EventFilter/Utilities/test/startFU.py b/EventFilter/Utilities/test/startFU.py index 03cf3e1798af2..b08755167d004 100644 --- a/EventFilter/Utilities/test/startFU.py +++ b/EventFilter/Utilities/test/startFU.py @@ -41,14 +41,22 @@ VarParsing.VarParsing.varType.int, # string, int, or float "Number of CMSSW streams") +options.register ('numEventsToWrite', + -1, # default value + VarParsing.VarParsing.multiplicity.singleton, + VarParsing.VarParsing.varType.int, # string, int, or float + "Number of Events to process. -1 means all.") + options.parseArguments() cmsswbase = os.path.expandvars("$CMSSW_BASE/") process = cms.Process("TESTFU") process.maxEvents = cms.untracked.PSet( - input = cms.untracked.int32(-1) + input = cms.untracked.int32(options.numEventsToWrite) ) +if options.numEventsToWrite == 0: + process.maxEvents.input = 1 process.options = cms.untracked.PSet( numberOfThreads = cms.untracked.uint32(options.numThreads), @@ -104,6 +112,12 @@ lvl1Labels = cms.vstring( 'Default' ) ) +#used in case where we write no events +process.pre = cms.EDFilter("PrescaleEventFilter", offset = cms.uint32(0), prescale=cms.uint32(1)) +if options.numEventsToWrite: + process.pre.offset = 2 + process.pre.prescale = 4 + process.filter1 = cms.EDFilter("HLTPrescaler", L1GtReadoutRecordTag = cms.InputTag( "hltGtDigis" ) ) @@ -124,8 +138,8 @@ process.tcdsRawToDigi = EventFilter.OnlineMetaDataRawToDigi.tcdsRawToDigi_cfi.tcdsRawToDigi.clone() process.tcdsRawToDigi.InputLabel = cms.InputTag("rawDataCollector") -process.HLT_Physics = cms.Path(process.a*process.tcdsRawToDigi*process.filter1) -process.HLT_Muon = cms.Path(process.b*process.filter2) +process.HLT_Physics = cms.Path(process.a*process.tcdsRawToDigi*process.filter1*process.pre) +process.HLT_Muon = cms.Path(process.b*process.filter2*process.pre) process.streamA = cms.OutputModule("GlobalEvFOutputModule", SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring( 'HLT_Physics' )) diff --git a/EventFilter/Utilities/test/test_dqmstream.py b/EventFilter/Utilities/test/test_dqmstream.py index 474b7d0bd3ec4..cd9d2e85742cc 100644 --- a/EventFilter/Utilities/test/test_dqmstream.py +++ b/EventFilter/Utilities/test/test_dqmstream.py @@ -19,6 +19,16 @@ VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.string, "Directory where the DQM files will appear.") +options.register('eventsPerLS', + 35, + VarParsing.VarParsing.multiplicity.singleton, + VarParsing.VarParsing.varType.int, # string, int, or float + "Max LS to generate (0 to disable limit)") +options.register ('maxLS', + 2, + VarParsing.VarParsing.multiplicity.singleton, + VarParsing.VarParsing.varType.int, # string, int, or float + "Max LS to generate (0 to disable limit)") options.parseArguments() @@ -42,3 +52,32 @@ SelectEvents = cms.untracked.vstring("HLT*Mu*","HLT_*Physics*") ) +#make a list of all the EventIDs that were seen by the previous job, +# given the filter is semi-random we do not know which of these will +# be the actual first event written +rn = options.runNumber +transitions = [cms.EventID(rn,0,0)] +evid = 1 +for lumi in range(1, options.maxLS+1): + transitions.append(cms.EventID(rn,lumi,0)) + for ev in range(0, options.eventsPerLS): + transitions.append(cms.EventID(rn,lumi,evid)) + evid += 1 + transitions.append(cms.EventID(rn,lumi,0)) #end lumi +transitions.append(cms.EventID(rn,0,0)) #end run + + +#only see 1 event as process.source.minEventsPerLumi == 1 +process.test = cms.EDAnalyzer("RunLumiEventChecker", + eventSequence = cms.untracked.VEventID(*transitions), + unorderedEvents = cms.untracked.bool(True), + minNumberOfEvents = cms.untracked.uint32(1+2+2), + maxNumberOfEvents = cms.untracked.uint32(1+2+2) +) +if options.eventsPerLS == 0: + process.test.eventSequence = [] + process.test.minNumberOfEvents = 0 + process.test.maxNumberOfEvents = 0 + +process.p = cms.Path(process.test) + diff --git a/EventFilter/Utilities/test/unittest_FU.py b/EventFilter/Utilities/test/unittest_FU.py index aa99c931a8b80..31cdacfd9b90e 100644 --- a/EventFilter/Utilities/test/unittest_FU.py +++ b/EventFilter/Utilities/test/unittest_FU.py @@ -41,6 +41,12 @@ VarParsing.VarParsing.varType.int, # string, int, or float "Number of CMSSW streams") +options.register ('numEventsToWrite', + -1, # default value + VarParsing.VarParsing.multiplicity.singleton, + VarParsing.VarParsing.varType.int, # string, int, or float + "Number of Events to process. -1 means all.") + options.parseArguments() @@ -48,8 +54,10 @@ process = cms.Process("TESTFU") process.maxEvents = cms.untracked.PSet( - input = cms.untracked.int32(-1) + input = cms.untracked.int32(options.numEventsToWrite) ) +if options.numEventsToWrite == 0: + process.maxEvents.input = 1 process.options = cms.untracked.PSet( numberOfThreads = cms.untracked.uint32(options.numThreads), @@ -115,6 +123,12 @@ lvl1Labels = cms.vstring( 'Default' ) ) +#used in case where we write no events +process.pre = cms.EDFilter("PrescaleEventFilter", offset = cms.uint32(0), prescale=cms.uint32(1)) +if options.numEventsToWrite: + process.pre.offset = 2 + process.pre.prescale = 4 + process.filter1 = cms.EDFilter("HLTPrescaler", L1GtReadoutRecordTag = cms.InputTag( "hltGtDigis" ) ) @@ -135,8 +149,8 @@ process.tcdsRawToDigi = EventFilter.OnlineMetaDataRawToDigi.tcdsRawToDigi_cfi.tcdsRawToDigi.clone() process.tcdsRawToDigi.InputLabel = cms.InputTag("rawDataCollector") -process.HLT_Physics = cms.Path(process.a*process.tcdsRawToDigi*process.filter1) -process.HLT_Muon = cms.Path(process.b*process.filter2) +process.HLT_Physics = cms.Path(process.a*process.tcdsRawToDigi*process.filter1*process.pre) +process.HLT_Muon = cms.Path(process.b*process.filter2*process.pre) process.streamA = cms.OutputModule("GlobalEvFOutputModule", SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring( 'HLT_Physics' )) diff --git a/EventFilter/Utilities/test/writeStreamerFile_cfg.py b/EventFilter/Utilities/test/writeStreamerFile_cfg.py new file mode 100644 index 0000000000000..7081bb8b1263e --- /dev/null +++ b/EventFilter/Utilities/test/writeStreamerFile_cfg.py @@ -0,0 +1,85 @@ +import FWCore.ParameterSet.Config as cms +from argparse import ArgumentParser + +parser = ArgumentParser(description='Test GlobalEvFOutputModule') +parser.add_argument("--numEvents", help="number of events to process", type=int, default=10) +parser.add_argument("--startEvent", help="start event number", type=int, default=1) +parser.add_argument("--runNumber", help="run number to use", type=int, default=100101) +parser.add_argument("--numThreads", help="number of threads to use", type=int, default=3) +parser.add_argument("--numFwkStreams", help="number of cmsRun streams", type=int, default=2) +parser.add_argument("--changeBranchIDLists", help="modify the branchIDLists", type=bool, default=False) + +parser.add_argument("--buBaseDir", help="BU base directory", type=str, default="ramdisk") +parser.add_argument("--fuBaseDir", help="FU base directory", type=str, default="data") +parser.add_argument("--fffBaseDir", help="FFF base directory", type=str, default=".") + +args = parser.parse_args() + +#try to create 'ramdisk' directory +try: + os.makedirs(args.fffBaseDir+"/"+args.buBaseDir+"/run"+str(args.runNumber).zfill(6)) +except:pass +#try to create 'data' directory +try: + os.makedirs(args.fffBaseDir+"/"+args.fuBaseDir+"/run"+str(args.runNumber).zfill(6)) +except Exception as ex: + print(str(ex)) + pass + + +process = cms.Process("WRITE") + +process.source = cms.Source("EmptySource", + firstRun=cms.untracked.uint32(args.runNumber), + firstEvent=cms.untracked.uint32(args.startEvent) +) + +if args.numEvents != 0: + process.maxEvents.input = args.numEvents +else: + process.maxEvents.input = 1 + +process.options = dict(numberOfThreads = args.numThreads, + numberOfStreams = args.numFwkStreams) + +process.intprod = cms.EDProducer("BranchIDListsModifierProducer", + makeExtraProduct=cms.untracked.bool(args.changeBranchIDLists)) + +process.thing = cms.EDProducer("ThingProducer") + +process.otherThing = cms.EDProducer("OtherThingProducer", + thingTag=cms.InputTag("thing")) + +process.t = cms.Task( + process.intprod, + process.thing, + process.otherThing +) + + +process.filter = cms.EDFilter("PrescaleEventFilter", offset = cms.uint32(0), prescale=cms.uint32(1)) +if args.numEvents == 0: + process.filter.offset = 2 + process.filter.prescale = 4 +process.p = cms.Path(process.filter) + +process.streamA = cms.OutputModule("GlobalEvFOutputModule", + SelectEvents = cms.untracked.PSet(SelectEvents = cms.vstring("p")), + outputCommands = cms.untracked.vstring("keep *") +) + +process.ep = cms.EndPath(process.streamA, process.t) + +process.EvFDaqDirector = cms.Service("EvFDaqDirector", + useFileBroker = cms.untracked.bool(False), + fileBrokerHostFromCfg = cms.untracked.bool(True), + fileBrokerHost = cms.untracked.string("htcp40.cern.ch"), + runNumber = cms.untracked.uint32(args.runNumber), + baseDir = cms.untracked.string(args.fffBaseDir+"/"+args.fuBaseDir), + buBaseDir = cms.untracked.string(args.fffBaseDir+"/"+args.buBaseDir), + directorIsBU = cms.untracked.bool(False), +) + +process.FastMonitoringService = cms.Service("FastMonitoringService", + sleepTime = cms.untracked.int32(1) +) diff --git a/FWCore/Modules/src/BranchIDListsModifierProducer.cc b/FWCore/Modules/src/BranchIDListsModifierProducer.cc new file mode 100644 index 0000000000000..5a9591b28e275 --- /dev/null +++ b/FWCore/Modules/src/BranchIDListsModifierProducer.cc @@ -0,0 +1,52 @@ +#include "FWCore/Framework/interface/global/EDProducer.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/MakerMacros.h" + +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" + +#include "FWCore/Utilities/interface/EDPutToken.h" + +#include "DataFormats/TestObjects/interface/ToyProducts.h" + +class BranchIDListsModifierProducer : public edm::global::EDProducer<> { +public: + BranchIDListsModifierProducer(edm::ParameterSet const& iPSet); + + void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const; + + static void fillDescriptions(edm::ConfigurationDescriptions& iDesc); + +private: + edm::EDPutTokenT const token_; + edm::EDPutTokenT extraToken_; + bool const extraProduct_; +}; + +BranchIDListsModifierProducer::BranchIDListsModifierProducer(edm::ParameterSet const& iPSet) + : token_(produces()), extraProduct_(iPSet.getUntrackedParameter("makeExtraProduct")) { + if (extraProduct_) { + extraToken_ = produces("extra"); + } +} + +void BranchIDListsModifierProducer::produce(edm::StreamID, edm::Event& iEvent, edm::EventSetup const&) const { + iEvent.emplace(token_, 1); + if (extraProduct_) { + iEvent.emplace(extraToken_, 2); + } +} + +void BranchIDListsModifierProducer::fillDescriptions(edm::ConfigurationDescriptions& iDesc) { + edm::ParameterSetDescription ps; + ps.setComment( + "Module which can cause the BranchIDLists to change even when the top level PSet remains the same.\n" + "Used for multi-file merge tests."); + + ps.addUntracked("makeExtraProduct", false)->setComment("If set to true will produce an extra product"); + + iDesc.addDefault(ps); +} + +DEFINE_FWK_MODULE(BranchIDListsModifierProducer); diff --git a/FWCore/Modules/src/RunLumiEventChecker.cc b/FWCore/Modules/src/RunLumiEventChecker.cc index e1da77f4fcd1c..15a00f5f3297e 100644 --- a/FWCore/Modules/src/RunLumiEventChecker.cc +++ b/FWCore/Modules/src/RunLumiEventChecker.cc @@ -67,6 +67,8 @@ class RunLumiEventChecker // ----------member data --------------------------- std::vector ids_; mutable std::atomic index_; + unsigned int minNEvents_ = 0; + unsigned int maxNEvents_ = 0; bool unorderedEvents_; }; @@ -84,6 +86,8 @@ class RunLumiEventChecker RunLumiEventChecker::RunLumiEventChecker(edm::ParameterSet const& iConfig) : ids_(iConfig.getUntrackedParameter>("eventSequence")), index_(0), + minNEvents_(iConfig.getUntrackedParameter("minNumberOfEvents")), + maxNEvents_(iConfig.getUntrackedParameter("maxNumberOfEvents")), unorderedEvents_(iConfig.getUntrackedParameter("unorderedEvents")) { //now do what ever initialization is needed } @@ -144,9 +148,16 @@ void RunLumiEventChecker::beginJob() {} // ------------ method called once each job just after ending the event loop ------------ void RunLumiEventChecker::endJob() { - if (index_ != ids_.size()) { + if (maxNEvents_ == 0 and index_ != ids_.size()) { throw cms::Exception("WrongNumberOfEvents") - << "Saw " << index_ << " events but was supposed to see " << ids_.size() << "\n"; + << "Saw " << index_ << " (begin runs)+(begin lumis)+events+(end lumis)+(end runs) but was supposed to see " + << ids_.size() << "\n"; + } + if (maxNEvents_ != 0 and (index_ < minNEvents_ or index_ > maxNEvents_)) { + throw cms::Exception("WrongNumberOfEvents") + << "Saw " << index_ + << " (begin runs)+(begin lumis)+events+(end lumis)+(end runs) but was supposed to see between " << minNEvents_ + << " and " << maxNEvents_; } } @@ -154,7 +165,14 @@ void RunLumiEventChecker::endJob() { void RunLumiEventChecker::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { edm::ParameterSetDescription desc; desc.addUntracked>("eventSequence"); - desc.addUntracked("unorderedEvents", false); + desc.addUntracked("minNumberOfEvents", 0) + ->setComment( + "minimum number of Events that must be seen. If max is 0 then this will be ignored and all Events must be " + "present"); + desc.addUntracked("maxNumberOfEvents", 0) + ->setComment("maximum number of Events that must be seen. If set to 0, min and max are ignored"); + desc.addUntracked("unorderedEvents", false) + ->setComment("set to true if events are not guaranteed to be in same order as 'eventSequence' specifies."); descriptions.add("runLumiEventIDChecker", desc); } diff --git a/IOPool/Streamer/bin/CalcAdler32.cpp b/IOPool/Streamer/bin/CalcAdler32.cpp index 2280a9e2110ce..21ec335bdc82c 100644 --- a/IOPool/Streamer/bin/CalcAdler32.cpp +++ b/IOPool/Streamer/bin/CalcAdler32.cpp @@ -8,6 +8,8 @@ #include int main(int argc, char* argv[]) { + using namespace edm::streamer; + if (argc < 2) { std::cerr << "No command line argument given, expected path/filename.\n"; return 1; diff --git a/IOPool/Streamer/bin/CatStreamerFiles.cpp b/IOPool/Streamer/bin/CatStreamerFiles.cpp index b42a9a8a35251..8902ba66cf936 100644 --- a/IOPool/Streamer/bin/CatStreamerFiles.cpp +++ b/IOPool/Streamer/bin/CatStreamerFiles.cpp @@ -9,6 +9,7 @@ // Utility to concatenate streamer files outside of the framework // Mimics the behavior of DAQ // Largely copied from DiagStreamerFile +using namespace edm::streamer; void help(); void mergefile(StreamerOutputFile&, std::string const&, bool); @@ -45,7 +46,7 @@ void mergefile(StreamerOutputFile& stream_output, std::string const& filename, b uint32 num_events(0); try { - edm::StreamerInputFile stream_reader(filename); + StreamerInputFile stream_reader(filename); std::cout << "Trying to Read The Init message from Streamer File: " << std::endl << filename << std::endl; InitMsgView const* init = stream_reader.startMessage(); @@ -56,7 +57,7 @@ void mergefile(StreamerOutputFile& stream_output, std::string const& filename, b std::cout << "Trying to read the Event messages" << std::endl; EventMsgView const* eview(nullptr); - while (edm::StreamerInputFile::Next::kEvent == stream_reader.next()) { + while (StreamerInputFile::Next::kEvent == stream_reader.next()) { eview = stream_reader.currentRecord(); ++num_events; stream_output.write(*eview); diff --git a/IOPool/Streamer/bin/DiagStreamerFile.cpp b/IOPool/Streamer/bin/DiagStreamerFile.cpp index bc60a60ada633..f701f077c10f3 100644 --- a/IOPool/Streamer/bin/DiagStreamerFile.cpp +++ b/IOPool/Streamer/bin/DiagStreamerFile.cpp @@ -36,16 +36,19 @@ #include #include -bool compares_bad(EventMsgView const* eview1, EventMsgView const* eview2); -bool uncompressBuffer(unsigned char* inputBuffer, - unsigned int inputSize, - std::vector& outputBuffer, - unsigned int expectedFullSize); -bool test_chksum(EventMsgView const* eview); -bool test_uncompress(EventMsgView const* eview, std::vector& dest); -void readfile(std::string filename, std::string outfile); -void help(); +using namespace edm::streamer; +namespace { + bool compares_bad(EventMsgView const* eview1, EventMsgView const* eview2); + bool uncompressBuffer(unsigned char* inputBuffer, + unsigned int inputSize, + std::vector& outputBuffer, + unsigned int expectedFullSize); + bool test_chksum(EventMsgView const* eview); + bool test_uncompress(EventMsgView const* eview, std::vector& dest); + void readfile(std::string filename, std::string outfile); + void help(); +} // namespace //========================================================================== int main(int argc, char* argv[]) { if (argc < 2) { @@ -66,231 +69,233 @@ int main(int argc, char* argv[]) { return 0; } -//========================================================================== -void help() { - std::cout << "Usage: DiagStreamerFile streamer_file_name" - << " [output_file_name]" << std::endl; -} -//========================================================================== -void readfile(std::string filename, std::string outfile) { - uint32 num_events(0); - uint32 num_badevents(0); - uint32 num_baduncompress(0); - uint32 num_badchksum(0); - uint32 num_goodevents(0); - uint32 num_duplevents(0); - std::vector compress_buffer(7000000); - std::map seenEventMap; - bool output(false); - if (outfile != "/dev/null") { - output = true; +namespace { + //========================================================================== + void help() { + std::cout << "Usage: DiagStreamerFile streamer_file_name" + << " [output_file_name]" << std::endl; } - StreamerOutputFile stream_output(outfile); - try { - // ----------- init - edm::StreamerInputFile stream_reader(filename); - //if(output) StreamerOutputFile stream_output(outfile); - - std::cout << "Trying to Read The Init message from Streamer File: " << std::endl << filename << std::endl; - InitMsgView const* init = stream_reader.startMessage(); - std::cout << "\n\n-------------INIT Message---------------------" << std::endl; - std::cout << "Dump the Init Message from Streamer:-" << std::endl; - dumpInitView(init); - if (output) { - stream_output.write(*init); + //========================================================================== + void readfile(std::string filename, std::string outfile) { + uint32 num_events(0); + uint32 num_badevents(0); + uint32 num_baduncompress(0); + uint32 num_badchksum(0); + uint32 num_goodevents(0); + uint32 num_duplevents(0); + std::vector compress_buffer(7000000); + std::map seenEventMap; + bool output(false); + if (outfile != "/dev/null") { + output = true; } + StreamerOutputFile stream_output(outfile); + try { + // ----------- init + StreamerInputFile stream_reader(filename); + //if(output) StreamerOutputFile stream_output(outfile); - // ------- event - std::cout << "\n\n-------------EVENT Messages-------------------" << std::endl; + std::cout << "Trying to Read The Init message from Streamer File: " << std::endl << filename << std::endl; + InitMsgView const* init = stream_reader.startMessage(); + std::cout << "\n\n-------------INIT Message---------------------" << std::endl; + std::cout << "Dump the Init Message from Streamer:-" << std::endl; + dumpInitView(init); + if (output) { + stream_output.write(*init); + } - bool first_event(true); - std::unique_ptr firstEvtView(nullptr); - std::vector savebuf(0); - EventMsgView const* eview(nullptr); - seenEventMap.clear(); + // ------- event + std::cout << "\n\n-------------EVENT Messages-------------------" << std::endl; - while (edm::StreamerInputFile::Next::kEvent == stream_reader.next()) { - eview = stream_reader.currentRecord(); - ++num_events; - bool good_event(true); - if (seenEventMap.find(eview->event()) == seenEventMap.end()) { - seenEventMap.insert(std::make_pair(eview->event(), 1)); - } else { - ++seenEventMap[eview->event()]; - ++num_duplevents; - std::cout << "??????? duplicate event Id for count " << num_events << " event number " << eview->event() - << " seen " << seenEventMap[eview->event()] << " times" << std::endl; - } - if (first_event) { - std::cout << "----------dumping first EVENT-----------" << std::endl; - dumpEventView(eview); - first_event = false; - unsigned char* src = (unsigned char*)eview->startAddress(); - unsigned int srcSize = eview->size(); - savebuf.resize(srcSize); - std::copy(src, src + srcSize, &(savebuf)[0]); - firstEvtView = std::make_unique(&(savebuf)[0]); - //firstEvtView, reset(new EventMsgView((void*)eview->startAddress())); - if (!test_chksum(eview)) { - std::cout << "checksum error for count " << num_events << " event number " << eview->event() - << " from host name " << eview->hostName() << std::endl; - ++num_badchksum; - std::cout << "----------dumping bad checksum EVENT-----------" << std::endl; - dumpEventView(eview); - good_event = false; - } - if (!test_uncompress(eview, compress_buffer)) { - std::cout << "uncompress error for count " << num_events << " event number " << firstEvtView->event() - << std::endl; - ++num_baduncompress; - std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl; - dumpEventView(firstEvtView.get()); - good_event = false; - } - } else { - if (compares_bad(firstEvtView.get(), eview)) { - std::cout << "Bad event at count " << num_events << " dumping event " << std::endl - << "----------dumping bad EVENT-----------" << std::endl; - dumpEventView(eview); - ++num_badevents; - good_event = false; + bool first_event(true); + std::unique_ptr firstEvtView(nullptr); + std::vector savebuf(0); + EventMsgView const* eview(nullptr); + seenEventMap.clear(); + + while (StreamerInputFile::Next::kEvent == stream_reader.next()) { + eview = stream_reader.currentRecord(); + ++num_events; + bool good_event(true); + if (seenEventMap.find(eview->event()) == seenEventMap.end()) { + seenEventMap.insert(std::make_pair(eview->event(), 1)); + } else { + ++seenEventMap[eview->event()]; + ++num_duplevents; + std::cout << "??????? duplicate event Id for count " << num_events << " event number " << eview->event() + << " seen " << seenEventMap[eview->event()] << " times" << std::endl; } - if (!test_chksum(eview)) { - std::cout << "checksum error for count " << num_events << " event number " << eview->event() - << " from host name " << eview->hostName() << std::endl; - ++num_badchksum; - std::cout << "----------dumping bad checksum EVENT-----------" << std::endl; + if (first_event) { + std::cout << "----------dumping first EVENT-----------" << std::endl; dumpEventView(eview); - good_event = false; + first_event = false; + unsigned char* src = (unsigned char*)eview->startAddress(); + unsigned int srcSize = eview->size(); + savebuf.resize(srcSize); + std::copy(src, src + srcSize, &(savebuf)[0]); + firstEvtView = std::make_unique(&(savebuf)[0]); + //firstEvtView, reset(new EventMsgView((void*)eview->startAddress())); + if (!test_chksum(eview)) { + std::cout << "checksum error for count " << num_events << " event number " << eview->event() + << " from host name " << eview->hostName() << std::endl; + ++num_badchksum; + std::cout << "----------dumping bad checksum EVENT-----------" << std::endl; + dumpEventView(eview); + good_event = false; + } + if (!test_uncompress(eview, compress_buffer)) { + std::cout << "uncompress error for count " << num_events << " event number " << firstEvtView->event() + << std::endl; + ++num_baduncompress; + std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl; + dumpEventView(firstEvtView.get()); + good_event = false; + } + } else { + if (compares_bad(firstEvtView.get(), eview)) { + std::cout << "Bad event at count " << num_events << " dumping event " << std::endl + << "----------dumping bad EVENT-----------" << std::endl; + dumpEventView(eview); + ++num_badevents; + good_event = false; + } + if (!test_chksum(eview)) { + std::cout << "checksum error for count " << num_events << " event number " << eview->event() + << " from host name " << eview->hostName() << std::endl; + ++num_badchksum; + std::cout << "----------dumping bad checksum EVENT-----------" << std::endl; + dumpEventView(eview); + good_event = false; + } + if (!test_uncompress(eview, compress_buffer)) { + std::cout << "uncompress error for count " << num_events << " event number " << eview->event() << std::endl; + ++num_baduncompress; + std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl; + dumpEventView(eview); + good_event = false; + } } - if (!test_uncompress(eview, compress_buffer)) { - std::cout << "uncompress error for count " << num_events << " event number " << eview->event() << std::endl; - ++num_baduncompress; - std::cout << "----------dumping bad uncompress EVENT-----------" << std::endl; - dumpEventView(eview); - good_event = false; + if (good_event) { + if (output) { + ++num_goodevents; + stream_output.write(*eview); + } + //dumpEventView(eview); } - } - if (good_event) { - if (output) { - ++num_goodevents; - stream_output.write(*eview); + if ((num_events % 50) == 0) { + std::cout << "Read " << num_events << " events, and " << num_badevents << " events with bad headers, and " + << num_badchksum << " events with bad check sum, and " << num_baduncompress + << " events with bad uncompress" << std::endl; + if (output) + std::cout << "Wrote " << num_goodevents << " good events " << std::endl; } - //dumpEventView(eview); - } - if ((num_events % 50) == 0) { - std::cout << "Read " << num_events << " events, and " << num_badevents << " events with bad headers, and " - << num_badchksum << " events with bad check sum, and " << num_baduncompress - << " events with bad uncompress" << std::endl; - if (output) - std::cout << "Wrote " << num_goodevents << " good events " << std::endl; } - } - std::cout << std::endl - << "------------END--------------" << std::endl - << "read " << num_events << " events" << std::endl - << "and " << num_badevents << " events with bad headers" << std::endl - << "and " << num_badchksum << " events with bad check sum" << std::endl - << "and " << num_baduncompress << " events with bad uncompress" << std::endl - << "and " << num_duplevents << " duplicated event Id" << std::endl; + std::cout << std::endl + << "------------END--------------" << std::endl + << "read " << num_events << " events" << std::endl + << "and " << num_badevents << " events with bad headers" << std::endl + << "and " << num_badchksum << " events with bad check sum" << std::endl + << "and " << num_baduncompress << " events with bad uncompress" << std::endl + << "and " << num_duplevents << " duplicated event Id" << std::endl; - if (output) { - std::cout << "Wrote " << num_goodevents << " good events " << std::endl; - } + if (output) { + std::cout << "Wrote " << num_goodevents << " good events " << std::endl; + } - } catch (cms::Exception& e) { - std::cerr << "Exception caught: " << e.what() << std::endl - << "After reading " << num_events << " events, and " << num_badevents << " events with bad headers" - << std::endl - << "and " << num_badchksum << " events with bad check sum" << std::endl - << "and " << num_baduncompress << " events with bad uncompress" << std::endl - << "and " << num_duplevents << " duplicated event Id" << std::endl; + } catch (cms::Exception& e) { + std::cerr << "Exception caught: " << e.what() << std::endl + << "After reading " << num_events << " events, and " << num_badevents << " events with bad headers" + << std::endl + << "and " << num_badchksum << " events with bad check sum" << std::endl + << "and " << num_baduncompress << " events with bad uncompress" << std::endl + << "and " << num_duplevents << " duplicated event Id" << std::endl; + } } -} -//========================================================================== -bool compares_bad(EventMsgView const* eview1, EventMsgView const* eview2) { - bool is_bad(false); - if (eview1->code() != eview2->code()) { - std::cout << "non-matching EVENT message code " << std::endl; - is_bad = true; - } - if (eview1->protocolVersion() != eview2->protocolVersion()) { - std::cout << "non-matching EVENT message protocol version" << std::endl; - is_bad = true; - } - if (eview1->run() != eview2->run()) { - std::cout << "non-matching run number " << std::endl; - is_bad = true; - } - if (eview1->lumi() != eview2->lumi()) { - std::cout << "non-matching lumi number" << std::endl; - is_bad = true; - } - if (eview1->outModId() != eview2->outModId()) { - std::cout << "non-matching output module id" << std::endl; - is_bad = true; - } - if (eview1->hltCount() != eview2->hltCount()) { - std::cout << "non-matching HLT count" << std::endl; - is_bad = true; - } - if (eview1->l1Count() != eview2->l1Count()) { - std::cout << "non-matching L1 count" << std::endl; - is_bad = true; + //========================================================================== + bool compares_bad(EventMsgView const* eview1, EventMsgView const* eview2) { + bool is_bad(false); + if (eview1->code() != eview2->code()) { + std::cout << "non-matching EVENT message code " << std::endl; + is_bad = true; + } + if (eview1->protocolVersion() != eview2->protocolVersion()) { + std::cout << "non-matching EVENT message protocol version" << std::endl; + is_bad = true; + } + if (eview1->run() != eview2->run()) { + std::cout << "non-matching run number " << std::endl; + is_bad = true; + } + if (eview1->lumi() != eview2->lumi()) { + std::cout << "non-matching lumi number" << std::endl; + is_bad = true; + } + if (eview1->outModId() != eview2->outModId()) { + std::cout << "non-matching output module id" << std::endl; + is_bad = true; + } + if (eview1->hltCount() != eview2->hltCount()) { + std::cout << "non-matching HLT count" << std::endl; + is_bad = true; + } + if (eview1->l1Count() != eview2->l1Count()) { + std::cout << "non-matching L1 count" << std::endl; + is_bad = true; + } + return is_bad; } - return is_bad; -} -//========================================================================== -bool test_chksum(EventMsgView const* eview) { - uint32_t adler32_chksum = cms::Adler32((char const*)eview->eventData(), eview->eventLength()); - //std::cout << "Adler32 checksum of event = " << adler32_chksum << std::endl; - //std::cout << "Adler32 checksum from header = " << eview->adler32_chksum() << std::endl; - //std::cout << "event from host name = " << eview->hostName() << std::endl; - if ((uint32)adler32_chksum != eview->adler32_chksum()) { - std::cout << "Bad chekcsum: Adler32 checksum of event data = " << adler32_chksum - << " from header = " << eview->adler32_chksum() << " host name = " << eview->hostName() << std::endl; - return false; + //========================================================================== + bool test_chksum(EventMsgView const* eview) { + uint32_t adler32_chksum = cms::Adler32((char const*)eview->eventData(), eview->eventLength()); + //std::cout << "Adler32 checksum of event = " << adler32_chksum << std::endl; + //std::cout << "Adler32 checksum from header = " << eview->adler32_chksum() << std::endl; + //std::cout << "event from host name = " << eview->hostName() << std::endl; + if ((uint32)adler32_chksum != eview->adler32_chksum()) { + std::cout << "Bad chekcsum: Adler32 checksum of event data = " << adler32_chksum + << " from header = " << eview->adler32_chksum() << " host name = " << eview->hostName() << std::endl; + return false; + } + return true; } - return true; -} -//========================================================================== -bool test_uncompress(EventMsgView const* eview, std::vector& dest) { - unsigned long origsize = eview->origDataSize(); - bool success = false; - if (origsize != 0 && origsize != 78) { - // compressed - success = uncompressBuffer( - const_cast((unsigned char const*)eview->eventData()), eview->eventLength(), dest, origsize); - } else { - // uncompressed anyway - success = true; + //========================================================================== + bool test_uncompress(EventMsgView const* eview, std::vector& dest) { + unsigned long origsize = eview->origDataSize(); + bool success = false; + if (origsize != 0 && origsize != 78) { + // compressed + success = uncompressBuffer( + const_cast((unsigned char const*)eview->eventData()), eview->eventLength(), dest, origsize); + } else { + // uncompressed anyway + success = true; + } + return success; } - return success; -} -//========================================================================== -bool uncompressBuffer(unsigned char* inputBuffer, - unsigned int inputSize, - std::vector& outputBuffer, - unsigned int expectedFullSize) { - unsigned long origSize = expectedFullSize; - unsigned long uncompressedSize = expectedFullSize * 1.1; - outputBuffer.resize(uncompressedSize); - int ret = uncompress(&outputBuffer[0], &uncompressedSize, inputBuffer, inputSize); - if (ret == Z_OK) { - // check the length against original uncompressed length - if (origSize != uncompressedSize) { - std::cout << "Problem with uncompress, original size = " << origSize << " uncompress size = " << uncompressedSize - << std::endl; + //========================================================================== + bool uncompressBuffer(unsigned char* inputBuffer, + unsigned int inputSize, + std::vector& outputBuffer, + unsigned int expectedFullSize) { + unsigned long origSize = expectedFullSize; + unsigned long uncompressedSize = expectedFullSize * 1.1; + outputBuffer.resize(uncompressedSize); + int ret = uncompress(&outputBuffer[0], &uncompressedSize, inputBuffer, inputSize); + if (ret == Z_OK) { + // check the length against original uncompressed length + if (origSize != uncompressedSize) { + std::cout << "Problem with uncompress, original size = " << origSize + << " uncompress size = " << uncompressedSize << std::endl; + return false; + } + } else { + std::cout << "Problem with uncompress, return value = " << ret << std::endl; return false; } - } else { - std::cout << "Problem with uncompress, return value = " << ret << std::endl; - return false; + return true; } - return true; -} +} // namespace diff --git a/IOPool/Streamer/interface/ClassFiller.h b/IOPool/Streamer/interface/ClassFiller.h index 7abe6a2298d8f..2820724d6acfd 100644 --- a/IOPool/Streamer/interface/ClassFiller.h +++ b/IOPool/Streamer/interface/ClassFiller.h @@ -11,7 +11,7 @@ #include #include -namespace edm { +namespace edm::streamer { class RootDebug { public: RootDebug(int flevel, int rlevel) : flevel_(flevel), rlevel_(rlevel), old_(gDebug) { @@ -33,6 +33,6 @@ namespace edm { TClass* getTClass(const std::type_info& ti); bool loadCap(const std::string& name, std::vector& missingDictionaries); void doBuildRealData(const std::string& name); -} // namespace edm +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/DumpTools.h b/IOPool/Streamer/interface/DumpTools.h index aa8cd5e4fbdae..61670c1f4595b 100644 --- a/IOPool/Streamer/interface/DumpTools.h +++ b/IOPool/Streamer/interface/DumpTools.h @@ -9,17 +9,17 @@ #include "IOPool/Streamer/interface/InitMessage.h" #include "IOPool/Streamer/interface/EventMessage.h" #include "IOPool/Streamer/interface/FRDEventMessage.h" - -void dumpInitHeader(const InitMsgView* view); -void dumpInitView(const InitMsgView* view); -void dumpStartMsg(const InitMsgView* view); -void dumpInitVerbose(const InitMsgView* view); -void dumpInit(uint8* buf); -void printBits(unsigned char c); -void dumpEventHeader(const EventMsgView* eview); -void dumpEventView(const EventMsgView* eview); -void dumpEventIndex(const EventMsgView* eview); -void dumpEvent(uint8* buf); -void dumpFRDEventView(const FRDEventMsgView* fview); - +namespace edm::streamer { + void dumpInitHeader(const InitMsgView* view); + void dumpInitView(const InitMsgView* view); + void dumpStartMsg(const InitMsgView* view); + void dumpInitVerbose(const InitMsgView* view); + void dumpInit(uint8* buf); + void printBits(unsigned char c); + void dumpEventHeader(const EventMsgView* eview); + void dumpEventView(const EventMsgView* eview); + void dumpEventIndex(const EventMsgView* eview); + void dumpEvent(uint8* buf); + void dumpFRDEventView(const FRDEventMsgView* fview); +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/EventMessage.h b/IOPool/Streamer/interface/EventMessage.h index be9b108fe20f6..bf12c1189430b 100644 --- a/IOPool/Streamer/interface/EventMessage.h +++ b/IOPool/Streamer/interface/EventMessage.h @@ -55,62 +55,63 @@ Protocol Version 11: identical to version 10, except event changed from 4 bytes #include "IOPool/Streamer/interface/MsgTools.h" #include "IOPool/Streamer/interface/MsgHeader.h" - -// ----------------------- event message ------------------------ - -struct EventHeader { - Header header_; - uint8 protocolVersion_; - char_uint32 run_; - char_uint64 event_; - char_uint32 lumi_; - char_uint32 origDataSize_; - char_uint32 outModId_; - char_uint32 droppedEventsCount_; -}; - -class EventMsgView { -public: - EventMsgView(void* buf); - - uint32 code() const { return head_.code(); } - uint32 size() const { return head_.size(); } - - const uint8* eventData() const { return event_start_; } - uint8* startAddress() const { return buf_; } - uint32 eventLength() const { return event_len_; } - uint32 headerSize() const { return event_start_ - buf_; } - uint32 protocolVersion() const; - uint32 run() const; - uint64 event() const; - uint32 lumi() const; - uint32 origDataSize() const; - uint32 outModId() const; - uint32 droppedEventsCount() const; - - void l1TriggerBits(std::vector& put_here) const; - void hltTriggerBits(uint8* put_here) const; - - uint32 hltCount() const { return hlt_bits_count_; } - uint32 l1Count() const { return l1_bits_count_; } - uint32 adler32_chksum() const { return adler32_chksum_; } - std::string hostName() const; - uint32 hostName_len() const { return host_name_len_; } - -private: - uint8* buf_; - HeaderView head_; - - uint8* hlt_bits_start_; - uint32 hlt_bits_count_; - uint8* l1_bits_start_; - uint32 l1_bits_count_; - uint8* event_start_; - uint32 event_len_; - uint32 adler32_chksum_; - uint8* host_name_start_; - uint32 host_name_len_; - bool v2Detected_; -}; - +namespace edm::streamer { + // ----------------------- event message ------------------------ + + struct EventHeader { + Header header_; + uint8 protocolVersion_; + char_uint32 run_; + char_uint64 event_; + char_uint32 lumi_; + char_uint32 origDataSize_; + char_uint32 outModId_; + char_uint32 droppedEventsCount_; + }; + + class EventMsgView { + public: + EventMsgView(void* buf); + + uint32 code() const { return head_.code(); } + uint32 size() const { return head_.size(); } + + const uint8* eventData() const { return event_start_; } + uint8* startAddress() const { return buf_; } + uint32 eventLength() const { return event_len_; } + uint32 headerSize() const { return event_start_ - buf_; } + uint32 protocolVersion() const; + uint32 run() const; + uint64 event() const; + uint32 lumi() const; + uint32 origDataSize() const; + uint32 outModId() const; + uint32 droppedEventsCount() const; + bool isEventMetaData() const; + + void l1TriggerBits(std::vector& put_here) const; + void hltTriggerBits(uint8* put_here) const; + + uint32 hltCount() const { return hlt_bits_count_; } + uint32 l1Count() const { return l1_bits_count_; } + uint32 adler32_chksum() const { return adler32_chksum_; } + std::string hostName() const; + uint32 hostName_len() const { return host_name_len_; } + + private: + uint8* buf_; + HeaderView head_; + + uint8* hlt_bits_start_; + uint32 hlt_bits_count_; + uint8* l1_bits_start_; + uint32 l1_bits_count_; + uint8* event_start_; + uint32 event_len_; + uint32 adler32_chksum_; + uint8* host_name_start_; + uint32 host_name_len_; + bool v2Detected_; + }; +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/EventMsgBuilder.h b/IOPool/Streamer/interface/EventMsgBuilder.h index d222e0790d629..04600ca99e79f 100644 --- a/IOPool/Streamer/interface/EventMsgBuilder.h +++ b/IOPool/Streamer/interface/EventMsgBuilder.h @@ -5,37 +5,38 @@ // ------------------ event message builder ---------------- -class EventMsgBuilder { -public: - EventMsgBuilder(void* buf, - uint32 size, - uint32 run, - uint64 event, - uint32 lumi, - uint32 outModId, - uint32 droppedEventsCount, - std::vector& l1_bits, - uint8* hlt_bits, - uint32 hlt_bit_count, - uint32 adler32_chksum, - const char* host_name); +namespace edm::streamer { + class EventMsgBuilder { + public: + EventMsgBuilder(void* buf, + uint32 size, + uint32 run, + uint64 event, + uint32 lumi, + uint32 outModId, + uint32 droppedEventsCount, + std::vector& l1_bits, + uint8* hlt_bits, + uint32 hlt_bit_count, + uint32 adler32_chksum, + const char* host_name); - void setOrigDataSize(uint32); - uint8* startAddress() const { return buf_; } - void setEventLength(uint32 len); - void setBufAddr(uint8* buf_addr) { buf_ = buf_addr; } - void setEventAddr(uint8* event_addr) { event_addr_ = event_addr; } - uint8* eventAddr() const { return event_addr_; } - uint32 headerSize() const { return event_addr_ - buf_; } - uint32 size() const; - uint32 bufferSize() const { return size_; } + void setOrigDataSize(uint32); + uint8* startAddress() const { return buf_; } + void setEventLength(uint32 len); + void setBufAddr(uint8* buf_addr) { buf_ = buf_addr; } + void setEventAddr(uint8* event_addr) { event_addr_ = event_addr; } + uint8* eventAddr() const { return event_addr_; } + uint32 headerSize() const { return event_addr_ - buf_; } + uint32 size() const; + uint32 bufferSize() const { return size_; } - static uint32 computeHeaderSize(uint32 l1t_bit_count, uint32 hlt_bit_count); - -private: - uint8* buf_; - uint32 size_; - uint8* event_addr_; -}; + static uint32 computeHeaderSize(uint32 l1t_bit_count, uint32 hlt_bit_count); + private: + uint8* buf_; + uint32 size_; + uint8* event_addr_; + }; +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/FRDEventMessage.h b/IOPool/Streamer/interface/FRDEventMessage.h index 8087389c42c61..729a958171b5e 100644 --- a/IOPool/Streamer/interface/FRDEventMessage.h +++ b/IOPool/Streamer/interface/FRDEventMessage.h @@ -71,102 +71,103 @@ #include -struct FRDEventHeader_V6 { - uint16 version_; - uint16 flags_; - uint32 run_; - uint32 lumi_; - uint32 event_; - uint32 eventSize_; - uint32 crc32c_; -}; - -struct FRDEventHeader_V5 { - uint32 version_; - uint32 run_; - uint32 lumi_; - uint32 event_; - uint32 eventSize_; - uint32 crc32c_; -}; - -struct FRDEventHeader_V4 { - uint32 version_; - uint32 run_; - uint32 lumi_; - uint32 eventLow_; - uint32 eventHigh_; - uint32 eventSize_; - uint32 paddingSize_; - uint32 adler32_; -}; - -struct FRDEventHeader_V3 { - uint32 version_; - uint32 run_; - uint32 lumi_; - uint32 event_; - uint32 eventSize_; - uint32 paddingSize_; - uint32 adler32_; -}; - -struct FRDEventHeader_V2 { - uint32 version_; - uint32 run_; - uint32 lumi_; - uint32 event_; -}; - -struct FRDEventHeader_V1 { - uint32 run_; - uint32 event_; -}; - -const uint16 FRDEVENT_MASK_ISGENDATA = 1; - -constexpr size_t FRDHeaderMaxVersion = 6; -constexpr std::array FRDHeaderVersionSize{{0, - 2 * sizeof(uint32), - (4 + 1024) * sizeof(uint32), - 7 * sizeof(uint32), - 8 * sizeof(uint32), - 6 * sizeof(uint32), - 6 * sizeof(uint32)}}; - -class FRDEventMsgView { -public: - FRDEventMsgView(void* buf); - - uint8* startAddress() const { return buf_; } - void* payload() const { return payload_; } - uint32 size() const { return size_; } - - uint16 version() const { return version_; } - uint16 flags() const { return flags_; } - uint32 run() const { return run_; } - uint32 lumi() const { return lumi_; } - uint64 event() const { return event_; } - uint32 eventSize() const { return eventSize_; } - uint32 paddingSize() const { return paddingSize_; } - uint32 adler32() const { return adler32_; } - uint32 crc32c() const { return crc32c_; } - - bool isRealData() const { return !(flags_ & FRDEVENT_MASK_ISGENDATA); } - -private: - uint8* buf_; - void* payload_; - uint32 size_; - uint16 version_; - uint16 flags_; - uint32 run_; - uint32 lumi_; - uint64 event_; - uint32 eventSize_; - uint32 paddingSize_; - uint32 adler32_; - uint32 crc32c_; -}; - +namespace edm::streamer { + struct FRDEventHeader_V6 { + uint16 version_; + uint16 flags_; + uint32 run_; + uint32 lumi_; + uint32 event_; + uint32 eventSize_; + uint32 crc32c_; + }; + + struct FRDEventHeader_V5 { + uint32 version_; + uint32 run_; + uint32 lumi_; + uint32 event_; + uint32 eventSize_; + uint32 crc32c_; + }; + + struct FRDEventHeader_V4 { + uint32 version_; + uint32 run_; + uint32 lumi_; + uint32 eventLow_; + uint32 eventHigh_; + uint32 eventSize_; + uint32 paddingSize_; + uint32 adler32_; + }; + + struct FRDEventHeader_V3 { + uint32 version_; + uint32 run_; + uint32 lumi_; + uint32 event_; + uint32 eventSize_; + uint32 paddingSize_; + uint32 adler32_; + }; + + struct FRDEventHeader_V2 { + uint32 version_; + uint32 run_; + uint32 lumi_; + uint32 event_; + }; + + struct FRDEventHeader_V1 { + uint32 run_; + uint32 event_; + }; + + const uint16 FRDEVENT_MASK_ISGENDATA = 1; + + constexpr size_t FRDHeaderMaxVersion = 6; + constexpr std::array FRDHeaderVersionSize{{0, + 2 * sizeof(uint32), + (4 + 1024) * sizeof(uint32), + 7 * sizeof(uint32), + 8 * sizeof(uint32), + 6 * sizeof(uint32), + 6 * sizeof(uint32)}}; + + class FRDEventMsgView { + public: + FRDEventMsgView(void* buf); + + uint8* startAddress() const { return buf_; } + void* payload() const { return payload_; } + uint32 size() const { return size_; } + + uint16 version() const { return version_; } + uint16 flags() const { return flags_; } + uint32 run() const { return run_; } + uint32 lumi() const { return lumi_; } + uint64 event() const { return event_; } + uint32 eventSize() const { return eventSize_; } + uint32 paddingSize() const { return paddingSize_; } + uint32 adler32() const { return adler32_; } + uint32 crc32c() const { return crc32c_; } + + bool isRealData() const { return !(flags_ & FRDEVENT_MASK_ISGENDATA); } + + private: + uint8* buf_; + void* payload_; + uint32 size_; + uint16 version_; + uint16 flags_; + uint32 run_; + uint32 lumi_; + uint64 event_; + uint32 eventSize_; + uint32 paddingSize_; + uint32 adler32_; + uint32 crc32c_; + }; +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/FRDFileHeader.h b/IOPool/Streamer/interface/FRDFileHeader.h index 32a5c5c629bbf..4ad388f8f5f2a 100644 --- a/IOPool/Streamer/interface/FRDFileHeader.h +++ b/IOPool/Streamer/interface/FRDFileHeader.h @@ -18,77 +18,78 @@ * * */ -constexpr std::array FRDFileHeader_id{{0x52, 0x41, 0x57, 0x5f}}; -constexpr std::array FRDFileVersion_1{{0x30, 0x30, 0x30, 0x31}}; -constexpr std::array FRDFileVersion_2{{0x30, 0x30, 0x30, 0x32}}; +namespace edm::streamer { + constexpr std::array FRDFileHeader_id{{0x52, 0x41, 0x57, 0x5f}}; + constexpr std::array FRDFileVersion_1{{0x30, 0x30, 0x30, 0x31}}; + constexpr std::array FRDFileVersion_2{{0x30, 0x30, 0x30, 0x32}}; -struct FRDFileHeaderIdentifier { - FRDFileHeaderIdentifier(const std::array& id, const std::array& version) - : id_(id), version_(version) {} + struct FRDFileHeaderIdentifier { + FRDFileHeaderIdentifier(const std::array& id, const std::array& version) + : id_(id), version_(version) {} - std::array id_; - std::array version_; -}; + std::array id_; + std::array version_; + }; -struct FRDFileHeaderContent_v1 { - FRDFileHeaderContent_v1(uint16_t eventCount, uint32_t lumiSection, uint64_t fileSize) - : headerSize_(sizeof(FRDFileHeaderContent_v1) + sizeof(FRDFileHeaderIdentifier)), - eventCount_(eventCount), - lumiSection_(lumiSection), - fileSize_(fileSize) {} + struct FRDFileHeaderContent_v1 { + FRDFileHeaderContent_v1(uint16_t eventCount, uint32_t lumiSection, uint64_t fileSize) + : headerSize_(sizeof(FRDFileHeaderContent_v1) + sizeof(FRDFileHeaderIdentifier)), + eventCount_(eventCount), + lumiSection_(lumiSection), + fileSize_(fileSize) {} - uint16_t headerSize_; - uint16_t eventCount_; - uint32_t lumiSection_; - uint64_t fileSize_; -}; + uint16_t headerSize_; + uint16_t eventCount_; + uint32_t lumiSection_; + uint64_t fileSize_; + }; -struct FRDFileHeader_v1 { - FRDFileHeader_v1(uint16_t eventCount, uint32_t lumiSection, uint64_t fileSize) - : id_(FRDFileHeader_id, FRDFileVersion_1), c_(eventCount, lumiSection, fileSize) {} + struct FRDFileHeader_v1 { + FRDFileHeader_v1(uint16_t eventCount, uint32_t lumiSection, uint64_t fileSize) + : id_(FRDFileHeader_id, FRDFileVersion_1), c_(eventCount, lumiSection, fileSize) {} - FRDFileHeaderIdentifier id_; - FRDFileHeaderContent_v1 c_; -}; + FRDFileHeaderIdentifier id_; + FRDFileHeaderContent_v1 c_; + }; -struct FRDFileHeaderContent_v2 { - FRDFileHeaderContent_v2( - uint16_t dataType, uint16_t eventCount, uint32_t runNumber, uint32_t lumiSection, uint64_t fileSize) - : headerSize_(sizeof(FRDFileHeaderContent_v2) + sizeof(FRDFileHeaderIdentifier)), - dataType_(dataType), - eventCount_(eventCount), - runNumber_(runNumber), - lumiSection_(lumiSection), - fileSize_(fileSize) {} + struct FRDFileHeaderContent_v2 { + FRDFileHeaderContent_v2( + uint16_t dataType, uint16_t eventCount, uint32_t runNumber, uint32_t lumiSection, uint64_t fileSize) + : headerSize_(sizeof(FRDFileHeaderContent_v2) + sizeof(FRDFileHeaderIdentifier)), + dataType_(dataType), + eventCount_(eventCount), + runNumber_(runNumber), + lumiSection_(lumiSection), + fileSize_(fileSize) {} - uint16_t headerSize_; - uint16_t dataType_; - uint32_t eventCount_; - uint32_t runNumber_; - uint32_t lumiSection_; - uint64_t fileSize_; -}; + uint16_t headerSize_; + uint16_t dataType_; + uint32_t eventCount_; + uint32_t runNumber_; + uint32_t lumiSection_; + uint64_t fileSize_; + }; -struct FRDFileHeader_v2 { - FRDFileHeader_v2(uint16_t dataType, uint16_t eventCount, uint32_t runNumber, uint32_t lumiSection, uint64_t fileSize) - : id_(FRDFileHeader_id, FRDFileVersion_2), c_(dataType, eventCount, runNumber, lumiSection, fileSize) {} + struct FRDFileHeader_v2 { + FRDFileHeader_v2(uint16_t dataType, uint16_t eventCount, uint32_t runNumber, uint32_t lumiSection, uint64_t fileSize) + : id_(FRDFileHeader_id, FRDFileVersion_2), c_(dataType, eventCount, runNumber, lumiSection, fileSize) {} - FRDFileHeaderIdentifier id_; - FRDFileHeaderContent_v2 c_; -}; + FRDFileHeaderIdentifier id_; + FRDFileHeaderContent_v2 c_; + }; -inline uint16_t getFRDFileHeaderVersion(const std::array& id, const std::array& version) { - size_t i; - for (i = 0; i < 4; i++) - if (id[i] != FRDFileHeader_id[i]) - return 0; //not FRD file header - uint16_t ret = 0; - for (i = 0; i < 4; i++) { - if (version[i] > '9' || version[i] < '0') - return 0; //NaN sequence - ret = ret * 10 + (uint16_t)(version[i] - '0'); + inline uint16_t getFRDFileHeaderVersion(const std::array& id, const std::array& version) { + size_t i; + for (i = 0; i < 4; i++) + if (id[i] != FRDFileHeader_id[i]) + return 0; //not FRD file header + uint16_t ret = 0; + for (i = 0; i < 4; i++) { + if (version[i] > '9' || version[i] < '0') + return 0; //NaN sequence + ret = ret * 10 + (uint16_t)(version[i] - '0'); + } + return ret; } - return ret; -} - +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/InitMessage.h b/IOPool/Streamer/interface/InitMessage.h index 2858d367927c8..08d3b945bbfd0 100644 --- a/IOPool/Streamer/interface/InitMessage.h +++ b/IOPool/Streamer/interface/InitMessage.h @@ -36,91 +36,92 @@ Protocol Version 11: identical to version 10, but incremented to keep in sync wi #include "IOPool/Streamer/interface/MsgTools.h" #include "IOPool/Streamer/interface/MsgHeader.h" -struct Version { - Version(const uint8* pset) : protocol_(11) { std::copy(pset, pset + sizeof(pset_id_), &pset_id_[0]); } - - uint8 protocol_; // version of the protocol - unsigned char pset_id_[16]; // parameter set ID -}; - -struct InitHeader { - InitHeader(const Header& h, uint32 run, const Version& v, uint32 init_header_size = 0, uint32 event_header_size = 0) - : header_(h), version_(v) { - convert(run, run_); - convert(init_header_size, init_header_size_); - convert(event_header_size, event_header_size_); - } - - Header header_; - Version version_; - char_uint32 run_; - char_uint32 init_header_size_; - char_uint32 event_header_size_; -}; - -class InitMsgView { -public: - InitMsgView(void* buf); - - uint32 code() const { return head_.code(); } - uint32 size() const { return head_.size(); } - uint8* startAddress() const { return buf_; } - - uint32 run() const; - uint32 protocolVersion() const; - void pset(uint8* put_here) const; - std::string releaseTag() const; - std::string processName() const; - std::string outputModuleLabel() const; - uint32 outputModuleId() const { return outputModuleId_; } - - void hltTriggerNames(Strings& save_here) const; - void hltTriggerSelections(Strings& save_here) const; - void l1TriggerNames(Strings& save_here) const; - - uint32 get_hlt_bit_cnt() const { return hlt_trig_count_; } - uint32 get_l1_bit_cnt() const { return l1_trig_count_; } - - // needed for streamer file - uint32 descLength() const { return desc_len_; } - const uint8* descData() const { return desc_start_; } - uint32 headerSize() const { return desc_start_ - buf_; } - uint32 eventHeaderSize() const; - uint32 adler32_chksum() const { return adler32_chksum_; } - std::string hostName() const; - uint32 hostName_len() const { return host_name_len_; } - -private: - uint8* buf_; - HeaderView head_; - - uint8* release_start_; // points to the string - uint32 release_len_; - - uint8* processName_start_; // points to the string - uint32 processName_len_; - - uint8* outputModuleLabel_start_; // points to the string - uint32 outputModuleLabel_len_; - uint32 outputModuleId_; - - uint8* hlt_trig_start_; // points to the string - uint32 hlt_trig_count_; // number of strings - uint32 hlt_trig_len_; // length of strings character array only - uint8* hlt_select_start_; // points to the string - uint32 hlt_select_count_; // number of strings - uint32 hlt_select_len_; // length of strings character array only - uint8* l1_trig_start_; // points to the string - uint32 l1_trig_count_; // number of strings - uint32 l1_trig_len_; // length of strings character array only - uint32 adler32_chksum_; - uint8* host_name_start_; - uint32 host_name_len_; - - // does not need to be present in the message sent over the network, - // but is needed for the index file - uint8* desc_start_; // point to the bytes - uint32 desc_len_; -}; - +namespace edm::streamer { + struct Version { + Version(const uint8* pset) : protocol_(11) { std::copy(pset, pset + sizeof(pset_id_), &pset_id_[0]); } + + uint8 protocol_; // version of the protocol + unsigned char pset_id_[16]; // parameter set ID + }; + + struct InitHeader { + InitHeader(const Header& h, uint32 run, const Version& v, uint32 init_header_size = 0, uint32 event_header_size = 0) + : header_(h), version_(v) { + convert(run, run_); + convert(init_header_size, init_header_size_); + convert(event_header_size, event_header_size_); + } + + Header header_; + Version version_; + char_uint32 run_; + char_uint32 init_header_size_; + char_uint32 event_header_size_; + }; + + class InitMsgView { + public: + InitMsgView(void* buf); + + uint32 code() const { return head_.code(); } + uint32 size() const { return head_.size(); } + uint8* startAddress() const { return buf_; } + + uint32 run() const; + uint32 protocolVersion() const; + void pset(uint8* put_here) const; + std::string releaseTag() const; + std::string processName() const; + std::string outputModuleLabel() const; + uint32 outputModuleId() const { return outputModuleId_; } + + void hltTriggerNames(Strings& save_here) const; + void hltTriggerSelections(Strings& save_here) const; + void l1TriggerNames(Strings& save_here) const; + + uint32 get_hlt_bit_cnt() const { return hlt_trig_count_; } + uint32 get_l1_bit_cnt() const { return l1_trig_count_; } + + // needed for streamer file + uint32 descLength() const { return desc_len_; } + const uint8* descData() const { return desc_start_; } + uint32 headerSize() const { return desc_start_ - buf_; } + uint32 eventHeaderSize() const; + uint32 adler32_chksum() const { return adler32_chksum_; } + std::string hostName() const; + uint32 hostName_len() const { return host_name_len_; } + + private: + uint8* buf_; + HeaderView head_; + + uint8* release_start_; // points to the string + uint32 release_len_; + + uint8* processName_start_; // points to the string + uint32 processName_len_; + + uint8* outputModuleLabel_start_; // points to the string + uint32 outputModuleLabel_len_; + uint32 outputModuleId_; + + uint8* hlt_trig_start_; // points to the string + uint32 hlt_trig_count_; // number of strings + uint32 hlt_trig_len_; // length of strings character array only + uint8* hlt_select_start_; // points to the string + uint32 hlt_select_count_; // number of strings + uint32 hlt_select_len_; // length of strings character array only + uint8* l1_trig_start_; // points to the string + uint32 l1_trig_count_; // number of strings + uint32 l1_trig_len_; // length of strings character array only + uint32 adler32_chksum_; + uint8* host_name_start_; + uint32 host_name_len_; + + // does not need to be present in the message sent over the network, + // but is needed for the index file + uint8* desc_start_; // point to the bytes + uint32 desc_len_; + }; +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/InitMsgBuilder.h b/IOPool/Streamer/interface/InitMsgBuilder.h index 90b15ab6c1435..56d4bbbfc22d0 100644 --- a/IOPool/Streamer/interface/InitMsgBuilder.h +++ b/IOPool/Streamer/interface/InitMsgBuilder.h @@ -5,34 +5,34 @@ #include "IOPool/Streamer/interface/InitMessage.h" // ----------------- init ------------------- +namespace edm::streamer { + class InitMsgBuilder { + public: + InitMsgBuilder(void* msg_mem, + uint32 size, + uint32 run, + const Version& v, + const char* release_tag, + const char* process_name, + const char* output_module_label, + uint32 output_module_id, + const Strings& hlt_names, + const Strings& hlt_selections, + const Strings& l1_names, + uint32 adler32_chksum); -class InitMsgBuilder { -public: - InitMsgBuilder(void* msg_mem, - uint32 size, - uint32 run, - const Version& v, - const char* release_tag, - const char* process_name, - const char* output_module_label, - uint32 output_module_id, - const Strings& hlt_names, - const Strings& hlt_selections, - const Strings& l1_names, - uint32 adler32_chksum); - - uint8* startAddress() const { return buf_; } - void setDataLength(uint32 registry_length); - uint8* dataAddress() const { return data_addr_; } - uint32 headerSize() const { return data_addr_ - buf_; } - uint32 size() const; - uint32 run() const; /** Required by EOF Record Builder */ - uint32 bufferSize() const { return size_; } - -private: - uint8* buf_; - uint32 size_; - uint8* data_addr_; -}; + uint8* startAddress() const { return buf_; } + void setDataLength(uint32 registry_length); + uint8* dataAddress() const { return data_addr_; } + uint32 headerSize() const { return data_addr_ - buf_; } + uint32 size() const; + uint32 run() const; /** Required by EOF Record Builder */ + uint32 bufferSize() const { return size_; } + private: + uint8* buf_; + uint32 size_; + uint8* data_addr_; + }; +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/MsgHeader.h b/IOPool/Streamer/interface/MsgHeader.h index eb84f7747cb36..6568174fedfe1 100644 --- a/IOPool/Streamer/interface/MsgHeader.h +++ b/IOPool/Streamer/interface/MsgHeader.h @@ -3,50 +3,51 @@ #include "IOPool/Streamer/interface/MsgTools.h" // as it is in memory of file -struct Header { - Header(uint32 code, uint32 size) : code_(code) { convert(size, size_); } +namespace edm::streamer { + struct Header { + Header(uint32 code, uint32 size) : code_(code) { convert(size, size_); } - uint8 code_; // type of the message - char_uint32 size_; // of entire message including all headers + uint8 code_; // type of the message + char_uint32 size_; // of entire message including all headers - // 20-Jul-2006, KAB: added enumeration for message types - enum Codes { - INVALID = 0, - INIT = 1, - EVENT = 2, - DONE = 3, // EOFRECORD = 4 is no longer used - HEADER_REQUEST = 5, - EVENT_REQUEST = 6, - CONS_REG_REQUEST = 7, - CONS_REG_RESPONSE = 8, - DQM_INIT = 9, - DQM_EVENT = 10, - DQMEVENT_REQUEST = 11, - INIT_SET = 12, - NEW_INIT_AVAILABLE = 13, - ERROR_EVENT = 14, - FILE_CLOSE_REQUEST = 15, - SPARE1 = 16, - SPARE2 = 17, - PADDING = 255 //reserved for padding + // 20-Jul-2006, KAB: added enumeration for message types + enum Codes { + INVALID = 0, + INIT = 1, + EVENT = 2, + DONE = 3, // EOFRECORD = 4 is no longer used + HEADER_REQUEST = 5, + EVENT_REQUEST = 6, + CONS_REG_REQUEST = 7, + CONS_REG_RESPONSE = 8, + DQM_INIT = 9, + DQM_EVENT = 10, + DQMEVENT_REQUEST = 11, + INIT_SET = 12, + NEW_INIT_AVAILABLE = 13, + ERROR_EVENT = 14, + FILE_CLOSE_REQUEST = 15, + SPARE1 = 16, + SPARE2 = 17, + PADDING = 255 //reserved for padding + }; }; -}; -// as we need to see it -class HeaderView { -public: - HeaderView(void* buf) { - Header* h = (Header*)buf; - code_ = h->code_; - size_ = convert32(h->size_); - } + // as we need to see it + class HeaderView { + public: + HeaderView(void* buf) { + Header* h = (Header*)buf; + code_ = h->code_; + size_ = convert32(h->size_); + } - uint32 code() const { return code_; } - uint32 size() const { return size_; } - -private: - uint32 code_; - uint32 size_; -}; + uint32 code() const { return code_; } + uint32 size() const { return size_; } + private: + uint32 code_; + uint32 size_; + }; +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/MsgTools.h b/IOPool/Streamer/interface/MsgTools.h index 7411b945c76de..75495a1072c60 100644 --- a/IOPool/Streamer/interface/MsgTools.h +++ b/IOPool/Streamer/interface/MsgTools.h @@ -7,88 +7,89 @@ #include #include "FWCore/Utilities/interface/Algorithms.h" -// could just use the c99 names here from stdint.h -typedef unsigned char uint8; -typedef unsigned short uint16; -typedef unsigned int uint32; -typedef unsigned long long uint64; -typedef unsigned char char_uint64[sizeof(uint64)]; -typedef unsigned char char_uint32[sizeof(uint32)]; -typedef unsigned char char_uint16[sizeof(uint16)]; -typedef std::vector Strings; +namespace edm::streamer { + // could just use the c99 names here from stdint.h + typedef unsigned char uint8; + typedef unsigned short uint16; + typedef unsigned int uint32; + typedef unsigned long long uint64; + typedef unsigned char char_uint64[sizeof(uint64)]; + typedef unsigned char char_uint32[sizeof(uint32)]; + typedef unsigned char char_uint16[sizeof(uint16)]; + typedef std::vector Strings; -inline uint64 convert64(char_uint64 v) { - // first four bytes are code, LSB first - unsigned long long a = v[0], b = v[1], c = v[2], d = v[3]; - unsigned long long e = v[4], f = v[5], g = v[6], h = v[7]; - a |= (b << 8) | (c << 16) | (d << 24) | (e << 32) | (f << 40) | (g << 48) | (h << 56); - return a; -} + inline uint64 convert64(char_uint64 v) { + // first four bytes are code, LSB first + unsigned long long a = v[0], b = v[1], c = v[2], d = v[3]; + unsigned long long e = v[4], f = v[5], g = v[6], h = v[7]; + a |= (b << 8) | (c << 16) | (d << 24) | (e << 32) | (f << 40) | (g << 48) | (h << 56); + return a; + } -inline uint32 convert32(char_uint32 v) { - // first four bytes are code, LSB first - unsigned int a = v[0], b = v[1], c = v[2], d = v[3]; - a |= (b << 8) | (c << 16) | (d << 24); - return a; -} + inline uint32 convert32(char_uint32 v) { + // first four bytes are code, LSB first + unsigned int a = v[0], b = v[1], c = v[2], d = v[3]; + a |= (b << 8) | (c << 16) | (d << 24); + return a; + } -inline uint16 convert16(char_uint16 v) { - // first four bytes are code, LSB first - unsigned int a = v[0], b = v[1]; - a |= (b << 8); - return a; -} + inline uint16 convert16(char_uint16 v) { + // first four bytes are code, LSB first + unsigned int a = v[0], b = v[1]; + a |= (b << 8); + return a; + } -inline void convert(uint32 i, char_uint32 v) { - v[0] = i & 0xff; - v[1] = (i >> 8) & 0xff; - v[2] = (i >> 16) & 0xff; - v[3] = (i >> 24) & 0xff; -} + inline void convert(uint32 i, char_uint32 v) { + v[0] = i & 0xff; + v[1] = (i >> 8) & 0xff; + v[2] = (i >> 16) & 0xff; + v[3] = (i >> 24) & 0xff; + } -inline void convert(uint16 i, char_uint16 v) { - v[0] = i & 0xff; - v[1] = (i >> 8) & 0xff; -} + inline void convert(uint16 i, char_uint16 v) { + v[0] = i & 0xff; + v[1] = (i >> 8) & 0xff; + } -inline void convert(uint64 li, char_uint64 v) { - v[0] = li & 0xff; - v[1] = (li >> 8) & 0xff; - v[2] = (li >> 16) & 0xff; - v[3] = (li >> 24) & 0xff; - v[4] = (li >> 32) & 0xff; - v[5] = (li >> 40) & 0xff; - v[6] = (li >> 48) & 0xff; - v[7] = (li >> 56) & 0xff; -} + inline void convert(uint64 li, char_uint64 v) { + v[0] = li & 0xff; + v[1] = (li >> 8) & 0xff; + v[2] = (li >> 16) & 0xff; + v[3] = (li >> 24) & 0xff; + v[4] = (li >> 32) & 0xff; + v[5] = (li >> 40) & 0xff; + v[6] = (li >> 48) & 0xff; + v[7] = (li >> 56) & 0xff; + } -namespace MsgTools { + namespace MsgTools { - inline uint8* fillNames(const Strings& names, uint8* pos) { - uint32 sz = names.size(); - convert(sz, pos); // save number of strings - uint8* len_pos = pos + sizeof(char_uint32); // area for length - pos = len_pos + sizeof(char_uint32); // area for full string of names - bool first = true; + inline uint8* fillNames(const Strings& names, uint8* pos) { + uint32 sz = names.size(); + convert(sz, pos); // save number of strings + uint8* len_pos = pos + sizeof(char_uint32); // area for length + pos = len_pos + sizeof(char_uint32); // area for full string of names + bool first = true; - for (Strings::const_iterator beg = names.begin(); beg != names.end(); ++beg) { - if (first) - first = false; - else - *pos++ = ' '; - pos = edm::copy_all(*beg, pos); + for (Strings::const_iterator beg = names.begin(); beg != names.end(); ++beg) { + if (first) + first = false; + else + *pos++ = ' '; + pos = edm::copy_all(*beg, pos); + } + convert((uint32)(pos - len_pos - sizeof(char_uint32)), len_pos); + return pos; } - convert((uint32)(pos - len_pos - sizeof(char_uint32)), len_pos); - return pos; - } - inline void getNames(uint8* from, uint32 from_len, Strings& to) { - // not the most efficient way to do this - std::istringstream ist(std::string(reinterpret_cast(from), from_len)); - typedef std::istream_iterator Iter; - std::copy(Iter(ist), Iter(), std::back_inserter(to)); - } - -} // namespace MsgTools + inline void getNames(uint8* from, uint32 from_len, Strings& to) { + // not the most efficient way to do this + std::istringstream ist(std::string(reinterpret_cast(from), from_len)); + typedef std::istream_iterator Iter; + std::copy(Iter(ist), Iter(), std::back_inserter(to)); + } + } // namespace MsgTools +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/StreamSerializer.h b/IOPool/Streamer/interface/StreamSerializer.h index 1e84f4651096f..adc83b09c34a8 100644 --- a/IOPool/Streamer/interface/StreamSerializer.h +++ b/IOPool/Streamer/interface/StreamSerializer.h @@ -19,76 +19,82 @@ #include "DataFormats/Streamer/interface/StreamedProducts.h" #include "FWCore/Utilities/interface/get_underlying_safe.h" -// Data structure to be shared by all output modules for event serialization -struct SerializeDataBuffer { - typedef std::vector SBuffer; - static constexpr int init_size = 0; //will be allocated on first event - static constexpr unsigned int reserve_size = 50000; - - SerializeDataBuffer() - : comp_buf_(reserve_size + init_size), - curr_event_size_(), - curr_space_used_(), - rootbuf_(TBuffer::kWrite, init_size), - ptr_((unsigned char *)rootbuf_.Buffer()), - header_buf_(), - adler32_chksum_(0) {} - - // This object caches the results of the last INIT or event - // serialization operation. You get access to the data using the - // following member functions. - - unsigned char const *bufferPointer() const { return get_underlying_safe(ptr_); } - unsigned char *&bufferPointer() { return get_underlying_safe(ptr_); } - unsigned int currentSpaceUsed() const { return curr_space_used_; } - unsigned int currentEventSize() const { return curr_event_size_; } - uint32_t adler32_chksum() const { return adler32_chksum_; } - - void clearHeaderBuffer() { - header_buf_.clear(); - header_buf_.shrink_to_fit(); - rootbuf_.Reset(); - rootbuf_.Expand(init_size); //shrink TBuffer to size 0 after resetting TBuffer length - } - - std::vector comp_buf_; // space for compressed data - unsigned int curr_event_size_; - unsigned int curr_space_used_; // less than curr_event_size_ if compressed - TBufferFile rootbuf_; - edm::propagate_const ptr_; // set to the place where the last event stored - SBuffer header_buf_; // place for INIT message creation and streamer event header - uint32_t adler32_chksum_; // adler32 check sum for the (compressed) data -}; - -class EventMsgBuilder; -class InitMsgBuilder; namespace edm { - enum StreamerCompressionAlgo { UNCOMPRESSED = 0, ZLIB = 1, LZMA = 2, ZSTD = 4 }; - class EventForOutput; class ModuleCallingContext; class ThinnedAssociationsHelper; +} // namespace edm + +// Data structure to be shared by all output modules for event serialization +namespace edm::streamer { + struct SerializeDataBuffer { + typedef std::vector SBuffer; + static constexpr int init_size = 0; //will be allocated on first event + static constexpr unsigned int reserve_size = 50000; + + SerializeDataBuffer() + : comp_buf_(reserve_size + init_size), + curr_event_size_(), + curr_space_used_(), + rootbuf_(TBuffer::kWrite, init_size), + ptr_((unsigned char *)rootbuf_.Buffer()), + header_buf_(), + adler32_chksum_(0) {} + + // This object caches the results of the last INIT or event + // serialization operation. You get access to the data using the + // following member functions. + + unsigned char const *bufferPointer() const { return get_underlying_safe(ptr_); } + unsigned char *&bufferPointer() { return get_underlying_safe(ptr_); } + unsigned int currentSpaceUsed() const { return curr_space_used_; } + unsigned int currentEventSize() const { return curr_event_size_; } + uint32_t adler32_chksum() const { return adler32_chksum_; } + + void clearHeaderBuffer() { + header_buf_.clear(); + header_buf_.shrink_to_fit(); + rootbuf_.Reset(); + rootbuf_.Expand(init_size); //shrink TBuffer to size 0 after resetting TBuffer length + } + + std::vector comp_buf_; // space for compressed data + unsigned int curr_event_size_; + unsigned int curr_space_used_; // less than curr_event_size_ if compressed + TBufferFile rootbuf_; + edm::propagate_const ptr_; // set to the place where the last event stored + SBuffer header_buf_; // place for INIT message creation and streamer event header + uint32_t adler32_chksum_; // adler32 check sum for the (compressed) data + }; + + class EventMsgBuilder; + class InitMsgBuilder; + enum StreamerCompressionAlgo { UNCOMPRESSED = 0, ZLIB = 1, LZMA = 2, ZSTD = 4 }; class StreamSerializer { public: StreamSerializer(SelectedProducts const *selections); - int serializeRegistry(SerializeDataBuffer &data_buffer, - const BranchIDLists &branchIDLists, - ThinnedAssociationsHelper const &thinnedAssociationsHelper); + int serializeRegistry(SerializeDataBuffer &data_buffer) const; - int serializeRegistry(SerializeDataBuffer &data_buffer, - const BranchIDLists &branchIDLists, - ThinnedAssociationsHelper const &thinnedAssociationsHelper, - SendJobHeader::ParameterSetMap const &psetMap); + int serializeRegistry(SerializeDataBuffer &data_buffer, SendJobHeader::ParameterSetMap const &psetMap) const; int serializeEvent(SerializeDataBuffer &data_buffer, EventForOutput const &event, ParameterSetID const &selectorConfig, + uint32_t metaDataChecksum, StreamerCompressionAlgo compressionAlgo, int compression_level, unsigned int reserveSize) const; + ///data_buffer.adler32_chksum_ is the meta data checksum to pass to subsequent events + int serializeEventMetaData(SerializeDataBuffer &data_buffer, + const BranchIDLists &branchIDLists, + ThinnedAssociationsHelper const &thinnedAssociationsHelper, + StreamerCompressionAlgo compressionAlgo, + int compression_level, + unsigned int reserveSize) const; + /** * Compresses the data in the specified input buffer into the * specified output buffer. Returns the size of the compressed data @@ -115,10 +121,16 @@ namespace edm { bool addHeader = true); private: + int serializeEventCommon(SerializeDataBuffer &data_buffer, + edm::SendEvent const &iEvent, + StreamerCompressionAlgo compressionAlgo, + int compression_level, + unsigned int reserveSize) const; + SelectedProducts const *selections_; edm::propagate_const tc_; }; -} // namespace edm +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/StreamerInputFile.h b/IOPool/Streamer/interface/StreamerInputFile.h index 033d558f07184..a969cecba2fbc 100644 --- a/IOPool/Streamer/interface/StreamerInputFile.h +++ b/IOPool/Streamer/interface/StreamerInputFile.h @@ -16,6 +16,8 @@ namespace edm { class EventSkipperByID; class FileCatalogItem; +} // namespace edm +namespace edm::streamer { class StreamerInputFile { public: /**Reads a Streamer file */ @@ -96,6 +98,6 @@ namespace edm { bool endOfFile_; }; -} // namespace edm +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/StreamerInputModule.h b/IOPool/Streamer/interface/StreamerInputModule.h index e43fb69d2cd1f..fd7fc86f042d9 100644 --- a/IOPool/Streamer/interface/StreamerInputModule.h +++ b/IOPool/Streamer/interface/StreamerInputModule.h @@ -15,7 +15,7 @@ #include #include -namespace edm { +namespace edm::streamer { template class StreamerInputModule : public StreamerInputSource { /** @@ -30,24 +30,49 @@ namespace edm { private: void genuineCloseFile() override { + if (didArtificialFile_) { + didArtificialFile_ = false; + + return; + } if (pr_.get() != nullptr) pr_->closeFile(); } + void setupMetaData() { + InitMsgView const* header = pr_->getHeader(); + assert(header); + deserializeAndMergeWithRegistry(*header); + + //NOTE: should read first Event to get the meta data + auto eview = pr_->getNextEvent(); + assert(eview); + assert(eview->isEventMetaData()); + deserializeEventMetaData(*eview); + updateEventMetaData(); + } + void genuineReadFile() override { if (isFirstFile_) { isFirstFile_ = false; return; } - InitMsgView const* header = pr_->getHeader(); - deserializeAndMergeWithRegistry(*header); + if (didArtificialFile_) { + //update the event meta data + didArtificialFile_ = false; + updateEventMetaData(); + + return; + } + setupMetaData(); } Next checkNext() override; edm::propagate_const> pr_; bool isFirstFile_ = true; + bool didArtificialFile_ = false; }; //end-of-class-def template @@ -59,25 +84,30 @@ namespace edm { //prod_reg_(&productRegistry()), pr_(new Producer(pset)) { //Get header/init from Producer - InitMsgView const* header = pr_->getHeader(); - deserializeAndMergeWithRegistry(*header); + setupMetaData(); } template StreamerInputSource::Next StreamerInputModule::checkNext() { EventMsgView const* eview = pr_->getNextEvent(); - if (pr_->newHeader()) { - FDEBUG(6) << "A new file has been opened and we must compare Headers here !!" << std::endl; - return Next::kFile; - } if (eview == nullptr) { + if (pr_->newHeader()) { + FDEBUG(6) << "A new file has been opened and we must compare Headers here !!" << std::endl; + return Next::kFile; + } return Next::kStop; } + if (eview->isEventMetaData()) { + //we lie and say there is a new file since we need to synchronize to update the meta data + deserializeEventMetaData(*eview); + didArtificialFile_ = true; + return Next::kFile; + } deserializeEvent(*eview); return Next::kEvent; } -} // namespace edm +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/StreamerInputSource.h b/IOPool/Streamer/interface/StreamerInputSource.h index 460c969c39b41..d3092bbabbac3 100644 --- a/IOPool/Streamer/interface/StreamerInputSource.h +++ b/IOPool/Streamer/interface/StreamerInputSource.h @@ -20,13 +20,15 @@ #include #include -class InitMsgView; -class EventMsgView; - namespace edm { class BranchIDListHelper; class ParameterSetDescription; class ThinnedAssociationsHelper; +} // namespace edm + +namespace edm::streamer { + class InitMsgView; + class EventMsgView; class StreamerInputSource : public RawInputSource { public: @@ -38,13 +40,18 @@ namespace edm { void deserializeAndMergeWithRegistry(InitMsgView const& initView, bool subsequent = false); + //If eventView is a meta data event then this returns its checksum + uint32_t eventMetaDataChecksum(EventMsgView const& eventView) const; + //Should be called right after this message has been read + void deserializeEventMetaData(EventMsgView const& eventView); void deserializeEvent(EventMsgView const& eventView); - static void mergeIntoRegistry(SendJobHeader const& header, - ProductRegistry&, - BranchIDListHelper&, - ThinnedAssociationsHelper&, - bool subsequent); + uint32_t presentEventMetaDataChecksum() const { return eventMetaDataChecksum_; } + //This can only be called during a new file transition as it updates state that requires + // framework synchronization. + void updateEventMetaData(); + + static void mergeIntoRegistry(SendJobHeader const& header, ProductRegistry&, bool subsequent); /** * Detect if buffer starts with "XZ\0" which means it is compressed in LZMA format @@ -87,6 +94,8 @@ namespace edm { void resetAfterEndRun(); private: + void deserializeEventCommon(EventMsgView const& eventView, bool isMetaData); + class EventPrincipalHolder : public EDProductGetter { public: EventPrincipalHolder(); @@ -123,7 +132,8 @@ namespace edm { std::string processName_; unsigned int protocolVersion_; + uint32_t eventMetaDataChecksum_ = 0; }; //end-of-class-def -} // namespace edm +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/StreamerOutputFile.h b/IOPool/Streamer/interface/StreamerOutputFile.h index 0bae650e7f83b..837e33ea1c496 100644 --- a/IOPool/Streamer/interface/StreamerOutputFile.h +++ b/IOPool/Streamer/interface/StreamerOutputFile.h @@ -20,48 +20,49 @@ #include #include -class StreamerOutputFile -/** +namespace edm::streamer { + class StreamerOutputFile + /** Class for doing Streamer Write operations */ -{ -public: - explicit StreamerOutputFile(const std::string& name, uint32 padding = 0); - /** + { + public: + explicit StreamerOutputFile(const std::string& name, uint32 padding = 0); + /** CTOR, takes file path name as argument */ - ~StreamerOutputFile(); + ~StreamerOutputFile(); - void write(const InitMsgBuilder&); - /** + void write(const InitMsgBuilder&); + /** Performs write on InitMsgBuilder type, Header + Blob, both are written out. */ - void write(const InitMsgView&); + void write(const InitMsgView&); - void writeInitFragment(uint32 fragIndex, uint32 fragCount, const char* dataPtr, uint32 dataSize); + void writeInitFragment(uint32 fragIndex, uint32 fragCount, const char* dataPtr, uint32 dataSize); - uint64 write(const EventMsgBuilder&); - /** + uint64 write(const EventMsgBuilder&); + /** Performs write on EventMsgBuilder type, Header + Blob, both are written out. RETURNS the Offset in Stream while at which EventForOutputwas written. */ - uint64 write(const EventMsgView&); - - uint64 writeEventFragment(uint32 fragIndex, uint32 fragCount, const char* dataPtr, uint32 dataSize); + uint64 write(const EventMsgView&); - uint32 adler32() const { return streamerfile_->adler32(); } + uint64 writeEventFragment(uint32 fragIndex, uint32 fragCount, const char* dataPtr, uint32 dataSize); - void close() { streamerfile_->close(); } + uint32 adler32() const { return streamerfile_->adler32(); } -private: - void writeEventHeader(const EventMsgView& ineview); - void writeStart(const InitMsgView& inview); + void close() { streamerfile_->close(); } -private: - edm::propagate_const> streamerfile_; -}; + private: + void writeEventHeader(const EventMsgView& ineview); + void writeStart(const InitMsgView& inview); + private: + edm::propagate_const> streamerfile_; + }; +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/StreamerOutputModule.h b/IOPool/Streamer/interface/StreamerOutputModule.h index 2359418e7e1b2..a03007bca82f6 100644 --- a/IOPool/Streamer/interface/StreamerOutputModule.h +++ b/IOPool/Streamer/interface/StreamerOutputModule.h @@ -5,7 +5,7 @@ #include "FWCore/Utilities/interface/propagate_const.h" #include "IOPool/Streamer/interface/StreamerOutputModuleBase.h" -namespace edm { +namespace edm::streamer { template class StreamerOutputModule : public StreamerOutputModuleBase { /** Consumers are suppose to provide @@ -74,6 +74,6 @@ namespace edm { Consumer::fillDescription(desc); descriptions.add("streamerOutput", desc); } -} // namespace edm +} // namespace edm::streamer #endif diff --git a/IOPool/Streamer/interface/StreamerOutputModuleBase.h b/IOPool/Streamer/interface/StreamerOutputModuleBase.h index 6e3b4f5a6e329..eedade18f4ac0 100644 --- a/IOPool/Streamer/interface/StreamerOutputModuleBase.h +++ b/IOPool/Streamer/interface/StreamerOutputModuleBase.h @@ -10,42 +10,45 @@ //#include //#include -class InitMsgBuilder; -class EventMsgBuilder; namespace edm { class ParameterSetDescription; typedef detail::TriggerResultsBasedEventSelector::handle_t Trig; - class StreamerOutputModuleBase : public one::OutputModule, - StreamerOutputModuleCommon { - public: - explicit StreamerOutputModuleBase(ParameterSet const& ps); - ~StreamerOutputModuleBase() override; - static void fillDescription(ParameterSetDescription& desc); - - private: - void beginRun(RunForOutput const&) override; - void endRun(RunForOutput const&) override; - void beginJob() override; - void endJob() override; - void writeRun(RunForOutput const&) override; - void writeLuminosityBlock(LuminosityBlockForOutput const&) override; - void write(EventForOutput const& e) override; - - virtual void start() = 0; - virtual void stop() = 0; - virtual void doOutputHeader(InitMsgBuilder const& init_message) = 0; - virtual void doOutputEvent(EventMsgBuilder const& msg) = 0; - - Trig getTriggerResults(EDGetTokenT const& token, EventForOutput const& e) const; - - private: - edm::EDGetTokenT trToken_; - edm::EDGetTokenT psetToken_; - - }; //end-of-class-def - + namespace streamer { + class InitMsgBuilder; + class EventMsgBuilder; + + class StreamerOutputModuleBase : public one::OutputModule, + StreamerOutputModuleCommon { + public: + explicit StreamerOutputModuleBase(ParameterSet const& ps); + ~StreamerOutputModuleBase() override; + static void fillDescription(ParameterSetDescription& desc); + + private: + void beginRun(RunForOutput const&) override; + void endRun(RunForOutput const&) override; + void beginJob() override; + void endJob() override; + void writeRun(RunForOutput const&) override; + void writeLuminosityBlock(LuminosityBlockForOutput const&) override; + void write(EventForOutput const& e) override; + + virtual void start() = 0; + virtual void stop() = 0; + virtual void doOutputHeader(InitMsgBuilder const& init_message) = 0; + virtual void doOutputEvent(EventMsgBuilder const& msg) = 0; + + Trig getTriggerResults(EDGetTokenT const& token, EventForOutput const& e) const; + + private: + edm::EDGetTokenT trToken_; + edm::EDGetTokenT psetToken_; + bool lastCallWasBeginRun_ = false; + + }; //end-of-class-def + } // namespace streamer } // namespace edm #endif diff --git a/IOPool/Streamer/interface/StreamerOutputModuleCommon.h b/IOPool/Streamer/interface/StreamerOutputModuleCommon.h index 68ecb1d8e208f..7c6a2fbb830b5 100644 --- a/IOPool/Streamer/interface/StreamerOutputModuleCommon.h +++ b/IOPool/Streamer/interface/StreamerOutputModuleCommon.h @@ -3,14 +3,12 @@ #include "IOPool/Streamer/interface/MsgTools.h" #include "FWCore/Common/interface/TriggerNames.h" -#include "IOPool/Streamer/interface/StreamSerializer.h" +#include "IOPool/Streamer/interface/StreamerOutputMsgBuilders.h" #include "DataFormats/Common/interface/Handle.h" #include "DataFormats/Streamer/interface/StreamedProducts.h" #include #include -class InitMsgBuilder; -class EventMsgBuilder; namespace edm { class ParameterSet; class ParameterSetDescription; @@ -18,74 +16,50 @@ namespace edm { class ThinnedAssociationsHelper; class TriggerResults; - class StreamerOutputModuleCommon { - public: - struct Parameters { - Strings hltTriggerSelections; - std::string compressionAlgoStr; - int compressionLevel; - int lumiSectionInterval; - bool useCompression; - }; + namespace streamer { + class InitMsgBuilder; + class EventMsgBuilder; - static Parameters parameters(ParameterSet const& ps); + class StreamerOutputModuleCommon { + public: + using Parameters = StreamerOutputMsgBuilders::Parameters; - explicit StreamerOutputModuleCommon(Parameters const& p, - SelectedProducts const* selections, - std::string const& moduleLabel); + static Parameters parameters(ParameterSet const& ps); - explicit StreamerOutputModuleCommon(ParameterSet const& ps, - SelectedProducts const* selections, - std::string const& moduleLabel) - : StreamerOutputModuleCommon(parameters(ps), selections, moduleLabel) {} + explicit StreamerOutputModuleCommon(Parameters const& p, + SelectedProducts const* selections, + std::string const& moduleLabel); - ~StreamerOutputModuleCommon(); - static void fillDescription(ParameterSetDescription& desc); + explicit StreamerOutputModuleCommon(ParameterSet const& ps, + SelectedProducts const* selections, + std::string const& moduleLabel) + : StreamerOutputModuleCommon(parameters(ps), selections, moduleLabel) {} - std::unique_ptr serializeRegistry(SerializeDataBuffer& sbuf, - BranchIDLists const& branchLists, - ThinnedAssociationsHelper const& helper, - std::string const& processName, - std::string const& moduleLabel, - ParameterSetID const& toplevel, - SendJobHeader::ParameterSetMap const* psetMap); + ~StreamerOutputModuleCommon(); + static void fillDescription(ParameterSetDescription& desc); - std::unique_ptr serializeEvent(SerializeDataBuffer& sbuf, - EventForOutput const& e, - Handle const& triggerResults, - ParameterSetID const& selectorCfg); + std::unique_ptr serializeRegistry(std::string const& processName, + std::string const& moduleLabel, + ParameterSetID const& toplevel, + SendJobHeader::ParameterSetMap const* psetMap); - SerializeDataBuffer* getSerializerBuffer(); + std::unique_ptr serializeEventMetaData(BranchIDLists const& branchLists, + ThinnedAssociationsHelper const& helper); - protected: - std::unique_ptr serializerBuffer_; + std::unique_ptr serializeEvent(EventForOutput const& e, + Handle const& triggerResults, + ParameterSetID const& selectorCfg); - private: - void setHltMask(EventForOutput const& e, - Handle const& triggerResults, - std::vector& hltbits) const; + protected: + void clearHeaderBuffer() { buffer_.clearHeaderBuffer(); } - StreamSerializer serializer_; - - int maxEventSize_; - bool useCompression_; - std::string compressionAlgoStr_; - int compressionLevel_; - - StreamerCompressionAlgo compressionAlgo_; - - // test luminosity sections - int lumiSectionInterval_; - double timeInSecSinceUTC; - - unsigned int hltsize_; - char host_name_[255]; - - Strings hltTriggerSelections_; - uint32 outputModuleId_; - - }; //end-of-class-def + private: + SerializeDataBuffer buffer_; + StreamerOutputMsgBuilders builders_; + uint32_t eventMetaDataChecksum_ = 0; + }; //end-of-class-def + } // namespace streamer } // namespace edm #endif diff --git a/IOPool/Streamer/interface/StreamerOutputMsgBuilders.h b/IOPool/Streamer/interface/StreamerOutputMsgBuilders.h new file mode 100644 index 0000000000000..caed77c0c4a9a --- /dev/null +++ b/IOPool/Streamer/interface/StreamerOutputMsgBuilders.h @@ -0,0 +1,93 @@ +#ifndef IOPool_Streamer_StreamerOutputMsgBuilders_h +#define IOPool_Streamer_StreamerOutputMsgBuilders_h + +#include "IOPool/Streamer/interface/MsgTools.h" +#include "FWCore/Common/interface/TriggerNames.h" +#include "IOPool/Streamer/interface/StreamSerializer.h" +#include "DataFormats/Common/interface/Handle.h" +#include "DataFormats/Streamer/interface/StreamedProducts.h" +#include +#include + +namespace edm { + class EventForOutput; + class ThinnedAssociationsHelper; + class TriggerResults; + class ParameterSet; + class ParameterSetDescription; + + namespace streamer { + class InitMsgBuilder; + class EventMsgBuilder; + + class StreamerOutputMsgBuilders { + public: + struct Parameters { + Strings hltTriggerSelections; + std::string compressionAlgoStr; + int compressionLevel; + int lumiSectionInterval; + bool useCompression; + }; + + static Parameters parameters(ParameterSet const& ps); + + explicit StreamerOutputMsgBuilders(Parameters const& p, + SelectedProducts const* selections, + std::string const& moduleLabel); + + ~StreamerOutputMsgBuilders(); + static void fillDescription(ParameterSetDescription& desc); + + std::unique_ptr serializeRegistry(SerializeDataBuffer& sbuf, + std::string const& processName, + std::string const& moduleLabel, + ParameterSetID const& toplevel, + SendJobHeader::ParameterSetMap const* psetMap) const; + + std::pair, uint32_t> serializeEventMetaData( + SerializeDataBuffer& sbuf, BranchIDLists const& branchLists, ThinnedAssociationsHelper const& helper) const; + + std::unique_ptr serializeEvent(SerializeDataBuffer& sbuf, + EventForOutput const& e, + Handle const& triggerResults, + ParameterSetID const& selectorCfg, + uint32_t eventMetaDataChecksum) const; + + private: + std::unique_ptr serializeEventCommon(uint32 run, + uint32 lumi, + uint64 event, + std::vector hltbits, + unsigned int hltsize, + SerializeDataBuffer& sbuf) const; + + void setHltMask(EventForOutput const& e, + Handle const& triggerResults, + std::vector& hltbits) const; + + StreamSerializer serializer_; + + int maxEventSize_; + bool useCompression_; + std::string compressionAlgoStr_; + int compressionLevel_; + + StreamerCompressionAlgo compressionAlgo_; + + // test luminosity sections + int lumiSectionInterval_; + double timeInSecSinceUTC; + + unsigned int hltsize_; + char host_name_[255]; + + Strings hltTriggerSelections_; + uint32 outputModuleId_; + + uint32_t eventMetaDataChecksum_ = 0; + }; //end-of-class-def + } // namespace streamer +} // namespace edm + +#endif diff --git a/IOPool/Streamer/plugins/Module.cc b/IOPool/Streamer/plugins/Module.cc index 1651269774805..9d3d1c786d99a 100644 --- a/IOPool/Streamer/plugins/Module.cc +++ b/IOPool/Streamer/plugins/Module.cc @@ -8,11 +8,8 @@ //new module to read events from Streamer files #include "IOPool/Streamer/src/StreamerFileReader.h" -typedef edm::StreamerOutputModule EventStreamFileWriter; -typedef edm::StreamerFileReader NewEventStreamFileReader; - -using edm::StreamerFileReader; -using edm::StreamerFileWriter; +using EventStreamFileWriter = edm::streamer::StreamerOutputModule; +using NewEventStreamFileReader = edm::streamer::StreamerFileReader; DEFINE_FWK_INPUT_SOURCE(NewEventStreamFileReader); diff --git a/IOPool/Streamer/src/ClassFiller.cc b/IOPool/Streamer/src/ClassFiller.cc index 4eaa9fe34bfa1..3313878ca1a06 100644 --- a/IOPool/Streamer/src/ClassFiller.cc +++ b/IOPool/Streamer/src/ClassFiller.cc @@ -11,7 +11,7 @@ #include #include -namespace edm { +namespace edm::streamer { bool loadCap(std::string const& name, std::vector& missingDictionaries) { FDEBUG(1) << "Loading dictionary for " << name << "\n"; @@ -64,4 +64,4 @@ namespace edm { TypeID const type(ti); return getRootClass(type.className()); } -} // namespace edm +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/DumpTools.cc b/IOPool/Streamer/src/DumpTools.cc index 977c7c393b975..eecbb7c49b13d 100644 --- a/IOPool/Streamer/src/DumpTools.cc +++ b/IOPool/Streamer/src/DumpTools.cc @@ -16,190 +16,192 @@ using namespace edm; -void dumpInitHeader(const InitMsgView* view) { - std::cout << "code = " << view->code() << ", " - << "size = " << view->size() << "\n" - << "run = " << view->run() << ", " - << "proto = " << view->protocolVersion() << "\n" - << "release = " << view->releaseTag() << "\n" - << "processName = " << view->processName() << "\n"; - if (view->protocolVersion() >= 5) { - std::cout << "outModuleLabel = " << view->outputModuleLabel() << "\n"; +namespace edm::streamer { + void dumpInitHeader(const InitMsgView* view) { + std::cout << "code = " << view->code() << ", " + << "size = " << view->size() << "\n" + << "run = " << view->run() << ", " + << "proto = " << view->protocolVersion() << "\n" + << "release = " << view->releaseTag() << "\n" + << "processName = " << view->processName() << "\n"; + if (view->protocolVersion() >= 5) { + std::cout << "outModuleLabel = " << view->outputModuleLabel() << "\n"; + } + if (view->protocolVersion() >= 6) { + std::cout << "outputModuleId=0x" << std::hex << view->outputModuleId() << std::dec << std::endl; + } + if (view->protocolVersion() >= 8) { + std::cout << "Checksum for Registry data = " << view->adler32_chksum() << " Hostname = " << view->hostName() + << std::endl; + } + + //PSet 16 byte non-printable representation, stored in message. + uint8 vpset[16]; + view->pset(vpset); + + //Lets convert it to printable hex form + std::string pset_str(vpset, vpset + sizeof(vpset)); + pset_str += '\0'; + cms::Digest dig(pset_str); + cms::MD5Result r1 = dig.digest(); + std::string hexy = r1.toString(); + std::cout << "PSetID= " << hexy << std::endl; + + Strings vhltnames, vhltselections, vl1names; + view->hltTriggerNames(vhltnames); + if (view->protocolVersion() >= 5) { + view->hltTriggerSelections(vhltselections); + } + view->l1TriggerNames(vl1names); + + std::cout << "HLT names :- \n "; + edm::copy_all(vhltnames, std::ostream_iterator(std::cout, "\n")); + + if (view->protocolVersion() >= 5) { + std::cout << "HLT selections :- \n "; + edm::copy_all(vhltselections, std::ostream_iterator(std::cout, "\n")); + } + + std::cout << "L1 names :- \n "; + edm::copy_all(vl1names, std::ostream_iterator(std::cout, "\n")); + std::cout << "\n"; + std::cout.flush(); } - if (view->protocolVersion() >= 6) { - std::cout << "outputModuleId=0x" << std::hex << view->outputModuleId() << std::dec << std::endl; + + void dumpInitView(const InitMsgView* view) { + dumpInitHeader(view); + std::cout << "desc len = " << view->descLength() << "\n"; + //const uint8* pos = view->descData(); + //std::copy(pos,pos+view->descLength(),std::ostream_iterator(std::cout,"")); + //std::cout << "\n"; + std::cout.flush(); } - if (view->protocolVersion() >= 8) { - std::cout << "Checksum for Registry data = " << view->adler32_chksum() << " Hostname = " << view->hostName() - << std::endl; + + void dumpStartMsg(const InitMsgView* view) { + dumpInitHeader(view); + std::cout.flush(); } - //PSet 16 byte non-printable representation, stored in message. - uint8 vpset[16]; - view->pset(vpset); - - //Lets convert it to printable hex form - std::string pset_str(vpset, vpset + sizeof(vpset)); - pset_str += '\0'; - cms::Digest dig(pset_str); - cms::MD5Result r1 = dig.digest(); - std::string hexy = r1.toString(); - std::cout << "PSetID= " << hexy << std::endl; - - Strings vhltnames, vhltselections, vl1names; - view->hltTriggerNames(vhltnames); - if (view->protocolVersion() >= 5) { - view->hltTriggerSelections(vhltselections); + void dumpInitVerbose(const InitMsgView* view) { + std::cout << ">>>>> INIT Message Dump (begin) >>>>>" << std::endl; + dumpInitHeader(view); + + TClass* desc = getTClass(typeid(SendJobHeader)); + TBufferFile xbuf(TBuffer::kRead, view->descLength(), (char*)view->descData(), kFALSE); + std::unique_ptr sd((SendJobHeader*)xbuf.ReadObjectAny(desc)); + + if (sd.get() == nullptr) { + std::cout << "Unable to determine the product registry - " + << "Registry deserialization error." << std::endl; + } else { + std::cout << "Branch Descriptions:" << std::endl; + SendDescs const& descs = sd->descs(); + SendDescs::const_iterator iDesc(descs.begin()), eDesc(descs.end()); + while (iDesc != eDesc) { + BranchDescription branchDesc = *iDesc; + branchDesc.init(); + //branchDesc.write(std::cout); + std::cout << branchDesc.branchName() << std::endl; + iDesc++; + } + } + + std::cout << "<<<<< INIT Message Dump (end) <<<<<" << std::endl; + std::cout.flush(); } - view->l1TriggerNames(vl1names); - std::cout << "HLT names :- \n "; - edm::copy_all(vhltnames, std::ostream_iterator(std::cout, "\n")); + void dumpInit(uint8* buf) { + InitMsgView view(buf); + dumpInitHeader(&view); - if (view->protocolVersion() >= 5) { - std::cout << "HLT selections :- \n "; - edm::copy_all(vhltselections, std::ostream_iterator(std::cout, "\n")); + std::cout << "desc len = " << view.descLength() << "\n"; + //const uint8* pos = view.descData(); + //std::copy(pos,pos+view.descLength(),std::ostream_iterator(std::cout,"")); + //std::cout << "\n"; + std::cout.flush(); } - std::cout << "L1 names :- \n "; - edm::copy_all(vl1names, std::ostream_iterator(std::cout, "\n")); - std::cout << "\n"; - std::cout.flush(); -} - -void dumpInitView(const InitMsgView* view) { - dumpInitHeader(view); - std::cout << "desc len = " << view->descLength() << "\n"; - //const uint8* pos = view->descData(); - //std::copy(pos,pos+view->descLength(),std::ostream_iterator(std::cout,"")); - //std::cout << "\n"; - std::cout.flush(); -} - -void dumpStartMsg(const InitMsgView* view) { - dumpInitHeader(view); - std::cout.flush(); -} - -void dumpInitVerbose(const InitMsgView* view) { - std::cout << ">>>>> INIT Message Dump (begin) >>>>>" << std::endl; - dumpInitHeader(view); - - TClass* desc = getTClass(typeid(SendJobHeader)); - TBufferFile xbuf(TBuffer::kRead, view->descLength(), (char*)view->descData(), kFALSE); - std::unique_ptr sd((SendJobHeader*)xbuf.ReadObjectAny(desc)); - - if (sd.get() == nullptr) { - std::cout << "Unable to determine the product registry - " - << "Registry deserialization error." << std::endl; - } else { - std::cout << "Branch Descriptions:" << std::endl; - SendDescs const& descs = sd->descs(); - SendDescs::const_iterator iDesc(descs.begin()), eDesc(descs.end()); - while (iDesc != eDesc) { - BranchDescription branchDesc = *iDesc; - branchDesc.init(); - //branchDesc.write(std::cout); - std::cout << branchDesc.branchName() << std::endl; - iDesc++; + void printBits(unsigned char c) { + for (int i = 7; i >= 0; --i) { + int bit = ((c >> i) & 1); + std::cout << " " << bit; } } - std::cout << "<<<<< INIT Message Dump (end) <<<<<" << std::endl; - std::cout.flush(); -} - -void dumpInit(uint8* buf) { - InitMsgView view(buf); - dumpInitHeader(&view); - - std::cout << "desc len = " << view.descLength() << "\n"; - //const uint8* pos = view.descData(); - //std::copy(pos,pos+view.descLength(),std::ostream_iterator(std::cout,"")); - //std::cout << "\n"; - std::cout.flush(); -} - -void printBits(unsigned char c) { - for (int i = 7; i >= 0; --i) { - int bit = ((c >> i) & 1); - std::cout << " " << bit; + void dumpEventHeader(const EventMsgView* eview) { + std::cout << "code=" << eview->code() << "\n" + << "size=" << eview->size() << "\n" + << "protocolVersion=" << eview->protocolVersion() << "\n" + << "run=" << eview->run() << "\n" + << "event=" << eview->event() << "\n" + << "lumi=" << eview->lumi() << "\n" + << "origDataSize=" << eview->origDataSize() << "\n" + << "outModId=0x" << std::hex << eview->outModId() << std::dec << "\n" + << "adler32 chksum= " << eview->adler32_chksum() << "\n" + << "host name= " << eview->hostName() << "\n" + << "event length=" << eview->eventLength() << "\n" + << "droppedEventsCount=" << eview->droppedEventsCount() << "\n"; + + std::vector l1_out; + eview->l1TriggerBits(l1_out); + + std::cout << "\nl1 size= " << l1_out.size() << "\n l1 bits=\n"; + edm::copy_all(l1_out, std::ostream_iterator(std::cout, " ")); + + std::vector hlt_out; + if (eview->hltCount() > 0) { + hlt_out.resize(1 + (eview->hltCount() - 1) / 4); + } + eview->hltTriggerBits(&hlt_out[0]); + + std::cout << "\nhlt Count:" << eview->hltCount(); + std::cout << "\nhlt bits=\n("; + for (int i = (hlt_out.size() - 1); i != -1; --i) + printBits(hlt_out[i]); + std::cout << ")\n"; + std::cout.flush(); + } + + void dumpEventView(const EventMsgView* eview) { + dumpEventHeader(eview); + //const uint8* edata = eview->eventData(); + //std::cout << "\nevent data=\n("; + //std::copy(&edata[0],&edata[0]+eview->eventLength(), + // std::ostream_iterator(std::cout,"")); + //std::cout << ")\n"; + std::cout.flush(); + } + + void dumpEventIndex(const EventMsgView* eview) { + dumpEventHeader(eview); + std::cout.flush(); + } + + void dumpEvent(uint8* buf) { + EventMsgView eview(buf); + + dumpEventHeader(&eview); + + //const uint8* edata = eview.eventData(); + //std::cout << "\nevent data=\n("; + //std::copy(&edata[0],&edata[0]+eview.eventLength(), + // std::ostream_iterator(std::cout,"")); + //std::cout << ")\n"; + std::cout.flush(); } -} - -void dumpEventHeader(const EventMsgView* eview) { - std::cout << "code=" << eview->code() << "\n" - << "size=" << eview->size() << "\n" - << "protocolVersion=" << eview->protocolVersion() << "\n" - << "run=" << eview->run() << "\n" - << "event=" << eview->event() << "\n" - << "lumi=" << eview->lumi() << "\n" - << "origDataSize=" << eview->origDataSize() << "\n" - << "outModId=0x" << std::hex << eview->outModId() << std::dec << "\n" - << "adler32 chksum= " << eview->adler32_chksum() << "\n" - << "host name= " << eview->hostName() << "\n" - << "event length=" << eview->eventLength() << "\n" - << "droppedEventsCount=" << eview->droppedEventsCount() << "\n"; - - std::vector l1_out; - eview->l1TriggerBits(l1_out); - - std::cout << "\nl1 size= " << l1_out.size() << "\n l1 bits=\n"; - edm::copy_all(l1_out, std::ostream_iterator(std::cout, " ")); - - std::vector hlt_out; - if (eview->hltCount() > 0) { - hlt_out.resize(1 + (eview->hltCount() - 1) / 4); + + void dumpFRDEventView(const FRDEventMsgView* fview) { + std::cout << "\n>>>>> FRDEvent Message Dump (begin) >>>>>" << std::endl; + std::cout.flush(); + + std::cout << "size = " << fview->size() << "\n" + << "version = " << fview->version() << "\n" + << "run = " << fview->run() << "\n" + << "lumi = " << fview->lumi() << "\n" + << "event = " << fview->event() << "\n"; + std::cout.flush(); + + std::cout << ">>>>> FRDEvent Message Dump (end) >>>>>" << std::endl; + std::cout.flush(); } - eview->hltTriggerBits(&hlt_out[0]); - - std::cout << "\nhlt Count:" << eview->hltCount(); - std::cout << "\nhlt bits=\n("; - for (int i = (hlt_out.size() - 1); i != -1; --i) - printBits(hlt_out[i]); - std::cout << ")\n"; - std::cout.flush(); -} - -void dumpEventView(const EventMsgView* eview) { - dumpEventHeader(eview); - //const uint8* edata = eview->eventData(); - //std::cout << "\nevent data=\n("; - //std::copy(&edata[0],&edata[0]+eview->eventLength(), - // std::ostream_iterator(std::cout,"")); - //std::cout << ")\n"; - std::cout.flush(); -} - -void dumpEventIndex(const EventMsgView* eview) { - dumpEventHeader(eview); - std::cout.flush(); -} - -void dumpEvent(uint8* buf) { - EventMsgView eview(buf); - - dumpEventHeader(&eview); - - //const uint8* edata = eview.eventData(); - //std::cout << "\nevent data=\n("; - //std::copy(&edata[0],&edata[0]+eview.eventLength(), - // std::ostream_iterator(std::cout,"")); - //std::cout << ")\n"; - std::cout.flush(); -} - -void dumpFRDEventView(const FRDEventMsgView* fview) { - std::cout << "\n>>>>> FRDEvent Message Dump (begin) >>>>>" << std::endl; - std::cout.flush(); - - std::cout << "size = " << fview->size() << "\n" - << "version = " << fview->version() << "\n" - << "run = " << fview->run() << "\n" - << "lumi = " << fview->lumi() << "\n" - << "event = " << fview->event() << "\n"; - std::cout.flush(); - - std::cout << ">>>>> FRDEvent Message Dump (end) >>>>>" << std::endl; - std::cout.flush(); -} +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/EventMessage.cc b/IOPool/Streamer/src/EventMessage.cc index 87e3530b5577c..570d1e8eb704a 100644 --- a/IOPool/Streamer/src/EventMessage.cc +++ b/IOPool/Streamer/src/EventMessage.cc @@ -1,6 +1,8 @@ #include "IOPool/Streamer/interface/EventMessage.h" #include "FWCore/Utilities/interface/Exception.h" +using namespace edm::streamer; + EventMsgView::EventMsgView(void* buf) : buf_((uint8*)buf), head_(buf), v2Detected_(false) { // 29-Jan-2008, KAB - adding an explicit version number. // We'll start with 5 to match the new version of the INIT message. @@ -64,41 +66,43 @@ EventMsgView::EventMsgView(void* buf) : buf_((uint8*)buf), head_(buf), v2Detecte } uint32 EventMsgView::protocolVersion() const { - EventHeader* h = (EventHeader*)buf_; + EventHeader* h = reinterpret_cast(buf_); return h->protocolVersion_; } uint32 EventMsgView::run() const { - EventHeader* h = (EventHeader*)buf_; + EventHeader* h = reinterpret_cast(buf_); return convert32(h->run_); } uint64 EventMsgView::event() const { - EventHeader* h = (EventHeader*)buf_; + EventHeader* h = reinterpret_cast(buf_); return convert64(h->event_); } uint32 EventMsgView::lumi() const { - EventHeader* h = (EventHeader*)buf_; + EventHeader* h = reinterpret_cast(buf_); return convert32(h->lumi_); } uint32 EventMsgView::origDataSize() const { - EventHeader* h = (EventHeader*)buf_; + EventHeader* h = reinterpret_cast(buf_); return convert32(h->origDataSize_); } uint32 EventMsgView::outModId() const { - EventHeader* h = (EventHeader*)buf_; + EventHeader* h = reinterpret_cast(buf_); return convert32(h->outModId_); } uint32 EventMsgView::droppedEventsCount() const { - EventHeader* h = (EventHeader*)buf_; + EventHeader* h = reinterpret_cast(buf_); return convert32(h->droppedEventsCount_); return 0; } +bool EventMsgView::isEventMetaData() const { return run() == 0; } + void EventMsgView::l1TriggerBits(std::vector& put_here) const { put_here.clear(); put_here.resize(l1_bits_count_); diff --git a/IOPool/Streamer/src/EventMsgBuilder.cc b/IOPool/Streamer/src/EventMsgBuilder.cc index 16a64e89d2cdf..31fcf478276c1 100644 --- a/IOPool/Streamer/src/EventMsgBuilder.cc +++ b/IOPool/Streamer/src/EventMsgBuilder.cc @@ -5,7 +5,9 @@ #include #include -#define MAX_HOSTNAME_LEN 25 +static constexpr int MAX_HOSTNAME_LEN = 25; + +using namespace edm::streamer; EventMsgBuilder::EventMsgBuilder(void* buf, uint32 size, diff --git a/IOPool/Streamer/src/FRDEventMessage.cc b/IOPool/Streamer/src/FRDEventMessage.cc index 0811b3e59fbbf..a6c586d0eda03 100644 --- a/IOPool/Streamer/src/FRDEventMessage.cc +++ b/IOPool/Streamer/src/FRDEventMessage.cc @@ -12,6 +12,8 @@ #include "IOPool/Streamer/interface/FRDEventMessage.h" #include "FWCore/Utilities/interface/Exception.h" +using namespace edm::streamer; + /** * Constructor for the FRD event message viewer. */ diff --git a/IOPool/Streamer/src/InitMessage.cc b/IOPool/Streamer/src/InitMessage.cc index 338019471d291..d731fc224fe5a 100644 --- a/IOPool/Streamer/src/InitMessage.cc +++ b/IOPool/Streamer/src/InitMessage.cc @@ -4,6 +4,8 @@ #include #include +using namespace edm::streamer; + InitMsgView::InitMsgView(void* buf) : buf_((uint8*)buf), head_(buf), diff --git a/IOPool/Streamer/src/InitMsgBuilder.cc b/IOPool/Streamer/src/InitMsgBuilder.cc index f41bc86205c73..904fc1b0ab9e9 100644 --- a/IOPool/Streamer/src/InitMsgBuilder.cc +++ b/IOPool/Streamer/src/InitMsgBuilder.cc @@ -6,6 +6,8 @@ #include #include +using namespace edm::streamer; + InitMsgBuilder::InitMsgBuilder(void* buf, uint32 size, uint32 run, diff --git a/IOPool/Streamer/src/StreamSerializer.cc b/IOPool/Streamer/src/StreamSerializer.cc index 69048593cfa2e..76b4cd2aaa411 100644 --- a/IOPool/Streamer/src/StreamSerializer.cc +++ b/IOPool/Streamer/src/StreamSerializer.cc @@ -30,7 +30,7 @@ #include #include -namespace edm { +namespace edm::streamer { /** * Creates a translator instance for the specified product registry. @@ -42,18 +42,14 @@ namespace edm { * Serializes the product registry (that was specified to the constructor) * into the specified InitMessage. */ - int StreamSerializer::serializeRegistry(SerializeDataBuffer &data_buffer, - const BranchIDLists &branchIDLists, - ThinnedAssociationsHelper const &thinnedAssociationsHelper) { + int StreamSerializer::serializeRegistry(SerializeDataBuffer &data_buffer) const { SendJobHeader::ParameterSetMap psetMap; pset::Registry::instance()->fillMap(psetMap); - return serializeRegistry(data_buffer, branchIDLists, thinnedAssociationsHelper, psetMap); + return serializeRegistry(data_buffer, psetMap); } int StreamSerializer::serializeRegistry(SerializeDataBuffer &data_buffer, - const BranchIDLists &branchIDLists, - ThinnedAssociationsHelper const &thinnedAssociationsHelper, - SendJobHeader::ParameterSetMap const &psetMap) { + SendJobHeader::ParameterSetMap const &psetMap) const { FDEBUG(6) << "StreamSerializer::serializeRegistry" << std::endl; SendJobHeader sd; @@ -64,8 +60,6 @@ namespace edm { FDEBUG(9) << "StreamOutput got product = " << selection.first->className() << std::endl; } Service reg; - sd.setBranchIDLists(branchIDLists); - sd.setThinnedAssociationsHelper(thinnedAssociationsHelper); sd.setParameterSetMap(psetMap); data_buffer.rootbuf_.Reset(); @@ -132,12 +126,19 @@ namespace edm { int StreamSerializer::serializeEvent(SerializeDataBuffer &data_buffer, EventForOutput const &event, ParameterSetID const &selectorConfig, + uint32_t metaDataChecksum, StreamerCompressionAlgo compressionAlgo, int compression_level, unsigned int reserveSize) const { EventSelectionIDVector selectionIDs = event.eventSelectionIDs(); selectionIDs.push_back(selectorConfig); - SendEvent se(event.eventAuxiliary(), event.processHistory(), selectionIDs, event.branchListIndexes()); + SendEvent se(event.eventAuxiliary(), + event.processHistory(), + selectionIDs, + event.branchListIndexes(), + {}, + {}, + metaDataChecksum); // Loop over EDProducts, fill the provenance, and write. @@ -172,7 +173,25 @@ namespace edm { } } } + return serializeEventCommon(data_buffer, se, compressionAlgo, compression_level, reserveSize); + } + + int StreamSerializer::serializeEventMetaData(SerializeDataBuffer &data_buffer, + const BranchIDLists &branchIDLists, + ThinnedAssociationsHelper const &thinnedAssociationsHelper, + StreamerCompressionAlgo compressionAlgo, + int compression_level, + unsigned int reserveSize) const { + SendEvent se({}, {}, {}, {}, branchIDLists, thinnedAssociationsHelper, 0); + + return serializeEventCommon(data_buffer, se, compressionAlgo, compression_level, reserveSize); + } + int StreamSerializer::serializeEventCommon(SerializeDataBuffer &data_buffer, + edm::SendEvent const &se, + StreamerCompressionAlgo compressionAlgo, + int compression_level, + unsigned int reserveSize) const { data_buffer.rootbuf_.Reset(); RootDebug tracer(10, 10); @@ -182,7 +201,7 @@ namespace edm { case 0: // failure { throw cms::Exception("StreamTranslation", "Event serialization failed") - << "StreamSerializer failed to serialize event: " << event.id(); + << "StreamSerializer failed to serialize event: " << se.aux().id(); break; } case 1: // succcess @@ -191,14 +210,14 @@ namespace edm { { throw cms::Exception("StreamTranslation", "Event serialization truncated") << "StreamSerializer module attempted to serialize an event\n" - << "that is to big for the allocated buffers: " << event.id(); + << "that is to big for the allocated buffers: " << se.aux().id(); break; } default: // unknown { throw cms::Exception("StreamTranslation", "Event serialization failed") << "StreamSerializer module got an unknown error code\n" - << " while attempting to serialize event: " << event.id(); + << " while attempting to serialize event: " << se.aux().id(); break; } } @@ -426,4 +445,4 @@ namespace edm { return resultSize; } -} // namespace edm +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/StreamerFileReader.cc b/IOPool/Streamer/src/StreamerFileReader.cc index 85c10a63c4279..e9b8a47d40b37 100644 --- a/IOPool/Streamer/src/StreamerFileReader.cc +++ b/IOPool/Streamer/src/StreamerFileReader.cc @@ -10,7 +10,8 @@ #include "FWCore/ParameterSet/interface/ParameterSetDescription.h" #include "FWCore/Sources/interface/EventSkipperByID.h" -namespace edm { +#include +namespace edm::streamer { StreamerFileReader::StreamerFileReader(ParameterSet const& pset, InputSourceDescription const& desc) : StreamerInputSource(pset, desc), @@ -39,13 +40,27 @@ namespace edm { << "No fileNames were specified\n"; } isFirstFile_ = true; - InitMsgView const* header = getHeader(); - deserializeAndMergeWithRegistry(*header, false); + updateMetaData(false); if (initialNumberOfEventsToSkip_) { skip(initialNumberOfEventsToSkip_); } } + void StreamerFileReader::updateMetaData(bool subsequent) { + InitMsgView const* header = getHeader(); + deserializeAndMergeWithRegistry(*header, subsequent); + //NOTE: should read first Event to get the meta data and then set 'artificial file' + auto eview = getNextEvent(); + + //if no events then file must be empty + if (eview == nullptr) + return; + + assert(eview->isEventMetaData()); + deserializeEventMetaData(*eview); + updateEventMetaData(); + } + StreamerFileReader::Next StreamerFileReader::checkNext() { EventMsgView const* eview = getNextEvent(); @@ -55,6 +70,23 @@ namespace edm { } return Next::kStop; } + if (eview->isEventMetaData()) { + if (presentEventMetaDataChecksum() != eventMetaDataChecksum(*eview)) { + //we lie and say there is a new file since we need to synchronize to update the meta data + didArtificialFile_ = true; + deserializeEventMetaData(*eview); + return Next::kFile; + } else { + //skip this meta data + eview = getNextEvent(); + if (eview == nullptr) { + if (newHeader()) { + return Next::kFile; + } + return Next::kStop; + } + } + } deserializeEvent(*eview); return Next::kEvent; } @@ -73,6 +105,9 @@ namespace edm { } void StreamerFileReader::genuineCloseFile() { + if (didArtificialFile_) { + return; + } if (streamReader_.get() != nullptr) streamReader_->closeStreamerFile(); } @@ -83,12 +118,17 @@ namespace edm { isFirstFile_ = false; return; } + if (didArtificialFile_) { + //update the event meta data + didArtificialFile_ = false; + updateEventMetaData(); + return; + } streamReader_->openNextFile(); // FDEBUG(6) << "A new file has been opened and we must compare Headers here !!" << std::endl; // A new file has been opened and we must compare Heraders here !! //Get header/init from reader - InitMsgView const* header = getHeader(); - deserializeAndMergeWithRegistry(*header, true); + updateMetaData(true); } bool StreamerFileReader::newHeader() { return streamReader_->newHeader(); } @@ -124,4 +164,4 @@ namespace edm { EventSkipperByID::fillDescription(desc); descriptions.add("source", desc); } -} // namespace edm +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/StreamerFileReader.h b/IOPool/Streamer/src/StreamerFileReader.h index 446bdd3cab3f4..0d734e458082e 100644 --- a/IOPool/Streamer/src/StreamerFileReader.h +++ b/IOPool/Streamer/src/StreamerFileReader.h @@ -18,35 +18,41 @@ namespace edm { class FileCatalogItem; struct InputSourceDescription; class ParameterSet; - class StreamerInputFile; - class StreamerFileReader : public StreamerInputSource { - public: - StreamerFileReader(ParameterSet const& pset, InputSourceDescription const& desc); - ~StreamerFileReader() override; - - static void fillDescriptions(ConfigurationDescriptions& descriptions); - - private: - InitMsgView const* getHeader(); - EventMsgView const* getNextEvent(); - bool newHeader(); - - Next checkNext() override; - void skip(int toSkip) override; - void genuineReadFile() override; - void genuineCloseFile() override; - void reset_() override; - - std::shared_ptr eventSkipperByID() const { return get_underlying_safe(eventSkipperByID_); } - std::shared_ptr& eventSkipperByID() { return get_underlying_safe(eventSkipperByID_); } - - std::vector streamerNames_; // names of Streamer files - edm::propagate_const> streamReader_; - edm::propagate_const> eventSkipperByID_; - int initialNumberOfEventsToSkip_; - int prefetchMBytes_; - bool isFirstFile_ = true; - }; + namespace streamer { + class StreamerInputFile; + class StreamerFileReader : public StreamerInputSource { + public: + StreamerFileReader(ParameterSet const& pset, InputSourceDescription const& desc); + ~StreamerFileReader() override; + + static void fillDescriptions(ConfigurationDescriptions& descriptions); + + private: + InitMsgView const* getHeader(); + EventMsgView const* getNextEvent(); + bool newHeader(); + void updateMetaData(bool subsequent); + + Next checkNext() override; + void skip(int toSkip) override; + void genuineReadFile() override; + void genuineCloseFile() override; + void reset_() override; + + std::shared_ptr eventSkipperByID() const { + return get_underlying_safe(eventSkipperByID_); + } + std::shared_ptr& eventSkipperByID() { return get_underlying_safe(eventSkipperByID_); } + + std::vector streamerNames_; // names of Streamer files + edm::propagate_const> streamReader_; + edm::propagate_const> eventSkipperByID_; + int initialNumberOfEventsToSkip_; + int prefetchMBytes_; + bool isFirstFile_ = true; + bool didArtificialFile_ = false; + }; + } // namespace streamer } // namespace edm #endif diff --git a/IOPool/Streamer/src/StreamerFileWriter.cc b/IOPool/Streamer/src/StreamerFileWriter.cc index 52234ff7d769b..eeac63e10af54 100644 --- a/IOPool/Streamer/src/StreamerFileWriter.cc +++ b/IOPool/Streamer/src/StreamerFileWriter.cc @@ -1,7 +1,8 @@ #include "IOPool/Streamer/src/StreamerFileWriter.h" #include "FWCore/ParameterSet/interface/ParameterSetDescription.h" -namespace edm { +namespace edm::streamer { + StreamerFileWriter::StreamerFileWriter(edm::ParameterSet const& ps) : stream_writer_(new StreamerOutputFile(ps.getUntrackedParameter("fileName"), ps.getUntrackedParameter("padding"))) {} @@ -38,4 +39,4 @@ namespace edm { desc.addUntracked("padding", 0) ->setComment("For testing: INIT and event block size will be rounded to this size padded with 0xff bytes."); } -} //namespace edm +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/StreamerFileWriter.h b/IOPool/Streamer/src/StreamerFileWriter.h index a0f75964cc598..e8213fe0fab12 100644 --- a/IOPool/Streamer/src/StreamerFileWriter.h +++ b/IOPool/Streamer/src/StreamerFileWriter.h @@ -18,27 +18,29 @@ namespace edm { class ParameterSetDescription; - class StreamerFileWriter { - public: - explicit StreamerFileWriter(edm::ParameterSet const& ps); - explicit StreamerFileWriter(std::string const& fileName); - ~StreamerFileWriter(); + namespace streamer { + class StreamerFileWriter { + public: + explicit StreamerFileWriter(edm::ParameterSet const& ps); + explicit StreamerFileWriter(std::string const& fileName); + ~StreamerFileWriter(); - static void fillDescription(ParameterSetDescription& desc); + static void fillDescription(ParameterSetDescription& desc); - void doOutputHeader(InitMsgBuilder const& init_message); - void doOutputHeader(InitMsgView const& init_message); + void doOutputHeader(InitMsgBuilder const& init_message); + void doOutputHeader(InitMsgView const& init_message); - void doOutputEvent(EventMsgBuilder const& msg); - void doOutputEvent(EventMsgView const& msg); + void doOutputEvent(EventMsgBuilder const& msg); + void doOutputEvent(EventMsgView const& msg); - void start() {} - void stop(){}; + void start() {} + void stop(){}; - uint32 get_adler32() const { return stream_writer_->adler32(); } + uint32 get_adler32() const { return stream_writer_->adler32(); } - private: - edm::propagate_const> stream_writer_; - }; + private: + edm::propagate_const> stream_writer_; + }; + } // namespace streamer } // namespace edm #endif diff --git a/IOPool/Streamer/src/StreamerInputFile.cc b/IOPool/Streamer/src/StreamerInputFile.cc index 09da2ea8d1ee0..ec88f1f969e92 100644 --- a/IOPool/Streamer/src/StreamerInputFile.cc +++ b/IOPool/Streamer/src/StreamerInputFile.cc @@ -14,7 +14,7 @@ #include #include -namespace edm { +namespace edm::streamer { StreamerInputFile::~StreamerInputFile() { closeStreamerFile(); } @@ -380,4 +380,4 @@ namespace edm { LogAbsolute("fileAction") << std::setprecision(0) << TimeOfDay() << msg << currentFileName_; FlushMessageLog(); } -} // namespace edm +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/StreamerInputSource.cc b/IOPool/Streamer/src/StreamerInputSource.cc index 9f9021c4a4962..160d3698ec993 100644 --- a/IOPool/Streamer/src/StreamerInputSource.cc +++ b/IOPool/Streamer/src/StreamerInputSource.cc @@ -37,7 +37,7 @@ #include #include -namespace edm { +namespace edm::streamer { namespace { int const init_size = 1024 * 1024; } @@ -56,11 +56,7 @@ namespace edm { StreamerInputSource::~StreamerInputSource() {} // --------------------------------------- - void StreamerInputSource::mergeIntoRegistry(SendJobHeader const& header, - ProductRegistry& reg, - BranchIDListHelper& branchIDListHelper, - ThinnedAssociationsHelper& thinnedHelper, - bool subsequent) { + void StreamerInputSource::mergeIntoRegistry(SendJobHeader const& header, ProductRegistry& reg, bool subsequent) { SendDescs const& descs = header.descs(); FDEBUG(6) << "mergeIntoRegistry: Product List: " << std::endl; @@ -72,8 +68,6 @@ namespace edm { if (!mergeInfo.empty()) { throw cms::Exception("MismatchedInput", "RootInputFileSequence::previousEvent()") << mergeInfo; } - branchIDListHelper.updateFromInput(header.branchIDLists()); - thinnedHelper.updateFromPrimaryInput(header.thinnedAssociationsHelper()); } else { declareStreamers(descs); buildClassCache(descs); @@ -81,8 +75,6 @@ namespace edm { if (!reg.frozen()) { reg.updateFromInput(descs); } - branchIDListHelper.updateFromInput(header.branchIDLists()); - thinnedHelper.updateFromPrimaryInput(header.thinnedAssociationsHelper()); } } @@ -166,7 +158,7 @@ namespace edm { */ void StreamerInputSource::deserializeAndMergeWithRegistry(InitMsgView const& initView, bool subsequent) { std::unique_ptr sd = deserializeRegistry(initView); - mergeIntoRegistry(*sd, productRegistryUpdate(), *branchIDListHelper(), *thinnedAssociationsHelper(), subsequent); + mergeIntoRegistry(*sd, productRegistryUpdate(), subsequent); if (subsequent) { adjustEventToNewProductRegistry_ = true; } @@ -179,10 +171,26 @@ namespace edm { } } + void StreamerInputSource::updateEventMetaData() { + branchIDListHelper()->updateFromInput(sendEvent_->branchIDLists()); + thinnedAssociationsHelper()->updateFromPrimaryInput(sendEvent_->thinnedAssociationsHelper()); + } + + uint32_t StreamerInputSource::eventMetaDataChecksum(EventMsgView const& eventView) const { + return eventView.adler32_chksum(); + } + + void StreamerInputSource::deserializeEventMetaData(EventMsgView const& eventView) { + deserializeEventCommon(eventView, true); + } /** * Deserializes the specified event message. */ void StreamerInputSource::deserializeEvent(EventMsgView const& eventView) { + deserializeEventCommon(eventView, false); + } + + void StreamerInputSource::deserializeEventCommon(EventMsgView const& eventView, bool isMetaData) { if (eventView.code() != Header::EVENT) throw cms::Exception("StreamTranslation", "Event deserialization error") << "received wrong message type: expected EVENT, got " << eventView.code() << "\n"; @@ -199,7 +207,7 @@ namespace edm { //std::cout << "Adler32 checksum of event = " << adler32_chksum << std::endl; //std::cout << "Adler32 checksum from header = " << eventView.adler32_chksum() << " " // << "host name = " << eventView.hostName() << " len = " << eventView.hostName_len() << std::endl; - if ((uint32)adler32_chksum != eventView.adler32_chksum()) { + if (static_cast(adler32_chksum) != eventView.adler32_chksum()) { // skip event (based on option?) or throw exception? throw cms::Exception("StreamDeserialization", "Checksum error") << " chksum from event = " << adler32_chksum << " from header = " << eventView.adler32_chksum() @@ -250,13 +258,24 @@ namespace edm { setRefCoreStreamer(); ; }); - sendEvent_ = std::unique_ptr((SendEvent*)xbuf_.ReadObjectAny(tc_)); + sendEvent_ = std::unique_ptr(reinterpret_cast(xbuf_.ReadObjectAny(tc_))); } if (sendEvent_.get() == nullptr) { throw cms::Exception("StreamTranslation", "Event deserialization error") << "got a null event from input stream\n"; } + + if (isMetaData) { + eventMetaDataChecksum_ = adler32_chksum; + return; + } + + if (sendEvent_->metaDataChecksum() != eventMetaDataChecksum_) { + throw cms::Exception("StreamTranslation") << " meta data checksum from event " << sendEvent_->metaDataChecksum() + << " does not match last read meta data " << eventMetaDataChecksum_; + } + processHistoryRegistryForUpdate().registerProcessHistory(sendEvent_->processHistory()); FDEBUG(5) << "Got event: " << sendEvent_->aux().id() << " " << sendEvent_->products().size() << std::endl; @@ -511,4 +530,4 @@ namespace edm { void StreamerInputSource::EventPrincipalHolder::setEventPrincipal(EventPrincipal* ep) { eventPrincipal_ = ep; } void StreamerInputSource::fillDescription(ParameterSetDescription& desc) { RawInputSource::fillDescription(desc); } -} // namespace edm +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/StreamerOutputFile.cc b/IOPool/Streamer/src/StreamerOutputFile.cc index 0ee1470cf26ab..65b67c74a8bdd 100644 --- a/IOPool/Streamer/src/StreamerOutputFile.cc +++ b/IOPool/Streamer/src/StreamerOutputFile.cc @@ -1,83 +1,88 @@ #include "IOPool/Streamer/interface/StreamerOutputFile.h" #include "FWCore/Utilities/interface/Exception.h" -StreamerOutputFile::~StreamerOutputFile() {} +namespace edm::streamer { + StreamerOutputFile::~StreamerOutputFile() {} -StreamerOutputFile::StreamerOutputFile(const std::string& name, uint32 padding) - : streamerfile_(std::make_shared(name, padding)) { - streamerfile_->set_do_adler(true); -} + StreamerOutputFile::StreamerOutputFile(const std::string& name, uint32 padding) + : streamerfile_(std::make_shared(name, padding)) { + streamerfile_->set_do_adler(true); + } -uint64 StreamerOutputFile::write(const EventMsgBuilder& ineview) { - EventMsgView tmpView(ineview.startAddress()); - return write(tmpView); -} + uint64 StreamerOutputFile::write(const EventMsgBuilder& ineview) { + EventMsgView tmpView(ineview.startAddress()); + return write(tmpView); + } -uint64 StreamerOutputFile::write(const EventMsgView& ineview) { - /** Offset where current event starts */ - uint64 offset_to_return = streamerfile_->current_offset(); + uint64 StreamerOutputFile::write(const EventMsgView& ineview) { + /** Offset where current event starts */ + uint64 offset_to_return = streamerfile_->current_offset(); - writeEventHeader(ineview); - bool ret = streamerfile_->write((const char*)ineview.eventData(), ineview.size() - ineview.headerSize(), true); - if (ret) { - throw cms::Exception("OutputFile", "write(EventMsgView)") - << "Error writing streamer event data to " << streamerfile_->fileName() << ". Possibly the output disk " - << "is full?" << std::endl; + writeEventHeader(ineview); + bool ret = streamerfile_->write((const char*)ineview.eventData(), ineview.size() - ineview.headerSize(), true); + if (ret) { + throw cms::Exception("OutputFile", "write(EventMsgView)") + << "Error writing streamer event data to " << streamerfile_->fileName() << ". Possibly the output disk " + << "is full?" << std::endl; + } + return offset_to_return; } - return offset_to_return; -} -uint64 StreamerOutputFile::writeEventFragment(uint32 fragIndex, uint32 fragCount, const char* dataPtr, uint32 dataSize) { - /** Offset where current event starts */ - uint64 offset_to_return = streamerfile_->current_offset(); + uint64 StreamerOutputFile::writeEventFragment(uint32 fragIndex, + uint32 fragCount, + const char* dataPtr, + uint32 dataSize) { + /** Offset where current event starts */ + uint64 offset_to_return = streamerfile_->current_offset(); - bool ret = streamerfile_->write(dataPtr, dataSize); - if (ret) { - throw cms::Exception("OutputFile", "writeEventFragment()") - << "Error writing streamer event data to " << streamerfile_->fileName() << ". Possibly the output disk " - << "is full?" << std::endl; + bool ret = streamerfile_->write(dataPtr, dataSize); + if (ret) { + throw cms::Exception("OutputFile", "writeEventFragment()") + << "Error writing streamer event data to " << streamerfile_->fileName() << ". Possibly the output disk " + << "is full?" << std::endl; + } + return offset_to_return; } - return offset_to_return; -} -void StreamerOutputFile::writeEventHeader(const EventMsgView& ineview) { - bool ret = streamerfile_->write((const char*)ineview.startAddress(), ineview.headerSize()); - if (ret) { - throw cms::Exception("OutputFile", "writeEventHeader") - << "Error writing streamer event data to " << streamerfile_->fileName() << ". Possibly the output disk " - << "is full?" << std::endl; + void StreamerOutputFile::writeEventHeader(const EventMsgView& ineview) { + bool ret = streamerfile_->write((const char*)ineview.startAddress(), ineview.headerSize()); + if (ret) { + throw cms::Exception("OutputFile", "writeEventHeader") + << "Error writing streamer event data to " << streamerfile_->fileName() << ". Possibly the output disk " + << "is full?" << std::endl; + } } -} -void StreamerOutputFile::write(const InitMsgBuilder& inview) { - InitMsgView tmpView(inview.startAddress()); - return write(tmpView); -} + void StreamerOutputFile::write(const InitMsgBuilder& inview) { + InitMsgView tmpView(inview.startAddress()); + return write(tmpView); + } -void StreamerOutputFile::write(const InitMsgView& inview) { - writeStart(inview); - bool ret = streamerfile_->write((const char*)inview.descData(), inview.size() - inview.headerSize(), true); - if (ret) { - throw cms::Exception("OutputFile", "write(InitMsgView)") - << "Error writing streamer header data to " << streamerfile_->fileName() << ". Possibly the output disk " - << "is full?" << std::endl; + void StreamerOutputFile::write(const InitMsgView& inview) { + writeStart(inview); + bool ret = streamerfile_->write((const char*)inview.descData(), inview.size() - inview.headerSize(), true); + if (ret) { + throw cms::Exception("OutputFile", "write(InitMsgView)") + << "Error writing streamer header data to " << streamerfile_->fileName() << ". Possibly the output disk " + << "is full?" << std::endl; + } } -} -void StreamerOutputFile::writeInitFragment(uint32 fragIndex, uint32 fragCount, const char* dataPtr, uint32 dataSize) { - bool ret = streamerfile_->write((const char*)dataPtr, dataSize); - if (ret) { - throw cms::Exception("OutputFile", "writeInitFragment()") - << "Error writing streamer header data to " << streamerfile_->fileName() << ". Possibly the output disk " - << "is full?" << std::endl; + void StreamerOutputFile::writeInitFragment(uint32 fragIndex, uint32 fragCount, const char* dataPtr, uint32 dataSize) { + bool ret = streamerfile_->write((const char*)dataPtr, dataSize); + if (ret) { + throw cms::Exception("OutputFile", "writeInitFragment()") + << "Error writing streamer header data to " << streamerfile_->fileName() << ". Possibly the output disk " + << "is full?" << std::endl; + } } -} -void StreamerOutputFile::writeStart(const InitMsgView& inview) { - bool ret = streamerfile_->write((const char*)inview.startAddress(), inview.headerSize()); - if (ret) { - throw cms::Exception("OutputFile", "writeStart") - << "Error writing streamer header data to " << streamerfile_->fileName() << ". Possibly the output disk " - << "is full?" << std::endl; + void StreamerOutputFile::writeStart(const InitMsgView& inview) { + bool ret = streamerfile_->write((const char*)inview.startAddress(), inview.headerSize()); + if (ret) { + throw cms::Exception("OutputFile", "writeStart") + << "Error writing streamer header data to " << streamerfile_->fileName() << ". Possibly the output disk " + << "is full?" << std::endl; + } } -} +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/StreamerOutputModuleBase.cc b/IOPool/Streamer/src/StreamerOutputModuleBase.cc index 6ba9fda6e917b..14afc99d7d30c 100644 --- a/IOPool/Streamer/src/StreamerOutputModuleBase.cc +++ b/IOPool/Streamer/src/StreamerOutputModuleBase.cc @@ -12,7 +12,7 @@ #include "zlib.h" -namespace edm { +namespace edm::streamer { StreamerOutputModuleBase::StreamerOutputModuleBase(ParameterSet const& ps) : one::OutputModuleBase::OutputModuleBase(ps), one::OutputModule(ps), @@ -29,16 +29,15 @@ namespace edm { auto psetMapHandle = iRun.getHandle(psetToken_); std::unique_ptr init_message = - serializeRegistry(*getSerializerBuffer(), - *branchIDLists(), - *thinnedAssociationsHelper(), - OutputModule::processName(), + serializeRegistry(OutputModule::processName(), description().moduleLabel(), moduleDescription().mainParameterSetID(), psetMapHandle.isValid() ? psetMapHandle.product() : nullptr); doOutputHeader(*init_message); - serializerBuffer_->clearHeaderBuffer(); + lastCallWasBeginRun_ = true; + + clearHeaderBuffer(); } void StreamerOutputModuleBase::endRun(RunForOutput const&) { stop(); } @@ -54,7 +53,13 @@ namespace edm { void StreamerOutputModuleBase::write(EventForOutput const& e) { Handle const& triggerResults = getTriggerResults(trToken_, e); - std::unique_ptr msg = serializeEvent(*getSerializerBuffer(), e, triggerResults, selectorConfig()); + if (lastCallWasBeginRun_) { + auto msg = serializeEventMetaData(*branchIDLists(), *thinnedAssociationsHelper()); + doOutputEvent(*msg); + lastCallWasBeginRun_ = false; + } + auto msg = serializeEvent(e, triggerResults, selectorConfig()); + doOutputEvent(*msg); // You can't use msg in StreamerOutputModuleBase after this point } @@ -71,4 +76,4 @@ namespace edm { desc.addUntracked("psetMap", {"hltPSetMap"}) ->setComment("Optionally allow the map of ParameterSets to be calculated externally."); } -} // namespace edm +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/StreamerOutputModuleCommon.cc b/IOPool/Streamer/src/StreamerOutputModuleCommon.cc index bc79452fcc75e..377906cbe69a9 100644 --- a/IOPool/Streamer/src/StreamerOutputModuleCommon.cc +++ b/IOPool/Streamer/src/StreamerOutputModuleCommon.cc @@ -22,282 +22,40 @@ #include #include -namespace edm { +namespace edm::streamer { StreamerOutputModuleCommon::Parameters StreamerOutputModuleCommon::parameters(ParameterSet const& ps) { - Parameters ret; - ret.hltTriggerSelections = EventSelector::getEventSelectionVString(ps); - ret.compressionAlgoStr = ps.getUntrackedParameter("compression_algorithm"); - ret.compressionLevel = ps.getUntrackedParameter("compression_level"); - ret.lumiSectionInterval = ps.getUntrackedParameter("lumiSection_interval"); - ret.useCompression = ps.getUntrackedParameter("use_compression"); - return ret; + return StreamerOutputMsgBuilders::parameters(ps); } StreamerOutputModuleCommon::StreamerOutputModuleCommon(Parameters const& p, SelectedProducts const* selections, std::string const& moduleLabel) - : - - serializer_(selections), - useCompression_(p.useCompression), - compressionAlgoStr_(p.compressionAlgoStr), - compressionLevel_(p.compressionLevel), - lumiSectionInterval_(p.lumiSectionInterval), - hltsize_(0), - host_name_(), - hltTriggerSelections_(), - outputModuleId_(0) { - //limits initially set for default ZLIB option - int minCompressionLevel = 1; - int maxCompressionLevel = 9; - - // test luminosity sections - struct timeval now; - struct timezone dummyTZ; - gettimeofday(&now, &dummyTZ); - timeInSecSinceUTC = static_cast(now.tv_sec) + (static_cast(now.tv_usec) / 1000000.0); - - if (useCompression_ == true) { - if (compressionAlgoStr_ == "ZLIB") { - compressionAlgo_ = ZLIB; - } else if (compressionAlgoStr_ == "LZMA") { - compressionAlgo_ = LZMA; - minCompressionLevel = 0; - } else if (compressionAlgoStr_ == "ZSTD") { - compressionAlgo_ = ZSTD; - maxCompressionLevel = 20; - } else if (compressionAlgoStr_ == "UNCOMPRESSED") { - compressionLevel_ = 0; - useCompression_ = false; - compressionAlgo_ = UNCOMPRESSED; - } else - throw cms::Exception("StreamerOutputModuleCommon", "Compression type unknown") - << "Unknown compression algorithm " << compressionAlgoStr_; - - if (compressionLevel_ < minCompressionLevel) { - FDEBUG(9) << "Compression Level = " << compressionLevel_ << " no compression" << std::endl; - compressionLevel_ = 0; - useCompression_ = false; - compressionAlgo_ = UNCOMPRESSED; - } else if (compressionLevel_ > maxCompressionLevel) { - FDEBUG(9) << "Compression Level = " << compressionLevel_ << " using max compression level " - << maxCompressionLevel << std::endl; - compressionLevel_ = maxCompressionLevel; - compressionAlgo_ = UNCOMPRESSED; - } - } else - compressionAlgo_ = UNCOMPRESSED; - - int got_host = gethostname(host_name_, 255); - if (got_host != 0) - strncpy(host_name_, "noHostNameFoundOrTooLong", sizeof(host_name_)); - //loadExtraClasses(); - - // 25-Jan-2008, KAB - pull out the trigger selection request - // which we need for the INIT message - hltTriggerSelections_ = p.hltTriggerSelections; - - Strings const& hltTriggerNames = edm::getAllTriggerNames(); - hltsize_ = hltTriggerNames.size(); - - //Checksum of the module label - uLong crc = crc32(0L, Z_NULL, 0); - Bytef const* buf = (Bytef const*)(moduleLabel.data()); - crc = crc32(crc, buf, moduleLabel.length()); - outputModuleId_ = static_cast(crc); - } + : builders_(p, selections, moduleLabel) {} StreamerOutputModuleCommon::~StreamerOutputModuleCommon() {} std::unique_ptr StreamerOutputModuleCommon::serializeRegistry( - SerializeDataBuffer& sbuf, - const BranchIDLists& branchLists, - ThinnedAssociationsHelper const& helper, std::string const& processName, std::string const& moduleLabel, ParameterSetID const& toplevel, SendJobHeader::ParameterSetMap const* psetMap) { - if (psetMap) { - serializer_.serializeRegistry(sbuf, branchLists, helper, *psetMap); - } else { - serializer_.serializeRegistry(sbuf, branchLists, helper); - } - // resize header_buf_ to reflect space used in serializer_ + header - // I just added an overhead for header of 50000 for now - unsigned int src_size = sbuf.currentSpaceUsed(); - unsigned int new_size = src_size + 50000; - if (sbuf.header_buf_.size() < new_size) - sbuf.header_buf_.resize(new_size); - - //Build the INIT Message - //Following values are strictly DUMMY and will be replaced - // once available with Utility function etc. - uint32 run = 1; - - //Get the Process PSet ID - - //In case we need to print it - // cms::Digest dig(toplevel.compactForm()); - // cms::MD5Result r1 = dig.digest(); - // std::string hexy = r1.toString(); - // std::cout << "HEX Representation of Process PSetID: " << hexy << std::endl; - - //L1 stays dummy as of today - Strings l1_names; //3 - l1_names.push_back("t1"); - l1_names.push_back("t10"); - l1_names.push_back("t2"); - - Strings const& hltTriggerNames = edm::getAllTriggerNames(); - - auto init_message = std::make_unique(&sbuf.header_buf_[0], - sbuf.header_buf_.size(), - run, - Version((uint8 const*)toplevel.compactForm().c_str()), - getReleaseVersion().c_str(), - processName.c_str(), - moduleLabel.c_str(), - outputModuleId_, - hltTriggerNames, - hltTriggerSelections_, - l1_names, - (uint32)sbuf.adler32_chksum()); - - // copy data into the destination message - unsigned char* src = sbuf.bufferPointer(); - std::copy(src, src + src_size, init_message->dataAddress()); - init_message->setDataLength(src_size); - return init_message; - } - - void StreamerOutputModuleCommon::setHltMask(EventForOutput const& e, - Handle const& triggerResults, - std::vector& hltbits) const { - hltbits.clear(); - - std::vector vHltState; - - if (triggerResults.isValid()) { - for (std::vector::size_type i = 0; i != hltsize_; ++i) { - vHltState.push_back(((triggerResults->at(i)).state())); - } - } else { - // We fill all Trigger bits to valid state. - for (std::vector::size_type i = 0; i != hltsize_; ++i) { - vHltState.push_back(hlt::Pass); - } - } - - //Pack into member hltbits - if (!vHltState.empty()) { - unsigned int packInOneByte = 4; - unsigned int sizeOfPackage = 1 + ((vHltState.size() - 1) / packInOneByte); //Two bits per HLT - - hltbits.resize(sizeOfPackage); - std::fill(hltbits.begin(), hltbits.end(), 0); - - for (std::vector::size_type i = 0; i != vHltState.size(); ++i) { - unsigned int whichByte = i / packInOneByte; - unsigned int indxWithinByte = i % packInOneByte; - hltbits[whichByte] = hltbits[whichByte] | (vHltState[i] << (indxWithinByte * 2)); - } - } - - //This is Just a printing code. - //std::cout << "Size of hltbits:" << hltbits_.size() << std::endl; - //for(unsigned int i=0; i != hltbits_.size() ; ++i) { - // printBits(hltbits_[i]); - //} - //std::cout << "\n"; + return builders_.serializeRegistry(buffer_, processName, moduleLabel, toplevel, psetMap); } std::unique_ptr StreamerOutputModuleCommon::serializeEvent( - SerializeDataBuffer& sbuf, - EventForOutput const& e, - Handle const& triggerResults, - ParameterSetID const& selectorCfg) { - constexpr unsigned int reserve_size = SerializeDataBuffer::reserve_size; - //Lets Build the Event Message first - - //Following is strictly DUMMY Data for L! Trig and will be replaced with actual - // once figured out, there is no logic involved here. - std::vector l1bit = {true, true, false}; - //End of dummy data - - std::vector hltbits; - setHltMask(e, triggerResults, hltbits); - - uint32 lumi = e.luminosityBlock(); - if (lumiSectionInterval_ != 0) { - struct timeval now; - struct timezone dummyTZ; - gettimeofday(&now, &dummyTZ); - double timeInSec = - static_cast(now.tv_sec) + (static_cast(now.tv_usec) / 1000000.0) - timeInSecSinceUTC; - // what about overflows? - lumi = static_cast(timeInSec / std::abs(lumiSectionInterval_)) + 1; - } - - serializer_.serializeEvent(sbuf, e, selectorCfg, compressionAlgo_, compressionLevel_, reserve_size); - - // resize header_buf_ to reserved size on first written event - if (sbuf.header_buf_.size() < reserve_size) - sbuf.header_buf_.resize(reserve_size); - - auto msg = std::make_unique(&sbuf.header_buf_[0], - sbuf.comp_buf_.size(), - e.id().run(), - e.id().event(), - lumi, - outputModuleId_, - 0, - l1bit, - (uint8*)&hltbits[0], - hltsize_, - (uint32)sbuf.adler32_chksum(), - host_name_); - - // 50000 bytes is reserved for header as has been the case with previous version which did one extra copy of event data - uint32 headerSize = msg->headerSize(); - if (headerSize > reserve_size) - throw cms::Exception("StreamerOutputModuleCommon", "Header Overflow") - << " header of size " << headerSize << "bytes is too big to fit into the reserved buffer space"; - - //set addresses to other buffer and copy constructed header there - msg->setBufAddr(&sbuf.comp_buf_[reserve_size - headerSize]); - msg->setEventAddr(sbuf.bufferPointer()); - std::copy(&sbuf.header_buf_[0], &sbuf.header_buf_[headerSize], (char*)(&sbuf.comp_buf_[reserve_size - headerSize])); - - unsigned int src_size = sbuf.currentSpaceUsed(); - msg->setEventLength(src_size); //compressed size - if (useCompression_) - msg->setOrigDataSize( - sbuf.currentEventSize()); //uncompressed size (or 0 if no compression -> streamer input source requires this) - else - msg->setOrigDataSize(0); + EventForOutput const& e, Handle const& triggerResults, ParameterSetID const& selectorCfg) { + return builders_.serializeEvent(buffer_, e, triggerResults, selectorCfg, eventMetaDataChecksum_); + } - return msg; + std::unique_ptr StreamerOutputModuleCommon::serializeEventMetaData( + BranchIDLists const& branchLists, ThinnedAssociationsHelper const& helper) { + auto ret = builders_.serializeEventMetaData(buffer_, branchLists, helper); + eventMetaDataChecksum_ = ret.second; + return std::move(ret.first); } void StreamerOutputModuleCommon::fillDescription(ParameterSetDescription& desc) { - desc.addUntracked("max_event_size", 7000000)->setComment("Obsolete parameter."); - desc.addUntracked("use_compression", true) - ->setComment("If True, compression will be used to write streamer file."); - desc.addUntracked("compression_algorithm", "ZLIB") - ->setComment("Compression algorithm to use: UNCOMPRESSED, ZLIB, LZMA or ZSTD"); - desc.addUntracked("compression_level", 1)->setComment("Compression level to use on serialized ROOT events"); - desc.addUntracked("lumiSection_interval", 0) - ->setComment( - "If 0, use lumi section number from event.\n" - "If not 0, the interval in seconds between fake lumi sections."); + StreamerOutputMsgBuilders::fillDescription(desc); } - SerializeDataBuffer* StreamerOutputModuleCommon::getSerializerBuffer() { - auto* ptr = serializerBuffer_.get(); - if (!ptr) { - serializerBuffer_ = std::make_unique(); - ptr = serializerBuffer_.get(); - } - return ptr; - } -} // namespace edm +} // namespace edm::streamer diff --git a/IOPool/Streamer/src/StreamerOutputMsgBuilders.cc b/IOPool/Streamer/src/StreamerOutputMsgBuilders.cc new file mode 100644 index 0000000000000..0abedde6cfe6d --- /dev/null +++ b/IOPool/Streamer/src/StreamerOutputMsgBuilders.cc @@ -0,0 +1,315 @@ +#include "IOPool/Streamer/interface/StreamerOutputMsgBuilders.h" + +#include "IOPool/Streamer/interface/InitMsgBuilder.h" +#include "IOPool/Streamer/interface/EventMsgBuilder.h" +#include "FWCore/Framework/interface/EventForOutput.h" +#include "FWCore/Framework/interface/EventSelector.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/Utilities/interface/DebugMacros.h" +#include "FWCore/Version/interface/GetReleaseVersion.h" +#include "DataFormats/Common/interface/TriggerResults.h" +#include "DataFormats/Provenance/interface/ModuleDescription.h" +#include "DataFormats/Provenance/interface/ParameterSetID.h" +#include "DataFormats/Provenance/interface/SelectedProducts.h" +#include "FWCore/Framework/interface/getAllTriggerNames.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace edm::streamer { + StreamerOutputMsgBuilders::Parameters StreamerOutputMsgBuilders::parameters(ParameterSet const& ps) { + Parameters ret; + ret.hltTriggerSelections = EventSelector::getEventSelectionVString(ps); + ret.compressionAlgoStr = ps.getUntrackedParameter("compression_algorithm"); + ret.compressionLevel = ps.getUntrackedParameter("compression_level"); + ret.lumiSectionInterval = ps.getUntrackedParameter("lumiSection_interval"); + ret.useCompression = ps.getUntrackedParameter("use_compression"); + return ret; + } + + StreamerOutputMsgBuilders::StreamerOutputMsgBuilders(Parameters const& p, + SelectedProducts const* selections, + std::string const& moduleLabel) + : + + serializer_(selections), + useCompression_(p.useCompression), + compressionAlgoStr_(p.compressionAlgoStr), + compressionLevel_(p.compressionLevel), + lumiSectionInterval_(p.lumiSectionInterval), + hltsize_(0), + host_name_(), + hltTriggerSelections_(), + outputModuleId_(0) { + //limits initially set for default ZLIB option + int minCompressionLevel = 1; + int maxCompressionLevel = 9; + + // test luminosity sections + struct timeval now; + struct timezone dummyTZ; + gettimeofday(&now, &dummyTZ); + timeInSecSinceUTC = static_cast(now.tv_sec) + (static_cast(now.tv_usec) / 1000000.0); + + if (useCompression_ == true) { + if (compressionAlgoStr_ == "ZLIB") { + compressionAlgo_ = ZLIB; + } else if (compressionAlgoStr_ == "LZMA") { + compressionAlgo_ = LZMA; + minCompressionLevel = 0; + } else if (compressionAlgoStr_ == "ZSTD") { + compressionAlgo_ = ZSTD; + maxCompressionLevel = 20; + } else if (compressionAlgoStr_ == "UNCOMPRESSED") { + compressionLevel_ = 0; + useCompression_ = false; + compressionAlgo_ = UNCOMPRESSED; + } else + throw cms::Exception("StreamerOutputMsgBuilders", "Compression type unknown") + << "Unknown compression algorithm " << compressionAlgoStr_; + + if (compressionLevel_ < minCompressionLevel) { + FDEBUG(9) << "Compression Level = " << compressionLevel_ << " no compression" << std::endl; + compressionLevel_ = 0; + useCompression_ = false; + compressionAlgo_ = UNCOMPRESSED; + } else if (compressionLevel_ > maxCompressionLevel) { + FDEBUG(9) << "Compression Level = " << compressionLevel_ << " using max compression level " + << maxCompressionLevel << std::endl; + compressionLevel_ = maxCompressionLevel; + compressionAlgo_ = UNCOMPRESSED; + } + } else + compressionAlgo_ = UNCOMPRESSED; + + int got_host = gethostname(host_name_, 255); + if (got_host != 0) + strncpy(host_name_, "noHostNameFoundOrTooLong", sizeof(host_name_)); + //loadExtraClasses(); + + // 25-Jan-2008, KAB - pull out the trigger selection request + // which we need for the INIT message + hltTriggerSelections_ = p.hltTriggerSelections; + + Strings const& hltTriggerNames = edm::getAllTriggerNames(); + hltsize_ = hltTriggerNames.size(); + + //Checksum of the module label + uLong crc = crc32(0L, Z_NULL, 0); + Bytef const* buf = (Bytef const*)(moduleLabel.data()); + crc = crc32(crc, buf, moduleLabel.length()); + outputModuleId_ = static_cast(crc); + } + + StreamerOutputMsgBuilders::~StreamerOutputMsgBuilders() {} + + std::unique_ptr StreamerOutputMsgBuilders::serializeRegistry( + SerializeDataBuffer& sbuf, + std::string const& processName, + std::string const& moduleLabel, + ParameterSetID const& toplevel, + SendJobHeader::ParameterSetMap const* psetMap) const { + if (psetMap) { + serializer_.serializeRegistry(sbuf, *psetMap); + } else { + serializer_.serializeRegistry(sbuf); + } + // resize header_buf_ to reflect space used in serializer_ + header + // I just added an overhead for header of 50000 for now + unsigned int src_size = sbuf.currentSpaceUsed(); + unsigned int new_size = src_size + 50000; + if (sbuf.header_buf_.size() < new_size) + sbuf.header_buf_.resize(new_size); + + //Build the INIT Message + //Following values are strictly DUMMY and will be replaced + // once available with Utility function etc. + uint32 run = 1; + + //Get the Process PSet ID + + //In case we need to print it + // cms::Digest dig(toplevel.compactForm()); + // cms::MD5Result r1 = dig.digest(); + // std::string hexy = r1.toString(); + // std::cout << "HEX Representation of Process PSetID: " << hexy << std::endl; + + //L1 stays dummy as of today + Strings l1_names; //3 + l1_names.push_back("t1"); + l1_names.push_back("t10"); + l1_names.push_back("t2"); + + Strings const& hltTriggerNames = edm::getAllTriggerNames(); + + auto init_message = std::make_unique(&sbuf.header_buf_[0], + sbuf.header_buf_.size(), + run, + Version((uint8 const*)toplevel.compactForm().c_str()), + getReleaseVersion().c_str(), + processName.c_str(), + moduleLabel.c_str(), + outputModuleId_, + hltTriggerNames, + hltTriggerSelections_, + l1_names, + (uint32)sbuf.adler32_chksum()); + + // copy data into the destination message + unsigned char* src = sbuf.bufferPointer(); + std::copy(src, src + src_size, init_message->dataAddress()); + init_message->setDataLength(src_size); + return init_message; + } + + void StreamerOutputMsgBuilders::setHltMask(EventForOutput const& e, + Handle const& triggerResults, + std::vector& hltbits) const { + hltbits.clear(); + + std::vector vHltState; + + if (triggerResults.isValid()) { + for (std::vector::size_type i = 0; i != hltsize_; ++i) { + vHltState.push_back(((triggerResults->at(i)).state())); + } + } else { + // We fill all Trigger bits to valid state. + for (std::vector::size_type i = 0; i != hltsize_; ++i) { + vHltState.push_back(hlt::Pass); + } + } + + //Pack into member hltbits + if (!vHltState.empty()) { + unsigned int packInOneByte = 4; + unsigned int sizeOfPackage = 1 + ((vHltState.size() - 1) / packInOneByte); //Two bits per HLT + + hltbits.resize(sizeOfPackage); + std::fill(hltbits.begin(), hltbits.end(), 0); + + for (std::vector::size_type i = 0; i != vHltState.size(); ++i) { + unsigned int whichByte = i / packInOneByte; + unsigned int indxWithinByte = i % packInOneByte; + hltbits[whichByte] = hltbits[whichByte] | (vHltState[i] << (indxWithinByte * 2)); + } + } + + //This is Just a printing code. + //std::cout << "Size of hltbits:" << hltbits_.size() << std::endl; + //for(unsigned int i=0; i != hltbits_.size() ; ++i) { + // printBits(hltbits_[i]); + //} + //std::cout << "\n"; + } + + std::unique_ptr StreamerOutputMsgBuilders::serializeEvent( + SerializeDataBuffer& sbuf, + EventForOutput const& e, + Handle const& triggerResults, + ParameterSetID const& selectorCfg, + uint32_t eventMetaDataChecksum) const { + constexpr unsigned int reserve_size = SerializeDataBuffer::reserve_size; + //Lets Build the Event Message first + + std::vector hltbits; + setHltMask(e, triggerResults, hltbits); + + uint32 lumi = e.luminosityBlock(); + if (lumiSectionInterval_ != 0) { + struct timeval now; + struct timezone dummyTZ; + gettimeofday(&now, &dummyTZ); + double timeInSec = + static_cast(now.tv_sec) + (static_cast(now.tv_usec) / 1000000.0) - timeInSecSinceUTC; + // what about overflows? + lumi = static_cast(timeInSec / std::abs(lumiSectionInterval_)) + 1; + } + serializer_.serializeEvent( + sbuf, e, selectorCfg, eventMetaDataChecksum, compressionAlgo_, compressionLevel_, reserve_size); + + return serializeEventCommon(e.id().run(), lumi, e.id().event(), hltbits, hltsize_, sbuf); + } + + std::pair, uint32_t> StreamerOutputMsgBuilders::serializeEventMetaData( + SerializeDataBuffer& sbuf, BranchIDLists const& branchLists, ThinnedAssociationsHelper const& helper) const { + constexpr unsigned int reserve_size = SerializeDataBuffer::reserve_size; + //Lets Build the Event Message first + + std::vector hltbits; + serializer_.serializeEventMetaData(sbuf, branchLists, helper, compressionAlgo_, compressionLevel_, reserve_size); + auto eventMetaDataChecksum = sbuf.adler32_chksum_; + + return std::make_pair(serializeEventCommon(0, 0, 0, hltbits, 0, sbuf), eventMetaDataChecksum); + } + + std::unique_ptr StreamerOutputMsgBuilders::serializeEventCommon(uint32 run, + uint32 lumi, + uint64 event, + std::vector hltbits, + unsigned int hltsize, + SerializeDataBuffer& sbuf) const { + // resize header_buf_ to reserved size on first written event + constexpr unsigned int reserve_size = SerializeDataBuffer::reserve_size; + if (sbuf.header_buf_.size() < reserve_size) + sbuf.header_buf_.resize(reserve_size); + + //Following is strictly DUMMY Data for L! Trig and will be replaced with actual + // once figured out, there is no logic involved here. + std::vector l1bit = {true, true, false}; + //End of dummy data + + auto msg = std::make_unique(&sbuf.header_buf_[0], + sbuf.comp_buf_.size(), + run, + event, + lumi, + outputModuleId_, + 0, + l1bit, + (uint8*)&hltbits[0], + hltsize, + (uint32)sbuf.adler32_chksum(), + host_name_); + + // 50000 bytes is reserved for header as has been the case with previous version which did one extra copy of event data + uint32 headerSize = msg->headerSize(); + if (headerSize > reserve_size) + throw cms::Exception("StreamerOutputMsgBuilders", "Header Overflow") + << " header of size " << headerSize << "bytes is too big to fit into the reserved buffer space"; + + //set addresses to other buffer and copy constructed header there + msg->setBufAddr(&sbuf.comp_buf_[reserve_size - headerSize]); + msg->setEventAddr(sbuf.bufferPointer()); + std::copy(&sbuf.header_buf_[0], &sbuf.header_buf_[headerSize], (char*)(&sbuf.comp_buf_[reserve_size - headerSize])); + + unsigned int src_size = sbuf.currentSpaceUsed(); + msg->setEventLength(src_size); //compressed size + if (useCompression_) + msg->setOrigDataSize( + sbuf.currentEventSize()); //uncompressed size (or 0 if no compression -> streamer input source requires this) + else + msg->setOrigDataSize(0); + + return msg; + } + + void StreamerOutputMsgBuilders::fillDescription(ParameterSetDescription& desc) { + desc.addUntracked("max_event_size", 7000000)->setComment("Obsolete parameter."); + desc.addUntracked("use_compression", true) + ->setComment("If True, compression will be used to write streamer file."); + desc.addUntracked("compression_algorithm", "ZLIB") + ->setComment("Compression algorithm to use: UNCOMPRESSED, ZLIB, LZMA or ZSTD"); + desc.addUntracked("compression_level", 1)->setComment("Compression level to use on serialized ROOT events"); + desc.addUntracked("lumiSection_interval", 0) + ->setComment( + "If 0, use lumi section number from event.\n" + "If not 0, the interval in seconds between fake lumi sections."); + } +} // namespace edm::streamer diff --git a/IOPool/Streamer/test/EventMessageTest.cpp b/IOPool/Streamer/test/EventMessageTest.cpp index 2004eecfb1f4f..e9512dae99b95 100644 --- a/IOPool/Streamer/test/EventMessageTest.cpp +++ b/IOPool/Streamer/test/EventMessageTest.cpp @@ -23,6 +23,8 @@ Disclaimer: Most of the code here is randomly written during #include "zlib.h" int main() try { + using namespace edm::streamer; + typedef std::vector Buffer; Buffer buf(2024); Buffer buf2(2024); diff --git a/IOPool/Streamer/test/ReadStreamerFile.cpp b/IOPool/Streamer/test/ReadStreamerFile.cpp index d9dcf17fbcc99..0e2eedbf19138 100644 --- a/IOPool/Streamer/test/ReadStreamerFile.cpp +++ b/IOPool/Streamer/test/ReadStreamerFile.cpp @@ -38,11 +38,13 @@ Disclaimer: Most of the code here is randomly written during #include +using namespace edm::streamer; + int readSingleStream(bool verbose) { try { // ----------- init std::string initfilename = "teststreamfile.dat"; - edm::StreamerInputFile stream_reader(initfilename); + StreamerInputFile stream_reader(initfilename); std::cout << "Trying to Read The Init message from Streamer File: " << initfilename << std::endl; InitMsgView const* init = stream_reader.startMessage(); @@ -54,7 +56,7 @@ int readSingleStream(bool verbose) { // ------- event - while (edm::StreamerInputFile::Next::kEvent == stream_reader.next()) { + while (StreamerInputFile::Next::kEvent == stream_reader.next()) { EventMsgView const* eview = stream_reader.currentRecord(); if (verbose) { std::cout << "----------EVENT-----------" << std::endl; @@ -80,7 +82,7 @@ int readMultipleStreams(bool verbose) { edm::InputFileCatalog catalog(streamFiles, ""); - edm::StreamerInputFile stream_reader(catalog.fileCatalogItems()); + StreamerInputFile stream_reader(catalog.fileCatalogItems()); std::cout << "Trying to Read The Init message from Streamer File: " << "teststreamfile.dat" << std::endl; @@ -92,7 +94,7 @@ int readMultipleStreams(bool verbose) { dumpInitView(init); } - while (edm::StreamerInputFile::Next::kStop != stream_reader.next()) { + while (StreamerInputFile::Next::kStop != stream_reader.next()) { if (stream_reader.newHeader()) { std::cout << "File Boundary has just been crossed, a new file is read" << std::endl; std::cout << "A new INIT Message is available" << std::endl; @@ -126,7 +128,7 @@ int readInvalidLFN(bool verbose) { edm::InputFileCatalog catalog(streamFiles, ""); - edm::StreamerInputFile stream_reader(catalog.fileCatalogItems()); + StreamerInputFile stream_reader(catalog.fileCatalogItems()); std::cout << "Trying to Read The Init message from Streamer File: " << "teststreamfile.dat" << std::endl; @@ -138,7 +140,7 @@ int readInvalidLFN(bool verbose) { dumpInitView(init); } - while (edm::StreamerInputFile::Next::kStop != stream_reader.next()) { + while (StreamerInputFile::Next::kStop != stream_reader.next()) { if (stream_reader.newHeader()) { std::cout << "File Boundary has just been crossed, a new file is read" << std::endl; std::cout << "A new INIT Message is available" << std::endl; diff --git a/IOPool/Streamer/test/WriteStreamerFile.cpp b/IOPool/Streamer/test/WriteStreamerFile.cpp index e6c2d57207ada..b9792a7d2750e 100644 --- a/IOPool/Streamer/test/WriteStreamerFile.cpp +++ b/IOPool/Streamer/test/WriteStreamerFile.cpp @@ -28,6 +28,8 @@ Disclaimer: Most of the code here is randomly written during #define NO_OF_EVENTS 10 +using namespace edm::streamer; + int main() try { typedef std::vector Buffer; Buffer buf(1024); diff --git a/IOPool/Streamer/test/run_TestRefProductIDMetadataConsistencyStreamer.sh b/IOPool/Streamer/test/run_TestRefProductIDMetadataConsistencyStreamer.sh index 7ba360b52f072..972f0250cb077 100755 --- a/IOPool/Streamer/test/run_TestRefProductIDMetadataConsistencyStreamer.sh +++ b/IOPool/Streamer/test/run_TestRefProductIDMetadataConsistencyStreamer.sh @@ -25,4 +25,4 @@ CatStreamerFiles refconsistency_cat.dat refconsistency_1.dat refconsistency_10.d echo # ... fails -runFailure ${SCRAM_TEST_PATH}/testModuleTypeResolverRefTest_cfg.py --input moduletyperesolver_ref_cat.dat +runSuccess ${SCRAM_TEST_PATH}/testRefProductIDMetadataConsistencyStreamerTest_cfg.py --input refconsistency_cat.dat