From 0bb08f3b92b771cf6af0f79a02d05818d2d707d2 Mon Sep 17 00:00:00 2001 From: Giovanni Date: Mon, 26 Jun 2023 11:34:21 +0200 Subject: [PATCH 1/2] DemonstratorTools: support transparent compression of pattern files --- L1Trigger/DemonstratorTools/BuildFile.xml | 1 + .../interface/BoardDataWriter.h | 2 ++ .../plugins/GTTFileWriter.cc | 7 +++++ .../python/GTTFileWriter_cff.py | 1 + .../DemonstratorTools/src/BoardDataWriter.cc | 7 +++-- L1Trigger/DemonstratorTools/src/utilities.cc | 26 ++++++++++++++++--- .../plugins/L1CTJetFileWriter.cc | 2 ++ .../plugins/L1TCtL2EgProducer.cc | 1 + .../python/l1ctLayer1_patternWriters_cff.py | 14 +++++++--- .../python/l1ctLayer2EG_cff.py | 11 +++++--- .../L1TCorrelatorLayer1PatternFileWriter.cc | 2 ++ 11 files changed, 60 insertions(+), 14 deletions(-) diff --git a/L1Trigger/DemonstratorTools/BuildFile.xml b/L1Trigger/DemonstratorTools/BuildFile.xml index 692d3965bb994..a0e9d37b75993 100644 --- a/L1Trigger/DemonstratorTools/BuildFile.xml +++ b/L1Trigger/DemonstratorTools/BuildFile.xml @@ -5,6 +5,7 @@ + diff --git a/L1Trigger/DemonstratorTools/interface/BoardDataWriter.h b/L1Trigger/DemonstratorTools/interface/BoardDataWriter.h index 0e069f6f62ec3..a3d54b0562065 100644 --- a/L1Trigger/DemonstratorTools/interface/BoardDataWriter.h +++ b/L1Trigger/DemonstratorTools/interface/BoardDataWriter.h @@ -24,6 +24,7 @@ namespace l1t::demo { BoardDataWriter(FileFormat, const std::string& filePath, + const std::string& fileExt, const size_t framesPerBX, const size_t tmux, const size_t maxFramesPerFile, @@ -31,6 +32,7 @@ namespace l1t::demo { BoardDataWriter(FileFormat, const std::string& filePath, + const std::string& fileExt, const size_t framesPerBX, const size_t tmux, const size_t maxFramesPerFile, diff --git a/L1Trigger/DemonstratorTools/plugins/GTTFileWriter.cc b/L1Trigger/DemonstratorTools/plugins/GTTFileWriter.cc index b28157833db59..a9fe05c34a399 100644 --- a/L1Trigger/DemonstratorTools/plugins/GTTFileWriter.cc +++ b/L1Trigger/DemonstratorTools/plugins/GTTFileWriter.cc @@ -164,6 +164,7 @@ GTTFileWriter::GTTFileWriter(const edm::ParameterSet& iConfig) etMissToken_(consumes>(iConfig.getUntrackedParameter("etmiss"))), fileWriterInputTracks_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter("format")), iConfig.getUntrackedParameter("inputFilename"), + iConfig.getUntrackedParameter("fileExtension"), kFramesPerTMUXPeriod, kGTTBoardTMUX, kMaxLinesPerFile, @@ -171,6 +172,7 @@ GTTFileWriter::GTTFileWriter(const edm::ParameterSet& iConfig) kChannelSpecsInput), fileWriterConvertedTracks_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter("format")), iConfig.getUntrackedParameter("inputConvertedFilename"), + iConfig.getUntrackedParameter("fileExtension"), kFramesPerTMUXPeriod, kGTTBoardTMUX, kMaxLinesPerFile, @@ -178,6 +180,7 @@ GTTFileWriter::GTTFileWriter(const edm::ParameterSet& iConfig) kChannelSpecsInput), fileWriterSelectedTracks_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter("format")), iConfig.getUntrackedParameter("selectedTracksFilename"), + iConfig.getUntrackedParameter("fileExtension"), kFramesPerTMUXPeriod, kGTTBoardTMUX, kMaxLinesPerFile, @@ -186,6 +189,7 @@ GTTFileWriter::GTTFileWriter(const edm::ParameterSet& iConfig) fileWriterVertexAssociatedTracks_( l1t::demo::parseFileFormat(iConfig.getUntrackedParameter("format")), iConfig.getUntrackedParameter("vertexAssociatedTracksFilename"), + iConfig.getUntrackedParameter("fileExtension"), kFramesPerTMUXPeriod, kGTTBoardTMUX, kMaxLinesPerFile, @@ -193,12 +197,14 @@ GTTFileWriter::GTTFileWriter(const edm::ParameterSet& iConfig) kChannelSpecsInput), fileWriterOutputToCorrelator_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter("format")), iConfig.getUntrackedParameter("outputCorrelatorFilename"), + iConfig.getUntrackedParameter("fileExtension"), kFramesPerTMUXPeriod, kGTTBoardTMUX, kMaxLinesPerFile, kChannelSpecsOutputToCorrelator), fileWriterOutputToGlobalTrigger_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter("format")), iConfig.getUntrackedParameter("outputGlobalTriggerFilename"), + iConfig.getUntrackedParameter("fileExtension"), kFramesPerTMUXPeriod, kGTTBoardTMUX, kMaxLinesPerFile, @@ -310,6 +316,7 @@ void GTTFileWriter::fillDescriptions(edm::ConfigurationDescriptions& description desc.addUntracked("outputCorrelatorFilename", "L1GTTOutputToCorrelatorFile"); desc.addUntracked("outputGlobalTriggerFilename", "L1GTTOutputToGlobalTriggerFile"); desc.addUntracked("format", "APx"); + desc.addUntracked("fileExtension", "txt"); descriptions.add("GTTFileWriter", desc); } diff --git a/L1Trigger/DemonstratorTools/python/GTTFileWriter_cff.py b/L1Trigger/DemonstratorTools/python/GTTFileWriter_cff.py index 61c4c46929041..c859d23bd5b45 100644 --- a/L1Trigger/DemonstratorTools/python/GTTFileWriter_cff.py +++ b/L1Trigger/DemonstratorTools/python/GTTFileWriter_cff.py @@ -15,5 +15,6 @@ vertexAssociatedTracksFilename = cms.untracked.string("L1GTTVertexAssociatedTracksFile"), outputCorrelatorFilename = cms.untracked.string("L1GTTOutputToCorrelatorFile"), outputGlobalTriggerFilename = cms.untracked.string("L1GTTOutputToGlobalTriggerFile"), + FileExtension = cms.untracked.string("txt"), format = cms.untracked.string("APx") ) diff --git a/L1Trigger/DemonstratorTools/src/BoardDataWriter.cc b/L1Trigger/DemonstratorTools/src/BoardDataWriter.cc index 557f0cdd0b929..1d63d9867c412 100644 --- a/L1Trigger/DemonstratorTools/src/BoardDataWriter.cc +++ b/L1Trigger/DemonstratorTools/src/BoardDataWriter.cc @@ -9,13 +9,14 @@ namespace l1t::demo { BoardDataWriter::BoardDataWriter(FileFormat format, const std::string& path, + const std::string& fileExt, const size_t framesPerBX, const size_t tmux, const size_t maxFramesPerFile, const ChannelMap_t& channelSpecs) : fileFormat_(format), boardDataFileID_("CMSSW"), - filePathGen_([=](const size_t i) { return path + "_" + std::to_string(i) + ".txt"; }), + filePathGen_([=](const size_t i) { return path + "_" + std::to_string(i) + "." + fileExt; }), framesPerBX_(framesPerBX), boardTMUX_(tmux), maxFramesPerFile_(maxFramesPerFile), @@ -51,12 +52,14 @@ namespace l1t::demo { BoardDataWriter::BoardDataWriter(FileFormat format, const std::string& path, + const std::string& fileExt, const size_t framesPerBX, const size_t tmux, const size_t maxFramesPerFile, const std::map>& channelMap, const std::map& channelSpecs) - : BoardDataWriter(format, path, framesPerBX, tmux, maxFramesPerFile, mergeMaps(channelMap, channelSpecs)) {} + : BoardDataWriter( + format, path, fileExt, framesPerBX, tmux, maxFramesPerFile, mergeMaps(channelMap, channelSpecs)) {} void BoardDataWriter::setBoardDataFileID(const std::string& aId) { boardDataFileID_ = aId; } diff --git a/L1Trigger/DemonstratorTools/src/utilities.cc b/L1Trigger/DemonstratorTools/src/utilities.cc index c72b36a2366f2..f9c13a3dcce54 100644 --- a/L1Trigger/DemonstratorTools/src/utilities.cc +++ b/L1Trigger/DemonstratorTools/src/utilities.cc @@ -4,6 +4,9 @@ #include #include #include +#include +#include +#include #ifdef CMSSW_GIT_HASH #include "FWCore/MessageLogger/interface/MessageLogger.h" @@ -93,12 +96,20 @@ namespace l1t::demo { BoardData readX2OFile(std::istream&); BoardData read(const std::string& filePath, const FileFormat format) { - std::ifstream file(filePath); + std::ifstream file(filePath, std::ios_base::in | std::ios_base::binary); if (not file.is_open()) throw std::runtime_error("Could not open file '" + filePath + "'"); - return read(file, format); + boost::iostreams::filtering_istream stream; + if (filePath.rfind(".gz") == filePath.length() - 3) + stream.push(boost::iostreams::gzip_decompressor()); + else if (filePath.rfind(".xz") == filePath.length() - 3) + stream.push(boost::iostreams::lzma_decompressor()); + + stream.push(file); + + return read(stream, format); } BoardData read(std::istream& file, const FileFormat format) { @@ -368,12 +379,19 @@ namespace l1t::demo { << "Writing board data (" << std::distance(data.begin(), data.end()) << " channels, " << data.begin()->second.size() << " frames) to file '" << filePath << "' (format: " << format << ")" << std::endl; - std::ofstream file(filePath); + + std::ofstream file(filePath, std::ios_base::out | std::ios_base::binary); if (not file.is_open()) throw std::runtime_error("Could not open file '" + filePath + "'"); - write(data, file, format); + boost::iostreams::filtering_ostream stream; + if (filePath.rfind(".gz") == filePath.length() - 3) + stream.push(boost::iostreams::gzip_compressor()); + else if (filePath.rfind(".xz") == filePath.length() - 3) + stream.push(boost::iostreams::lzma_compressor()); + stream.push(file); + write(data, stream, format); } void write(const BoardData& data, std::ostream& file, const FileFormat format) { diff --git a/L1Trigger/Phase2L1ParticleFlow/plugins/L1CTJetFileWriter.cc b/L1Trigger/Phase2L1ParticleFlow/plugins/L1CTJetFileWriter.cc index f3d21a7c85e1f..4696bfa2fe70f 100644 --- a/L1Trigger/Phase2L1ParticleFlow/plugins/L1CTJetFileWriter.cc +++ b/L1Trigger/Phase2L1ParticleFlow/plugins/L1CTJetFileWriter.cc @@ -55,6 +55,7 @@ L1CTJetFileWriter::L1CTJetFileWriter(const edm::ParameterSet& iConfig) jetsToken_(consumes>(iConfig.getParameter("jets"))), fileWriterOutputToGT_(l1t::demo::parseFileFormat(iConfig.getParameter("format")), iConfig.getParameter("outputFilename"), + iConfig.getParameter("outputFileExtension"), nFramesPerBX_, ctl2BoardTMUX_, maxLinesPerFile_, @@ -108,6 +109,7 @@ void L1CTJetFileWriter::fillDescriptions(edm::ConfigurationDescriptions& descrip edm::ParameterSetDescription desc; desc.add("jets"); desc.add("outputFilename"); + desc.add("outputFileExtension", "txt"); desc.add("nJets", 12); desc.add("nFramesPerBX", 9); desc.add("TMUX", 6); diff --git a/L1Trigger/Phase2L1ParticleFlow/plugins/L1TCtL2EgProducer.cc b/L1Trigger/Phase2L1ParticleFlow/plugins/L1TCtL2EgProducer.cc index 54399d8a497a3..6097408bf20e3 100644 --- a/L1Trigger/Phase2L1ParticleFlow/plugins/L1TCtL2EgProducer.cc +++ b/L1Trigger/Phase2L1ParticleFlow/plugins/L1TCtL2EgProducer.cc @@ -97,6 +97,7 @@ class L1TCtL2EgProducer : public edm::global::EDProducer<> { dataWriter_ = std::make_unique( l1t::demo::parseFileFormat(conf.getParameter("format")), conf.getParameter("outputFilename"), + conf.getParameter("outputFileExtension"), nFramesPerBX, conf.getParameter("TMUX"), conf.getParameter("maxLinesPerFile"), diff --git a/L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer1_patternWriters_cff.py b/L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer1_patternWriters_cff.py index e79e4bae5f6c8..9d9affc831d1c 100644 --- a/L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer1_patternWriters_cff.py +++ b/L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer1_patternWriters_cff.py @@ -12,7 +12,8 @@ outputLinkEgamma = cms.int32(6), nEgammaObjectsOut = cms.uint32(16), nOutputFramesPerBX = cms.uint32(9), - fileFormat = cms.string("EMP"), + fileFormat = cms.string("EMPv2"), + outputFileExtension = cms.string("txt.gz"), maxLinesPerOutputFile = cms.uint32(1024), eventsPerFile = cms.uint32(eventsPerFile_), ) @@ -55,7 +56,9 @@ nEgammaObjectsOut = cms.uint32(16), nInputFramesPerBX = cms.uint32(9), nOutputFramesPerBX = cms.uint32(9), - fileFormat = cms.string("EMP"), + fileFormat = cms.string("EMPv2"), + inputFileExtension = cms.string("txt.gz"), + outputFileExtension = cms.string("txt.gz"), maxLinesPerInputFile = cms.uint32(1024), maxLinesPerOutputFile = cms.uint32(1024), eventsPerFile = cms.uint32(eventsPerFile_), @@ -146,7 +149,9 @@ partition = cms.string("HGCalNoTk"), outputRegions = cms.vuint32(*range(18)), nOutputFramesPerBX = cms.uint32(9), - fileFormat = cms.string("EMP"), + fileFormat = cms.string("EMPv2"), + inputFileExtension = cms.string("txt.gz"), + outputFileExtension = cms.string("txt.gz"), maxLinesPerOutputFile = cms.uint32(1024), eventsPerFile = cms.uint32(eventsPerFile_), outputLinksPuppi = cms.vuint32(0,1,2,4), @@ -165,7 +170,8 @@ partition = cms.string("HF"), outputLinksPuppi = cms.vuint32(*range(3)), nOutputFramesPerBX = cms.uint32(9), - fileFormat = cms.string("EMP"), + fileFormat = cms.string("EMPv2"), + outputFileExtension = cms.string("txt.gz"), maxLinesPerOutputFile = cms.uint32(1024), eventsPerFile = cms.uint32(eventsPerFile_), ) diff --git a/L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer2EG_cff.py b/L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer2EG_cff.py index e42be9d3b8211..18e28d46752f0 100644 --- a/L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer2EG_cff.py +++ b/L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer2EG_cff.py @@ -72,8 +72,9 @@ writeOutPattern=cms.bool(False), inPatternFile=cms.PSet( nFramesPerBX=cms.uint32(9), # 360 MHz clock or 25 Gb/s link - format=cms.string("EMP"), + format=cms.string("EMPv2"), outputFilename=cms.string("L1TCTL2EG_InPattern"), + outputFileExtension=cms.string("txt.gz"), TMUX=cms.uint32(6), maxLinesPerFile=cms.uint32(1024), channels=cms.VPSet( @@ -117,14 +118,15 @@ ), outPatternFile=cms.PSet( nFramesPerBX=cms.uint32(9), # 360 MHz clock or 25 Gb/s link - format=cms.string("EMP"), + format=cms.string("EMPv2"), outputFilename=cms.string("L1TCTL2EG_OutPattern"), + outputFileExtension=cms.string("txt.gz"), TMUX=cms.uint32(6), maxLinesPerFile=cms.uint32(1024), channels=cms.VPSet( cms.PSet( TMUX=cms.uint32(6), - nWords=cms.uint32(36), # 36 = 12*3/2words ele + 12*3/2words phhotons + nWords=cms.uint32(36), # 36 = 12*3/2words ele + 12*3/2words photons interface=cms.string("eglayer2"), id=cms.uint32(0), channels=cms.vuint32(0) @@ -134,8 +136,9 @@ # NOTE: to write out the GT input from 6TS # outPatternFile=cms.PSet( # nFramesPerBX=cms.uint32(9), # 360 MHz clock or 25 Gb/s link - # format=cms.string("EMP"), + # format=cms.string("EMPv2"), # outputFilename=cms.string("L1TCTL2EG_ToGTPattern"), + # outputFileExtension=cms.string("txt.gz"), # TMUX=cms.uint32(1), # maxLinesPerFile=cms.uint32(1024), # channels=cms.VPSet( diff --git a/L1Trigger/Phase2L1ParticleFlow/src/L1TCorrelatorLayer1PatternFileWriter.cc b/L1Trigger/Phase2L1ParticleFlow/src/L1TCorrelatorLayer1PatternFileWriter.cc index 018f156978ca0..6c6043241f5d9 100644 --- a/L1Trigger/Phase2L1ParticleFlow/src/L1TCorrelatorLayer1PatternFileWriter.cc +++ b/L1Trigger/Phase2L1ParticleFlow/src/L1TCorrelatorLayer1PatternFileWriter.cc @@ -61,6 +61,7 @@ L1TCorrelatorLayer1PatternFileWriter::L1TCorrelatorLayer1PatternFileWriter(const inputFileWriter_ = std::make_unique(l1t::demo::parseFileFormat(fileFormat_), iConfig.getParameter("inputFileName"), + iConfig.getParameter("inputFileExtension"), nInputFramesPerBX_, tmuxFactor_, iConfig.getParameter("maxLinesPerInputFile"), @@ -93,6 +94,7 @@ L1TCorrelatorLayer1PatternFileWriter::L1TCorrelatorLayer1PatternFileWriter(const outputFileWriter_ = std::make_unique(l1t::demo::parseFileFormat(fileFormat_), iConfig.getParameter("outputFileName"), + iConfig.getParameter("outputFileExtension"), nOutputFramesPerBX_, tmuxFactor_, iConfig.getParameter("maxLinesPerOutputFile"), From b58fcf1ed5be070678777188ae3b9b4c60c64dac Mon Sep 17 00:00:00 2001 From: Giovanni Date: Thu, 29 Jun 2023 11:43:35 +0200 Subject: [PATCH 2/2] Add check on file extension (txt, txt.gz, txt.xz) --- L1Trigger/DemonstratorTools/src/BoardDataWriter.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/L1Trigger/DemonstratorTools/src/BoardDataWriter.cc b/L1Trigger/DemonstratorTools/src/BoardDataWriter.cc index 1d63d9867c412..7dcd95af15696 100644 --- a/L1Trigger/DemonstratorTools/src/BoardDataWriter.cc +++ b/L1Trigger/DemonstratorTools/src/BoardDataWriter.cc @@ -26,6 +26,8 @@ namespace l1t::demo { channelMap_(channelSpecs) { if (channelMap_.empty()) throw std::runtime_error("BoardDataWriter channel map cannnot be empty"); + if (fileExt != "txt" && fileExt != "txt.gz" && fileExt != "txt.xz") + throw std::runtime_error("BoardDataWriter fileExt must be one of txt, txt.gz, txt.xz"); for (const auto& [id, value] : channelMap_) { const auto& [spec, indices] = value;