Skip to content

Commit

Permalink
Merge pull request #42115 from gpetruc/DemonstratorTools_compression_…
Browse files Browse the repository at this point in the history 
…master

L1T DemonstratorTools: support transparent compression of pattern files
  • Loading branch information
@cmsbuild
cmsbuild authored Jul 3, 2023
2 parents 92e6055 + b58fcf1 commit 8956656
Show file tree
Hide file tree
Showing 11 changed files with 62 additions and 14 deletions.
1 change: 1 addition & 0 deletions L1Trigger/DemonstratorTools/BuildFile.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
<use name="hls"/>
<use name="root"/>
<use name="rootrflx"/>
<use name="boost_iostreams"/>
<export>
<lib name="1"/>
</export>
2 changes: 2 additions & 0 deletions L1Trigger/DemonstratorTools/interface/BoardDataWriter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,15 @@ namespace l1t::demo {

BoardDataWriter(FileFormat,
const std::string& filePath,
const std::string& fileExt,
const size_t framesPerBX,
const size_t tmux,
const size_t maxFramesPerFile,
const ChannelMap_t&);

BoardDataWriter(FileFormat,
const std::string& filePath,
const std::string& fileExt,
const size_t framesPerBX,
const size_t tmux,
const size_t maxFramesPerFile,
Expand Down
7 changes: 7 additions & 0 deletions L1Trigger/DemonstratorTools/plugins/GTTFileWriter.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,20 +164,23 @@ GTTFileWriter::GTTFileWriter(const edm::ParameterSet& iConfig)
etMissToken_(consumes<edm::View<l1t::EtSum>>(iConfig.getUntrackedParameter<edm::InputTag>("etmiss"))),
fileWriterInputTracks_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter<std::string>("format")),
iConfig.getUntrackedParameter<std::string>("inputFilename"),
iConfig.getUntrackedParameter<std::string>("fileExtension"),
kFramesPerTMUXPeriod,
kGTTBoardTMUX,
kMaxLinesPerFile,
kChannelIdsInput,
kChannelSpecsInput),
fileWriterConvertedTracks_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter<std::string>("format")),
iConfig.getUntrackedParameter<std::string>("inputConvertedFilename"),
iConfig.getUntrackedParameter<std::string>("fileExtension"),
kFramesPerTMUXPeriod,
kGTTBoardTMUX,
kMaxLinesPerFile,
kChannelIdsInput,
kChannelSpecsInput),
fileWriterSelectedTracks_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter<std::string>("format")),
iConfig.getUntrackedParameter<std::string>("selectedTracksFilename"),
iConfig.getUntrackedParameter<std::string>("fileExtension"),
kFramesPerTMUXPeriod,
kGTTBoardTMUX,
kMaxLinesPerFile,
Expand All @@ -186,19 +189,22 @@ GTTFileWriter::GTTFileWriter(const edm::ParameterSet& iConfig)
fileWriterVertexAssociatedTracks_(
l1t::demo::parseFileFormat(iConfig.getUntrackedParameter<std::string>("format")),
iConfig.getUntrackedParameter<std::string>("vertexAssociatedTracksFilename"),
iConfig.getUntrackedParameter<std::string>("fileExtension"),
kFramesPerTMUXPeriod,
kGTTBoardTMUX,
kMaxLinesPerFile,
kChannelIdsInput,
kChannelSpecsInput),
fileWriterOutputToCorrelator_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter<std::string>("format")),
iConfig.getUntrackedParameter<std::string>("outputCorrelatorFilename"),
iConfig.getUntrackedParameter<std::string>("fileExtension"),
kFramesPerTMUXPeriod,
kGTTBoardTMUX,
kMaxLinesPerFile,
kChannelSpecsOutputToCorrelator),
fileWriterOutputToGlobalTrigger_(l1t::demo::parseFileFormat(iConfig.getUntrackedParameter<std::string>("format")),
iConfig.getUntrackedParameter<std::string>("outputGlobalTriggerFilename"),
iConfig.getUntrackedParameter<std::string>("fileExtension"),
kFramesPerTMUXPeriod,
kGTTBoardTMUX,
kMaxLinesPerFile,
Expand Down Expand Up @@ -310,6 +316,7 @@ void GTTFileWriter::fillDescriptions(edm::ConfigurationDescriptions& description
desc.addUntracked<std::string>("outputCorrelatorFilename", "L1GTTOutputToCorrelatorFile");
desc.addUntracked<std::string>("outputGlobalTriggerFilename", "L1GTTOutputToGlobalTriggerFile");
desc.addUntracked<std::string>("format", "APx");
desc.addUntracked<std::string>("fileExtension", "txt");
descriptions.add("GTTFileWriter", desc);
}

Expand Down
1 change: 1 addition & 0 deletions L1Trigger/DemonstratorTools/python/GTTFileWriter_cff.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,6 @@
vertexAssociatedTracksFilename = cms.untracked.string("L1GTTVertexAssociatedTracksFile"),
outputCorrelatorFilename = cms.untracked.string("L1GTTOutputToCorrelatorFile"),
outputGlobalTriggerFilename = cms.untracked.string("L1GTTOutputToGlobalTriggerFile"),
FileExtension = cms.untracked.string("txt"),
format = cms.untracked.string("APx")
)
9 changes: 7 additions & 2 deletions L1Trigger/DemonstratorTools/src/BoardDataWriter.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@ namespace l1t::demo {

BoardDataWriter::BoardDataWriter(FileFormat format,
const std::string& path,
const std::string& fileExt,
const size_t framesPerBX,
const size_t tmux,
const size_t maxFramesPerFile,
const ChannelMap_t& channelSpecs)
: fileFormat_(format),
boardDataFileID_("CMSSW"),
filePathGen_([=](const size_t i) { return path + "_" + std::to_string(i) + ".txt"; }),
filePathGen_([=](const size_t i) { return path + "_" + std::to_string(i) + "." + fileExt; }),
framesPerBX_(framesPerBX),
boardTMUX_(tmux),
maxFramesPerFile_(maxFramesPerFile),
Expand All @@ -25,6 +26,8 @@ namespace l1t::demo {
channelMap_(channelSpecs) {
if (channelMap_.empty())
throw std::runtime_error("BoardDataWriter channel map cannnot be empty");
if (fileExt != "txt" && fileExt != "txt.gz" && fileExt != "txt.xz")
throw std::runtime_error("BoardDataWriter fileExt must be one of txt, txt.gz, txt.xz");

for (const auto& [id, value] : channelMap_) {
const auto& [spec, indices] = value;
Expand All @@ -51,12 +54,14 @@ namespace l1t::demo {

BoardDataWriter::BoardDataWriter(FileFormat format,
const std::string& path,
const std::string& fileExt,
const size_t framesPerBX,
const size_t tmux,
const size_t maxFramesPerFile,
const std::map<LinkId, std::vector<size_t>>& channelMap,
const std::map<std::string, ChannelSpec>& channelSpecs)
: BoardDataWriter(format, path, framesPerBX, tmux, maxFramesPerFile, mergeMaps(channelMap, channelSpecs)) {}
: BoardDataWriter(
format, path, fileExt, framesPerBX, tmux, maxFramesPerFile, mergeMaps(channelMap, channelSpecs)) {}

void BoardDataWriter::setBoardDataFileID(const std::string& aId) { boardDataFileID_ = aId; }

Expand Down
26 changes: 22 additions & 4 deletions L1Trigger/DemonstratorTools/src/utilities.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
#include <fstream>
#include <regex>
#include <unordered_map>
#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/filter/lzma.hpp>
#include <boost/iostreams/filtering_stream.hpp>

#ifdef CMSSW_GIT_HASH
#include "FWCore/MessageLogger/interface/MessageLogger.h"
Expand Down Expand Up @@ -93,12 +96,20 @@ namespace l1t::demo {
BoardData readX2OFile(std::istream&);

BoardData read(const std::string& filePath, const FileFormat format) {
std::ifstream file(filePath);
std::ifstream file(filePath, std::ios_base::in | std::ios_base::binary);

if (not file.is_open())
throw std::runtime_error("Could not open file '" + filePath + "'");

return read(file, format);
boost::iostreams::filtering_istream stream;
if (filePath.rfind(".gz") == filePath.length() - 3)
stream.push(boost::iostreams::gzip_decompressor());
else if (filePath.rfind(".xz") == filePath.length() - 3)
stream.push(boost::iostreams::lzma_decompressor());

stream.push(file);

return read(stream, format);
}

BoardData read(std::istream& file, const FileFormat format) {
Expand Down Expand Up @@ -368,12 +379,19 @@ namespace l1t::demo {
<< "Writing board data (" << std::distance(data.begin(), data.end()) << " channels, "
<< data.begin()->second.size() << " frames) to file '" << filePath << "' (format: " << format << ")"
<< std::endl;
std::ofstream file(filePath);

std::ofstream file(filePath, std::ios_base::out | std::ios_base::binary);

if (not file.is_open())
throw std::runtime_error("Could not open file '" + filePath + "'");

write(data, file, format);
boost::iostreams::filtering_ostream stream;
if (filePath.rfind(".gz") == filePath.length() - 3)
stream.push(boost::iostreams::gzip_compressor());
else if (filePath.rfind(".xz") == filePath.length() - 3)
stream.push(boost::iostreams::lzma_compressor());
stream.push(file);
write(data, stream, format);
}

void write(const BoardData& data, std::ostream& file, const FileFormat format) {
Expand Down
2 changes: 2 additions & 0 deletions L1Trigger/Phase2L1ParticleFlow/plugins/L1CTJetFileWriter.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ L1CTJetFileWriter::L1CTJetFileWriter(const edm::ParameterSet& iConfig)
jetsToken_(consumes<edm::View<l1t::PFJet>>(iConfig.getParameter<edm::InputTag>("jets"))),
fileWriterOutputToGT_(l1t::demo::parseFileFormat(iConfig.getParameter<std::string>("format")),
iConfig.getParameter<std::string>("outputFilename"),
iConfig.getParameter<std::string>("outputFileExtension"),
nFramesPerBX_,
ctl2BoardTMUX_,
maxLinesPerFile_,
Expand Down Expand Up @@ -108,6 +109,7 @@ void L1CTJetFileWriter::fillDescriptions(edm::ConfigurationDescriptions& descrip
edm::ParameterSetDescription desc;
desc.add<edm::InputTag>("jets");
desc.add<std::string>("outputFilename");
desc.add<std::string>("outputFileExtension", "txt");
desc.add<uint32_t>("nJets", 12);
desc.add<uint32_t>("nFramesPerBX", 9);
desc.add<uint32_t>("TMUX", 6);
Expand Down
1 change: 1 addition & 0 deletions L1Trigger/Phase2L1ParticleFlow/plugins/L1TCtL2EgProducer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ class L1TCtL2EgProducer : public edm::global::EDProducer<> {
dataWriter_ = std::make_unique<l1t::demo::BoardDataWriter>(
l1t::demo::parseFileFormat(conf.getParameter<std::string>("format")),
conf.getParameter<std::string>("outputFilename"),
conf.getParameter<std::string>("outputFileExtension"),
nFramesPerBX,
conf.getParameter<uint32_t>("TMUX"),
conf.getParameter<uint32_t>("maxLinesPerFile"),
Expand Down
14 changes: 10 additions & 4 deletions L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer1_patternWriters_cff.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
outputLinkEgamma = cms.int32(6),
nEgammaObjectsOut = cms.uint32(16),
nOutputFramesPerBX = cms.uint32(9),
fileFormat = cms.string("EMP"),
fileFormat = cms.string("EMPv2"),
outputFileExtension = cms.string("txt.gz"),
maxLinesPerOutputFile = cms.uint32(1024),
eventsPerFile = cms.uint32(eventsPerFile_),
)
Expand Down Expand Up @@ -55,7 +56,9 @@
nEgammaObjectsOut = cms.uint32(16),
nInputFramesPerBX = cms.uint32(9),
nOutputFramesPerBX = cms.uint32(9),
fileFormat = cms.string("EMP"),
fileFormat = cms.string("EMPv2"),
inputFileExtension = cms.string("txt.gz"),
outputFileExtension = cms.string("txt.gz"),
maxLinesPerInputFile = cms.uint32(1024),
maxLinesPerOutputFile = cms.uint32(1024),
eventsPerFile = cms.uint32(eventsPerFile_),
Expand Down Expand Up @@ -146,7 +149,9 @@
partition = cms.string("HGCalNoTk"),
outputRegions = cms.vuint32(*range(18)),
nOutputFramesPerBX = cms.uint32(9),
fileFormat = cms.string("EMP"),
fileFormat = cms.string("EMPv2"),
inputFileExtension = cms.string("txt.gz"),
outputFileExtension = cms.string("txt.gz"),
maxLinesPerOutputFile = cms.uint32(1024),
eventsPerFile = cms.uint32(eventsPerFile_),
outputLinksPuppi = cms.vuint32(0,1,2,4),
Expand All @@ -165,7 +170,8 @@
partition = cms.string("HF"),
outputLinksPuppi = cms.vuint32(*range(3)),
nOutputFramesPerBX = cms.uint32(9),
fileFormat = cms.string("EMP"),
fileFormat = cms.string("EMPv2"),
outputFileExtension = cms.string("txt.gz"),
maxLinesPerOutputFile = cms.uint32(1024),
eventsPerFile = cms.uint32(eventsPerFile_),
)
Expand Down
11 changes: 7 additions & 4 deletions L1Trigger/Phase2L1ParticleFlow/python/l1ctLayer2EG_cff.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,9 @@
writeOutPattern=cms.bool(False),
inPatternFile=cms.PSet(
nFramesPerBX=cms.uint32(9), # 360 MHz clock or 25 Gb/s link
format=cms.string("EMP"),
format=cms.string("EMPv2"),
outputFilename=cms.string("L1TCTL2EG_InPattern"),
outputFileExtension=cms.string("txt.gz"),
TMUX=cms.uint32(6),
maxLinesPerFile=cms.uint32(1024),
channels=cms.VPSet(
Expand Down Expand Up @@ -117,14 +118,15 @@
),
outPatternFile=cms.PSet(
nFramesPerBX=cms.uint32(9), # 360 MHz clock or 25 Gb/s link
format=cms.string("EMP"),
format=cms.string("EMPv2"),
outputFilename=cms.string("L1TCTL2EG_OutPattern"),
outputFileExtension=cms.string("txt.gz"),
TMUX=cms.uint32(6),
maxLinesPerFile=cms.uint32(1024),
channels=cms.VPSet(
cms.PSet(
TMUX=cms.uint32(6),
nWords=cms.uint32(36), # 36 = 12*3/2words ele + 12*3/2words phhotons
nWords=cms.uint32(36), # 36 = 12*3/2words ele + 12*3/2words photons
interface=cms.string("eglayer2"),
id=cms.uint32(0),
channels=cms.vuint32(0)
Expand All @@ -134,8 +136,9 @@
# NOTE: to write out the GT input from 6TS
# outPatternFile=cms.PSet(
# nFramesPerBX=cms.uint32(9), # 360 MHz clock or 25 Gb/s link
# format=cms.string("EMP"),
# format=cms.string("EMPv2"),
# outputFilename=cms.string("L1TCTL2EG_ToGTPattern"),
# outputFileExtension=cms.string("txt.gz"),
# TMUX=cms.uint32(1),
# maxLinesPerFile=cms.uint32(1024),
# channels=cms.VPSet(
Expand Down
2 changes: 2 additions & 0 deletions L1Trigger/Phase2L1ParticleFlow/src/L1TCorrelatorLayer1PatternFileWriter.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ L1TCorrelatorLayer1PatternFileWriter::L1TCorrelatorLayer1PatternFileWriter(const
inputFileWriter_ =
std::make_unique<l1t::demo::BoardDataWriter>(l1t::demo::parseFileFormat(fileFormat_),
iConfig.getParameter<std::string>("inputFileName"),
iConfig.getParameter<std::string>("inputFileExtension"),
nInputFramesPerBX_,
tmuxFactor_,
iConfig.getParameter<uint32_t>("maxLinesPerInputFile"),
Expand Down Expand Up @@ -93,6 +94,7 @@ L1TCorrelatorLayer1PatternFileWriter::L1TCorrelatorLayer1PatternFileWriter(const
outputFileWriter_ =
std::make_unique<l1t::demo::BoardDataWriter>(l1t::demo::parseFileFormat(fileFormat_),
iConfig.getParameter<std::string>("outputFileName"),
iConfig.getParameter<std::string>("outputFileExtension"),
nOutputFramesPerBX_,
tmuxFactor_,
iConfig.getParameter<uint32_t>("maxLinesPerOutputFile"),
Expand Down

0 comments on commit 8956656

Please sign in to comment.